@hyperframes/aws-lambda 0.7.72 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handler.d.ts +2 -2
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +232 -111
- package/dist/handler.js.map +4 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +241 -119
- package/dist/index.js.map +4 -4
- package/dist/s3PlanV2Publisher.d.ts +26 -0
- package/dist/s3PlanV2Publisher.d.ts.map +1 -0
- package/dist/s3Transport.d.ts.map +1 -1
- package/dist/sdk/index.js.map +2 -2
- package/package.json +2 -2
package/dist/handler.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* primitive → S3 upload → return small JSON result.
|
|
12
12
|
*/
|
|
13
13
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
14
|
-
import { assemble, plan,
|
|
14
|
+
import { assemble, plan, planV2WithPublisher, renderChunk } from "@hyperframes/producer/distributed";
|
|
15
15
|
import type { AssembleEvent, LambdaEvent, LambdaResult, PlanEvent, RenderChunkEvent } from "./events.js";
|
|
16
16
|
/**
|
|
17
17
|
* Optional injection points used by the handler's unit tests. Production
|
|
@@ -24,7 +24,7 @@ export interface HandlerDeps {
|
|
|
24
24
|
s3?: S3Client;
|
|
25
25
|
primitives?: {
|
|
26
26
|
plan: typeof plan;
|
|
27
|
-
|
|
27
|
+
planV2WithPublisher?: typeof planV2WithPublisher;
|
|
28
28
|
renderChunk: typeof renderChunk;
|
|
29
29
|
assemble: typeof assemble;
|
|
30
30
|
};
|
package/dist/handler.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,
|
|
1
|
+
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,mBAAmB,EAMnB,WAAW,EACZ,MAAM,mCAAmC,CAAC;AAG3C,OAAO,KAAK,EACV,aAAa,EAGb,WAAW,EACX,YAAY,EACZ,SAAS,EAET,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AAsBrB;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,QAAQ,CAAC;IACd,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,OAAO,IAAI,CAAC;QAClB,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;QACjD,WAAW,EAAE,OAAO,WAAW,CAAC;QAChC,QAAQ,EAAE,OAAO,QAAQ,CAAC;KAC3B,CAAC;IACF,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;;GAIG;AACH,wBAAsB,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CA0C3F;AA8BD,wBAAgB,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,aAAa,CAsB5F"}
|
package/dist/handler.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// src/handler.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, join as join2 } from "node:path";
|
|
5
5
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
6
6
|
import {
|
|
7
7
|
assemble,
|
|
8
8
|
listPlanV2ArtifactsForTarget,
|
|
9
9
|
materializePlanV2Target,
|
|
10
10
|
plan,
|
|
11
|
-
|
|
11
|
+
planV2WithPublisher,
|
|
12
12
|
readPlanV2Manifest,
|
|
13
13
|
renderChunk
|
|
14
14
|
} from "@hyperframes/producer/distributed";
|
|
@@ -173,33 +173,36 @@ async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSh
|
|
|
173
173
|
}
|
|
174
174
|
const { bucket, key } = parseS3Uri(uri);
|
|
175
175
|
const size = statSync(localPath).size;
|
|
176
|
+
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
177
|
+
if (existing === "matching") return "reused";
|
|
178
|
+
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
|
179
|
+
const body = createReadStream(localPath);
|
|
176
180
|
try {
|
|
177
|
-
|
|
178
|
-
new
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
181
|
+
await client.send(
|
|
182
|
+
new PutObjectCommand({
|
|
183
|
+
Bucket: bucket,
|
|
184
|
+
Key: key,
|
|
185
|
+
Body: body,
|
|
186
|
+
ContentType: contentType,
|
|
187
|
+
ContentLength: size,
|
|
188
|
+
Metadata: { sha256: expectedSha256 },
|
|
189
|
+
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
|
|
190
|
+
// HEAD followed by an unconditional PUT can overwrite a conflicting
|
|
191
|
+
// object published by a concurrent planner. Conditional create makes
|
|
192
|
+
// immutable CAS and fixed-key manifest publication race-safe.
|
|
193
|
+
IfNoneMatch: "*"
|
|
194
|
+
})
|
|
185
195
|
);
|
|
186
|
-
|
|
187
|
-
throw error;
|
|
196
|
+
return "uploaded";
|
|
188
197
|
} catch (error) {
|
|
189
|
-
if (!
|
|
198
|
+
if (!isS3PreconditionFailed(error)) throw error;
|
|
199
|
+
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
200
|
+
if (raced === "matching") return "reused";
|
|
201
|
+
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
|
202
|
+
throw error;
|
|
203
|
+
} finally {
|
|
204
|
+
body.destroy();
|
|
190
205
|
}
|
|
191
|
-
await client.send(
|
|
192
|
-
new PutObjectCommand({
|
|
193
|
-
Bucket: bucket,
|
|
194
|
-
Key: key,
|
|
195
|
-
Body: createReadStream(localPath),
|
|
196
|
-
ContentType: contentType,
|
|
197
|
-
ContentLength: size,
|
|
198
|
-
Metadata: { sha256: expectedSha256 },
|
|
199
|
-
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64")
|
|
200
|
-
})
|
|
201
|
-
);
|
|
202
|
-
return "uploaded";
|
|
203
206
|
}
|
|
204
207
|
async function sha256File(path) {
|
|
205
208
|
const hash = createHash("sha256");
|
|
@@ -215,10 +218,36 @@ function assertSha256(value) {
|
|
|
215
218
|
);
|
|
216
219
|
}
|
|
217
220
|
}
|
|
221
|
+
async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
|
|
222
|
+
try {
|
|
223
|
+
const existing = await client.send(
|
|
224
|
+
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
|
|
225
|
+
);
|
|
226
|
+
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? "matching" : "conflict";
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (isS3NotFound(error)) return "missing";
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function throwImmutableObjectConflict(uri) {
|
|
233
|
+
const error = new Error(
|
|
234
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
|
|
235
|
+
);
|
|
236
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
218
239
|
function isS3NotFound(error) {
|
|
219
|
-
if (!error
|
|
220
|
-
const
|
|
221
|
-
return
|
|
240
|
+
if (!isRecord(error)) return false;
|
|
241
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
242
|
+
return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
|
|
243
|
+
}
|
|
244
|
+
function isS3PreconditionFailed(error) {
|
|
245
|
+
if (!isRecord(error)) return false;
|
|
246
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
247
|
+
return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
|
|
248
|
+
}
|
|
249
|
+
function isRecord(value) {
|
|
250
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
222
251
|
}
|
|
223
252
|
async function tarDirectory(sourceDir, destTarball) {
|
|
224
253
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
@@ -238,6 +267,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
238
267
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
239
268
|
}
|
|
240
269
|
|
|
270
|
+
// src/s3PlanV2Publisher.ts
|
|
271
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
272
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
273
|
+
import { tmpdir } from "node:os";
|
|
274
|
+
import { join } from "node:path";
|
|
275
|
+
import {
|
|
276
|
+
PlanV2IntegrityError
|
|
277
|
+
} from "@hyperframes/producer/distributed";
|
|
278
|
+
function isRecord2(value) {
|
|
279
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
280
|
+
}
|
|
281
|
+
function assertSha2562(value, label) {
|
|
282
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
283
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
284
|
+
}
|
|
285
|
+
return value;
|
|
286
|
+
}
|
|
287
|
+
function manifestDigests(manifestBytes) {
|
|
288
|
+
let value;
|
|
289
|
+
try {
|
|
290
|
+
value = JSON.parse(manifestBytes);
|
|
291
|
+
} catch {
|
|
292
|
+
throw new PlanV2IntegrityError("S3 publisher received invalid manifest JSON");
|
|
293
|
+
}
|
|
294
|
+
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
295
|
+
throw new PlanV2IntegrityError("S3 publisher manifest requires an artifacts array");
|
|
296
|
+
}
|
|
297
|
+
return new Set(
|
|
298
|
+
value.artifacts.map((artifact, index) => {
|
|
299
|
+
if (!isRecord2(artifact)) {
|
|
300
|
+
throw new PlanV2IntegrityError(`S3 publisher artifacts[${index}] must be an object`);
|
|
301
|
+
}
|
|
302
|
+
return assertSha2562(artifact.sha256, `S3 publisher artifacts[${index}].sha256`);
|
|
303
|
+
})
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
function trimTrailingSlash(value) {
|
|
307
|
+
let end = value.length;
|
|
308
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
309
|
+
return value.slice(0, end);
|
|
310
|
+
}
|
|
311
|
+
var S3PlanV2ArtifactPublisher = class {
|
|
312
|
+
artifactPrefix;
|
|
313
|
+
manifestUri;
|
|
314
|
+
#s3;
|
|
315
|
+
#temporaryRoot;
|
|
316
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
317
|
+
#state = "open";
|
|
318
|
+
constructor(options) {
|
|
319
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
|
320
|
+
parseS3Uri(outputPrefix);
|
|
321
|
+
this.#s3 = options.s3;
|
|
322
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
323
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
324
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
325
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
326
|
+
}
|
|
327
|
+
async putBlob(blob) {
|
|
328
|
+
this.#assertOpen("publish a blob");
|
|
329
|
+
const digest = assertSha2562(blob.sha256, "S3 published blob sha256");
|
|
330
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
331
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
332
|
+
throw new PlanV2IntegrityError(
|
|
333
|
+
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
337
|
+
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
|
338
|
+
this.#publishedDigests.add(digest);
|
|
339
|
+
}
|
|
340
|
+
async commitManifest(manifestBytes) {
|
|
341
|
+
this.#assertOpen("commit a manifest");
|
|
342
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
343
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
344
|
+
throw new PlanV2IntegrityError(
|
|
345
|
+
`cannot commit S3 manifest before referenced blob is durable: ${digest}`
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
350
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
351
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
352
|
+
try {
|
|
353
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
354
|
+
await uploadContentAddressedFileToS3(
|
|
355
|
+
this.#s3,
|
|
356
|
+
manifestPath,
|
|
357
|
+
this.manifestUri,
|
|
358
|
+
manifestDigest,
|
|
359
|
+
"application/json"
|
|
360
|
+
);
|
|
361
|
+
this.#state = "committed";
|
|
362
|
+
} finally {
|
|
363
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async abort() {
|
|
367
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
368
|
+
}
|
|
369
|
+
#assertOpen(operation) {
|
|
370
|
+
if (this.#state !== "open") {
|
|
371
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
241
376
|
// src/handler.ts
|
|
242
377
|
var cachedS3Client = null;
|
|
243
378
|
function getS3Client() {
|
|
@@ -349,7 +484,7 @@ function primeRuntimeEnv() {
|
|
|
349
484
|
if (runtimeEnvPrimed) return;
|
|
350
485
|
runtimeEnvPrimed = true;
|
|
351
486
|
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
|
352
|
-
const bin =
|
|
487
|
+
const bin = join2(taskRoot, "bin");
|
|
353
488
|
if (existsSync3(bin)) {
|
|
354
489
|
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
|
355
490
|
}
|
|
@@ -365,10 +500,10 @@ async function handlePlan(event, deps) {
|
|
|
365
500
|
const chromePath = await resolveChromeExecutablePath();
|
|
366
501
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
367
502
|
}
|
|
368
|
-
const work =
|
|
369
|
-
const projectArchive =
|
|
370
|
-
const projectDir =
|
|
371
|
-
const planDir =
|
|
503
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-"));
|
|
504
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
505
|
+
const projectDir = join2(work, "project");
|
|
506
|
+
const planDir = join2(work, "plan");
|
|
372
507
|
try {
|
|
373
508
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
374
509
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -376,12 +511,12 @@ async function handlePlan(event, deps) {
|
|
|
376
511
|
...event.Config
|
|
377
512
|
};
|
|
378
513
|
const result = await primitive(projectDir, config, planDir);
|
|
379
|
-
const planTar =
|
|
514
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
380
515
|
await tarDirectory(planDir, planTar);
|
|
381
|
-
const planTarUri = `${
|
|
382
|
-
const audioPath =
|
|
383
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
384
|
-
const audioUri = hasAudio ? `${
|
|
516
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
|
517
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
518
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
519
|
+
const audioUri = hasAudio ? `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/audio.aac` : null;
|
|
385
520
|
await Promise.all([
|
|
386
521
|
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
|
387
522
|
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null
|
|
@@ -409,60 +544,40 @@ async function handlePlan(event, deps) {
|
|
|
409
544
|
async function handlePlanV2(event, deps) {
|
|
410
545
|
const started = Date.now();
|
|
411
546
|
const s3 = deps?.s3 ?? getS3Client();
|
|
412
|
-
const primitive = deps?.primitives?.
|
|
547
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
413
548
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
414
549
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
415
550
|
}
|
|
416
|
-
const work =
|
|
417
|
-
const projectArchive =
|
|
418
|
-
const projectDir =
|
|
419
|
-
const planV2Dir = join(work, "plan-v2");
|
|
551
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-v2-"));
|
|
552
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
553
|
+
const projectDir = join2(work, "project");
|
|
420
554
|
try {
|
|
421
555
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
422
556
|
await untarDirectory(projectArchive, projectDir);
|
|
423
|
-
const
|
|
424
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
425
|
-
if (manifest.planHash !== result.planHash) {
|
|
426
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
427
|
-
}
|
|
428
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/v2`;
|
|
429
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
430
|
-
const uniqueArtifacts = [
|
|
431
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
432
|
-
];
|
|
433
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
434
|
-
const localPath = planV2BlobPath(planV2Dir, artifact.sha256);
|
|
435
|
-
await uploadContentAddressedFileToS3(
|
|
436
|
-
s3,
|
|
437
|
-
localPath,
|
|
438
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
439
|
-
artifact.sha256
|
|
440
|
-
);
|
|
441
|
-
});
|
|
442
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
443
|
-
await uploadContentAddressedFileToS3(
|
|
557
|
+
const publisher = new S3PlanV2ArtifactPublisher({
|
|
444
558
|
s3,
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
559
|
+
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
560
|
+
temporaryRoot: work
|
|
561
|
+
});
|
|
562
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
563
|
+
stagingParentDir: work
|
|
564
|
+
});
|
|
450
565
|
return {
|
|
451
566
|
Action: "plan",
|
|
452
567
|
PlanProtocol: "v2",
|
|
453
|
-
PlanV2ManifestS3Uri: manifestUri,
|
|
454
|
-
PlanV2ArtifactS3Prefix: artifactPrefix,
|
|
455
|
-
PlanHash:
|
|
456
|
-
ChunkCount:
|
|
457
|
-
TotalFrames:
|
|
458
|
-
Fps:
|
|
459
|
-
Width:
|
|
460
|
-
Height:
|
|
461
|
-
Format:
|
|
568
|
+
PlanV2ManifestS3Uri: publisher.manifestUri,
|
|
569
|
+
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
|
|
570
|
+
PlanHash: manifest.planHash,
|
|
571
|
+
ChunkCount: manifest.chunkCount,
|
|
572
|
+
TotalFrames: manifest.totalFrames,
|
|
573
|
+
Fps: manifest.fps,
|
|
574
|
+
Width: manifest.width,
|
|
575
|
+
Height: manifest.height,
|
|
576
|
+
Format: manifest.format,
|
|
462
577
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
463
578
|
AudioS3Uri: null,
|
|
464
|
-
FfmpegVersion:
|
|
465
|
-
ProducerVersion:
|
|
579
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
580
|
+
ProducerVersion: manifest.producerVersion,
|
|
466
581
|
DurationMs: Date.now() - started
|
|
467
582
|
};
|
|
468
583
|
} finally {
|
|
@@ -480,14 +595,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
480
595
|
const chromePath = await resolveChromeExecutablePath();
|
|
481
596
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
482
597
|
}
|
|
483
|
-
const work =
|
|
484
|
-
const planTar =
|
|
485
|
-
const planDir =
|
|
598
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-"));
|
|
599
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
600
|
+
const planDir = join2(work, "plan");
|
|
486
601
|
try {
|
|
487
602
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
488
603
|
await untarDirectory(planTar, planDir);
|
|
489
604
|
verifyPlanHash(planDir, event.PlanHash);
|
|
490
|
-
const chunkOutputBase =
|
|
605
|
+
const chunkOutputBase = join2(
|
|
491
606
|
work,
|
|
492
607
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
493
608
|
);
|
|
@@ -517,7 +632,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
517
632
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
518
633
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
519
634
|
}
|
|
520
|
-
const work =
|
|
635
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-v2-"));
|
|
521
636
|
try {
|
|
522
637
|
const planDir = await downloadAndMaterializePlanV2(
|
|
523
638
|
s3,
|
|
@@ -525,7 +640,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
525
640
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
526
641
|
work
|
|
527
642
|
);
|
|
528
|
-
const chunkOutputBase =
|
|
643
|
+
const chunkOutputBase = join2(
|
|
529
644
|
work,
|
|
530
645
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
531
646
|
);
|
|
@@ -549,7 +664,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
549
664
|
}
|
|
550
665
|
}
|
|
551
666
|
async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
552
|
-
const trimmed =
|
|
667
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
553
668
|
if (result.outputKind === "file") {
|
|
554
669
|
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
|
555
670
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -569,19 +684,19 @@ async function handleAssemble(event, deps) {
|
|
|
569
684
|
const started = Date.now();
|
|
570
685
|
const s3 = deps?.s3 ?? getS3Client();
|
|
571
686
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
572
|
-
const work =
|
|
573
|
-
const planTar =
|
|
574
|
-
const planDir =
|
|
687
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-"));
|
|
688
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
689
|
+
const planDir = join2(work, "plan");
|
|
575
690
|
try {
|
|
576
691
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
577
692
|
await untarDirectory(planTar, planDir);
|
|
578
693
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
579
694
|
let audioPath = null;
|
|
580
695
|
if (event.AudioS3Uri) {
|
|
581
|
-
audioPath =
|
|
696
|
+
audioPath = join2(planDir, "audio.aac");
|
|
582
697
|
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
|
583
698
|
}
|
|
584
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
699
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
585
700
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
586
701
|
cfr: event.Cfr === true
|
|
587
702
|
});
|
|
@@ -607,12 +722,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
607
722
|
const started = Date.now();
|
|
608
723
|
const s3 = deps?.s3 ?? getS3Client();
|
|
609
724
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
610
|
-
const work =
|
|
725
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-v2-"));
|
|
611
726
|
try {
|
|
612
727
|
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
|
|
613
|
-
const audioPath = existsSync3(
|
|
728
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
614
729
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
615
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
730
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
616
731
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
617
732
|
cfr: event.Cfr === true
|
|
618
733
|
});
|
|
@@ -635,9 +750,9 @@ async function handleAssembleV2(event, deps) {
|
|
|
635
750
|
}
|
|
636
751
|
}
|
|
637
752
|
async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
638
|
-
const transportDir =
|
|
639
|
-
|
|
640
|
-
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri,
|
|
753
|
+
const transportDir = join2(work, "plan-v2");
|
|
754
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
755
|
+
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join2(transportDir, "plan.json"));
|
|
641
756
|
const manifest = readPlanV2Manifest(transportDir);
|
|
642
757
|
if (manifest.planHash !== event.PlanHash) {
|
|
643
758
|
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
@@ -649,7 +764,7 @@ async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
|
649
764
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
650
765
|
await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
|
|
651
766
|
});
|
|
652
|
-
const planDir =
|
|
767
|
+
const planDir = join2(work, "plan");
|
|
653
768
|
materializePlanV2Target(transportDir, target, planDir);
|
|
654
769
|
return planDir;
|
|
655
770
|
}
|
|
@@ -662,10 +777,10 @@ async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
|
|
|
662
777
|
);
|
|
663
778
|
}
|
|
664
779
|
function planV2BlobPath(planV2Dir, digest) {
|
|
665
|
-
return
|
|
780
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
666
781
|
}
|
|
667
782
|
function planV2BlobUri(prefix, digest) {
|
|
668
|
-
return `${
|
|
783
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
669
784
|
}
|
|
670
785
|
function throwPlanHashMismatch(expected, actual) {
|
|
671
786
|
const error = new Error(
|
|
@@ -682,11 +797,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
682
797
|
await fn(values[index]);
|
|
683
798
|
}
|
|
684
799
|
}
|
|
685
|
-
|
|
800
|
+
const results = await Promise.allSettled(
|
|
801
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
802
|
+
);
|
|
803
|
+
const failure = results.find(
|
|
804
|
+
(result) => result.status === "rejected"
|
|
805
|
+
);
|
|
806
|
+
if (failure) throw failure.reason;
|
|
686
807
|
}
|
|
687
808
|
async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
688
|
-
const chunksDir =
|
|
689
|
-
|
|
809
|
+
const chunksDir = join2(workDir, "chunks");
|
|
810
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
690
811
|
const local = new Array(uris.length);
|
|
691
812
|
await Promise.all(
|
|
692
813
|
uris.map(async (uri, i) => {
|
|
@@ -694,10 +815,10 @@ async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
|
694
815
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
695
816
|
}
|
|
696
817
|
const { key } = parseS3Uri(uri);
|
|
697
|
-
const localPath =
|
|
818
|
+
const localPath = join2(chunksDir, basename(key));
|
|
698
819
|
await downloadS3ObjectToFile(s3, uri, localPath);
|
|
699
820
|
if (format === "png-sequence") {
|
|
700
|
-
const dirPath =
|
|
821
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
701
822
|
await untarDirectory(localPath, dirPath);
|
|
702
823
|
local[i] = dirPath;
|
|
703
824
|
} else {
|
|
@@ -739,17 +860,17 @@ function validateEventS3Uris(event) {
|
|
|
739
860
|
function pad(n) {
|
|
740
861
|
return n.toString().padStart(4, "0");
|
|
741
862
|
}
|
|
742
|
-
function
|
|
863
|
+
function trimTrailingSlash2(prefix) {
|
|
743
864
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
744
865
|
}
|
|
745
866
|
function cleanupDir(dir) {
|
|
746
867
|
try {
|
|
747
|
-
|
|
868
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
748
869
|
} catch {
|
|
749
870
|
}
|
|
750
871
|
}
|
|
751
872
|
function verifyPlanHash(planDir, expected) {
|
|
752
|
-
const planJsonPath =
|
|
873
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
753
874
|
let parsed;
|
|
754
875
|
try {
|
|
755
876
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|