@hyperframes/aws-lambda 0.7.72 → 0.7.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/events.d.ts +2 -0
- package/dist/events.d.ts.map +1 -1
- package/dist/handler.d.ts +2 -2
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +234 -111
- package/dist/handler.js.map +4 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +243 -119
- package/dist/index.js.map +4 -4
- package/dist/s3PlanV2Publisher.d.ts +26 -0
- package/dist/s3PlanV2Publisher.d.ts.map +1 -0
- package/dist/s3Transport.d.ts.map +1 -1
- package/dist/sdk/index.js.map +2 -2
- package/package.json +2 -2
- package/scripts/probe-beginframe.test.ts +80 -0
- package/scripts/probe-beginframe.ts +246 -51
package/dist/index.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// src/handler.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, join as join2 } from "node:path";
|
|
5
5
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
6
6
|
import {
|
|
7
7
|
assemble,
|
|
8
8
|
listPlanV2ArtifactsForTarget,
|
|
9
9
|
materializePlanV2Target,
|
|
10
10
|
plan,
|
|
11
|
-
|
|
11
|
+
planV2WithPublisher,
|
|
12
12
|
readPlanV2Manifest,
|
|
13
13
|
renderChunk
|
|
14
14
|
} from "@hyperframes/producer/distributed";
|
|
@@ -181,33 +181,36 @@ async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSh
|
|
|
181
181
|
}
|
|
182
182
|
const { bucket, key } = parseS3Uri(uri);
|
|
183
183
|
const size = statSync(localPath).size;
|
|
184
|
+
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
185
|
+
if (existing === "matching") return "reused";
|
|
186
|
+
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
|
187
|
+
const body = createReadStream(localPath);
|
|
184
188
|
try {
|
|
185
|
-
|
|
186
|
-
new
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
189
|
+
await client.send(
|
|
190
|
+
new PutObjectCommand({
|
|
191
|
+
Bucket: bucket,
|
|
192
|
+
Key: key,
|
|
193
|
+
Body: body,
|
|
194
|
+
ContentType: contentType,
|
|
195
|
+
ContentLength: size,
|
|
196
|
+
Metadata: { sha256: expectedSha256 },
|
|
197
|
+
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
|
|
198
|
+
// HEAD followed by an unconditional PUT can overwrite a conflicting
|
|
199
|
+
// object published by a concurrent planner. Conditional create makes
|
|
200
|
+
// immutable CAS and fixed-key manifest publication race-safe.
|
|
201
|
+
IfNoneMatch: "*"
|
|
202
|
+
})
|
|
193
203
|
);
|
|
194
|
-
|
|
195
|
-
throw error;
|
|
204
|
+
return "uploaded";
|
|
196
205
|
} catch (error) {
|
|
197
|
-
if (!
|
|
206
|
+
if (!isS3PreconditionFailed(error)) throw error;
|
|
207
|
+
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
208
|
+
if (raced === "matching") return "reused";
|
|
209
|
+
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
|
210
|
+
throw error;
|
|
211
|
+
} finally {
|
|
212
|
+
body.destroy();
|
|
198
213
|
}
|
|
199
|
-
await client.send(
|
|
200
|
-
new PutObjectCommand({
|
|
201
|
-
Bucket: bucket,
|
|
202
|
-
Key: key,
|
|
203
|
-
Body: createReadStream(localPath),
|
|
204
|
-
ContentType: contentType,
|
|
205
|
-
ContentLength: size,
|
|
206
|
-
Metadata: { sha256: expectedSha256 },
|
|
207
|
-
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64")
|
|
208
|
-
})
|
|
209
|
-
);
|
|
210
|
-
return "uploaded";
|
|
211
214
|
}
|
|
212
215
|
async function sha256File(path) {
|
|
213
216
|
const hash = createHash("sha256");
|
|
@@ -223,10 +226,36 @@ function assertSha256(value) {
|
|
|
223
226
|
);
|
|
224
227
|
}
|
|
225
228
|
}
|
|
229
|
+
async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
|
|
230
|
+
try {
|
|
231
|
+
const existing = await client.send(
|
|
232
|
+
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
|
|
233
|
+
);
|
|
234
|
+
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? "matching" : "conflict";
|
|
235
|
+
} catch (error) {
|
|
236
|
+
if (isS3NotFound(error)) return "missing";
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
function throwImmutableObjectConflict(uri) {
|
|
241
|
+
const error = new Error(
|
|
242
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
|
|
243
|
+
);
|
|
244
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
226
247
|
function isS3NotFound(error) {
|
|
227
|
-
if (!error
|
|
228
|
-
const
|
|
229
|
-
return
|
|
248
|
+
if (!isRecord(error)) return false;
|
|
249
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
250
|
+
return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
|
|
251
|
+
}
|
|
252
|
+
function isS3PreconditionFailed(error) {
|
|
253
|
+
if (!isRecord(error)) return false;
|
|
254
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
255
|
+
return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
|
|
256
|
+
}
|
|
257
|
+
function isRecord(value) {
|
|
258
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
230
259
|
}
|
|
231
260
|
async function tarDirectory(sourceDir, destTarball) {
|
|
232
261
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
@@ -246,6 +275,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
246
275
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
247
276
|
}
|
|
248
277
|
|
|
278
|
+
// src/s3PlanV2Publisher.ts
|
|
279
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
280
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
281
|
+
import { tmpdir } from "node:os";
|
|
282
|
+
import { join } from "node:path";
|
|
283
|
+
import {
|
|
284
|
+
PlanV2IntegrityError
|
|
285
|
+
} from "@hyperframes/producer/distributed";
|
|
286
|
+
function isRecord2(value) {
|
|
287
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
288
|
+
}
|
|
289
|
+
function assertSha2562(value, label) {
|
|
290
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
291
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
292
|
+
}
|
|
293
|
+
return value;
|
|
294
|
+
}
|
|
295
|
+
function manifestDigests(manifestBytes) {
|
|
296
|
+
let value;
|
|
297
|
+
try {
|
|
298
|
+
value = JSON.parse(manifestBytes);
|
|
299
|
+
} catch {
|
|
300
|
+
throw new PlanV2IntegrityError("S3 publisher received invalid manifest JSON");
|
|
301
|
+
}
|
|
302
|
+
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
303
|
+
throw new PlanV2IntegrityError("S3 publisher manifest requires an artifacts array");
|
|
304
|
+
}
|
|
305
|
+
return new Set(
|
|
306
|
+
value.artifacts.map((artifact, index) => {
|
|
307
|
+
if (!isRecord2(artifact)) {
|
|
308
|
+
throw new PlanV2IntegrityError(`S3 publisher artifacts[${index}] must be an object`);
|
|
309
|
+
}
|
|
310
|
+
return assertSha2562(artifact.sha256, `S3 publisher artifacts[${index}].sha256`);
|
|
311
|
+
})
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
function trimTrailingSlash(value) {
|
|
315
|
+
let end = value.length;
|
|
316
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
317
|
+
return value.slice(0, end);
|
|
318
|
+
}
|
|
319
|
+
var S3PlanV2ArtifactPublisher = class {
|
|
320
|
+
artifactPrefix;
|
|
321
|
+
manifestUri;
|
|
322
|
+
#s3;
|
|
323
|
+
#temporaryRoot;
|
|
324
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
325
|
+
#state = "open";
|
|
326
|
+
constructor(options) {
|
|
327
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
|
328
|
+
parseS3Uri(outputPrefix);
|
|
329
|
+
this.#s3 = options.s3;
|
|
330
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
331
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
332
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
333
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
334
|
+
}
|
|
335
|
+
async putBlob(blob) {
|
|
336
|
+
this.#assertOpen("publish a blob");
|
|
337
|
+
const digest = assertSha2562(blob.sha256, "S3 published blob sha256");
|
|
338
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
339
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
340
|
+
throw new PlanV2IntegrityError(
|
|
341
|
+
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
345
|
+
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
|
346
|
+
this.#publishedDigests.add(digest);
|
|
347
|
+
}
|
|
348
|
+
async commitManifest(manifestBytes) {
|
|
349
|
+
this.#assertOpen("commit a manifest");
|
|
350
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
351
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
352
|
+
throw new PlanV2IntegrityError(
|
|
353
|
+
`cannot commit S3 manifest before referenced blob is durable: ${digest}`
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
358
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
359
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
360
|
+
try {
|
|
361
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
362
|
+
await uploadContentAddressedFileToS3(
|
|
363
|
+
this.#s3,
|
|
364
|
+
manifestPath,
|
|
365
|
+
this.manifestUri,
|
|
366
|
+
manifestDigest,
|
|
367
|
+
"application/json"
|
|
368
|
+
);
|
|
369
|
+
this.#state = "committed";
|
|
370
|
+
} finally {
|
|
371
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
async abort() {
|
|
375
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
376
|
+
}
|
|
377
|
+
#assertOpen(operation) {
|
|
378
|
+
if (this.#state !== "open") {
|
|
379
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
|
|
249
384
|
// src/handler.ts
|
|
250
385
|
var cachedS3Client = null;
|
|
251
386
|
function getS3Client() {
|
|
@@ -357,7 +492,7 @@ function primeRuntimeEnv() {
|
|
|
357
492
|
if (runtimeEnvPrimed) return;
|
|
358
493
|
runtimeEnvPrimed = true;
|
|
359
494
|
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
|
360
|
-
const bin =
|
|
495
|
+
const bin = join2(taskRoot, "bin");
|
|
361
496
|
if (existsSync3(bin)) {
|
|
362
497
|
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
|
363
498
|
}
|
|
@@ -373,10 +508,10 @@ async function handlePlan(event, deps) {
|
|
|
373
508
|
const chromePath = await resolveChromeExecutablePath();
|
|
374
509
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
375
510
|
}
|
|
376
|
-
const work =
|
|
377
|
-
const projectArchive =
|
|
378
|
-
const projectDir =
|
|
379
|
-
const planDir =
|
|
511
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-"));
|
|
512
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
513
|
+
const projectDir = join2(work, "project");
|
|
514
|
+
const planDir = join2(work, "plan");
|
|
380
515
|
try {
|
|
381
516
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
382
517
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -384,12 +519,12 @@ async function handlePlan(event, deps) {
|
|
|
384
519
|
...event.Config
|
|
385
520
|
};
|
|
386
521
|
const result = await primitive(projectDir, config, planDir);
|
|
387
|
-
const planTar =
|
|
522
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
388
523
|
await tarDirectory(planDir, planTar);
|
|
389
|
-
const planTarUri = `${
|
|
390
|
-
const audioPath =
|
|
391
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
392
|
-
const audioUri = hasAudio ? `${
|
|
524
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
|
525
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
526
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
527
|
+
const audioUri = hasAudio ? `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/audio.aac` : null;
|
|
393
528
|
await Promise.all([
|
|
394
529
|
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
|
395
530
|
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null
|
|
@@ -417,60 +552,40 @@ async function handlePlan(event, deps) {
|
|
|
417
552
|
async function handlePlanV2(event, deps) {
|
|
418
553
|
const started = Date.now();
|
|
419
554
|
const s3 = deps?.s3 ?? getS3Client();
|
|
420
|
-
const primitive = deps?.primitives?.
|
|
555
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
421
556
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
422
557
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
423
558
|
}
|
|
424
|
-
const work =
|
|
425
|
-
const projectArchive =
|
|
426
|
-
const projectDir =
|
|
427
|
-
const planV2Dir = join(work, "plan-v2");
|
|
559
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-v2-"));
|
|
560
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
561
|
+
const projectDir = join2(work, "project");
|
|
428
562
|
try {
|
|
429
563
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
430
564
|
await untarDirectory(projectArchive, projectDir);
|
|
431
|
-
const
|
|
432
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
433
|
-
if (manifest.planHash !== result.planHash) {
|
|
434
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
435
|
-
}
|
|
436
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/v2`;
|
|
437
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
438
|
-
const uniqueArtifacts = [
|
|
439
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
440
|
-
];
|
|
441
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
442
|
-
const localPath = planV2BlobPath(planV2Dir, artifact.sha256);
|
|
443
|
-
await uploadContentAddressedFileToS3(
|
|
444
|
-
s3,
|
|
445
|
-
localPath,
|
|
446
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
447
|
-
artifact.sha256
|
|
448
|
-
);
|
|
449
|
-
});
|
|
450
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
451
|
-
await uploadContentAddressedFileToS3(
|
|
565
|
+
const publisher = new S3PlanV2ArtifactPublisher({
|
|
452
566
|
s3,
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
567
|
+
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
568
|
+
temporaryRoot: work
|
|
569
|
+
});
|
|
570
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
571
|
+
stagingParentDir: work
|
|
572
|
+
});
|
|
458
573
|
return {
|
|
459
574
|
Action: "plan",
|
|
460
575
|
PlanProtocol: "v2",
|
|
461
|
-
PlanV2ManifestS3Uri: manifestUri,
|
|
462
|
-
PlanV2ArtifactS3Prefix: artifactPrefix,
|
|
463
|
-
PlanHash:
|
|
464
|
-
ChunkCount:
|
|
465
|
-
TotalFrames:
|
|
466
|
-
Fps:
|
|
467
|
-
Width:
|
|
468
|
-
Height:
|
|
469
|
-
Format:
|
|
576
|
+
PlanV2ManifestS3Uri: publisher.manifestUri,
|
|
577
|
+
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
|
|
578
|
+
PlanHash: manifest.planHash,
|
|
579
|
+
ChunkCount: manifest.chunkCount,
|
|
580
|
+
TotalFrames: manifest.totalFrames,
|
|
581
|
+
Fps: manifest.fps,
|
|
582
|
+
Width: manifest.width,
|
|
583
|
+
Height: manifest.height,
|
|
584
|
+
Format: manifest.format,
|
|
470
585
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
471
586
|
AudioS3Uri: null,
|
|
472
|
-
FfmpegVersion:
|
|
473
|
-
ProducerVersion:
|
|
587
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
588
|
+
ProducerVersion: manifest.producerVersion,
|
|
474
589
|
DurationMs: Date.now() - started
|
|
475
590
|
};
|
|
476
591
|
} finally {
|
|
@@ -488,14 +603,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
488
603
|
const chromePath = await resolveChromeExecutablePath();
|
|
489
604
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
490
605
|
}
|
|
491
|
-
const work =
|
|
492
|
-
const planTar =
|
|
493
|
-
const planDir =
|
|
606
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-"));
|
|
607
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
608
|
+
const planDir = join2(work, "plan");
|
|
494
609
|
try {
|
|
495
610
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
496
611
|
await untarDirectory(planTar, planDir);
|
|
497
612
|
verifyPlanHash(planDir, event.PlanHash);
|
|
498
|
-
const chunkOutputBase =
|
|
613
|
+
const chunkOutputBase = join2(
|
|
499
614
|
work,
|
|
500
615
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
501
616
|
);
|
|
@@ -512,6 +627,7 @@ async function handleRenderChunk(event, deps) {
|
|
|
512
627
|
ChunkIndex: event.ChunkIndex,
|
|
513
628
|
Sha256: result.sha256,
|
|
514
629
|
FramesEncoded: result.framesEncoded,
|
|
630
|
+
CaptureMode: result.captureMode,
|
|
515
631
|
DurationMs: Date.now() - started
|
|
516
632
|
};
|
|
517
633
|
} finally {
|
|
@@ -525,7 +641,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
525
641
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
526
642
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
527
643
|
}
|
|
528
|
-
const work =
|
|
644
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-v2-"));
|
|
529
645
|
try {
|
|
530
646
|
const planDir = await downloadAndMaterializePlanV2(
|
|
531
647
|
s3,
|
|
@@ -533,7 +649,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
533
649
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
534
650
|
work
|
|
535
651
|
);
|
|
536
|
-
const chunkOutputBase =
|
|
652
|
+
const chunkOutputBase = join2(
|
|
537
653
|
work,
|
|
538
654
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
539
655
|
);
|
|
@@ -550,6 +666,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
550
666
|
ChunkIndex: event.ChunkIndex,
|
|
551
667
|
Sha256: result.sha256,
|
|
552
668
|
FramesEncoded: result.framesEncoded,
|
|
669
|
+
CaptureMode: result.captureMode,
|
|
553
670
|
DurationMs: Date.now() - started
|
|
554
671
|
};
|
|
555
672
|
} finally {
|
|
@@ -557,7 +674,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
557
674
|
}
|
|
558
675
|
}
|
|
559
676
|
async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
560
|
-
const trimmed =
|
|
677
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
561
678
|
if (result.outputKind === "file") {
|
|
562
679
|
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
|
563
680
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -577,19 +694,19 @@ async function handleAssemble(event, deps) {
|
|
|
577
694
|
const started = Date.now();
|
|
578
695
|
const s3 = deps?.s3 ?? getS3Client();
|
|
579
696
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
580
|
-
const work =
|
|
581
|
-
const planTar =
|
|
582
|
-
const planDir =
|
|
697
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-"));
|
|
698
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
699
|
+
const planDir = join2(work, "plan");
|
|
583
700
|
try {
|
|
584
701
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
585
702
|
await untarDirectory(planTar, planDir);
|
|
586
703
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
587
704
|
let audioPath = null;
|
|
588
705
|
if (event.AudioS3Uri) {
|
|
589
|
-
audioPath =
|
|
706
|
+
audioPath = join2(planDir, "audio.aac");
|
|
590
707
|
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
|
591
708
|
}
|
|
592
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
709
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
593
710
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
594
711
|
cfr: event.Cfr === true
|
|
595
712
|
});
|
|
@@ -615,12 +732,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
615
732
|
const started = Date.now();
|
|
616
733
|
const s3 = deps?.s3 ?? getS3Client();
|
|
617
734
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
618
|
-
const work =
|
|
735
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-v2-"));
|
|
619
736
|
try {
|
|
620
737
|
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
|
|
621
|
-
const audioPath = existsSync3(
|
|
738
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
622
739
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
623
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
740
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
624
741
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
625
742
|
cfr: event.Cfr === true
|
|
626
743
|
});
|
|
@@ -643,9 +760,9 @@ async function handleAssembleV2(event, deps) {
|
|
|
643
760
|
}
|
|
644
761
|
}
|
|
645
762
|
async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
646
|
-
const transportDir =
|
|
647
|
-
|
|
648
|
-
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri,
|
|
763
|
+
const transportDir = join2(work, "plan-v2");
|
|
764
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
765
|
+
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join2(transportDir, "plan.json"));
|
|
649
766
|
const manifest = readPlanV2Manifest(transportDir);
|
|
650
767
|
if (manifest.planHash !== event.PlanHash) {
|
|
651
768
|
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
@@ -657,7 +774,7 @@ async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
|
657
774
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
658
775
|
await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
|
|
659
776
|
});
|
|
660
|
-
const planDir =
|
|
777
|
+
const planDir = join2(work, "plan");
|
|
661
778
|
materializePlanV2Target(transportDir, target, planDir);
|
|
662
779
|
return planDir;
|
|
663
780
|
}
|
|
@@ -670,10 +787,10 @@ async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
|
|
|
670
787
|
);
|
|
671
788
|
}
|
|
672
789
|
function planV2BlobPath(planV2Dir, digest) {
|
|
673
|
-
return
|
|
790
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
674
791
|
}
|
|
675
792
|
function planV2BlobUri(prefix, digest) {
|
|
676
|
-
return `${
|
|
793
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
677
794
|
}
|
|
678
795
|
function throwPlanHashMismatch(expected, actual) {
|
|
679
796
|
const error = new Error(
|
|
@@ -690,11 +807,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
690
807
|
await fn(values[index]);
|
|
691
808
|
}
|
|
692
809
|
}
|
|
693
|
-
|
|
810
|
+
const results = await Promise.allSettled(
|
|
811
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
812
|
+
);
|
|
813
|
+
const failure = results.find(
|
|
814
|
+
(result) => result.status === "rejected"
|
|
815
|
+
);
|
|
816
|
+
if (failure) throw failure.reason;
|
|
694
817
|
}
|
|
695
818
|
async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
696
|
-
const chunksDir =
|
|
697
|
-
|
|
819
|
+
const chunksDir = join2(workDir, "chunks");
|
|
820
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
698
821
|
const local = new Array(uris.length);
|
|
699
822
|
await Promise.all(
|
|
700
823
|
uris.map(async (uri, i) => {
|
|
@@ -702,10 +825,10 @@ async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
|
702
825
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
703
826
|
}
|
|
704
827
|
const { key } = parseS3Uri(uri);
|
|
705
|
-
const localPath =
|
|
828
|
+
const localPath = join2(chunksDir, basename(key));
|
|
706
829
|
await downloadS3ObjectToFile(s3, uri, localPath);
|
|
707
830
|
if (format === "png-sequence") {
|
|
708
|
-
const dirPath =
|
|
831
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
709
832
|
await untarDirectory(localPath, dirPath);
|
|
710
833
|
local[i] = dirPath;
|
|
711
834
|
} else {
|
|
@@ -747,17 +870,17 @@ function validateEventS3Uris(event) {
|
|
|
747
870
|
function pad(n) {
|
|
748
871
|
return n.toString().padStart(4, "0");
|
|
749
872
|
}
|
|
750
|
-
function
|
|
873
|
+
function trimTrailingSlash2(prefix) {
|
|
751
874
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
752
875
|
}
|
|
753
876
|
function cleanupDir(dir) {
|
|
754
877
|
try {
|
|
755
|
-
|
|
878
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
756
879
|
} catch {
|
|
757
880
|
}
|
|
758
881
|
}
|
|
759
882
|
function verifyPlanHash(planDir, expected) {
|
|
760
|
-
const planJsonPath =
|
|
883
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
761
884
|
let parsed;
|
|
762
885
|
try {
|
|
763
886
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
@@ -778,13 +901,13 @@ function verifyPlanHash(planDir, expected) {
|
|
|
778
901
|
}
|
|
779
902
|
|
|
780
903
|
// src/sdk/deploySite.ts
|
|
781
|
-
import { mkdtempSync as
|
|
782
|
-
import { tmpdir as
|
|
783
|
-
import { join as
|
|
904
|
+
import { mkdtempSync as mkdtempSync3, rmSync as rmSync4, statSync as statSync4 } from "node:fs";
|
|
905
|
+
import { tmpdir as tmpdir3 } from "node:os";
|
|
906
|
+
import { join as join3 } from "node:path";
|
|
784
907
|
import { HeadObjectCommand as HeadObjectCommand2, S3Client as S3Client2 } from "@aws-sdk/client-s3";
|
|
785
908
|
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
|
786
909
|
async function deploySite(opts) {
|
|
787
|
-
if (!
|
|
910
|
+
if (!statSync4(opts.projectDir).isDirectory()) {
|
|
788
911
|
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
|
789
912
|
}
|
|
790
913
|
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
|
@@ -802,11 +925,11 @@ async function deploySite(opts) {
|
|
|
802
925
|
uploaded: false
|
|
803
926
|
};
|
|
804
927
|
}
|
|
805
|
-
const workdir =
|
|
928
|
+
const workdir = mkdtempSync3(join3(tmpdir3(), "hf-deploy-site-"));
|
|
806
929
|
try {
|
|
807
|
-
const tarball =
|
|
930
|
+
const tarball = join3(workdir, "project.tar.gz");
|
|
808
931
|
await tarDirectory(opts.projectDir, tarball);
|
|
809
|
-
const size =
|
|
932
|
+
const size = statSync4(tarball).size;
|
|
810
933
|
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
|
|
811
934
|
return {
|
|
812
935
|
siteId,
|
|
@@ -817,7 +940,7 @@ async function deploySite(opts) {
|
|
|
817
940
|
uploaded: true
|
|
818
941
|
};
|
|
819
942
|
} finally {
|
|
820
|
-
|
|
943
|
+
rmSync4(workdir, { recursive: true, force: true });
|
|
821
944
|
}
|
|
822
945
|
}
|
|
823
946
|
async function headObject(s3, bucket, key) {
|
|
@@ -1209,6 +1332,7 @@ function isTerminalFailure(status) {
|
|
|
1209
1332
|
export {
|
|
1210
1333
|
ChromeBinaryUnavailableError,
|
|
1211
1334
|
InvalidConfigError2 as InvalidConfigError,
|
|
1335
|
+
S3PlanV2ArtifactPublisher,
|
|
1212
1336
|
computeRenderCost,
|
|
1213
1337
|
deploySite,
|
|
1214
1338
|
downloadS3ObjectToFile,
|