@hyperframes/aws-lambda 0.7.71 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/cdk/HyperframesRenderStack.d.ts.map +1 -1
- package/dist/cdk/index.js +130 -2
- package/dist/cdk/index.js.map +2 -2
- package/dist/events.d.ts +61 -14
- package/dist/events.d.ts.map +1 -1
- package/dist/handler.d.ts +2 -1
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +448 -37
- package/dist/handler.js.map +4 -4
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +471 -50
- package/dist/index.js.map +4 -4
- package/dist/s3PlanV2Publisher.d.ts +26 -0
- package/dist/s3PlanV2Publisher.d.ts.map +1 -0
- package/dist/s3Transport.d.ts +11 -0
- package/dist/s3Transport.d.ts.map +1 -1
- package/dist/sdk/index.js +17 -6
- package/dist/sdk/index.js.map +3 -3
- package/dist/sdk/renderToLambda.d.ts +6 -1
- package/dist/sdk/renderToLambda.d.ts.map +1 -1
- package/package.json +4 -3
package/dist/handler.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
// src/handler.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, join as join2 } from "node:path";
|
|
5
5
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
6
6
|
import {
|
|
7
7
|
assemble,
|
|
8
|
+
listPlanV2ArtifactsForTarget,
|
|
9
|
+
materializePlanV2Target,
|
|
8
10
|
plan,
|
|
11
|
+
planV2WithPublisher,
|
|
12
|
+
readPlanV2Manifest,
|
|
9
13
|
renderChunk
|
|
10
14
|
} from "@hyperframes/producer/distributed";
|
|
11
15
|
|
|
@@ -90,9 +94,14 @@ import {
|
|
|
90
94
|
rmSync,
|
|
91
95
|
statSync
|
|
92
96
|
} from "node:fs";
|
|
97
|
+
import { createHash } from "node:crypto";
|
|
93
98
|
import { dirname } from "node:path";
|
|
94
99
|
import { pipeline } from "node:stream/promises";
|
|
95
|
-
import {
|
|
100
|
+
import {
|
|
101
|
+
GetObjectCommand,
|
|
102
|
+
HeadObjectCommand,
|
|
103
|
+
PutObjectCommand
|
|
104
|
+
} from "@aws-sdk/client-s3";
|
|
96
105
|
import * as tar from "tar";
|
|
97
106
|
function parseS3Uri(uri) {
|
|
98
107
|
if (!uri.startsWith("s3://")) {
|
|
@@ -120,6 +129,19 @@ async function downloadS3ObjectToFile(client, uri, destPath) {
|
|
|
120
129
|
mkdirSync(dirname(destPath), { recursive: true });
|
|
121
130
|
await pipeline(body, createWriteStream(destPath));
|
|
122
131
|
}
|
|
132
|
+
async function downloadS3ObjectToFileVerified(client, uri, destPath, expectedSha256) {
|
|
133
|
+
assertSha256(expectedSha256);
|
|
134
|
+
await downloadS3ObjectToFile(client, uri, destPath);
|
|
135
|
+
const actual = await sha256File(destPath);
|
|
136
|
+
if (actual !== expectedSha256) {
|
|
137
|
+
rmSync(destPath, { force: true });
|
|
138
|
+
const error = new Error(
|
|
139
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
|
|
140
|
+
);
|
|
141
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
123
145
|
async function uploadFileToS3(client, localPath, uri, contentType) {
|
|
124
146
|
if (!existsSync2(localPath)) {
|
|
125
147
|
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
|
@@ -136,6 +158,97 @@ async function uploadFileToS3(client, localPath, uri, contentType) {
|
|
|
136
158
|
})
|
|
137
159
|
);
|
|
138
160
|
}
|
|
161
|
+
async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSha256, contentType) {
|
|
162
|
+
assertSha256(expectedSha256);
|
|
163
|
+
if (!existsSync2(localPath)) {
|
|
164
|
+
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
|
165
|
+
}
|
|
166
|
+
const actualSha256 = await sha256File(localPath);
|
|
167
|
+
if (actualSha256 !== expectedSha256) {
|
|
168
|
+
const error = new Error(
|
|
169
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`
|
|
170
|
+
);
|
|
171
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
174
|
+
const { bucket, key } = parseS3Uri(uri);
|
|
175
|
+
const size = statSync(localPath).size;
|
|
176
|
+
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
177
|
+
if (existing === "matching") return "reused";
|
|
178
|
+
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
|
179
|
+
const body = createReadStream(localPath);
|
|
180
|
+
try {
|
|
181
|
+
await client.send(
|
|
182
|
+
new PutObjectCommand({
|
|
183
|
+
Bucket: bucket,
|
|
184
|
+
Key: key,
|
|
185
|
+
Body: body,
|
|
186
|
+
ContentType: contentType,
|
|
187
|
+
ContentLength: size,
|
|
188
|
+
Metadata: { sha256: expectedSha256 },
|
|
189
|
+
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
|
|
190
|
+
// HEAD followed by an unconditional PUT can overwrite a conflicting
|
|
191
|
+
// object published by a concurrent planner. Conditional create makes
|
|
192
|
+
// immutable CAS and fixed-key manifest publication race-safe.
|
|
193
|
+
IfNoneMatch: "*"
|
|
194
|
+
})
|
|
195
|
+
);
|
|
196
|
+
return "uploaded";
|
|
197
|
+
} catch (error) {
|
|
198
|
+
if (!isS3PreconditionFailed(error)) throw error;
|
|
199
|
+
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
200
|
+
if (raced === "matching") return "reused";
|
|
201
|
+
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
|
202
|
+
throw error;
|
|
203
|
+
} finally {
|
|
204
|
+
body.destroy();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
async function sha256File(path) {
|
|
208
|
+
const hash = createHash("sha256");
|
|
209
|
+
for await (const chunk of createReadStream(path)) {
|
|
210
|
+
hash.update(chunk);
|
|
211
|
+
}
|
|
212
|
+
return hash.digest("hex");
|
|
213
|
+
}
|
|
214
|
+
function assertSha256(value) {
|
|
215
|
+
if (!/^[a-f0-9]{64}$/.test(value)) {
|
|
216
|
+
throw new Error(
|
|
217
|
+
`[s3Transport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
|
|
222
|
+
try {
|
|
223
|
+
const existing = await client.send(
|
|
224
|
+
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
|
|
225
|
+
);
|
|
226
|
+
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? "matching" : "conflict";
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (isS3NotFound(error)) return "missing";
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function throwImmutableObjectConflict(uri) {
|
|
233
|
+
const error = new Error(
|
|
234
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
|
|
235
|
+
);
|
|
236
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
function isS3NotFound(error) {
|
|
240
|
+
if (!isRecord(error)) return false;
|
|
241
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
242
|
+
return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
|
|
243
|
+
}
|
|
244
|
+
function isS3PreconditionFailed(error) {
|
|
245
|
+
if (!isRecord(error)) return false;
|
|
246
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
247
|
+
return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
|
|
248
|
+
}
|
|
249
|
+
function isRecord(value) {
|
|
250
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
251
|
+
}
|
|
139
252
|
async function tarDirectory(sourceDir, destTarball) {
|
|
140
253
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
141
254
|
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
|
|
@@ -154,6 +267,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
154
267
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
155
268
|
}
|
|
156
269
|
|
|
270
|
+
// src/s3PlanV2Publisher.ts
|
|
271
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
272
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
273
|
+
import { tmpdir } from "node:os";
|
|
274
|
+
import { join } from "node:path";
|
|
275
|
+
import {
|
|
276
|
+
PlanV2IntegrityError
|
|
277
|
+
} from "@hyperframes/producer/distributed";
|
|
278
|
+
function isRecord2(value) {
|
|
279
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
280
|
+
}
|
|
281
|
+
function assertSha2562(value, label) {
|
|
282
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
283
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
284
|
+
}
|
|
285
|
+
return value;
|
|
286
|
+
}
|
|
287
|
+
function manifestDigests(manifestBytes) {
|
|
288
|
+
let value;
|
|
289
|
+
try {
|
|
290
|
+
value = JSON.parse(manifestBytes);
|
|
291
|
+
} catch {
|
|
292
|
+
throw new PlanV2IntegrityError("S3 publisher received invalid manifest JSON");
|
|
293
|
+
}
|
|
294
|
+
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
295
|
+
throw new PlanV2IntegrityError("S3 publisher manifest requires an artifacts array");
|
|
296
|
+
}
|
|
297
|
+
return new Set(
|
|
298
|
+
value.artifacts.map((artifact, index) => {
|
|
299
|
+
if (!isRecord2(artifact)) {
|
|
300
|
+
throw new PlanV2IntegrityError(`S3 publisher artifacts[${index}] must be an object`);
|
|
301
|
+
}
|
|
302
|
+
return assertSha2562(artifact.sha256, `S3 publisher artifacts[${index}].sha256`);
|
|
303
|
+
})
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
function trimTrailingSlash(value) {
|
|
307
|
+
let end = value.length;
|
|
308
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
309
|
+
return value.slice(0, end);
|
|
310
|
+
}
|
|
311
|
+
var S3PlanV2ArtifactPublisher = class {
|
|
312
|
+
artifactPrefix;
|
|
313
|
+
manifestUri;
|
|
314
|
+
#s3;
|
|
315
|
+
#temporaryRoot;
|
|
316
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
317
|
+
#state = "open";
|
|
318
|
+
constructor(options) {
|
|
319
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
|
320
|
+
parseS3Uri(outputPrefix);
|
|
321
|
+
this.#s3 = options.s3;
|
|
322
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
323
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
324
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
325
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
326
|
+
}
|
|
327
|
+
async putBlob(blob) {
|
|
328
|
+
this.#assertOpen("publish a blob");
|
|
329
|
+
const digest = assertSha2562(blob.sha256, "S3 published blob sha256");
|
|
330
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
331
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
332
|
+
throw new PlanV2IntegrityError(
|
|
333
|
+
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
337
|
+
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
|
338
|
+
this.#publishedDigests.add(digest);
|
|
339
|
+
}
|
|
340
|
+
async commitManifest(manifestBytes) {
|
|
341
|
+
this.#assertOpen("commit a manifest");
|
|
342
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
343
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
344
|
+
throw new PlanV2IntegrityError(
|
|
345
|
+
`cannot commit S3 manifest before referenced blob is durable: ${digest}`
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
350
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
351
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
352
|
+
try {
|
|
353
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
354
|
+
await uploadContentAddressedFileToS3(
|
|
355
|
+
this.#s3,
|
|
356
|
+
manifestPath,
|
|
357
|
+
this.manifestUri,
|
|
358
|
+
manifestDigest,
|
|
359
|
+
"application/json"
|
|
360
|
+
);
|
|
361
|
+
this.#state = "committed";
|
|
362
|
+
} finally {
|
|
363
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async abort() {
|
|
367
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
368
|
+
}
|
|
369
|
+
#assertOpen(operation) {
|
|
370
|
+
if (this.#state !== "open") {
|
|
371
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
|
|
157
376
|
// src/handler.ts
|
|
158
377
|
var cachedS3Client = null;
|
|
159
378
|
function getS3Client() {
|
|
@@ -184,15 +403,24 @@ async function handler(event, deps) {
|
|
|
184
403
|
}
|
|
185
404
|
}
|
|
186
405
|
} catch (err) {
|
|
406
|
+
normalizeTerminalErrorName(err);
|
|
187
407
|
logEvent({
|
|
188
408
|
event: "handler_error",
|
|
189
409
|
action: unwrapped.Action,
|
|
410
|
+
input: summarizeEvent(unwrapped),
|
|
190
411
|
message: err instanceof Error ? err.message : String(err),
|
|
191
412
|
name: err instanceof Error ? err.name : void 0
|
|
192
413
|
});
|
|
193
414
|
throw err;
|
|
194
415
|
}
|
|
195
416
|
}
|
|
417
|
+
function normalizeTerminalErrorName(error) {
|
|
418
|
+
if (!error || typeof error !== "object") return;
|
|
419
|
+
const candidate = error;
|
|
420
|
+
if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE") {
|
|
421
|
+
candidate.name = candidate.code;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
196
424
|
var MAX_ENVELOPE_DEPTH = 4;
|
|
197
425
|
function unwrapEvent(event) {
|
|
198
426
|
let cursor = event;
|
|
@@ -229,18 +457,21 @@ function summarizeEvent(event) {
|
|
|
229
457
|
return {
|
|
230
458
|
projectS3Uri: event.ProjectS3Uri,
|
|
231
459
|
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
460
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
232
461
|
format: event.Config.format,
|
|
233
462
|
fps: event.Config.fps
|
|
234
463
|
};
|
|
235
464
|
case "renderChunk":
|
|
236
465
|
return {
|
|
237
|
-
|
|
466
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
467
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
|
|
238
468
|
chunkIndex: event.ChunkIndex,
|
|
239
469
|
format: event.Format
|
|
240
470
|
};
|
|
241
471
|
case "assemble":
|
|
242
472
|
return {
|
|
243
|
-
|
|
473
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
474
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
|
|
244
475
|
chunkCount: event.ChunkS3Uris.length,
|
|
245
476
|
hasAudio: event.AudioS3Uri !== null,
|
|
246
477
|
outputS3Uri: event.OutputS3Uri,
|
|
@@ -253,12 +484,15 @@ function primeRuntimeEnv() {
|
|
|
253
484
|
if (runtimeEnvPrimed) return;
|
|
254
485
|
runtimeEnvPrimed = true;
|
|
255
486
|
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
|
256
|
-
const bin =
|
|
487
|
+
const bin = join2(taskRoot, "bin");
|
|
257
488
|
if (existsSync3(bin)) {
|
|
258
489
|
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
|
259
490
|
}
|
|
260
491
|
}
|
|
261
492
|
async function handlePlan(event, deps) {
|
|
493
|
+
if (event.PlanProtocol === "v2") {
|
|
494
|
+
return handlePlanV2(event, deps);
|
|
495
|
+
}
|
|
262
496
|
const started = Date.now();
|
|
263
497
|
const s3 = deps?.s3 ?? getS3Client();
|
|
264
498
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
@@ -266,10 +500,10 @@ async function handlePlan(event, deps) {
|
|
|
266
500
|
const chromePath = await resolveChromeExecutablePath();
|
|
267
501
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
268
502
|
}
|
|
269
|
-
const work =
|
|
270
|
-
const projectArchive =
|
|
271
|
-
const projectDir =
|
|
272
|
-
const planDir =
|
|
503
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-"));
|
|
504
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
505
|
+
const projectDir = join2(work, "project");
|
|
506
|
+
const planDir = join2(work, "plan");
|
|
273
507
|
try {
|
|
274
508
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
275
509
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -277,12 +511,12 @@ async function handlePlan(event, deps) {
|
|
|
277
511
|
...event.Config
|
|
278
512
|
};
|
|
279
513
|
const result = await primitive(projectDir, config, planDir);
|
|
280
|
-
const planTar =
|
|
514
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
281
515
|
await tarDirectory(planDir, planTar);
|
|
282
|
-
const planTarUri = `${
|
|
283
|
-
const audioPath =
|
|
284
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
285
|
-
const audioUri = hasAudio ? `${
|
|
516
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
|
517
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
518
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
519
|
+
const audioUri = hasAudio ? `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/audio.aac` : null;
|
|
286
520
|
await Promise.all([
|
|
287
521
|
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
|
288
522
|
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null
|
|
@@ -307,7 +541,53 @@ async function handlePlan(event, deps) {
|
|
|
307
541
|
cleanupDir(work);
|
|
308
542
|
}
|
|
309
543
|
}
|
|
544
|
+
async function handlePlanV2(event, deps) {
|
|
545
|
+
const started = Date.now();
|
|
546
|
+
const s3 = deps?.s3 ?? getS3Client();
|
|
547
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
548
|
+
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
549
|
+
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
550
|
+
}
|
|
551
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-v2-"));
|
|
552
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
553
|
+
const projectDir = join2(work, "project");
|
|
554
|
+
try {
|
|
555
|
+
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
556
|
+
await untarDirectory(projectArchive, projectDir);
|
|
557
|
+
const publisher = new S3PlanV2ArtifactPublisher({
|
|
558
|
+
s3,
|
|
559
|
+
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
560
|
+
temporaryRoot: work
|
|
561
|
+
});
|
|
562
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
563
|
+
stagingParentDir: work
|
|
564
|
+
});
|
|
565
|
+
return {
|
|
566
|
+
Action: "plan",
|
|
567
|
+
PlanProtocol: "v2",
|
|
568
|
+
PlanV2ManifestS3Uri: publisher.manifestUri,
|
|
569
|
+
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
|
|
570
|
+
PlanHash: manifest.planHash,
|
|
571
|
+
ChunkCount: manifest.chunkCount,
|
|
572
|
+
TotalFrames: manifest.totalFrames,
|
|
573
|
+
Fps: manifest.fps,
|
|
574
|
+
Width: manifest.width,
|
|
575
|
+
Height: manifest.height,
|
|
576
|
+
Format: manifest.format,
|
|
577
|
+
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
578
|
+
AudioS3Uri: null,
|
|
579
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
580
|
+
ProducerVersion: manifest.producerVersion,
|
|
581
|
+
DurationMs: Date.now() - started
|
|
582
|
+
};
|
|
583
|
+
} finally {
|
|
584
|
+
cleanupDir(work);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
310
587
|
async function handleRenderChunk(event, deps) {
|
|
588
|
+
if (event.PlanProtocol === "v2") {
|
|
589
|
+
return handleRenderChunkV2(event, deps);
|
|
590
|
+
}
|
|
311
591
|
const started = Date.now();
|
|
312
592
|
const s3 = deps?.s3 ?? getS3Client();
|
|
313
593
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
@@ -315,14 +595,52 @@ async function handleRenderChunk(event, deps) {
|
|
|
315
595
|
const chromePath = await resolveChromeExecutablePath();
|
|
316
596
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
317
597
|
}
|
|
318
|
-
const work =
|
|
319
|
-
const planTar =
|
|
320
|
-
const planDir =
|
|
598
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-"));
|
|
599
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
600
|
+
const planDir = join2(work, "plan");
|
|
321
601
|
try {
|
|
322
602
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
323
603
|
await untarDirectory(planTar, planDir);
|
|
324
604
|
verifyPlanHash(planDir, event.PlanHash);
|
|
325
|
-
const chunkOutputBase =
|
|
605
|
+
const chunkOutputBase = join2(
|
|
606
|
+
work,
|
|
607
|
+
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
608
|
+
);
|
|
609
|
+
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
|
610
|
+
const chunkUri = await uploadChunkOutput(
|
|
611
|
+
s3,
|
|
612
|
+
result,
|
|
613
|
+
event.ChunkOutputS3Prefix,
|
|
614
|
+
event.ChunkIndex
|
|
615
|
+
);
|
|
616
|
+
return {
|
|
617
|
+
Action: "renderChunk",
|
|
618
|
+
ChunkS3Uri: chunkUri,
|
|
619
|
+
ChunkIndex: event.ChunkIndex,
|
|
620
|
+
Sha256: result.sha256,
|
|
621
|
+
FramesEncoded: result.framesEncoded,
|
|
622
|
+
DurationMs: Date.now() - started
|
|
623
|
+
};
|
|
624
|
+
} finally {
|
|
625
|
+
cleanupDir(work);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
async function handleRenderChunkV2(event, deps) {
|
|
629
|
+
const started = Date.now();
|
|
630
|
+
const s3 = deps?.s3 ?? getS3Client();
|
|
631
|
+
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
632
|
+
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
633
|
+
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
634
|
+
}
|
|
635
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-v2-"));
|
|
636
|
+
try {
|
|
637
|
+
const planDir = await downloadAndMaterializePlanV2(
|
|
638
|
+
s3,
|
|
639
|
+
event,
|
|
640
|
+
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
641
|
+
work
|
|
642
|
+
);
|
|
643
|
+
const chunkOutputBase = join2(
|
|
326
644
|
work,
|
|
327
645
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
328
646
|
);
|
|
@@ -346,7 +664,7 @@ async function handleRenderChunk(event, deps) {
|
|
|
346
664
|
}
|
|
347
665
|
}
|
|
348
666
|
async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
349
|
-
const trimmed =
|
|
667
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
350
668
|
if (result.outputKind === "file") {
|
|
351
669
|
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
|
352
670
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -360,22 +678,25 @@ async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
|
360
678
|
return uri;
|
|
361
679
|
}
|
|
362
680
|
async function handleAssemble(event, deps) {
|
|
681
|
+
if (event.PlanProtocol === "v2") {
|
|
682
|
+
return handleAssembleV2(event, deps);
|
|
683
|
+
}
|
|
363
684
|
const started = Date.now();
|
|
364
685
|
const s3 = deps?.s3 ?? getS3Client();
|
|
365
686
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
366
|
-
const work =
|
|
367
|
-
const planTar =
|
|
368
|
-
const planDir =
|
|
687
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-"));
|
|
688
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
689
|
+
const planDir = join2(work, "plan");
|
|
369
690
|
try {
|
|
370
691
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
371
692
|
await untarDirectory(planTar, planDir);
|
|
372
693
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
373
694
|
let audioPath = null;
|
|
374
695
|
if (event.AudioS3Uri) {
|
|
375
|
-
audioPath =
|
|
696
|
+
audioPath = join2(planDir, "audio.aac");
|
|
376
697
|
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
|
377
698
|
}
|
|
378
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
699
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
379
700
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
380
701
|
cfr: event.Cfr === true
|
|
381
702
|
});
|
|
@@ -397,9 +718,96 @@ async function handleAssemble(event, deps) {
|
|
|
397
718
|
cleanupDir(work);
|
|
398
719
|
}
|
|
399
720
|
}
|
|
721
|
+
async function handleAssembleV2(event, deps) {
|
|
722
|
+
const started = Date.now();
|
|
723
|
+
const s3 = deps?.s3 ?? getS3Client();
|
|
724
|
+
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
725
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-v2-"));
|
|
726
|
+
try {
|
|
727
|
+
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
|
|
728
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
729
|
+
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
730
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
731
|
+
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
732
|
+
cfr: event.Cfr === true
|
|
733
|
+
});
|
|
734
|
+
if (event.Format === "png-sequence") {
|
|
735
|
+
const tarball = `${finalOutput}.tar.gz`;
|
|
736
|
+
await tarDirectory(finalOutput, tarball);
|
|
737
|
+
await uploadFileToS3(s3, tarball, event.OutputS3Uri, "application/gzip");
|
|
738
|
+
} else {
|
|
739
|
+
await uploadFileToS3(s3, finalOutput, event.OutputS3Uri);
|
|
740
|
+
}
|
|
741
|
+
return {
|
|
742
|
+
Action: "assemble",
|
|
743
|
+
OutputS3Uri: event.OutputS3Uri,
|
|
744
|
+
FramesEncoded: result.framesEncoded,
|
|
745
|
+
FileSize: result.fileSize,
|
|
746
|
+
DurationMs: Date.now() - started
|
|
747
|
+
};
|
|
748
|
+
} finally {
|
|
749
|
+
cleanupDir(work);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
753
|
+
const transportDir = join2(work, "plan-v2");
|
|
754
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
755
|
+
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join2(transportDir, "plan.json"));
|
|
756
|
+
const manifest = readPlanV2Manifest(transportDir);
|
|
757
|
+
if (manifest.planHash !== event.PlanHash) {
|
|
758
|
+
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
759
|
+
}
|
|
760
|
+
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
761
|
+
const uniqueArtifacts = [
|
|
762
|
+
...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
763
|
+
];
|
|
764
|
+
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
765
|
+
await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
|
|
766
|
+
});
|
|
767
|
+
const planDir = join2(work, "plan");
|
|
768
|
+
materializePlanV2Target(transportDir, target, planDir);
|
|
769
|
+
return planDir;
|
|
770
|
+
}
|
|
771
|
+
async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
|
|
772
|
+
await downloadS3ObjectToFileVerified(
|
|
773
|
+
s3,
|
|
774
|
+
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
775
|
+
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
776
|
+
artifact.sha256
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
function planV2BlobPath(planV2Dir, digest) {
|
|
780
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
781
|
+
}
|
|
782
|
+
function planV2BlobUri(prefix, digest) {
|
|
783
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
784
|
+
}
|
|
785
|
+
function throwPlanHashMismatch(expected, actual) {
|
|
786
|
+
const error = new Error(
|
|
787
|
+
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
|
|
788
|
+
);
|
|
789
|
+
error.name = "PLAN_HASH_MISMATCH";
|
|
790
|
+
throw error;
|
|
791
|
+
}
|
|
792
|
+
async function mapConcurrent(values, concurrency, fn) {
|
|
793
|
+
let cursor = 0;
|
|
794
|
+
async function worker() {
|
|
795
|
+
while (cursor < values.length) {
|
|
796
|
+
const index = cursor++;
|
|
797
|
+
await fn(values[index]);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
const results = await Promise.allSettled(
|
|
801
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
802
|
+
);
|
|
803
|
+
const failure = results.find(
|
|
804
|
+
(result) => result.status === "rejected"
|
|
805
|
+
);
|
|
806
|
+
if (failure) throw failure.reason;
|
|
807
|
+
}
|
|
400
808
|
async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
401
|
-
const chunksDir =
|
|
402
|
-
|
|
809
|
+
const chunksDir = join2(workDir, "chunks");
|
|
810
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
403
811
|
const local = new Array(uris.length);
|
|
404
812
|
await Promise.all(
|
|
405
813
|
uris.map(async (uri, i) => {
|
|
@@ -407,10 +815,10 @@ async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
|
407
815
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
408
816
|
}
|
|
409
817
|
const { key } = parseS3Uri(uri);
|
|
410
|
-
const localPath =
|
|
818
|
+
const localPath = join2(chunksDir, basename(key));
|
|
411
819
|
await downloadS3ObjectToFile(s3, uri, localPath);
|
|
412
820
|
if (format === "png-sequence") {
|
|
413
|
-
const dirPath =
|
|
821
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
414
822
|
await untarDirectory(localPath, dirPath);
|
|
415
823
|
local[i] = dirPath;
|
|
416
824
|
} else {
|
|
@@ -425,11 +833,14 @@ function getEventS3Uris(event) {
|
|
|
425
833
|
case "plan":
|
|
426
834
|
return [event.ProjectS3Uri, event.PlanOutputS3Prefix];
|
|
427
835
|
case "renderChunk":
|
|
428
|
-
return [event.PlanS3Uri, event.ChunkOutputS3Prefix];
|
|
836
|
+
return event.PlanProtocol === "v2" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix, event.ChunkOutputS3Prefix] : [event.PlanS3Uri, event.ChunkOutputS3Prefix];
|
|
429
837
|
case "assemble":
|
|
430
|
-
return [
|
|
431
|
-
|
|
432
|
-
|
|
838
|
+
return [
|
|
839
|
+
...event.PlanProtocol === "v2" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix] : [event.PlanS3Uri],
|
|
840
|
+
...event.ChunkS3Uris,
|
|
841
|
+
event.OutputS3Uri,
|
|
842
|
+
event.AudioS3Uri
|
|
843
|
+
].filter((u) => u != null);
|
|
433
844
|
}
|
|
434
845
|
}
|
|
435
846
|
function validateEventS3Uris(event) {
|
|
@@ -449,17 +860,17 @@ function validateEventS3Uris(event) {
|
|
|
449
860
|
function pad(n) {
|
|
450
861
|
return n.toString().padStart(4, "0");
|
|
451
862
|
}
|
|
452
|
-
function
|
|
863
|
+
function trimTrailingSlash2(prefix) {
|
|
453
864
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
454
865
|
}
|
|
455
866
|
function cleanupDir(dir) {
|
|
456
867
|
try {
|
|
457
|
-
|
|
868
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
458
869
|
} catch {
|
|
459
870
|
}
|
|
460
871
|
}
|
|
461
872
|
function verifyPlanHash(planDir, expected) {
|
|
462
|
-
const planJsonPath =
|
|
873
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
463
874
|
let parsed;
|
|
464
875
|
try {
|
|
465
876
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|