@hyperframes/aws-lambda 0.7.72 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handler.d.ts +2 -2
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +232 -111
- package/dist/handler.js.map +4 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +241 -119
- package/dist/index.js.map +4 -4
- package/dist/s3PlanV2Publisher.d.ts +26 -0
- package/dist/s3PlanV2Publisher.d.ts.map +1 -0
- package/dist/s3Transport.d.ts.map +1 -1
- package/dist/sdk/index.js.map +2 -2
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// src/handler.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, join as join2 } from "node:path";
|
|
5
5
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
6
6
|
import {
|
|
7
7
|
assemble,
|
|
8
8
|
listPlanV2ArtifactsForTarget,
|
|
9
9
|
materializePlanV2Target,
|
|
10
10
|
plan,
|
|
11
|
-
|
|
11
|
+
planV2WithPublisher,
|
|
12
12
|
readPlanV2Manifest,
|
|
13
13
|
renderChunk
|
|
14
14
|
} from "@hyperframes/producer/distributed";
|
|
@@ -181,33 +181,36 @@ async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSh
|
|
|
181
181
|
}
|
|
182
182
|
const { bucket, key } = parseS3Uri(uri);
|
|
183
183
|
const size = statSync(localPath).size;
|
|
184
|
+
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
185
|
+
if (existing === "matching") return "reused";
|
|
186
|
+
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
|
187
|
+
const body = createReadStream(localPath);
|
|
184
188
|
try {
|
|
185
|
-
|
|
186
|
-
new
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
189
|
+
await client.send(
|
|
190
|
+
new PutObjectCommand({
|
|
191
|
+
Bucket: bucket,
|
|
192
|
+
Key: key,
|
|
193
|
+
Body: body,
|
|
194
|
+
ContentType: contentType,
|
|
195
|
+
ContentLength: size,
|
|
196
|
+
Metadata: { sha256: expectedSha256 },
|
|
197
|
+
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
|
|
198
|
+
// HEAD followed by an unconditional PUT can overwrite a conflicting
|
|
199
|
+
// object published by a concurrent planner. Conditional create makes
|
|
200
|
+
// immutable CAS and fixed-key manifest publication race-safe.
|
|
201
|
+
IfNoneMatch: "*"
|
|
202
|
+
})
|
|
193
203
|
);
|
|
194
|
-
|
|
195
|
-
throw error;
|
|
204
|
+
return "uploaded";
|
|
196
205
|
} catch (error) {
|
|
197
|
-
if (!
|
|
206
|
+
if (!isS3PreconditionFailed(error)) throw error;
|
|
207
|
+
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
208
|
+
if (raced === "matching") return "reused";
|
|
209
|
+
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
|
210
|
+
throw error;
|
|
211
|
+
} finally {
|
|
212
|
+
body.destroy();
|
|
198
213
|
}
|
|
199
|
-
await client.send(
|
|
200
|
-
new PutObjectCommand({
|
|
201
|
-
Bucket: bucket,
|
|
202
|
-
Key: key,
|
|
203
|
-
Body: createReadStream(localPath),
|
|
204
|
-
ContentType: contentType,
|
|
205
|
-
ContentLength: size,
|
|
206
|
-
Metadata: { sha256: expectedSha256 },
|
|
207
|
-
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64")
|
|
208
|
-
})
|
|
209
|
-
);
|
|
210
|
-
return "uploaded";
|
|
211
214
|
}
|
|
212
215
|
async function sha256File(path) {
|
|
213
216
|
const hash = createHash("sha256");
|
|
@@ -223,10 +226,36 @@ function assertSha256(value) {
|
|
|
223
226
|
);
|
|
224
227
|
}
|
|
225
228
|
}
|
|
229
|
+
async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
|
|
230
|
+
try {
|
|
231
|
+
const existing = await client.send(
|
|
232
|
+
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
|
|
233
|
+
);
|
|
234
|
+
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? "matching" : "conflict";
|
|
235
|
+
} catch (error) {
|
|
236
|
+
if (isS3NotFound(error)) return "missing";
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
function throwImmutableObjectConflict(uri) {
|
|
241
|
+
const error = new Error(
|
|
242
|
+
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
|
|
243
|
+
);
|
|
244
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
226
247
|
function isS3NotFound(error) {
|
|
227
|
-
if (!error
|
|
228
|
-
const
|
|
229
|
-
return
|
|
248
|
+
if (!isRecord(error)) return false;
|
|
249
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
250
|
+
return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
|
|
251
|
+
}
|
|
252
|
+
function isS3PreconditionFailed(error) {
|
|
253
|
+
if (!isRecord(error)) return false;
|
|
254
|
+
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
255
|
+
return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
|
|
256
|
+
}
|
|
257
|
+
function isRecord(value) {
|
|
258
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
230
259
|
}
|
|
231
260
|
async function tarDirectory(sourceDir, destTarball) {
|
|
232
261
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
@@ -246,6 +275,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
246
275
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
247
276
|
}
|
|
248
277
|
|
|
278
|
+
// src/s3PlanV2Publisher.ts
|
|
279
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
280
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
281
|
+
import { tmpdir } from "node:os";
|
|
282
|
+
import { join } from "node:path";
|
|
283
|
+
import {
|
|
284
|
+
PlanV2IntegrityError
|
|
285
|
+
} from "@hyperframes/producer/distributed";
|
|
286
|
+
function isRecord2(value) {
|
|
287
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
288
|
+
}
|
|
289
|
+
function assertSha2562(value, label) {
|
|
290
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
291
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
292
|
+
}
|
|
293
|
+
return value;
|
|
294
|
+
}
|
|
295
|
+
function manifestDigests(manifestBytes) {
|
|
296
|
+
let value;
|
|
297
|
+
try {
|
|
298
|
+
value = JSON.parse(manifestBytes);
|
|
299
|
+
} catch {
|
|
300
|
+
throw new PlanV2IntegrityError("S3 publisher received invalid manifest JSON");
|
|
301
|
+
}
|
|
302
|
+
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
303
|
+
throw new PlanV2IntegrityError("S3 publisher manifest requires an artifacts array");
|
|
304
|
+
}
|
|
305
|
+
return new Set(
|
|
306
|
+
value.artifacts.map((artifact, index) => {
|
|
307
|
+
if (!isRecord2(artifact)) {
|
|
308
|
+
throw new PlanV2IntegrityError(`S3 publisher artifacts[${index}] must be an object`);
|
|
309
|
+
}
|
|
310
|
+
return assertSha2562(artifact.sha256, `S3 publisher artifacts[${index}].sha256`);
|
|
311
|
+
})
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
function trimTrailingSlash(value) {
|
|
315
|
+
let end = value.length;
|
|
316
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
317
|
+
return value.slice(0, end);
|
|
318
|
+
}
|
|
319
|
+
var S3PlanV2ArtifactPublisher = class {
|
|
320
|
+
artifactPrefix;
|
|
321
|
+
manifestUri;
|
|
322
|
+
#s3;
|
|
323
|
+
#temporaryRoot;
|
|
324
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
325
|
+
#state = "open";
|
|
326
|
+
constructor(options) {
|
|
327
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
|
328
|
+
parseS3Uri(outputPrefix);
|
|
329
|
+
this.#s3 = options.s3;
|
|
330
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
331
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
332
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
333
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
334
|
+
}
|
|
335
|
+
async putBlob(blob) {
|
|
336
|
+
this.#assertOpen("publish a blob");
|
|
337
|
+
const digest = assertSha2562(blob.sha256, "S3 published blob sha256");
|
|
338
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
339
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
340
|
+
throw new PlanV2IntegrityError(
|
|
341
|
+
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
345
|
+
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
|
346
|
+
this.#publishedDigests.add(digest);
|
|
347
|
+
}
|
|
348
|
+
async commitManifest(manifestBytes) {
|
|
349
|
+
this.#assertOpen("commit a manifest");
|
|
350
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
351
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
352
|
+
throw new PlanV2IntegrityError(
|
|
353
|
+
`cannot commit S3 manifest before referenced blob is durable: ${digest}`
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
358
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
359
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
360
|
+
try {
|
|
361
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
362
|
+
await uploadContentAddressedFileToS3(
|
|
363
|
+
this.#s3,
|
|
364
|
+
manifestPath,
|
|
365
|
+
this.manifestUri,
|
|
366
|
+
manifestDigest,
|
|
367
|
+
"application/json"
|
|
368
|
+
);
|
|
369
|
+
this.#state = "committed";
|
|
370
|
+
} finally {
|
|
371
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
async abort() {
|
|
375
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
376
|
+
}
|
|
377
|
+
#assertOpen(operation) {
|
|
378
|
+
if (this.#state !== "open") {
|
|
379
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
|
|
249
384
|
// src/handler.ts
|
|
250
385
|
var cachedS3Client = null;
|
|
251
386
|
function getS3Client() {
|
|
@@ -357,7 +492,7 @@ function primeRuntimeEnv() {
|
|
|
357
492
|
if (runtimeEnvPrimed) return;
|
|
358
493
|
runtimeEnvPrimed = true;
|
|
359
494
|
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
|
360
|
-
const bin =
|
|
495
|
+
const bin = join2(taskRoot, "bin");
|
|
361
496
|
if (existsSync3(bin)) {
|
|
362
497
|
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
|
363
498
|
}
|
|
@@ -373,10 +508,10 @@ async function handlePlan(event, deps) {
|
|
|
373
508
|
const chromePath = await resolveChromeExecutablePath();
|
|
374
509
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
375
510
|
}
|
|
376
|
-
const work =
|
|
377
|
-
const projectArchive =
|
|
378
|
-
const projectDir =
|
|
379
|
-
const planDir =
|
|
511
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-"));
|
|
512
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
513
|
+
const projectDir = join2(work, "project");
|
|
514
|
+
const planDir = join2(work, "plan");
|
|
380
515
|
try {
|
|
381
516
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
382
517
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -384,12 +519,12 @@ async function handlePlan(event, deps) {
|
|
|
384
519
|
...event.Config
|
|
385
520
|
};
|
|
386
521
|
const result = await primitive(projectDir, config, planDir);
|
|
387
|
-
const planTar =
|
|
522
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
388
523
|
await tarDirectory(planDir, planTar);
|
|
389
|
-
const planTarUri = `${
|
|
390
|
-
const audioPath =
|
|
391
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
392
|
-
const audioUri = hasAudio ? `${
|
|
524
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
|
525
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
526
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
527
|
+
const audioUri = hasAudio ? `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/audio.aac` : null;
|
|
393
528
|
await Promise.all([
|
|
394
529
|
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
|
395
530
|
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null
|
|
@@ -417,60 +552,40 @@ async function handlePlan(event, deps) {
|
|
|
417
552
|
async function handlePlanV2(event, deps) {
|
|
418
553
|
const started = Date.now();
|
|
419
554
|
const s3 = deps?.s3 ?? getS3Client();
|
|
420
|
-
const primitive = deps?.primitives?.
|
|
555
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
421
556
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
422
557
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
423
558
|
}
|
|
424
|
-
const work =
|
|
425
|
-
const projectArchive =
|
|
426
|
-
const projectDir =
|
|
427
|
-
const planV2Dir = join(work, "plan-v2");
|
|
559
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-v2-"));
|
|
560
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
561
|
+
const projectDir = join2(work, "project");
|
|
428
562
|
try {
|
|
429
563
|
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
430
564
|
await untarDirectory(projectArchive, projectDir);
|
|
431
|
-
const
|
|
432
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
433
|
-
if (manifest.planHash !== result.planHash) {
|
|
434
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
435
|
-
}
|
|
436
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/v2`;
|
|
437
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
438
|
-
const uniqueArtifacts = [
|
|
439
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
440
|
-
];
|
|
441
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
442
|
-
const localPath = planV2BlobPath(planV2Dir, artifact.sha256);
|
|
443
|
-
await uploadContentAddressedFileToS3(
|
|
444
|
-
s3,
|
|
445
|
-
localPath,
|
|
446
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
447
|
-
artifact.sha256
|
|
448
|
-
);
|
|
449
|
-
});
|
|
450
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
451
|
-
await uploadContentAddressedFileToS3(
|
|
565
|
+
const publisher = new S3PlanV2ArtifactPublisher({
|
|
452
566
|
s3,
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
567
|
+
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
568
|
+
temporaryRoot: work
|
|
569
|
+
});
|
|
570
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
571
|
+
stagingParentDir: work
|
|
572
|
+
});
|
|
458
573
|
return {
|
|
459
574
|
Action: "plan",
|
|
460
575
|
PlanProtocol: "v2",
|
|
461
|
-
PlanV2ManifestS3Uri: manifestUri,
|
|
462
|
-
PlanV2ArtifactS3Prefix: artifactPrefix,
|
|
463
|
-
PlanHash:
|
|
464
|
-
ChunkCount:
|
|
465
|
-
TotalFrames:
|
|
466
|
-
Fps:
|
|
467
|
-
Width:
|
|
468
|
-
Height:
|
|
469
|
-
Format:
|
|
576
|
+
PlanV2ManifestS3Uri: publisher.manifestUri,
|
|
577
|
+
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
|
|
578
|
+
PlanHash: manifest.planHash,
|
|
579
|
+
ChunkCount: manifest.chunkCount,
|
|
580
|
+
TotalFrames: manifest.totalFrames,
|
|
581
|
+
Fps: manifest.fps,
|
|
582
|
+
Width: manifest.width,
|
|
583
|
+
Height: manifest.height,
|
|
584
|
+
Format: manifest.format,
|
|
470
585
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
471
586
|
AudioS3Uri: null,
|
|
472
|
-
FfmpegVersion:
|
|
473
|
-
ProducerVersion:
|
|
587
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
588
|
+
ProducerVersion: manifest.producerVersion,
|
|
474
589
|
DurationMs: Date.now() - started
|
|
475
590
|
};
|
|
476
591
|
} finally {
|
|
@@ -488,14 +603,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
488
603
|
const chromePath = await resolveChromeExecutablePath();
|
|
489
604
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
490
605
|
}
|
|
491
|
-
const work =
|
|
492
|
-
const planTar =
|
|
493
|
-
const planDir =
|
|
606
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-"));
|
|
607
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
608
|
+
const planDir = join2(work, "plan");
|
|
494
609
|
try {
|
|
495
610
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
496
611
|
await untarDirectory(planTar, planDir);
|
|
497
612
|
verifyPlanHash(planDir, event.PlanHash);
|
|
498
|
-
const chunkOutputBase =
|
|
613
|
+
const chunkOutputBase = join2(
|
|
499
614
|
work,
|
|
500
615
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
501
616
|
);
|
|
@@ -525,7 +640,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
525
640
|
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
526
641
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
527
642
|
}
|
|
528
|
-
const work =
|
|
643
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-v2-"));
|
|
529
644
|
try {
|
|
530
645
|
const planDir = await downloadAndMaterializePlanV2(
|
|
531
646
|
s3,
|
|
@@ -533,7 +648,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
533
648
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
534
649
|
work
|
|
535
650
|
);
|
|
536
|
-
const chunkOutputBase =
|
|
651
|
+
const chunkOutputBase = join2(
|
|
537
652
|
work,
|
|
538
653
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
539
654
|
);
|
|
@@ -557,7 +672,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
557
672
|
}
|
|
558
673
|
}
|
|
559
674
|
async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
560
|
-
const trimmed =
|
|
675
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
561
676
|
if (result.outputKind === "file") {
|
|
562
677
|
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
|
563
678
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -577,19 +692,19 @@ async function handleAssemble(event, deps) {
|
|
|
577
692
|
const started = Date.now();
|
|
578
693
|
const s3 = deps?.s3 ?? getS3Client();
|
|
579
694
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
580
|
-
const work =
|
|
581
|
-
const planTar =
|
|
582
|
-
const planDir =
|
|
695
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-"));
|
|
696
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
697
|
+
const planDir = join2(work, "plan");
|
|
583
698
|
try {
|
|
584
699
|
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
585
700
|
await untarDirectory(planTar, planDir);
|
|
586
701
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
587
702
|
let audioPath = null;
|
|
588
703
|
if (event.AudioS3Uri) {
|
|
589
|
-
audioPath =
|
|
704
|
+
audioPath = join2(planDir, "audio.aac");
|
|
590
705
|
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
|
591
706
|
}
|
|
592
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
707
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
593
708
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
594
709
|
cfr: event.Cfr === true
|
|
595
710
|
});
|
|
@@ -615,12 +730,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
615
730
|
const started = Date.now();
|
|
616
731
|
const s3 = deps?.s3 ?? getS3Client();
|
|
617
732
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
618
|
-
const work =
|
|
733
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-v2-"));
|
|
619
734
|
try {
|
|
620
735
|
const planDir = await downloadAndMaterializePlanV2(s3, event, { role: "assembler" }, work);
|
|
621
|
-
const audioPath = existsSync3(
|
|
736
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
622
737
|
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
623
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
738
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
624
739
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
625
740
|
cfr: event.Cfr === true
|
|
626
741
|
});
|
|
@@ -643,9 +758,9 @@ async function handleAssembleV2(event, deps) {
|
|
|
643
758
|
}
|
|
644
759
|
}
|
|
645
760
|
async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
646
|
-
const transportDir =
|
|
647
|
-
|
|
648
|
-
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri,
|
|
761
|
+
const transportDir = join2(work, "plan-v2");
|
|
762
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
763
|
+
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join2(transportDir, "plan.json"));
|
|
649
764
|
const manifest = readPlanV2Manifest(transportDir);
|
|
650
765
|
if (manifest.planHash !== event.PlanHash) {
|
|
651
766
|
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
@@ -657,7 +772,7 @@ async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
|
657
772
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
658
773
|
await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
|
|
659
774
|
});
|
|
660
|
-
const planDir =
|
|
775
|
+
const planDir = join2(work, "plan");
|
|
661
776
|
materializePlanV2Target(transportDir, target, planDir);
|
|
662
777
|
return planDir;
|
|
663
778
|
}
|
|
@@ -670,10 +785,10 @@ async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
|
|
|
670
785
|
);
|
|
671
786
|
}
|
|
672
787
|
function planV2BlobPath(planV2Dir, digest) {
|
|
673
|
-
return
|
|
788
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
674
789
|
}
|
|
675
790
|
function planV2BlobUri(prefix, digest) {
|
|
676
|
-
return `${
|
|
791
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
677
792
|
}
|
|
678
793
|
function throwPlanHashMismatch(expected, actual) {
|
|
679
794
|
const error = new Error(
|
|
@@ -690,11 +805,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
690
805
|
await fn(values[index]);
|
|
691
806
|
}
|
|
692
807
|
}
|
|
693
|
-
|
|
808
|
+
const results = await Promise.allSettled(
|
|
809
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
810
|
+
);
|
|
811
|
+
const failure = results.find(
|
|
812
|
+
(result) => result.status === "rejected"
|
|
813
|
+
);
|
|
814
|
+
if (failure) throw failure.reason;
|
|
694
815
|
}
|
|
695
816
|
async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
696
|
-
const chunksDir =
|
|
697
|
-
|
|
817
|
+
const chunksDir = join2(workDir, "chunks");
|
|
818
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
698
819
|
const local = new Array(uris.length);
|
|
699
820
|
await Promise.all(
|
|
700
821
|
uris.map(async (uri, i) => {
|
|
@@ -702,10 +823,10 @@ async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
|
702
823
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
703
824
|
}
|
|
704
825
|
const { key } = parseS3Uri(uri);
|
|
705
|
-
const localPath =
|
|
826
|
+
const localPath = join2(chunksDir, basename(key));
|
|
706
827
|
await downloadS3ObjectToFile(s3, uri, localPath);
|
|
707
828
|
if (format === "png-sequence") {
|
|
708
|
-
const dirPath =
|
|
829
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
709
830
|
await untarDirectory(localPath, dirPath);
|
|
710
831
|
local[i] = dirPath;
|
|
711
832
|
} else {
|
|
@@ -747,17 +868,17 @@ function validateEventS3Uris(event) {
|
|
|
747
868
|
function pad(n) {
|
|
748
869
|
return n.toString().padStart(4, "0");
|
|
749
870
|
}
|
|
750
|
-
function
|
|
871
|
+
function trimTrailingSlash2(prefix) {
|
|
751
872
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
752
873
|
}
|
|
753
874
|
function cleanupDir(dir) {
|
|
754
875
|
try {
|
|
755
|
-
|
|
876
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
756
877
|
} catch {
|
|
757
878
|
}
|
|
758
879
|
}
|
|
759
880
|
function verifyPlanHash(planDir, expected) {
|
|
760
|
-
const planJsonPath =
|
|
881
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
761
882
|
let parsed;
|
|
762
883
|
try {
|
|
763
884
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
@@ -778,13 +899,13 @@ function verifyPlanHash(planDir, expected) {
|
|
|
778
899
|
}
|
|
779
900
|
|
|
780
901
|
// src/sdk/deploySite.ts
|
|
781
|
-
import { mkdtempSync as
|
|
782
|
-
import { tmpdir as
|
|
783
|
-
import { join as
|
|
902
|
+
import { mkdtempSync as mkdtempSync3, rmSync as rmSync4, statSync as statSync4 } from "node:fs";
|
|
903
|
+
import { tmpdir as tmpdir3 } from "node:os";
|
|
904
|
+
import { join as join3 } from "node:path";
|
|
784
905
|
import { HeadObjectCommand as HeadObjectCommand2, S3Client as S3Client2 } from "@aws-sdk/client-s3";
|
|
785
906
|
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
|
786
907
|
async function deploySite(opts) {
|
|
787
|
-
if (!
|
|
908
|
+
if (!statSync4(opts.projectDir).isDirectory()) {
|
|
788
909
|
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
|
789
910
|
}
|
|
790
911
|
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
|
@@ -802,11 +923,11 @@ async function deploySite(opts) {
|
|
|
802
923
|
uploaded: false
|
|
803
924
|
};
|
|
804
925
|
}
|
|
805
|
-
const workdir =
|
|
926
|
+
const workdir = mkdtempSync3(join3(tmpdir3(), "hf-deploy-site-"));
|
|
806
927
|
try {
|
|
807
|
-
const tarball =
|
|
928
|
+
const tarball = join3(workdir, "project.tar.gz");
|
|
808
929
|
await tarDirectory(opts.projectDir, tarball);
|
|
809
|
-
const size =
|
|
930
|
+
const size = statSync4(tarball).size;
|
|
810
931
|
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
|
|
811
932
|
return {
|
|
812
933
|
siteId,
|
|
@@ -817,7 +938,7 @@ async function deploySite(opts) {
|
|
|
817
938
|
uploaded: true
|
|
818
939
|
};
|
|
819
940
|
} finally {
|
|
820
|
-
|
|
941
|
+
rmSync4(workdir, { recursive: true, force: true });
|
|
821
942
|
}
|
|
822
943
|
}
|
|
823
944
|
async function headObject(s3, bucket, key) {
|
|
@@ -1209,6 +1330,7 @@ function isTerminalFailure(status) {
|
|
|
1209
1330
|
export {
|
|
1210
1331
|
ChromeBinaryUnavailableError,
|
|
1211
1332
|
InvalidConfigError2 as InvalidConfigError,
|
|
1333
|
+
S3PlanV2ArtifactPublisher,
|
|
1212
1334
|
computeRenderCost,
|
|
1213
1335
|
deploySite,
|
|
1214
1336
|
downloadS3ObjectToFile,
|