@hyperframes/gcp-cloud-run 0.7.72 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gcsPlanV2Publisher.d.ts +25 -0
- package/dist/gcsPlanV2Publisher.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +185 -91
- package/dist/index.js.map +4 -4
- package/dist/server.d.ts +2 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +176 -83
- package/dist/server.js.map +4 -4
- package/package.json +2 -2
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,mBAAmB,EAMnB,WAAW,EACZ,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,EACV,aAAa,EAGb,aAAa,EACb,cAAc,EACd,SAAS,EAET,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AAwBrB;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,OAAO,IAAI,CAAC;QAClB,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;QACjD,WAAW,EAAE,OAAO,WAAW,CAAC;QAChC,QAAQ,EAAE,OAAO,QAAQ,CAAC;KAC3B,CAAC;IACF,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;;GAIG;AAEH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,CAmChG;AA0DD,wBAAgB,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,GAAG,gBAAgB,GAAG,aAAa,CAsB9F;AA2rBD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CA2BlD;AAED,sEAAsE;AACtE,wBAAgB,WAAW,IAAI,IAAI,CAMlC"}
|
package/dist/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// src/server.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, extname, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, extname, join as join2 } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { Storage } from "@google-cloud/storage";
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
listPlanV2ArtifactsForTarget,
|
|
12
12
|
materializePlanV2Target,
|
|
13
13
|
plan,
|
|
14
|
-
|
|
14
|
+
planV2WithPublisher,
|
|
15
15
|
readPlanV2Manifest,
|
|
16
16
|
renderChunk
|
|
17
17
|
} from "@hyperframes/producer/distributed";
|
|
@@ -224,6 +224,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
224
224
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
+
// src/gcsPlanV2Publisher.ts
|
|
228
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
229
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
230
|
+
import { tmpdir } from "node:os";
|
|
231
|
+
import { join } from "node:path";
|
|
232
|
+
import {
|
|
233
|
+
PlanV2IntegrityError
|
|
234
|
+
} from "@hyperframes/producer/distributed";
|
|
235
|
+
function isRecord(value) {
|
|
236
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
237
|
+
}
|
|
238
|
+
function assertSha2562(value, label) {
|
|
239
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
240
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
241
|
+
}
|
|
242
|
+
return value;
|
|
243
|
+
}
|
|
244
|
+
function manifestDigests(manifestBytes) {
|
|
245
|
+
let value;
|
|
246
|
+
try {
|
|
247
|
+
value = JSON.parse(manifestBytes);
|
|
248
|
+
} catch {
|
|
249
|
+
throw new PlanV2IntegrityError("GCS publisher received invalid manifest JSON");
|
|
250
|
+
}
|
|
251
|
+
if (!isRecord(value) || !Array.isArray(value.artifacts)) {
|
|
252
|
+
throw new PlanV2IntegrityError("GCS publisher manifest requires an artifacts array");
|
|
253
|
+
}
|
|
254
|
+
return new Set(
|
|
255
|
+
value.artifacts.map((artifact, index) => {
|
|
256
|
+
if (!isRecord(artifact)) {
|
|
257
|
+
throw new PlanV2IntegrityError(`GCS publisher artifacts[${index}] must be an object`);
|
|
258
|
+
}
|
|
259
|
+
return assertSha2562(artifact.sha256, `GCS publisher artifacts[${index}].sha256`);
|
|
260
|
+
})
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
function trimTrailingSlash(value) {
|
|
264
|
+
let end = value.length;
|
|
265
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
266
|
+
return value.slice(0, end);
|
|
267
|
+
}
|
|
268
|
+
var GcsPlanV2ArtifactPublisher = class {
|
|
269
|
+
artifactPrefix;
|
|
270
|
+
manifestUri;
|
|
271
|
+
#storage;
|
|
272
|
+
#temporaryRoot;
|
|
273
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
274
|
+
#state = "open";
|
|
275
|
+
constructor(options) {
|
|
276
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputGcsPrefix)}/v2`;
|
|
277
|
+
parseGcsUri(outputPrefix);
|
|
278
|
+
this.#storage = options.storage;
|
|
279
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
280
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
281
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
282
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
283
|
+
}
|
|
284
|
+
async putBlob(blob) {
|
|
285
|
+
this.#assertOpen("publish a blob");
|
|
286
|
+
const digest = assertSha2562(blob.sha256, "GCS published blob sha256");
|
|
287
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
288
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
289
|
+
throw new PlanV2IntegrityError(
|
|
290
|
+
`GCS published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
294
|
+
await uploadContentAddressedFileToGcs(this.#storage, blob.sourcePath, uri, digest);
|
|
295
|
+
this.#publishedDigests.add(digest);
|
|
296
|
+
}
|
|
297
|
+
async commitManifest(manifestBytes) {
|
|
298
|
+
this.#assertOpen("commit a manifest");
|
|
299
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
300
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
301
|
+
throw new PlanV2IntegrityError(
|
|
302
|
+
`cannot commit GCS manifest before referenced blob is durable: ${digest}`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
307
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
308
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
309
|
+
try {
|
|
310
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
311
|
+
await uploadContentAddressedFileToGcs(
|
|
312
|
+
this.#storage,
|
|
313
|
+
manifestPath,
|
|
314
|
+
this.manifestUri,
|
|
315
|
+
manifestDigest,
|
|
316
|
+
"application/json"
|
|
317
|
+
);
|
|
318
|
+
this.#state = "committed";
|
|
319
|
+
} finally {
|
|
320
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async abort() {
|
|
324
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
325
|
+
}
|
|
326
|
+
#assertOpen(operation) {
|
|
327
|
+
if (this.#state !== "open") {
|
|
328
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
|
|
227
333
|
// src/server.ts
|
|
228
334
|
var cachedStorage = null;
|
|
229
335
|
function getStorage() {
|
|
@@ -371,10 +477,10 @@ async function handlePlan(event, deps) {
|
|
|
371
477
|
const storage = deps?.storage ?? getStorage();
|
|
372
478
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
373
479
|
primeChrome(deps);
|
|
374
|
-
const work =
|
|
375
|
-
const projectArchive =
|
|
376
|
-
const projectDir =
|
|
377
|
-
const planDir =
|
|
480
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-"));
|
|
481
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
482
|
+
const projectDir = join2(work, "project");
|
|
483
|
+
const planDir = join2(work, "plan");
|
|
378
484
|
try {
|
|
379
485
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
380
486
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -382,11 +488,11 @@ async function handlePlan(event, deps) {
|
|
|
382
488
|
...event.Config
|
|
383
489
|
};
|
|
384
490
|
const result = await primitive(projectDir, config, planDir);
|
|
385
|
-
const planTar =
|
|
491
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
386
492
|
await tarDirectory(planDir, planTar);
|
|
387
|
-
const planTarUri = `${
|
|
388
|
-
const audioPath =
|
|
389
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
493
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
|
494
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
495
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
390
496
|
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
|
391
497
|
return {
|
|
392
498
|
Action: "plan",
|
|
@@ -411,57 +517,38 @@ async function handlePlan(event, deps) {
|
|
|
411
517
|
async function handlePlanV2(event, deps) {
|
|
412
518
|
const started = Date.now();
|
|
413
519
|
const storage = deps?.storage ?? getStorage();
|
|
414
|
-
const primitive = deps?.primitives?.
|
|
520
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
415
521
|
primeChrome(deps);
|
|
416
|
-
const work =
|
|
417
|
-
const projectArchive =
|
|
418
|
-
const projectDir =
|
|
419
|
-
const planV2Dir = join(work, "plan-v2");
|
|
522
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-v2-"));
|
|
523
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
524
|
+
const projectDir = join2(work, "project");
|
|
420
525
|
try {
|
|
421
526
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
422
527
|
await untarDirectory(projectArchive, projectDir);
|
|
423
|
-
const
|
|
424
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
425
|
-
if (manifest.planHash !== result.planHash) {
|
|
426
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
427
|
-
}
|
|
428
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/v2`;
|
|
429
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
430
|
-
const uniqueArtifacts = [
|
|
431
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
432
|
-
];
|
|
433
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
434
|
-
await uploadContentAddressedFileToGcs(
|
|
435
|
-
storage,
|
|
436
|
-
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
437
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
438
|
-
artifact.sha256
|
|
439
|
-
);
|
|
440
|
-
});
|
|
441
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
442
|
-
await uploadContentAddressedFileToGcs(
|
|
528
|
+
const publisher = new GcsPlanV2ArtifactPublisher({
|
|
443
529
|
storage,
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
530
|
+
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
531
|
+
temporaryRoot: work
|
|
532
|
+
});
|
|
533
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
534
|
+
stagingParentDir: work
|
|
535
|
+
});
|
|
449
536
|
return {
|
|
450
537
|
Action: "plan",
|
|
451
538
|
PlanProtocol: "v2",
|
|
452
|
-
PlanV2ManifestGcsUri: manifestUri,
|
|
453
|
-
PlanV2ArtifactGcsPrefix: artifactPrefix,
|
|
454
|
-
PlanHash:
|
|
455
|
-
ChunkCount:
|
|
456
|
-
TotalFrames:
|
|
457
|
-
Fps:
|
|
458
|
-
Width:
|
|
459
|
-
Height:
|
|
460
|
-
Format:
|
|
539
|
+
PlanV2ManifestGcsUri: publisher.manifestUri,
|
|
540
|
+
PlanV2ArtifactGcsPrefix: publisher.artifactPrefix,
|
|
541
|
+
PlanHash: manifest.planHash,
|
|
542
|
+
ChunkCount: manifest.chunkCount,
|
|
543
|
+
TotalFrames: manifest.totalFrames,
|
|
544
|
+
Fps: manifest.fps,
|
|
545
|
+
Width: manifest.width,
|
|
546
|
+
Height: manifest.height,
|
|
547
|
+
Format: manifest.format,
|
|
461
548
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
462
549
|
AudioGcsUri: null,
|
|
463
|
-
FfmpegVersion:
|
|
464
|
-
ProducerVersion:
|
|
550
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
551
|
+
ProducerVersion: manifest.producerVersion,
|
|
465
552
|
DurationMs: Date.now() - started
|
|
466
553
|
};
|
|
467
554
|
} finally {
|
|
@@ -476,14 +563,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
476
563
|
const storage = deps?.storage ?? getStorage();
|
|
477
564
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
478
565
|
primeChrome(deps);
|
|
479
|
-
const work =
|
|
480
|
-
const planTar =
|
|
481
|
-
const planDir =
|
|
566
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-"));
|
|
567
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
568
|
+
const planDir = join2(work, "plan");
|
|
482
569
|
try {
|
|
483
570
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
484
571
|
await untarDirectory(planTar, planDir);
|
|
485
572
|
verifyPlanHash(planDir, event.PlanHash);
|
|
486
|
-
const chunkOutputBase =
|
|
573
|
+
const chunkOutputBase = join2(
|
|
487
574
|
work,
|
|
488
575
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
489
576
|
);
|
|
@@ -511,7 +598,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
511
598
|
const storage = deps?.storage ?? getStorage();
|
|
512
599
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
513
600
|
primeChrome(deps);
|
|
514
|
-
const work =
|
|
601
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-v2-"));
|
|
515
602
|
try {
|
|
516
603
|
const planDir = await downloadAndMaterializePlanV2(
|
|
517
604
|
storage,
|
|
@@ -519,7 +606,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
519
606
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
520
607
|
work
|
|
521
608
|
);
|
|
522
|
-
const chunkOutputBase =
|
|
609
|
+
const chunkOutputBase = join2(
|
|
523
610
|
work,
|
|
524
611
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
525
612
|
);
|
|
@@ -543,7 +630,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
543
630
|
}
|
|
544
631
|
}
|
|
545
632
|
async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
546
|
-
const trimmed =
|
|
633
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
547
634
|
if (result.outputKind === "file") {
|
|
548
635
|
const ext = extname(result.outputPath);
|
|
549
636
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -563,22 +650,22 @@ async function handleAssemble(event, deps) {
|
|
|
563
650
|
const started = Date.now();
|
|
564
651
|
const storage = deps?.storage ?? getStorage();
|
|
565
652
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
566
|
-
const work =
|
|
567
|
-
const planTar =
|
|
568
|
-
const planDir =
|
|
653
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-"));
|
|
654
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
655
|
+
const planDir = join2(work, "plan");
|
|
569
656
|
try {
|
|
570
657
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
571
658
|
await untarDirectory(planTar, planDir);
|
|
572
659
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
573
660
|
let audioPath = null;
|
|
574
|
-
const planAudio =
|
|
575
|
-
if (existsSync3(planAudio) &&
|
|
661
|
+
const planAudio = join2(planDir, "audio.aac");
|
|
662
|
+
if (existsSync3(planAudio) && statSync3(planAudio).size > 0) {
|
|
576
663
|
audioPath = planAudio;
|
|
577
664
|
} else if (event.AudioGcsUri) {
|
|
578
665
|
audioPath = planAudio;
|
|
579
666
|
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
|
580
667
|
}
|
|
581
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
668
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
582
669
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
583
670
|
cfr: event.Cfr === true
|
|
584
671
|
});
|
|
@@ -604,12 +691,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
604
691
|
const started = Date.now();
|
|
605
692
|
const storage = deps?.storage ?? getStorage();
|
|
606
693
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
607
|
-
const work =
|
|
694
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-v2-"));
|
|
608
695
|
try {
|
|
609
696
|
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
|
|
610
|
-
const audioPath = existsSync3(
|
|
697
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
611
698
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
612
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
699
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
613
700
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
614
701
|
cfr: event.Cfr === true
|
|
615
702
|
});
|
|
@@ -632,12 +719,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
632
719
|
}
|
|
633
720
|
}
|
|
634
721
|
async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
635
|
-
const transportDir =
|
|
636
|
-
|
|
722
|
+
const transportDir = join2(work, "plan-v2");
|
|
723
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
637
724
|
await downloadGcsObjectToFile(
|
|
638
725
|
storage,
|
|
639
726
|
event.PlanV2ManifestGcsUri,
|
|
640
|
-
|
|
727
|
+
join2(transportDir, "plan.json")
|
|
641
728
|
);
|
|
642
729
|
const manifest = readPlanV2Manifest(transportDir);
|
|
643
730
|
if (manifest.planHash !== event.PlanHash) {
|
|
@@ -650,7 +737,7 @@ async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
|
650
737
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
651
738
|
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
|
|
652
739
|
});
|
|
653
|
-
const planDir =
|
|
740
|
+
const planDir = join2(work, "plan");
|
|
654
741
|
materializePlanV2Target(transportDir, target, planDir);
|
|
655
742
|
return planDir;
|
|
656
743
|
}
|
|
@@ -663,10 +750,10 @@ async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifa
|
|
|
663
750
|
);
|
|
664
751
|
}
|
|
665
752
|
function planV2BlobPath(planV2Dir, digest) {
|
|
666
|
-
return
|
|
753
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
667
754
|
}
|
|
668
755
|
function planV2BlobUri(prefix, digest) {
|
|
669
|
-
return `${
|
|
756
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
670
757
|
}
|
|
671
758
|
function throwPlanHashMismatch(expected, actual) {
|
|
672
759
|
const error = new Error(
|
|
@@ -683,11 +770,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
683
770
|
await fn(values[index]);
|
|
684
771
|
}
|
|
685
772
|
}
|
|
686
|
-
|
|
773
|
+
const results = await Promise.allSettled(
|
|
774
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
775
|
+
);
|
|
776
|
+
const failure = results.find(
|
|
777
|
+
(result) => result.status === "rejected"
|
|
778
|
+
);
|
|
779
|
+
if (failure) throw failure.reason;
|
|
687
780
|
}
|
|
688
781
|
async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
689
|
-
const chunksDir =
|
|
690
|
-
|
|
782
|
+
const chunksDir = join2(workDir, "chunks");
|
|
783
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
691
784
|
const local = new Array(uris.length);
|
|
692
785
|
await Promise.all(
|
|
693
786
|
uris.map(async (uri, i) => {
|
|
@@ -695,10 +788,10 @@ async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
|
695
788
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
696
789
|
}
|
|
697
790
|
const { key } = parseGcsUri(uri);
|
|
698
|
-
const localPath =
|
|
791
|
+
const localPath = join2(chunksDir, basename(key));
|
|
699
792
|
await downloadGcsObjectToFile(storage, uri, localPath);
|
|
700
793
|
if (format === "png-sequence") {
|
|
701
|
-
const dirPath =
|
|
794
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
702
795
|
await untarDirectory(localPath, dirPath);
|
|
703
796
|
local[i] = dirPath;
|
|
704
797
|
} else {
|
|
@@ -752,17 +845,17 @@ function validateEventGcsUris(event) {
|
|
|
752
845
|
function pad(n) {
|
|
753
846
|
return n.toString().padStart(4, "0");
|
|
754
847
|
}
|
|
755
|
-
function
|
|
848
|
+
function trimTrailingSlash2(prefix) {
|
|
756
849
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
757
850
|
}
|
|
758
851
|
function cleanupDir(dir) {
|
|
759
852
|
try {
|
|
760
|
-
|
|
853
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
761
854
|
} catch {
|
|
762
855
|
}
|
|
763
856
|
}
|
|
764
857
|
function verifyPlanHash(planDir, expected) {
|
|
765
|
-
const planJsonPath =
|
|
858
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
766
859
|
let parsed;
|
|
767
860
|
try {
|
|
768
861
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|