@hyperframes/gcp-cloud-run 0.7.72 → 0.7.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +21 -4
- package/README.md +5 -1
- package/dist/chromium.d.ts +4 -3
- package/dist/chromium.d.ts.map +1 -1
- package/dist/events.d.ts +2 -0
- package/dist/events.d.ts.map +1 -1
- package/dist/gcsPlanV2Publisher.d.ts +25 -0
- package/dist/gcsPlanV2Publisher.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +187 -91
- package/dist/index.js.map +4 -4
- package/dist/server.d.ts +2 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +178 -83
- package/dist/server.js.map +4 -4
- package/package.json +2 -2
- package/terraform/smoke-safety.test.ts +3 -0
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EACL,QAAQ,EAMR,IAAI,EACJ,mBAAmB,EAMnB,WAAW,EACZ,MAAM,mCAAmC,CAAC;AAE3C,OAAO,KAAK,EACV,aAAa,EAGb,aAAa,EACb,cAAc,EACd,SAAS,EAET,gBAAgB,EAEjB,MAAM,aAAa,CAAC;AAwBrB;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,OAAO,IAAI,CAAC;QAClB,mBAAmB,CAAC,EAAE,OAAO,mBAAmB,CAAC;QACjD,WAAW,EAAE,OAAO,WAAW,CAAC;QAChC,QAAQ,EAAE,OAAO,QAAQ,CAAC;KAC3B,CAAC;IACF,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;;GAIG;AAEH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,CAmChG;AA0DD,wBAAgB,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,GAAG,gBAAgB,GAAG,aAAa,CAsB9F;AA6rBD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CA2BlD;AAED,sEAAsE;AACtE,wBAAgB,WAAW,IAAI,IAAI,CAMlC"}
|
package/dist/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// src/server.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, extname, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, extname, join as join2 } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { Storage } from "@google-cloud/storage";
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
listPlanV2ArtifactsForTarget,
|
|
12
12
|
materializePlanV2Target,
|
|
13
13
|
plan,
|
|
14
|
-
|
|
14
|
+
planV2WithPublisher,
|
|
15
15
|
readPlanV2Manifest,
|
|
16
16
|
renderChunk
|
|
17
17
|
} from "@hyperframes/producer/distributed";
|
|
@@ -224,6 +224,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
224
224
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
+
// src/gcsPlanV2Publisher.ts
|
|
228
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
229
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
230
|
+
import { tmpdir } from "node:os";
|
|
231
|
+
import { join } from "node:path";
|
|
232
|
+
import {
|
|
233
|
+
PlanV2IntegrityError
|
|
234
|
+
} from "@hyperframes/producer/distributed";
|
|
235
|
+
function isRecord(value) {
|
|
236
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
237
|
+
}
|
|
238
|
+
function assertSha2562(value, label) {
|
|
239
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
240
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
241
|
+
}
|
|
242
|
+
return value;
|
|
243
|
+
}
|
|
244
|
+
function manifestDigests(manifestBytes) {
|
|
245
|
+
let value;
|
|
246
|
+
try {
|
|
247
|
+
value = JSON.parse(manifestBytes);
|
|
248
|
+
} catch {
|
|
249
|
+
throw new PlanV2IntegrityError("GCS publisher received invalid manifest JSON");
|
|
250
|
+
}
|
|
251
|
+
if (!isRecord(value) || !Array.isArray(value.artifacts)) {
|
|
252
|
+
throw new PlanV2IntegrityError("GCS publisher manifest requires an artifacts array");
|
|
253
|
+
}
|
|
254
|
+
return new Set(
|
|
255
|
+
value.artifacts.map((artifact, index) => {
|
|
256
|
+
if (!isRecord(artifact)) {
|
|
257
|
+
throw new PlanV2IntegrityError(`GCS publisher artifacts[${index}] must be an object`);
|
|
258
|
+
}
|
|
259
|
+
return assertSha2562(artifact.sha256, `GCS publisher artifacts[${index}].sha256`);
|
|
260
|
+
})
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
function trimTrailingSlash(value) {
|
|
264
|
+
let end = value.length;
|
|
265
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
266
|
+
return value.slice(0, end);
|
|
267
|
+
}
|
|
268
|
+
var GcsPlanV2ArtifactPublisher = class {
|
|
269
|
+
artifactPrefix;
|
|
270
|
+
manifestUri;
|
|
271
|
+
#storage;
|
|
272
|
+
#temporaryRoot;
|
|
273
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
274
|
+
#state = "open";
|
|
275
|
+
constructor(options) {
|
|
276
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputGcsPrefix)}/v2`;
|
|
277
|
+
parseGcsUri(outputPrefix);
|
|
278
|
+
this.#storage = options.storage;
|
|
279
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
280
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
281
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
282
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
283
|
+
}
|
|
284
|
+
async putBlob(blob) {
|
|
285
|
+
this.#assertOpen("publish a blob");
|
|
286
|
+
const digest = assertSha2562(blob.sha256, "GCS published blob sha256");
|
|
287
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
288
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
289
|
+
throw new PlanV2IntegrityError(
|
|
290
|
+
`GCS published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
294
|
+
await uploadContentAddressedFileToGcs(this.#storage, blob.sourcePath, uri, digest);
|
|
295
|
+
this.#publishedDigests.add(digest);
|
|
296
|
+
}
|
|
297
|
+
async commitManifest(manifestBytes) {
|
|
298
|
+
this.#assertOpen("commit a manifest");
|
|
299
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
300
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
301
|
+
throw new PlanV2IntegrityError(
|
|
302
|
+
`cannot commit GCS manifest before referenced blob is durable: ${digest}`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
307
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
308
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
309
|
+
try {
|
|
310
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
311
|
+
await uploadContentAddressedFileToGcs(
|
|
312
|
+
this.#storage,
|
|
313
|
+
manifestPath,
|
|
314
|
+
this.manifestUri,
|
|
315
|
+
manifestDigest,
|
|
316
|
+
"application/json"
|
|
317
|
+
);
|
|
318
|
+
this.#state = "committed";
|
|
319
|
+
} finally {
|
|
320
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async abort() {
|
|
324
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
325
|
+
}
|
|
326
|
+
#assertOpen(operation) {
|
|
327
|
+
if (this.#state !== "open") {
|
|
328
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
|
|
227
333
|
// src/server.ts
|
|
228
334
|
var cachedStorage = null;
|
|
229
335
|
function getStorage() {
|
|
@@ -371,10 +477,10 @@ async function handlePlan(event, deps) {
|
|
|
371
477
|
const storage = deps?.storage ?? getStorage();
|
|
372
478
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
373
479
|
primeChrome(deps);
|
|
374
|
-
const work =
|
|
375
|
-
const projectArchive =
|
|
376
|
-
const projectDir =
|
|
377
|
-
const planDir =
|
|
480
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-"));
|
|
481
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
482
|
+
const projectDir = join2(work, "project");
|
|
483
|
+
const planDir = join2(work, "plan");
|
|
378
484
|
try {
|
|
379
485
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
380
486
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -382,11 +488,11 @@ async function handlePlan(event, deps) {
|
|
|
382
488
|
...event.Config
|
|
383
489
|
};
|
|
384
490
|
const result = await primitive(projectDir, config, planDir);
|
|
385
|
-
const planTar =
|
|
491
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
386
492
|
await tarDirectory(planDir, planTar);
|
|
387
|
-
const planTarUri = `${
|
|
388
|
-
const audioPath =
|
|
389
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
493
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
|
494
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
495
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
390
496
|
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
|
391
497
|
return {
|
|
392
498
|
Action: "plan",
|
|
@@ -411,57 +517,38 @@ async function handlePlan(event, deps) {
|
|
|
411
517
|
async function handlePlanV2(event, deps) {
|
|
412
518
|
const started = Date.now();
|
|
413
519
|
const storage = deps?.storage ?? getStorage();
|
|
414
|
-
const primitive = deps?.primitives?.
|
|
520
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
415
521
|
primeChrome(deps);
|
|
416
|
-
const work =
|
|
417
|
-
const projectArchive =
|
|
418
|
-
const projectDir =
|
|
419
|
-
const planV2Dir = join(work, "plan-v2");
|
|
522
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-v2-"));
|
|
523
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
524
|
+
const projectDir = join2(work, "project");
|
|
420
525
|
try {
|
|
421
526
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
422
527
|
await untarDirectory(projectArchive, projectDir);
|
|
423
|
-
const
|
|
424
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
425
|
-
if (manifest.planHash !== result.planHash) {
|
|
426
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
427
|
-
}
|
|
428
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/v2`;
|
|
429
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
430
|
-
const uniqueArtifacts = [
|
|
431
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
432
|
-
];
|
|
433
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
434
|
-
await uploadContentAddressedFileToGcs(
|
|
435
|
-
storage,
|
|
436
|
-
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
437
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
438
|
-
artifact.sha256
|
|
439
|
-
);
|
|
440
|
-
});
|
|
441
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
442
|
-
await uploadContentAddressedFileToGcs(
|
|
528
|
+
const publisher = new GcsPlanV2ArtifactPublisher({
|
|
443
529
|
storage,
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
530
|
+
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
531
|
+
temporaryRoot: work
|
|
532
|
+
});
|
|
533
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
534
|
+
stagingParentDir: work
|
|
535
|
+
});
|
|
449
536
|
return {
|
|
450
537
|
Action: "plan",
|
|
451
538
|
PlanProtocol: "v2",
|
|
452
|
-
PlanV2ManifestGcsUri: manifestUri,
|
|
453
|
-
PlanV2ArtifactGcsPrefix: artifactPrefix,
|
|
454
|
-
PlanHash:
|
|
455
|
-
ChunkCount:
|
|
456
|
-
TotalFrames:
|
|
457
|
-
Fps:
|
|
458
|
-
Width:
|
|
459
|
-
Height:
|
|
460
|
-
Format:
|
|
539
|
+
PlanV2ManifestGcsUri: publisher.manifestUri,
|
|
540
|
+
PlanV2ArtifactGcsPrefix: publisher.artifactPrefix,
|
|
541
|
+
PlanHash: manifest.planHash,
|
|
542
|
+
ChunkCount: manifest.chunkCount,
|
|
543
|
+
TotalFrames: manifest.totalFrames,
|
|
544
|
+
Fps: manifest.fps,
|
|
545
|
+
Width: manifest.width,
|
|
546
|
+
Height: manifest.height,
|
|
547
|
+
Format: manifest.format,
|
|
461
548
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
462
549
|
AudioGcsUri: null,
|
|
463
|
-
FfmpegVersion:
|
|
464
|
-
ProducerVersion:
|
|
550
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
551
|
+
ProducerVersion: manifest.producerVersion,
|
|
465
552
|
DurationMs: Date.now() - started
|
|
466
553
|
};
|
|
467
554
|
} finally {
|
|
@@ -476,14 +563,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
476
563
|
const storage = deps?.storage ?? getStorage();
|
|
477
564
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
478
565
|
primeChrome(deps);
|
|
479
|
-
const work =
|
|
480
|
-
const planTar =
|
|
481
|
-
const planDir =
|
|
566
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-"));
|
|
567
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
568
|
+
const planDir = join2(work, "plan");
|
|
482
569
|
try {
|
|
483
570
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
484
571
|
await untarDirectory(planTar, planDir);
|
|
485
572
|
verifyPlanHash(planDir, event.PlanHash);
|
|
486
|
-
const chunkOutputBase =
|
|
573
|
+
const chunkOutputBase = join2(
|
|
487
574
|
work,
|
|
488
575
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
489
576
|
);
|
|
@@ -500,6 +587,7 @@ async function handleRenderChunk(event, deps) {
|
|
|
500
587
|
ChunkIndex: event.ChunkIndex,
|
|
501
588
|
Sha256: result.sha256,
|
|
502
589
|
FramesEncoded: result.framesEncoded,
|
|
590
|
+
CaptureMode: result.captureMode,
|
|
503
591
|
DurationMs: Date.now() - started
|
|
504
592
|
};
|
|
505
593
|
} finally {
|
|
@@ -511,7 +599,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
511
599
|
const storage = deps?.storage ?? getStorage();
|
|
512
600
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
513
601
|
primeChrome(deps);
|
|
514
|
-
const work =
|
|
602
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-v2-"));
|
|
515
603
|
try {
|
|
516
604
|
const planDir = await downloadAndMaterializePlanV2(
|
|
517
605
|
storage,
|
|
@@ -519,7 +607,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
519
607
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
520
608
|
work
|
|
521
609
|
);
|
|
522
|
-
const chunkOutputBase =
|
|
610
|
+
const chunkOutputBase = join2(
|
|
523
611
|
work,
|
|
524
612
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
525
613
|
);
|
|
@@ -536,6 +624,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
536
624
|
ChunkIndex: event.ChunkIndex,
|
|
537
625
|
Sha256: result.sha256,
|
|
538
626
|
FramesEncoded: result.framesEncoded,
|
|
627
|
+
CaptureMode: result.captureMode,
|
|
539
628
|
DurationMs: Date.now() - started
|
|
540
629
|
};
|
|
541
630
|
} finally {
|
|
@@ -543,7 +632,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
543
632
|
}
|
|
544
633
|
}
|
|
545
634
|
async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
546
|
-
const trimmed =
|
|
635
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
547
636
|
if (result.outputKind === "file") {
|
|
548
637
|
const ext = extname(result.outputPath);
|
|
549
638
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -563,22 +652,22 @@ async function handleAssemble(event, deps) {
|
|
|
563
652
|
const started = Date.now();
|
|
564
653
|
const storage = deps?.storage ?? getStorage();
|
|
565
654
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
566
|
-
const work =
|
|
567
|
-
const planTar =
|
|
568
|
-
const planDir =
|
|
655
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-"));
|
|
656
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
657
|
+
const planDir = join2(work, "plan");
|
|
569
658
|
try {
|
|
570
659
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
571
660
|
await untarDirectory(planTar, planDir);
|
|
572
661
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
573
662
|
let audioPath = null;
|
|
574
|
-
const planAudio =
|
|
575
|
-
if (existsSync3(planAudio) &&
|
|
663
|
+
const planAudio = join2(planDir, "audio.aac");
|
|
664
|
+
if (existsSync3(planAudio) && statSync3(planAudio).size > 0) {
|
|
576
665
|
audioPath = planAudio;
|
|
577
666
|
} else if (event.AudioGcsUri) {
|
|
578
667
|
audioPath = planAudio;
|
|
579
668
|
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
|
580
669
|
}
|
|
581
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
670
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
582
671
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
583
672
|
cfr: event.Cfr === true
|
|
584
673
|
});
|
|
@@ -604,12 +693,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
604
693
|
const started = Date.now();
|
|
605
694
|
const storage = deps?.storage ?? getStorage();
|
|
606
695
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
607
|
-
const work =
|
|
696
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-v2-"));
|
|
608
697
|
try {
|
|
609
698
|
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
|
|
610
|
-
const audioPath = existsSync3(
|
|
699
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
611
700
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
612
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
701
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
613
702
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
614
703
|
cfr: event.Cfr === true
|
|
615
704
|
});
|
|
@@ -632,12 +721,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
632
721
|
}
|
|
633
722
|
}
|
|
634
723
|
async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
635
|
-
const transportDir =
|
|
636
|
-
|
|
724
|
+
const transportDir = join2(work, "plan-v2");
|
|
725
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
637
726
|
await downloadGcsObjectToFile(
|
|
638
727
|
storage,
|
|
639
728
|
event.PlanV2ManifestGcsUri,
|
|
640
|
-
|
|
729
|
+
join2(transportDir, "plan.json")
|
|
641
730
|
);
|
|
642
731
|
const manifest = readPlanV2Manifest(transportDir);
|
|
643
732
|
if (manifest.planHash !== event.PlanHash) {
|
|
@@ -650,7 +739,7 @@ async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
|
650
739
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
651
740
|
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
|
|
652
741
|
});
|
|
653
|
-
const planDir =
|
|
742
|
+
const planDir = join2(work, "plan");
|
|
654
743
|
materializePlanV2Target(transportDir, target, planDir);
|
|
655
744
|
return planDir;
|
|
656
745
|
}
|
|
@@ -663,10 +752,10 @@ async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifa
|
|
|
663
752
|
);
|
|
664
753
|
}
|
|
665
754
|
function planV2BlobPath(planV2Dir, digest) {
|
|
666
|
-
return
|
|
755
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
667
756
|
}
|
|
668
757
|
function planV2BlobUri(prefix, digest) {
|
|
669
|
-
return `${
|
|
758
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
670
759
|
}
|
|
671
760
|
function throwPlanHashMismatch(expected, actual) {
|
|
672
761
|
const error = new Error(
|
|
@@ -683,11 +772,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
683
772
|
await fn(values[index]);
|
|
684
773
|
}
|
|
685
774
|
}
|
|
686
|
-
|
|
775
|
+
const results = await Promise.allSettled(
|
|
776
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
777
|
+
);
|
|
778
|
+
const failure = results.find(
|
|
779
|
+
(result) => result.status === "rejected"
|
|
780
|
+
);
|
|
781
|
+
if (failure) throw failure.reason;
|
|
687
782
|
}
|
|
688
783
|
async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
689
|
-
const chunksDir =
|
|
690
|
-
|
|
784
|
+
const chunksDir = join2(workDir, "chunks");
|
|
785
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
691
786
|
const local = new Array(uris.length);
|
|
692
787
|
await Promise.all(
|
|
693
788
|
uris.map(async (uri, i) => {
|
|
@@ -695,10 +790,10 @@ async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
|
695
790
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
696
791
|
}
|
|
697
792
|
const { key } = parseGcsUri(uri);
|
|
698
|
-
const localPath =
|
|
793
|
+
const localPath = join2(chunksDir, basename(key));
|
|
699
794
|
await downloadGcsObjectToFile(storage, uri, localPath);
|
|
700
795
|
if (format === "png-sequence") {
|
|
701
|
-
const dirPath =
|
|
796
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
702
797
|
await untarDirectory(localPath, dirPath);
|
|
703
798
|
local[i] = dirPath;
|
|
704
799
|
} else {
|
|
@@ -752,17 +847,17 @@ function validateEventGcsUris(event) {
|
|
|
752
847
|
function pad(n) {
|
|
753
848
|
return n.toString().padStart(4, "0");
|
|
754
849
|
}
|
|
755
|
-
function
|
|
850
|
+
function trimTrailingSlash2(prefix) {
|
|
756
851
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
757
852
|
}
|
|
758
853
|
function cleanupDir(dir) {
|
|
759
854
|
try {
|
|
760
|
-
|
|
855
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
761
856
|
} catch {
|
|
762
857
|
}
|
|
763
858
|
}
|
|
764
859
|
function verifyPlanHash(planDir, expected) {
|
|
765
|
-
const planJsonPath =
|
|
860
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
766
861
|
let parsed;
|
|
767
862
|
try {
|
|
768
863
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|