@hyperframes/gcp-cloud-run 0.7.72 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gcsPlanV2Publisher.d.ts +25 -0
- package/dist/gcsPlanV2Publisher.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +185 -91
- package/dist/index.js.map +4 -4
- package/dist/server.d.ts +2 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +176 -83
- package/dist/server.js.map +4 -4
- package/package.json +2 -2
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Storage } from "@google-cloud/storage";
|
|
2
|
+
import { type PlanV2ArtifactPublisher, type PlanV2PublishBlob } from "@hyperframes/producer/distributed";
|
|
3
|
+
export interface GcsPlanV2ArtifactPublisherOptions {
|
|
4
|
+
readonly storage: Storage;
|
|
5
|
+
/** Validated render output prefix from which all v2 object keys are derived. */
|
|
6
|
+
readonly planOutputGcsPrefix: string;
|
|
7
|
+
/** Planner-local scratch parent for the small manifest upload file. */
|
|
8
|
+
readonly temporaryRoot?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Manifest-last GCS implementation of the producer's plan-v2 publication seam.
|
|
12
|
+
*
|
|
13
|
+
* Every path remains private to the planner container. Remote workers receive
|
|
14
|
+
* only the manifest URI and artifact prefix and materialize their own target.
|
|
15
|
+
*/
|
|
16
|
+
export declare class GcsPlanV2ArtifactPublisher implements PlanV2ArtifactPublisher {
|
|
17
|
+
#private;
|
|
18
|
+
readonly artifactPrefix: string;
|
|
19
|
+
readonly manifestUri: string;
|
|
20
|
+
constructor(options: Readonly<GcsPlanV2ArtifactPublisherOptions>);
|
|
21
|
+
putBlob(blob: Readonly<PlanV2PublishBlob>): Promise<void>;
|
|
22
|
+
commitManifest(manifestBytes: string): Promise<void>;
|
|
23
|
+
abort(): Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=gcsPlanV2Publisher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gcsPlanV2Publisher.d.ts","sourceRoot":"","sources":["../src/gcsPlanV2Publisher.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAEL,KAAK,uBAAuB,EAC5B,KAAK,iBAAiB,EACvB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,WAAW,iCAAiC;IAChD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,gFAAgF;IAChF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,uEAAuE;IACvE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAuCD;;;;;GAKG;AACH,qBAAa,0BAA2B,YAAW,uBAAuB;;IACxE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;gBAMjB,OAAO,EAAE,QAAQ,CAAC,iCAAiC,CAAC;IAU1D,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAczD,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BpD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAW7B"}
|
package/dist/index.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ export { createApp, dispatch, type HandlerDeps, startServer, unwrapEvent } from
|
|
|
21
21
|
export { type AssembleEvent, type AssembleV1Event, type AssembleV2Event, type AssembleResultBody, type CloudRunAction, type CloudRunEvent, type CloudRunPlanProtocol, type CloudRunResult, type PlanEvent, type PlanResultBody, type PlanV1Event, type PlanV1ResultBody, type PlanV2Event, type PlanV2ResultBody, type RenderChunkEvent, type RenderChunkResultBody, type RenderChunkV1Event, type RenderChunkV2Event, type SerializableDistributedRenderConfig, } from "./events.js";
|
|
22
22
|
export { ChromeBinaryUnavailableError, resolveChromeExecutablePath } from "./chromium.js";
|
|
23
23
|
export { downloadGcsObjectToFile, downloadGcsObjectToFileVerified, formatGcsUri, type GcsLocation, parseGcsUri, sha256File, tarDirectory, untarDirectory, uploadContentAddressedFileToGcs, uploadFileToGcs, } from "./gcsTransport.js";
|
|
24
|
+
export { GcsPlanV2ArtifactPublisher, type GcsPlanV2ArtifactPublisherOptions, } from "./gcsPlanV2Publisher.js";
|
|
24
25
|
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./sdk/deploySite.js";
|
|
25
26
|
export { renderToCloudRun, type RenderHandle, type RenderToCloudRunOptions, } from "./sdk/renderToCloudRun.js";
|
|
26
27
|
export { getRenderProgress, type GetRenderProgressOptions, type RenderError, type RenderProgress, type RenderStatus, } from "./sdk/getRenderProgress.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,mCAAmC,GACzC,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EACL,uBAAuB,EACvB,+BAA+B,EAC/B,YAAY,EACZ,KAAK,WAAW,EAChB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,cAAc,EACd,+BAA+B,EAC/B,eAAe,GAChB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC9F,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,mCAAmC,GACzC,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EACL,uBAAuB,EACvB,+BAA+B,EAC/B,YAAY,EACZ,KAAK,WAAW,EAChB,WAAW,EACX,UAAU,EACV,YAAY,EACZ,cAAc,EACd,+BAA+B,EAC/B,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,0BAA0B,EAC1B,KAAK,iCAAiC,GACvC,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,KAAK,iBAAiB,EAAE,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EACL,gBAAgB,EAChB,KAAK,YAAY,EACjB,KAAK,uBAAuB,GAC7B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,iBAAiB,EACjB,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,YAAY,GAClB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,KAAK,UAAU,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,kBAAkB,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// src/server.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, extname, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, extname, join as join2 } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { Storage } from "@google-cloud/storage";
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
listPlanV2ArtifactsForTarget,
|
|
12
12
|
materializePlanV2Target,
|
|
13
13
|
plan,
|
|
14
|
-
|
|
14
|
+
planV2WithPublisher,
|
|
15
15
|
readPlanV2Manifest,
|
|
16
16
|
renderChunk
|
|
17
17
|
} from "@hyperframes/producer/distributed";
|
|
@@ -227,6 +227,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
227
227
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
228
228
|
}
|
|
229
229
|
|
|
230
|
+
// src/gcsPlanV2Publisher.ts
|
|
231
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
232
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
233
|
+
import { tmpdir } from "node:os";
|
|
234
|
+
import { join } from "node:path";
|
|
235
|
+
import {
|
|
236
|
+
PlanV2IntegrityError
|
|
237
|
+
} from "@hyperframes/producer/distributed";
|
|
238
|
+
function isRecord(value) {
|
|
239
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
240
|
+
}
|
|
241
|
+
function assertSha2562(value, label) {
|
|
242
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
243
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
244
|
+
}
|
|
245
|
+
return value;
|
|
246
|
+
}
|
|
247
|
+
function manifestDigests(manifestBytes) {
|
|
248
|
+
let value;
|
|
249
|
+
try {
|
|
250
|
+
value = JSON.parse(manifestBytes);
|
|
251
|
+
} catch {
|
|
252
|
+
throw new PlanV2IntegrityError("GCS publisher received invalid manifest JSON");
|
|
253
|
+
}
|
|
254
|
+
if (!isRecord(value) || !Array.isArray(value.artifacts)) {
|
|
255
|
+
throw new PlanV2IntegrityError("GCS publisher manifest requires an artifacts array");
|
|
256
|
+
}
|
|
257
|
+
return new Set(
|
|
258
|
+
value.artifacts.map((artifact, index) => {
|
|
259
|
+
if (!isRecord(artifact)) {
|
|
260
|
+
throw new PlanV2IntegrityError(`GCS publisher artifacts[${index}] must be an object`);
|
|
261
|
+
}
|
|
262
|
+
return assertSha2562(artifact.sha256, `GCS publisher artifacts[${index}].sha256`);
|
|
263
|
+
})
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
function trimTrailingSlash(value) {
|
|
267
|
+
let end = value.length;
|
|
268
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
269
|
+
return value.slice(0, end);
|
|
270
|
+
}
|
|
271
|
+
var GcsPlanV2ArtifactPublisher = class {
|
|
272
|
+
artifactPrefix;
|
|
273
|
+
manifestUri;
|
|
274
|
+
#storage;
|
|
275
|
+
#temporaryRoot;
|
|
276
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
277
|
+
#state = "open";
|
|
278
|
+
constructor(options) {
|
|
279
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputGcsPrefix)}/v2`;
|
|
280
|
+
parseGcsUri(outputPrefix);
|
|
281
|
+
this.#storage = options.storage;
|
|
282
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
283
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
284
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
285
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
286
|
+
}
|
|
287
|
+
async putBlob(blob) {
|
|
288
|
+
this.#assertOpen("publish a blob");
|
|
289
|
+
const digest = assertSha2562(blob.sha256, "GCS published blob sha256");
|
|
290
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
291
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
292
|
+
throw new PlanV2IntegrityError(
|
|
293
|
+
`GCS published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
297
|
+
await uploadContentAddressedFileToGcs(this.#storage, blob.sourcePath, uri, digest);
|
|
298
|
+
this.#publishedDigests.add(digest);
|
|
299
|
+
}
|
|
300
|
+
async commitManifest(manifestBytes) {
|
|
301
|
+
this.#assertOpen("commit a manifest");
|
|
302
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
303
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
304
|
+
throw new PlanV2IntegrityError(
|
|
305
|
+
`cannot commit GCS manifest before referenced blob is durable: ${digest}`
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
310
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
311
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
312
|
+
try {
|
|
313
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
314
|
+
await uploadContentAddressedFileToGcs(
|
|
315
|
+
this.#storage,
|
|
316
|
+
manifestPath,
|
|
317
|
+
this.manifestUri,
|
|
318
|
+
manifestDigest,
|
|
319
|
+
"application/json"
|
|
320
|
+
);
|
|
321
|
+
this.#state = "committed";
|
|
322
|
+
} finally {
|
|
323
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
async abort() {
|
|
327
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
328
|
+
}
|
|
329
|
+
#assertOpen(operation) {
|
|
330
|
+
if (this.#state !== "open") {
|
|
331
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
};
|
|
335
|
+
|
|
230
336
|
// src/server.ts
|
|
231
337
|
var cachedStorage = null;
|
|
232
338
|
function getStorage() {
|
|
@@ -374,10 +480,10 @@ async function handlePlan(event, deps) {
|
|
|
374
480
|
const storage = deps?.storage ?? getStorage();
|
|
375
481
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
376
482
|
primeChrome(deps);
|
|
377
|
-
const work =
|
|
378
|
-
const projectArchive =
|
|
379
|
-
const projectDir =
|
|
380
|
-
const planDir =
|
|
483
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-"));
|
|
484
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
485
|
+
const projectDir = join2(work, "project");
|
|
486
|
+
const planDir = join2(work, "plan");
|
|
381
487
|
try {
|
|
382
488
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
383
489
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -385,11 +491,11 @@ async function handlePlan(event, deps) {
|
|
|
385
491
|
...event.Config
|
|
386
492
|
};
|
|
387
493
|
const result = await primitive(projectDir, config, planDir);
|
|
388
|
-
const planTar =
|
|
494
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
389
495
|
await tarDirectory(planDir, planTar);
|
|
390
|
-
const planTarUri = `${
|
|
391
|
-
const audioPath =
|
|
392
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
496
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
|
497
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
498
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
393
499
|
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
|
394
500
|
return {
|
|
395
501
|
Action: "plan",
|
|
@@ -414,57 +520,38 @@ async function handlePlan(event, deps) {
|
|
|
414
520
|
async function handlePlanV2(event, deps) {
|
|
415
521
|
const started = Date.now();
|
|
416
522
|
const storage = deps?.storage ?? getStorage();
|
|
417
|
-
const primitive = deps?.primitives?.
|
|
523
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
418
524
|
primeChrome(deps);
|
|
419
|
-
const work =
|
|
420
|
-
const projectArchive =
|
|
421
|
-
const projectDir =
|
|
422
|
-
const planV2Dir = join(work, "plan-v2");
|
|
525
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-v2-"));
|
|
526
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
527
|
+
const projectDir = join2(work, "project");
|
|
423
528
|
try {
|
|
424
529
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
425
530
|
await untarDirectory(projectArchive, projectDir);
|
|
426
|
-
const
|
|
427
|
-
const manifest = readPlanV2Manifest(planV2Dir);
|
|
428
|
-
if (manifest.planHash !== result.planHash) {
|
|
429
|
-
throwPlanHashMismatch(result.planHash, manifest.planHash);
|
|
430
|
-
}
|
|
431
|
-
const outputPrefix = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/v2`;
|
|
432
|
-
const artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
433
|
-
const uniqueArtifacts = [
|
|
434
|
-
...new Map(manifest.artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
435
|
-
];
|
|
436
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
437
|
-
await uploadContentAddressedFileToGcs(
|
|
438
|
-
storage,
|
|
439
|
-
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
440
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
441
|
-
artifact.sha256
|
|
442
|
-
);
|
|
443
|
-
});
|
|
444
|
-
const manifestUri = `${outputPrefix}/manifest.json`;
|
|
445
|
-
await uploadContentAddressedFileToGcs(
|
|
531
|
+
const publisher = new GcsPlanV2ArtifactPublisher({
|
|
446
532
|
storage,
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
533
|
+
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
534
|
+
temporaryRoot: work
|
|
535
|
+
});
|
|
536
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
537
|
+
stagingParentDir: work
|
|
538
|
+
});
|
|
452
539
|
return {
|
|
453
540
|
Action: "plan",
|
|
454
541
|
PlanProtocol: "v2",
|
|
455
|
-
PlanV2ManifestGcsUri: manifestUri,
|
|
456
|
-
PlanV2ArtifactGcsPrefix: artifactPrefix,
|
|
457
|
-
PlanHash:
|
|
458
|
-
ChunkCount:
|
|
459
|
-
TotalFrames:
|
|
460
|
-
Fps:
|
|
461
|
-
Width:
|
|
462
|
-
Height:
|
|
463
|
-
Format:
|
|
542
|
+
PlanV2ManifestGcsUri: publisher.manifestUri,
|
|
543
|
+
PlanV2ArtifactGcsPrefix: publisher.artifactPrefix,
|
|
544
|
+
PlanHash: manifest.planHash,
|
|
545
|
+
ChunkCount: manifest.chunkCount,
|
|
546
|
+
TotalFrames: manifest.totalFrames,
|
|
547
|
+
Fps: manifest.fps,
|
|
548
|
+
Width: manifest.width,
|
|
549
|
+
Height: manifest.height,
|
|
550
|
+
Format: manifest.format,
|
|
464
551
|
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
465
552
|
AudioGcsUri: null,
|
|
466
|
-
FfmpegVersion:
|
|
467
|
-
ProducerVersion:
|
|
553
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
554
|
+
ProducerVersion: manifest.producerVersion,
|
|
468
555
|
DurationMs: Date.now() - started
|
|
469
556
|
};
|
|
470
557
|
} finally {
|
|
@@ -479,14 +566,14 @@ async function handleRenderChunk(event, deps) {
|
|
|
479
566
|
const storage = deps?.storage ?? getStorage();
|
|
480
567
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
481
568
|
primeChrome(deps);
|
|
482
|
-
const work =
|
|
483
|
-
const planTar =
|
|
484
|
-
const planDir =
|
|
569
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-"));
|
|
570
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
571
|
+
const planDir = join2(work, "plan");
|
|
485
572
|
try {
|
|
486
573
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
487
574
|
await untarDirectory(planTar, planDir);
|
|
488
575
|
verifyPlanHash(planDir, event.PlanHash);
|
|
489
|
-
const chunkOutputBase =
|
|
576
|
+
const chunkOutputBase = join2(
|
|
490
577
|
work,
|
|
491
578
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
492
579
|
);
|
|
@@ -514,7 +601,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
514
601
|
const storage = deps?.storage ?? getStorage();
|
|
515
602
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
516
603
|
primeChrome(deps);
|
|
517
|
-
const work =
|
|
604
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-v2-"));
|
|
518
605
|
try {
|
|
519
606
|
const planDir = await downloadAndMaterializePlanV2(
|
|
520
607
|
storage,
|
|
@@ -522,7 +609,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
522
609
|
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
523
610
|
work
|
|
524
611
|
);
|
|
525
|
-
const chunkOutputBase =
|
|
612
|
+
const chunkOutputBase = join2(
|
|
526
613
|
work,
|
|
527
614
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
528
615
|
);
|
|
@@ -546,7 +633,7 @@ async function handleRenderChunkV2(event, deps) {
|
|
|
546
633
|
}
|
|
547
634
|
}
|
|
548
635
|
async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
549
|
-
const trimmed =
|
|
636
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
550
637
|
if (result.outputKind === "file") {
|
|
551
638
|
const ext = extname(result.outputPath);
|
|
552
639
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -566,22 +653,22 @@ async function handleAssemble(event, deps) {
|
|
|
566
653
|
const started = Date.now();
|
|
567
654
|
const storage = deps?.storage ?? getStorage();
|
|
568
655
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
569
|
-
const work =
|
|
570
|
-
const planTar =
|
|
571
|
-
const planDir =
|
|
656
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-"));
|
|
657
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
658
|
+
const planDir = join2(work, "plan");
|
|
572
659
|
try {
|
|
573
660
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
574
661
|
await untarDirectory(planTar, planDir);
|
|
575
662
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
576
663
|
let audioPath = null;
|
|
577
|
-
const planAudio =
|
|
578
|
-
if (existsSync3(planAudio) &&
|
|
664
|
+
const planAudio = join2(planDir, "audio.aac");
|
|
665
|
+
if (existsSync3(planAudio) && statSync3(planAudio).size > 0) {
|
|
579
666
|
audioPath = planAudio;
|
|
580
667
|
} else if (event.AudioGcsUri) {
|
|
581
668
|
audioPath = planAudio;
|
|
582
669
|
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
|
583
670
|
}
|
|
584
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
671
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
585
672
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
586
673
|
cfr: event.Cfr === true
|
|
587
674
|
});
|
|
@@ -607,12 +694,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
607
694
|
const started = Date.now();
|
|
608
695
|
const storage = deps?.storage ?? getStorage();
|
|
609
696
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
610
|
-
const work =
|
|
697
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-v2-"));
|
|
611
698
|
try {
|
|
612
699
|
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
|
|
613
|
-
const audioPath = existsSync3(
|
|
700
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
614
701
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
615
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
702
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
616
703
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
617
704
|
cfr: event.Cfr === true
|
|
618
705
|
});
|
|
@@ -635,12 +722,12 @@ async function handleAssembleV2(event, deps) {
|
|
|
635
722
|
}
|
|
636
723
|
}
|
|
637
724
|
async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
638
|
-
const transportDir =
|
|
639
|
-
|
|
725
|
+
const transportDir = join2(work, "plan-v2");
|
|
726
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
640
727
|
await downloadGcsObjectToFile(
|
|
641
728
|
storage,
|
|
642
729
|
event.PlanV2ManifestGcsUri,
|
|
643
|
-
|
|
730
|
+
join2(transportDir, "plan.json")
|
|
644
731
|
);
|
|
645
732
|
const manifest = readPlanV2Manifest(transportDir);
|
|
646
733
|
if (manifest.planHash !== event.PlanHash) {
|
|
@@ -653,7 +740,7 @@ async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
|
653
740
|
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
654
741
|
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
|
|
655
742
|
});
|
|
656
|
-
const planDir =
|
|
743
|
+
const planDir = join2(work, "plan");
|
|
657
744
|
materializePlanV2Target(transportDir, target, planDir);
|
|
658
745
|
return planDir;
|
|
659
746
|
}
|
|
@@ -666,10 +753,10 @@ async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifa
|
|
|
666
753
|
);
|
|
667
754
|
}
|
|
668
755
|
function planV2BlobPath(planV2Dir, digest) {
|
|
669
|
-
return
|
|
756
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
670
757
|
}
|
|
671
758
|
function planV2BlobUri(prefix, digest) {
|
|
672
|
-
return `${
|
|
759
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
673
760
|
}
|
|
674
761
|
function throwPlanHashMismatch(expected, actual) {
|
|
675
762
|
const error = new Error(
|
|
@@ -686,11 +773,17 @@ async function mapConcurrent(values, concurrency, fn) {
|
|
|
686
773
|
await fn(values[index]);
|
|
687
774
|
}
|
|
688
775
|
}
|
|
689
|
-
|
|
776
|
+
const results = await Promise.allSettled(
|
|
777
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
778
|
+
);
|
|
779
|
+
const failure = results.find(
|
|
780
|
+
(result) => result.status === "rejected"
|
|
781
|
+
);
|
|
782
|
+
if (failure) throw failure.reason;
|
|
690
783
|
}
|
|
691
784
|
async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
692
|
-
const chunksDir =
|
|
693
|
-
|
|
785
|
+
const chunksDir = join2(workDir, "chunks");
|
|
786
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
694
787
|
const local = new Array(uris.length);
|
|
695
788
|
await Promise.all(
|
|
696
789
|
uris.map(async (uri, i) => {
|
|
@@ -698,10 +791,10 @@ async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
|
698
791
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
699
792
|
}
|
|
700
793
|
const { key } = parseGcsUri(uri);
|
|
701
|
-
const localPath =
|
|
794
|
+
const localPath = join2(chunksDir, basename(key));
|
|
702
795
|
await downloadGcsObjectToFile(storage, uri, localPath);
|
|
703
796
|
if (format === "png-sequence") {
|
|
704
|
-
const dirPath =
|
|
797
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
705
798
|
await untarDirectory(localPath, dirPath);
|
|
706
799
|
local[i] = dirPath;
|
|
707
800
|
} else {
|
|
@@ -755,17 +848,17 @@ function validateEventGcsUris(event) {
|
|
|
755
848
|
function pad(n) {
|
|
756
849
|
return n.toString().padStart(4, "0");
|
|
757
850
|
}
|
|
758
|
-
function
|
|
851
|
+
function trimTrailingSlash2(prefix) {
|
|
759
852
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
760
853
|
}
|
|
761
854
|
function cleanupDir(dir) {
|
|
762
855
|
try {
|
|
763
|
-
|
|
856
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
764
857
|
} catch {
|
|
765
858
|
}
|
|
766
859
|
}
|
|
767
860
|
function verifyPlanHash(planDir, expected) {
|
|
768
|
-
const planJsonPath =
|
|
861
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
769
862
|
let parsed;
|
|
770
863
|
try {
|
|
771
864
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
@@ -841,13 +934,13 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
|
|
841
934
|
}
|
|
842
935
|
|
|
843
936
|
// src/sdk/deploySite.ts
|
|
844
|
-
import { mkdtempSync as
|
|
845
|
-
import { tmpdir as
|
|
846
|
-
import { join as
|
|
937
|
+
import { mkdtempSync as mkdtempSync3, rmSync as rmSync4, statSync as statSync4 } from "node:fs";
|
|
938
|
+
import { tmpdir as tmpdir3 } from "node:os";
|
|
939
|
+
import { join as join3 } from "node:path";
|
|
847
940
|
import { Storage as Storage2 } from "@google-cloud/storage";
|
|
848
941
|
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
|
849
942
|
async function deploySite(opts) {
|
|
850
|
-
if (!
|
|
943
|
+
if (!statSync4(opts.projectDir).isDirectory()) {
|
|
851
944
|
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
|
852
945
|
}
|
|
853
946
|
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
|
@@ -866,11 +959,11 @@ async function deploySite(opts) {
|
|
|
866
959
|
uploaded: false
|
|
867
960
|
};
|
|
868
961
|
}
|
|
869
|
-
const workdir =
|
|
962
|
+
const workdir = mkdtempSync3(join3(tmpdir3(), "hf-deploy-site-"));
|
|
870
963
|
try {
|
|
871
|
-
const tarball =
|
|
964
|
+
const tarball = join3(workdir, "project.tar.gz");
|
|
872
965
|
await tarDirectory(opts.projectDir, tarball);
|
|
873
|
-
const size =
|
|
966
|
+
const size = statSync4(tarball).size;
|
|
874
967
|
await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");
|
|
875
968
|
return {
|
|
876
969
|
siteId,
|
|
@@ -881,7 +974,7 @@ async function deploySite(opts) {
|
|
|
881
974
|
uploaded: true
|
|
882
975
|
};
|
|
883
976
|
} finally {
|
|
884
|
-
|
|
977
|
+
rmSync4(workdir, { recursive: true, force: true });
|
|
885
978
|
}
|
|
886
979
|
}
|
|
887
980
|
async function headObject(file) {
|
|
@@ -1170,6 +1263,7 @@ function getTerraformModuleDir() {
|
|
|
1170
1263
|
}
|
|
1171
1264
|
export {
|
|
1172
1265
|
ChromeBinaryUnavailableError,
|
|
1266
|
+
GcsPlanV2ArtifactPublisher,
|
|
1173
1267
|
InvalidConfigError2 as InvalidConfigError,
|
|
1174
1268
|
computeRenderCost,
|
|
1175
1269
|
createApp,
|