@hyperframes/gcp-cloud-run 0.7.71 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -0
- package/dist/events.d.ts +71 -11
- package/dist/events.d.ts.map +1 -1
- package/dist/gcsPlanV2Publisher.d.ts +25 -0
- package/dist/gcsPlanV2Publisher.d.ts.map +1 -0
- package/dist/gcsTransport.d.ts +12 -0
- package/dist/gcsTransport.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +469 -43
- package/dist/index.js.map +4 -4
- package/dist/sdk/index.js +10 -2
- package/dist/sdk/index.js.map +2 -2
- package/dist/sdk/renderToCloudRun.d.ts +6 -1
- package/dist/sdk/renderToCloudRun.d.ts.map +1 -1
- package/dist/server.d.ts +2 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +455 -34
- package/dist/server.js.map +4 -4
- package/package.json +4 -3
- package/terraform/main.tf +5 -0
- package/terraform/outputs.tf +10 -0
- package/terraform/smoke-safety.test.ts +71 -0
- package/terraform/variables.tf +9 -0
- package/terraform/workflow.test.ts +148 -0
- package/terraform/workflow.yaml +149 -10
package/dist/server.js
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
// src/server.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, extname, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, extname, join as join2 } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { Storage } from "@google-cloud/storage";
|
|
8
8
|
import { Hono } from "hono";
|
|
9
9
|
import {
|
|
10
10
|
assemble,
|
|
11
|
+
listPlanV2ArtifactsForTarget,
|
|
12
|
+
materializePlanV2Target,
|
|
11
13
|
plan,
|
|
14
|
+
planV2WithPublisher,
|
|
15
|
+
readPlanV2Manifest,
|
|
12
16
|
renderChunk
|
|
13
17
|
} from "@hyperframes/producer/distributed";
|
|
14
18
|
|
|
@@ -74,7 +78,15 @@ function formatExtension(format) {
|
|
|
74
78
|
}
|
|
75
79
|
|
|
76
80
|
// src/gcsTransport.ts
|
|
77
|
-
import {
|
|
81
|
+
import {
|
|
82
|
+
createReadStream,
|
|
83
|
+
createWriteStream,
|
|
84
|
+
existsSync as existsSync2,
|
|
85
|
+
mkdirSync,
|
|
86
|
+
rmSync,
|
|
87
|
+
statSync
|
|
88
|
+
} from "node:fs";
|
|
89
|
+
import { createHash } from "node:crypto";
|
|
78
90
|
import { dirname } from "node:path";
|
|
79
91
|
import { pipeline } from "node:stream/promises";
|
|
80
92
|
import * as tar from "tar";
|
|
@@ -100,6 +112,19 @@ async function downloadGcsObjectToFile(storage, uri, destPath) {
|
|
|
100
112
|
const file = storage.bucket(bucket).file(key);
|
|
101
113
|
await pipeline(file.createReadStream(), createWriteStream(destPath));
|
|
102
114
|
}
|
|
115
|
+
async function downloadGcsObjectToFileVerified(storage, uri, destPath, expectedSha256) {
|
|
116
|
+
assertSha256(expectedSha256);
|
|
117
|
+
await downloadGcsObjectToFile(storage, uri, destPath);
|
|
118
|
+
const actual = await sha256File(destPath);
|
|
119
|
+
if (actual !== expectedSha256) {
|
|
120
|
+
rmSync(destPath, { force: true });
|
|
121
|
+
const error = new Error(
|
|
122
|
+
`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
|
|
123
|
+
);
|
|
124
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
103
128
|
async function uploadFileToGcs(storage, localPath, uri, contentType) {
|
|
104
129
|
if (!existsSync2(localPath)) {
|
|
105
130
|
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
|
|
@@ -114,6 +139,73 @@ async function uploadFileToGcs(storage, localPath, uri, contentType) {
|
|
|
114
139
|
contentType
|
|
115
140
|
});
|
|
116
141
|
}
|
|
142
|
+
async function uploadContentAddressedFileToGcs(storage, localPath, uri, expectedSha256, contentType) {
|
|
143
|
+
assertSha256(expectedSha256);
|
|
144
|
+
if (!existsSync2(localPath)) {
|
|
145
|
+
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
|
|
146
|
+
}
|
|
147
|
+
const actualSha256 = await sha256File(localPath);
|
|
148
|
+
if (actualSha256 !== expectedSha256) {
|
|
149
|
+
throwDigestMismatch(
|
|
150
|
+
`local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
const { bucket, key } = parseGcsUri(uri);
|
|
154
|
+
const bucketHandle = storage.bucket(bucket);
|
|
155
|
+
const file = bucketHandle.file(key);
|
|
156
|
+
const size = statSync(localPath).size;
|
|
157
|
+
if (await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
|
|
158
|
+
return "reused";
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
await bucketHandle.upload(localPath, {
|
|
162
|
+
destination: key,
|
|
163
|
+
contentType,
|
|
164
|
+
metadata: { metadata: { sha256: expectedSha256 } },
|
|
165
|
+
preconditionOpts: { ifGenerationMatch: 0 }
|
|
166
|
+
});
|
|
167
|
+
return "uploaded";
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (isGcsPreconditionFailed(error) && await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
|
|
170
|
+
return "reused";
|
|
171
|
+
}
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
async function isReusableContentAddressedObject(file, uri, expectedSize, expectedSha256) {
|
|
176
|
+
const [exists] = await file.exists();
|
|
177
|
+
if (!exists) return false;
|
|
178
|
+
const [metadata] = await file.getMetadata();
|
|
179
|
+
if (Number(metadata.size) === expectedSize && metadata.metadata?.sha256 === expectedSha256) {
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
throwDigestMismatch(
|
|
183
|
+
`immutable object ${uri} already exists with different digest metadata or size`
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
async function sha256File(path) {
|
|
187
|
+
const hash = createHash("sha256");
|
|
188
|
+
for await (const chunk of createReadStream(path)) {
|
|
189
|
+
hash.update(chunk);
|
|
190
|
+
}
|
|
191
|
+
return hash.digest("hex");
|
|
192
|
+
}
|
|
193
|
+
function assertSha256(value) {
|
|
194
|
+
if (!/^[a-f0-9]{64}$/.test(value)) {
|
|
195
|
+
throw new Error(
|
|
196
|
+
`[gcsTransport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function throwDigestMismatch(detail) {
|
|
201
|
+
const error = new Error(`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${detail}`);
|
|
202
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
205
|
+
function isGcsPreconditionFailed(error) {
|
|
206
|
+
if (!error || typeof error !== "object") return false;
|
|
207
|
+
return error.code === 412;
|
|
208
|
+
}
|
|
117
209
|
async function tarDirectory(sourceDir, destTarball) {
|
|
118
210
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
119
211
|
throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
|
|
@@ -132,6 +224,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
132
224
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
133
225
|
}
|
|
134
226
|
|
|
227
|
+
// src/gcsPlanV2Publisher.ts
|
|
228
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
229
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
230
|
+
import { tmpdir } from "node:os";
|
|
231
|
+
import { join } from "node:path";
|
|
232
|
+
import {
|
|
233
|
+
PlanV2IntegrityError
|
|
234
|
+
} from "@hyperframes/producer/distributed";
|
|
235
|
+
function isRecord(value) {
|
|
236
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
237
|
+
}
|
|
238
|
+
function assertSha2562(value, label) {
|
|
239
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
240
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
241
|
+
}
|
|
242
|
+
return value;
|
|
243
|
+
}
|
|
244
|
+
function manifestDigests(manifestBytes) {
|
|
245
|
+
let value;
|
|
246
|
+
try {
|
|
247
|
+
value = JSON.parse(manifestBytes);
|
|
248
|
+
} catch {
|
|
249
|
+
throw new PlanV2IntegrityError("GCS publisher received invalid manifest JSON");
|
|
250
|
+
}
|
|
251
|
+
if (!isRecord(value) || !Array.isArray(value.artifacts)) {
|
|
252
|
+
throw new PlanV2IntegrityError("GCS publisher manifest requires an artifacts array");
|
|
253
|
+
}
|
|
254
|
+
return new Set(
|
|
255
|
+
value.artifacts.map((artifact, index) => {
|
|
256
|
+
if (!isRecord(artifact)) {
|
|
257
|
+
throw new PlanV2IntegrityError(`GCS publisher artifacts[${index}] must be an object`);
|
|
258
|
+
}
|
|
259
|
+
return assertSha2562(artifact.sha256, `GCS publisher artifacts[${index}].sha256`);
|
|
260
|
+
})
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
function trimTrailingSlash(value) {
|
|
264
|
+
let end = value.length;
|
|
265
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
266
|
+
return value.slice(0, end);
|
|
267
|
+
}
|
|
268
|
+
var GcsPlanV2ArtifactPublisher = class {
|
|
269
|
+
artifactPrefix;
|
|
270
|
+
manifestUri;
|
|
271
|
+
#storage;
|
|
272
|
+
#temporaryRoot;
|
|
273
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
274
|
+
#state = "open";
|
|
275
|
+
constructor(options) {
|
|
276
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputGcsPrefix)}/v2`;
|
|
277
|
+
parseGcsUri(outputPrefix);
|
|
278
|
+
this.#storage = options.storage;
|
|
279
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
280
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
281
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
282
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
283
|
+
}
|
|
284
|
+
async putBlob(blob) {
|
|
285
|
+
this.#assertOpen("publish a blob");
|
|
286
|
+
const digest = assertSha2562(blob.sha256, "GCS published blob sha256");
|
|
287
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
288
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
289
|
+
throw new PlanV2IntegrityError(
|
|
290
|
+
`GCS published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
294
|
+
await uploadContentAddressedFileToGcs(this.#storage, blob.sourcePath, uri, digest);
|
|
295
|
+
this.#publishedDigests.add(digest);
|
|
296
|
+
}
|
|
297
|
+
async commitManifest(manifestBytes) {
|
|
298
|
+
this.#assertOpen("commit a manifest");
|
|
299
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
300
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
301
|
+
throw new PlanV2IntegrityError(
|
|
302
|
+
`cannot commit GCS manifest before referenced blob is durable: ${digest}`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
307
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
308
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
309
|
+
try {
|
|
310
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
311
|
+
await uploadContentAddressedFileToGcs(
|
|
312
|
+
this.#storage,
|
|
313
|
+
manifestPath,
|
|
314
|
+
this.manifestUri,
|
|
315
|
+
manifestDigest,
|
|
316
|
+
"application/json"
|
|
317
|
+
);
|
|
318
|
+
this.#state = "committed";
|
|
319
|
+
} finally {
|
|
320
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async abort() {
|
|
324
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
325
|
+
}
|
|
326
|
+
#assertOpen(operation) {
|
|
327
|
+
if (this.#state !== "open") {
|
|
328
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
|
|
135
333
|
// src/server.ts
|
|
136
334
|
var cachedStorage = null;
|
|
137
335
|
function getStorage() {
|
|
@@ -141,6 +339,7 @@ function getStorage() {
|
|
|
141
339
|
}
|
|
142
340
|
async function dispatch(event, deps) {
|
|
143
341
|
const unwrapped = unwrapEvent(event);
|
|
342
|
+
validatePlanProtocolShape(unwrapped);
|
|
144
343
|
validateEventGcsUris(unwrapped);
|
|
145
344
|
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
|
|
146
345
|
try {
|
|
@@ -161,15 +360,52 @@ async function dispatch(event, deps) {
|
|
|
161
360
|
}
|
|
162
361
|
}
|
|
163
362
|
} catch (err) {
|
|
363
|
+
normalizeTerminalErrorName(err);
|
|
164
364
|
logEvent({
|
|
165
365
|
event: "handler_error",
|
|
166
366
|
action: unwrapped.Action,
|
|
367
|
+
input: summarizeEvent(unwrapped),
|
|
167
368
|
message: err instanceof Error ? err.message : String(err),
|
|
168
369
|
name: err instanceof Error ? err.name : void 0
|
|
169
370
|
});
|
|
170
371
|
throw err;
|
|
171
372
|
}
|
|
172
373
|
}
|
|
374
|
+
function validatePlanProtocolShape(event) {
|
|
375
|
+
const raw = event;
|
|
376
|
+
const protocol = raw.PlanProtocol;
|
|
377
|
+
if (protocol !== void 0 && protocol !== "v1" && protocol !== "v2") {
|
|
378
|
+
const error = new Error(
|
|
379
|
+
`[handler] unsupported PlanProtocol ${JSON.stringify(protocol)}; expected "v1", "v2", or absent`
|
|
380
|
+
);
|
|
381
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
382
|
+
throw error;
|
|
383
|
+
}
|
|
384
|
+
if (event.Action === "plan") return;
|
|
385
|
+
const hasV1Locator = typeof raw.PlanGcsUri === "string";
|
|
386
|
+
const hasV2Manifest = typeof raw.PlanV2ManifestGcsUri === "string";
|
|
387
|
+
const hasV2Prefix = typeof raw.PlanV2ArtifactGcsPrefix === "string";
|
|
388
|
+
const valid = protocol === "v2" ? !hasV1Locator && hasV2Manifest && hasV2Prefix : hasV1Locator && !hasV2Manifest && !hasV2Prefix;
|
|
389
|
+
if (!valid) {
|
|
390
|
+
const error = new Error(
|
|
391
|
+
`[handler] ${protocol === "v2" ? "v2" : "v1"} ${event.Action} event has mixed or missing plan locators`
|
|
392
|
+
);
|
|
393
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
394
|
+
throw error;
|
|
395
|
+
}
|
|
396
|
+
if (protocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
|
|
397
|
+
const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
|
|
398
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
399
|
+
throw error;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
function normalizeTerminalErrorName(error) {
|
|
403
|
+
if (!error || typeof error !== "object") return;
|
|
404
|
+
const candidate = error;
|
|
405
|
+
if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE") {
|
|
406
|
+
candidate.name = candidate.code;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
173
409
|
var MAX_ENVELOPE_DEPTH = 4;
|
|
174
410
|
function unwrapEvent(event) {
|
|
175
411
|
let cursor = event;
|
|
@@ -206,18 +442,21 @@ function summarizeEvent(event) {
|
|
|
206
442
|
return {
|
|
207
443
|
projectGcsUri: event.ProjectGcsUri,
|
|
208
444
|
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
445
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
209
446
|
format: event.Config.format,
|
|
210
447
|
fps: event.Config.fps
|
|
211
448
|
};
|
|
212
449
|
case "renderChunk":
|
|
213
450
|
return {
|
|
214
|
-
|
|
451
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
452
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
|
|
215
453
|
chunkIndex: event.ChunkIndex,
|
|
216
454
|
format: event.Format
|
|
217
455
|
};
|
|
218
456
|
case "assemble":
|
|
219
457
|
return {
|
|
220
|
-
|
|
458
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
459
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
|
|
221
460
|
chunkCount: event.ChunkGcsUris.length,
|
|
222
461
|
hasAudio: event.AudioGcsUri !== null,
|
|
223
462
|
outputGcsUri: event.OutputGcsUri,
|
|
@@ -231,14 +470,17 @@ function primeChrome(deps) {
|
|
|
231
470
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = resolveChromeExecutablePath();
|
|
232
471
|
}
|
|
233
472
|
async function handlePlan(event, deps) {
|
|
473
|
+
if (event.PlanProtocol === "v2") {
|
|
474
|
+
return handlePlanV2(event, deps);
|
|
475
|
+
}
|
|
234
476
|
const started = Date.now();
|
|
235
477
|
const storage = deps?.storage ?? getStorage();
|
|
236
478
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
237
479
|
primeChrome(deps);
|
|
238
|
-
const work =
|
|
239
|
-
const projectArchive =
|
|
240
|
-
const projectDir =
|
|
241
|
-
const planDir =
|
|
480
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-"));
|
|
481
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
482
|
+
const projectDir = join2(work, "project");
|
|
483
|
+
const planDir = join2(work, "plan");
|
|
242
484
|
try {
|
|
243
485
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
244
486
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -246,11 +488,11 @@ async function handlePlan(event, deps) {
|
|
|
246
488
|
...event.Config
|
|
247
489
|
};
|
|
248
490
|
const result = await primitive(projectDir, config, planDir);
|
|
249
|
-
const planTar =
|
|
491
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
250
492
|
await tarDirectory(planDir, planTar);
|
|
251
|
-
const planTarUri = `${
|
|
252
|
-
const audioPath =
|
|
253
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
493
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
|
494
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
495
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
254
496
|
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
|
255
497
|
return {
|
|
256
498
|
Action: "plan",
|
|
@@ -272,19 +514,99 @@ async function handlePlan(event, deps) {
|
|
|
272
514
|
cleanupDir(work);
|
|
273
515
|
}
|
|
274
516
|
}
|
|
517
|
+
async function handlePlanV2(event, deps) {
|
|
518
|
+
const started = Date.now();
|
|
519
|
+
const storage = deps?.storage ?? getStorage();
|
|
520
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
521
|
+
primeChrome(deps);
|
|
522
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-v2-"));
|
|
523
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
524
|
+
const projectDir = join2(work, "project");
|
|
525
|
+
try {
|
|
526
|
+
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
527
|
+
await untarDirectory(projectArchive, projectDir);
|
|
528
|
+
const publisher = new GcsPlanV2ArtifactPublisher({
|
|
529
|
+
storage,
|
|
530
|
+
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
531
|
+
temporaryRoot: work
|
|
532
|
+
});
|
|
533
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
534
|
+
stagingParentDir: work
|
|
535
|
+
});
|
|
536
|
+
return {
|
|
537
|
+
Action: "plan",
|
|
538
|
+
PlanProtocol: "v2",
|
|
539
|
+
PlanV2ManifestGcsUri: publisher.manifestUri,
|
|
540
|
+
PlanV2ArtifactGcsPrefix: publisher.artifactPrefix,
|
|
541
|
+
PlanHash: manifest.planHash,
|
|
542
|
+
ChunkCount: manifest.chunkCount,
|
|
543
|
+
TotalFrames: manifest.totalFrames,
|
|
544
|
+
Fps: manifest.fps,
|
|
545
|
+
Width: manifest.width,
|
|
546
|
+
Height: manifest.height,
|
|
547
|
+
Format: manifest.format,
|
|
548
|
+
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
549
|
+
AudioGcsUri: null,
|
|
550
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
551
|
+
ProducerVersion: manifest.producerVersion,
|
|
552
|
+
DurationMs: Date.now() - started
|
|
553
|
+
};
|
|
554
|
+
} finally {
|
|
555
|
+
cleanupDir(work);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
275
558
|
async function handleRenderChunk(event, deps) {
|
|
559
|
+
if (event.PlanProtocol === "v2") {
|
|
560
|
+
return handleRenderChunkV2(event, deps);
|
|
561
|
+
}
|
|
276
562
|
const started = Date.now();
|
|
277
563
|
const storage = deps?.storage ?? getStorage();
|
|
278
564
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
279
565
|
primeChrome(deps);
|
|
280
|
-
const work =
|
|
281
|
-
const planTar =
|
|
282
|
-
const planDir =
|
|
566
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-"));
|
|
567
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
568
|
+
const planDir = join2(work, "plan");
|
|
283
569
|
try {
|
|
284
570
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
285
571
|
await untarDirectory(planTar, planDir);
|
|
286
572
|
verifyPlanHash(planDir, event.PlanHash);
|
|
287
|
-
const chunkOutputBase =
|
|
573
|
+
const chunkOutputBase = join2(
|
|
574
|
+
work,
|
|
575
|
+
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
576
|
+
);
|
|
577
|
+
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
|
578
|
+
const chunkUri = await uploadChunkOutput(
|
|
579
|
+
storage,
|
|
580
|
+
result,
|
|
581
|
+
event.ChunkOutputGcsPrefix,
|
|
582
|
+
event.ChunkIndex
|
|
583
|
+
);
|
|
584
|
+
return {
|
|
585
|
+
Action: "renderChunk",
|
|
586
|
+
ChunkGcsUri: chunkUri,
|
|
587
|
+
ChunkIndex: event.ChunkIndex,
|
|
588
|
+
Sha256: result.sha256,
|
|
589
|
+
FramesEncoded: result.framesEncoded,
|
|
590
|
+
DurationMs: Date.now() - started
|
|
591
|
+
};
|
|
592
|
+
} finally {
|
|
593
|
+
cleanupDir(work);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
async function handleRenderChunkV2(event, deps) {
|
|
597
|
+
const started = Date.now();
|
|
598
|
+
const storage = deps?.storage ?? getStorage();
|
|
599
|
+
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
600
|
+
primeChrome(deps);
|
|
601
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-v2-"));
|
|
602
|
+
try {
|
|
603
|
+
const planDir = await downloadAndMaterializePlanV2(
|
|
604
|
+
storage,
|
|
605
|
+
event,
|
|
606
|
+
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
607
|
+
work
|
|
608
|
+
);
|
|
609
|
+
const chunkOutputBase = join2(
|
|
288
610
|
work,
|
|
289
611
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
290
612
|
);
|
|
@@ -308,7 +630,7 @@ async function handleRenderChunk(event, deps) {
|
|
|
308
630
|
}
|
|
309
631
|
}
|
|
310
632
|
async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
311
|
-
const trimmed =
|
|
633
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
312
634
|
if (result.outputKind === "file") {
|
|
313
635
|
const ext = extname(result.outputPath);
|
|
314
636
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -322,25 +644,28 @@ async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
|
322
644
|
return uri;
|
|
323
645
|
}
|
|
324
646
|
async function handleAssemble(event, deps) {
|
|
647
|
+
if (event.PlanProtocol === "v2") {
|
|
648
|
+
return handleAssembleV2(event, deps);
|
|
649
|
+
}
|
|
325
650
|
const started = Date.now();
|
|
326
651
|
const storage = deps?.storage ?? getStorage();
|
|
327
652
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
328
|
-
const work =
|
|
329
|
-
const planTar =
|
|
330
|
-
const planDir =
|
|
653
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-"));
|
|
654
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
655
|
+
const planDir = join2(work, "plan");
|
|
331
656
|
try {
|
|
332
657
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
333
658
|
await untarDirectory(planTar, planDir);
|
|
334
659
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
335
660
|
let audioPath = null;
|
|
336
|
-
const planAudio =
|
|
337
|
-
if (existsSync3(planAudio) &&
|
|
661
|
+
const planAudio = join2(planDir, "audio.aac");
|
|
662
|
+
if (existsSync3(planAudio) && statSync3(planAudio).size > 0) {
|
|
338
663
|
audioPath = planAudio;
|
|
339
664
|
} else if (event.AudioGcsUri) {
|
|
340
665
|
audioPath = planAudio;
|
|
341
666
|
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
|
342
667
|
}
|
|
343
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
668
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
344
669
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
345
670
|
cfr: event.Cfr === true
|
|
346
671
|
});
|
|
@@ -362,9 +687,100 @@ async function handleAssemble(event, deps) {
|
|
|
362
687
|
cleanupDir(work);
|
|
363
688
|
}
|
|
364
689
|
}
|
|
690
|
+
async function handleAssembleV2(event, deps) {
|
|
691
|
+
const started = Date.now();
|
|
692
|
+
const storage = deps?.storage ?? getStorage();
|
|
693
|
+
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
694
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-v2-"));
|
|
695
|
+
try {
|
|
696
|
+
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
|
|
697
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
698
|
+
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
699
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
700
|
+
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
701
|
+
cfr: event.Cfr === true
|
|
702
|
+
});
|
|
703
|
+
if (event.Format === "png-sequence") {
|
|
704
|
+
const tarball = `${finalOutput}.tar.gz`;
|
|
705
|
+
await tarDirectory(finalOutput, tarball);
|
|
706
|
+
await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
|
|
707
|
+
} else {
|
|
708
|
+
await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
|
|
709
|
+
}
|
|
710
|
+
return {
|
|
711
|
+
Action: "assemble",
|
|
712
|
+
OutputGcsUri: event.OutputGcsUri,
|
|
713
|
+
FramesEncoded: result.framesEncoded,
|
|
714
|
+
FileSize: result.fileSize,
|
|
715
|
+
DurationMs: Date.now() - started
|
|
716
|
+
};
|
|
717
|
+
} finally {
|
|
718
|
+
cleanupDir(work);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
722
|
+
const transportDir = join2(work, "plan-v2");
|
|
723
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
724
|
+
await downloadGcsObjectToFile(
|
|
725
|
+
storage,
|
|
726
|
+
event.PlanV2ManifestGcsUri,
|
|
727
|
+
join2(transportDir, "plan.json")
|
|
728
|
+
);
|
|
729
|
+
const manifest = readPlanV2Manifest(transportDir);
|
|
730
|
+
if (manifest.planHash !== event.PlanHash) {
|
|
731
|
+
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
732
|
+
}
|
|
733
|
+
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
734
|
+
const uniqueArtifacts = [
|
|
735
|
+
...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
736
|
+
];
|
|
737
|
+
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
738
|
+
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
|
|
739
|
+
});
|
|
740
|
+
const planDir = join2(work, "plan");
|
|
741
|
+
materializePlanV2Target(transportDir, target, planDir);
|
|
742
|
+
return planDir;
|
|
743
|
+
}
|
|
744
|
+
async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifact) {
|
|
745
|
+
await downloadGcsObjectToFileVerified(
|
|
746
|
+
storage,
|
|
747
|
+
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
748
|
+
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
749
|
+
artifact.sha256
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
function planV2BlobPath(planV2Dir, digest) {
|
|
753
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
754
|
+
}
|
|
755
|
+
function planV2BlobUri(prefix, digest) {
|
|
756
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
757
|
+
}
|
|
758
|
+
function throwPlanHashMismatch(expected, actual) {
|
|
759
|
+
const error = new Error(
|
|
760
|
+
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
|
|
761
|
+
);
|
|
762
|
+
error.name = "PLAN_HASH_MISMATCH";
|
|
763
|
+
throw error;
|
|
764
|
+
}
|
|
765
|
+
async function mapConcurrent(values, concurrency, fn) {
|
|
766
|
+
let cursor = 0;
|
|
767
|
+
async function worker() {
|
|
768
|
+
while (cursor < values.length) {
|
|
769
|
+
const index = cursor++;
|
|
770
|
+
await fn(values[index]);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
const results = await Promise.allSettled(
|
|
774
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
775
|
+
);
|
|
776
|
+
const failure = results.find(
|
|
777
|
+
(result) => result.status === "rejected"
|
|
778
|
+
);
|
|
779
|
+
if (failure) throw failure.reason;
|
|
780
|
+
}
|
|
365
781
|
async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
366
|
-
const chunksDir =
|
|
367
|
-
|
|
782
|
+
const chunksDir = join2(workDir, "chunks");
|
|
783
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
368
784
|
const local = new Array(uris.length);
|
|
369
785
|
await Promise.all(
|
|
370
786
|
uris.map(async (uri, i) => {
|
|
@@ -372,10 +788,10 @@ async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
|
372
788
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
373
789
|
}
|
|
374
790
|
const { key } = parseGcsUri(uri);
|
|
375
|
-
const localPath =
|
|
791
|
+
const localPath = join2(chunksDir, basename(key));
|
|
376
792
|
await downloadGcsObjectToFile(storage, uri, localPath);
|
|
377
793
|
if (format === "png-sequence") {
|
|
378
|
-
const dirPath =
|
|
794
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
379
795
|
await untarDirectory(localPath, dirPath);
|
|
380
796
|
local[i] = dirPath;
|
|
381
797
|
} else {
|
|
@@ -390,10 +806,10 @@ function getEventGcsUris(event) {
|
|
|
390
806
|
case "plan":
|
|
391
807
|
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
|
|
392
808
|
case "renderChunk":
|
|
393
|
-
return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
|
|
809
|
+
return event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix, event.ChunkOutputGcsPrefix] : [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
|
|
394
810
|
case "assemble":
|
|
395
811
|
return [
|
|
396
|
-
event.PlanGcsUri,
|
|
812
|
+
...event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix] : [event.PlanGcsUri],
|
|
397
813
|
...event.ChunkGcsUris,
|
|
398
814
|
event.OutputGcsUri,
|
|
399
815
|
event.AudioGcsUri
|
|
@@ -429,17 +845,17 @@ function validateEventGcsUris(event) {
|
|
|
429
845
|
function pad(n) {
|
|
430
846
|
return n.toString().padStart(4, "0");
|
|
431
847
|
}
|
|
432
|
-
function
|
|
848
|
+
function trimTrailingSlash2(prefix) {
|
|
433
849
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
434
850
|
}
|
|
435
851
|
function cleanupDir(dir) {
|
|
436
852
|
try {
|
|
437
|
-
|
|
853
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
438
854
|
} catch {
|
|
439
855
|
}
|
|
440
856
|
}
|
|
441
857
|
function verifyPlanHash(planDir, expected) {
|
|
442
|
-
const planJsonPath =
|
|
858
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
443
859
|
let parsed;
|
|
444
860
|
try {
|
|
445
861
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
@@ -462,12 +878,17 @@ var NON_RETRYABLE_ERROR_NAMES = /* @__PURE__ */ new Set([
|
|
|
462
878
|
// Handler-boundary guards.
|
|
463
879
|
"GCS_URI_NOT_ALLOWED",
|
|
464
880
|
"PLAN_HASH_MISMATCH",
|
|
881
|
+
"PLAN_ARTIFACT_DIGEST_MISMATCH",
|
|
882
|
+
"PLAN_PROTOCOL_UNSUPPORTED",
|
|
883
|
+
"PLAN_V2_INTEGRITY_UNRECOVERABLE",
|
|
465
884
|
// Producer error class names (`.name`) + their string code aliases — the
|
|
466
885
|
// class sets `.name` to the class name but wraps a `code`; cover both so a
|
|
467
886
|
// raw-code throw is caught too. Mirrors the AWS state machine's
|
|
468
887
|
// non-retryable list.
|
|
469
888
|
"FormatNotSupportedInDistributedError",
|
|
470
889
|
"PlanTooLargeError",
|
|
890
|
+
"PlanProtocolUnsupportedError",
|
|
891
|
+
"PlanV2IntegrityError",
|
|
471
892
|
"RenderChunkValidationError",
|
|
472
893
|
"FFMPEG_VERSION_MISMATCH",
|
|
473
894
|
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|