@hyperframes/gcp-cloud-run 0.7.71 → 0.7.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -0
- package/dist/events.d.ts +71 -11
- package/dist/events.d.ts.map +1 -1
- package/dist/gcsPlanV2Publisher.d.ts +25 -0
- package/dist/gcsPlanV2Publisher.d.ts.map +1 -0
- package/dist/gcsTransport.d.ts +12 -0
- package/dist/gcsTransport.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +469 -43
- package/dist/index.js.map +4 -4
- package/dist/sdk/index.js +10 -2
- package/dist/sdk/index.js.map +2 -2
- package/dist/sdk/renderToCloudRun.d.ts +6 -1
- package/dist/sdk/renderToCloudRun.d.ts.map +1 -1
- package/dist/server.d.ts +2 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +455 -34
- package/dist/server.js.map +4 -4
- package/package.json +4 -3
- package/terraform/main.tf +5 -0
- package/terraform/outputs.tf +10 -0
- package/terraform/smoke-safety.test.ts +71 -0
- package/terraform/variables.tf +9 -0
- package/terraform/workflow.test.ts +148 -0
- package/terraform/workflow.yaml +149 -10
package/dist/index.js
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
// src/server.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { basename, extname, join } from "node:path";
|
|
2
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
+
import { basename, extname, join as join2 } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { serve } from "@hono/node-server";
|
|
7
7
|
import { Storage } from "@google-cloud/storage";
|
|
8
8
|
import { Hono } from "hono";
|
|
9
9
|
import {
|
|
10
10
|
assemble,
|
|
11
|
+
listPlanV2ArtifactsForTarget,
|
|
12
|
+
materializePlanV2Target,
|
|
11
13
|
plan,
|
|
14
|
+
planV2WithPublisher,
|
|
15
|
+
readPlanV2Manifest,
|
|
12
16
|
renderChunk
|
|
13
17
|
} from "@hyperframes/producer/distributed";
|
|
14
18
|
|
|
@@ -74,7 +78,15 @@ function formatExtension(format) {
|
|
|
74
78
|
}
|
|
75
79
|
|
|
76
80
|
// src/gcsTransport.ts
|
|
77
|
-
import {
|
|
81
|
+
import {
|
|
82
|
+
createReadStream,
|
|
83
|
+
createWriteStream,
|
|
84
|
+
existsSync as existsSync2,
|
|
85
|
+
mkdirSync,
|
|
86
|
+
rmSync,
|
|
87
|
+
statSync
|
|
88
|
+
} from "node:fs";
|
|
89
|
+
import { createHash } from "node:crypto";
|
|
78
90
|
import { dirname } from "node:path";
|
|
79
91
|
import { pipeline } from "node:stream/promises";
|
|
80
92
|
import * as tar from "tar";
|
|
@@ -103,6 +115,19 @@ async function downloadGcsObjectToFile(storage, uri, destPath) {
|
|
|
103
115
|
const file = storage.bucket(bucket).file(key);
|
|
104
116
|
await pipeline(file.createReadStream(), createWriteStream(destPath));
|
|
105
117
|
}
|
|
118
|
+
async function downloadGcsObjectToFileVerified(storage, uri, destPath, expectedSha256) {
|
|
119
|
+
assertSha256(expectedSha256);
|
|
120
|
+
await downloadGcsObjectToFile(storage, uri, destPath);
|
|
121
|
+
const actual = await sha256File(destPath);
|
|
122
|
+
if (actual !== expectedSha256) {
|
|
123
|
+
rmSync(destPath, { force: true });
|
|
124
|
+
const error = new Error(
|
|
125
|
+
`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
|
|
126
|
+
);
|
|
127
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
128
|
+
throw error;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
106
131
|
async function uploadFileToGcs(storage, localPath, uri, contentType) {
|
|
107
132
|
if (!existsSync2(localPath)) {
|
|
108
133
|
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
|
|
@@ -117,6 +142,73 @@ async function uploadFileToGcs(storage, localPath, uri, contentType) {
|
|
|
117
142
|
contentType
|
|
118
143
|
});
|
|
119
144
|
}
|
|
145
|
+
async function uploadContentAddressedFileToGcs(storage, localPath, uri, expectedSha256, contentType) {
|
|
146
|
+
assertSha256(expectedSha256);
|
|
147
|
+
if (!existsSync2(localPath)) {
|
|
148
|
+
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
|
|
149
|
+
}
|
|
150
|
+
const actualSha256 = await sha256File(localPath);
|
|
151
|
+
if (actualSha256 !== expectedSha256) {
|
|
152
|
+
throwDigestMismatch(
|
|
153
|
+
`local artifact ${localPath} expected ${expectedSha256}, got ${actualSha256}`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
const { bucket, key } = parseGcsUri(uri);
|
|
157
|
+
const bucketHandle = storage.bucket(bucket);
|
|
158
|
+
const file = bucketHandle.file(key);
|
|
159
|
+
const size = statSync(localPath).size;
|
|
160
|
+
if (await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
|
|
161
|
+
return "reused";
|
|
162
|
+
}
|
|
163
|
+
try {
|
|
164
|
+
await bucketHandle.upload(localPath, {
|
|
165
|
+
destination: key,
|
|
166
|
+
contentType,
|
|
167
|
+
metadata: { metadata: { sha256: expectedSha256 } },
|
|
168
|
+
preconditionOpts: { ifGenerationMatch: 0 }
|
|
169
|
+
});
|
|
170
|
+
return "uploaded";
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (isGcsPreconditionFailed(error) && await isReusableContentAddressedObject(file, uri, size, expectedSha256)) {
|
|
173
|
+
return "reused";
|
|
174
|
+
}
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async function isReusableContentAddressedObject(file, uri, expectedSize, expectedSha256) {
|
|
179
|
+
const [exists] = await file.exists();
|
|
180
|
+
if (!exists) return false;
|
|
181
|
+
const [metadata] = await file.getMetadata();
|
|
182
|
+
if (Number(metadata.size) === expectedSize && metadata.metadata?.sha256 === expectedSha256) {
|
|
183
|
+
return true;
|
|
184
|
+
}
|
|
185
|
+
throwDigestMismatch(
|
|
186
|
+
`immutable object ${uri} already exists with different digest metadata or size`
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
async function sha256File(path) {
|
|
190
|
+
const hash = createHash("sha256");
|
|
191
|
+
for await (const chunk of createReadStream(path)) {
|
|
192
|
+
hash.update(chunk);
|
|
193
|
+
}
|
|
194
|
+
return hash.digest("hex");
|
|
195
|
+
}
|
|
196
|
+
function assertSha256(value) {
|
|
197
|
+
if (!/^[a-f0-9]{64}$/.test(value)) {
|
|
198
|
+
throw new Error(
|
|
199
|
+
`[gcsTransport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
function throwDigestMismatch(detail) {
|
|
204
|
+
const error = new Error(`[gcsTransport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${detail}`);
|
|
205
|
+
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
function isGcsPreconditionFailed(error) {
|
|
209
|
+
if (!error || typeof error !== "object") return false;
|
|
210
|
+
return error.code === 412;
|
|
211
|
+
}
|
|
120
212
|
async function tarDirectory(sourceDir, destTarball) {
|
|
121
213
|
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
122
214
|
throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
|
|
@@ -135,6 +227,112 @@ async function untarDirectory(tarballPath, destDir) {
|
|
|
135
227
|
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
136
228
|
}
|
|
137
229
|
|
|
230
|
+
// src/gcsPlanV2Publisher.ts
|
|
231
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
232
|
+
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
233
|
+
import { tmpdir } from "node:os";
|
|
234
|
+
import { join } from "node:path";
|
|
235
|
+
import {
|
|
236
|
+
PlanV2IntegrityError
|
|
237
|
+
} from "@hyperframes/producer/distributed";
|
|
238
|
+
function isRecord(value) {
|
|
239
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
240
|
+
}
|
|
241
|
+
function assertSha2562(value, label) {
|
|
242
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
243
|
+
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
244
|
+
}
|
|
245
|
+
return value;
|
|
246
|
+
}
|
|
247
|
+
function manifestDigests(manifestBytes) {
|
|
248
|
+
let value;
|
|
249
|
+
try {
|
|
250
|
+
value = JSON.parse(manifestBytes);
|
|
251
|
+
} catch {
|
|
252
|
+
throw new PlanV2IntegrityError("GCS publisher received invalid manifest JSON");
|
|
253
|
+
}
|
|
254
|
+
if (!isRecord(value) || !Array.isArray(value.artifacts)) {
|
|
255
|
+
throw new PlanV2IntegrityError("GCS publisher manifest requires an artifacts array");
|
|
256
|
+
}
|
|
257
|
+
return new Set(
|
|
258
|
+
value.artifacts.map((artifact, index) => {
|
|
259
|
+
if (!isRecord(artifact)) {
|
|
260
|
+
throw new PlanV2IntegrityError(`GCS publisher artifacts[${index}] must be an object`);
|
|
261
|
+
}
|
|
262
|
+
return assertSha2562(artifact.sha256, `GCS publisher artifacts[${index}].sha256`);
|
|
263
|
+
})
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
function trimTrailingSlash(value) {
|
|
267
|
+
let end = value.length;
|
|
268
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
269
|
+
return value.slice(0, end);
|
|
270
|
+
}
|
|
271
|
+
var GcsPlanV2ArtifactPublisher = class {
|
|
272
|
+
artifactPrefix;
|
|
273
|
+
manifestUri;
|
|
274
|
+
#storage;
|
|
275
|
+
#temporaryRoot;
|
|
276
|
+
#publishedDigests = /* @__PURE__ */ new Set();
|
|
277
|
+
#state = "open";
|
|
278
|
+
constructor(options) {
|
|
279
|
+
const outputPrefix = `${trimTrailingSlash(options.planOutputGcsPrefix)}/v2`;
|
|
280
|
+
parseGcsUri(outputPrefix);
|
|
281
|
+
this.#storage = options.storage;
|
|
282
|
+
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
283
|
+
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
284
|
+
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
285
|
+
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
286
|
+
}
|
|
287
|
+
async putBlob(blob) {
|
|
288
|
+
this.#assertOpen("publish a blob");
|
|
289
|
+
const digest = assertSha2562(blob.sha256, "GCS published blob sha256");
|
|
290
|
+
const sourceSize = statSync2(blob.sourcePath).size;
|
|
291
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
292
|
+
throw new PlanV2IntegrityError(
|
|
293
|
+
`GCS published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
297
|
+
await uploadContentAddressedFileToGcs(this.#storage, blob.sourcePath, uri, digest);
|
|
298
|
+
this.#publishedDigests.add(digest);
|
|
299
|
+
}
|
|
300
|
+
async commitManifest(manifestBytes) {
|
|
301
|
+
this.#assertOpen("commit a manifest");
|
|
302
|
+
for (const digest of manifestDigests(manifestBytes)) {
|
|
303
|
+
if (!this.#publishedDigests.has(digest)) {
|
|
304
|
+
throw new PlanV2IntegrityError(
|
|
305
|
+
`cannot commit GCS manifest before referenced blob is durable: ${digest}`
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
310
|
+
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
311
|
+
const manifestPath = join(stagingDir, "manifest.json");
|
|
312
|
+
try {
|
|
313
|
+
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
314
|
+
await uploadContentAddressedFileToGcs(
|
|
315
|
+
this.#storage,
|
|
316
|
+
manifestPath,
|
|
317
|
+
this.manifestUri,
|
|
318
|
+
manifestDigest,
|
|
319
|
+
"application/json"
|
|
320
|
+
);
|
|
321
|
+
this.#state = "committed";
|
|
322
|
+
} finally {
|
|
323
|
+
rmSync2(stagingDir, { recursive: true, force: true });
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
async abort() {
|
|
327
|
+
if (this.#state === "open") this.#state = "aborted";
|
|
328
|
+
}
|
|
329
|
+
#assertOpen(operation) {
|
|
330
|
+
if (this.#state !== "open") {
|
|
331
|
+
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
};
|
|
335
|
+
|
|
138
336
|
// src/server.ts
|
|
139
337
|
var cachedStorage = null;
|
|
140
338
|
function getStorage() {
|
|
@@ -144,6 +342,7 @@ function getStorage() {
|
|
|
144
342
|
}
|
|
145
343
|
async function dispatch(event, deps) {
|
|
146
344
|
const unwrapped = unwrapEvent(event);
|
|
345
|
+
validatePlanProtocolShape(unwrapped);
|
|
147
346
|
validateEventGcsUris(unwrapped);
|
|
148
347
|
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
|
|
149
348
|
try {
|
|
@@ -164,15 +363,52 @@ async function dispatch(event, deps) {
|
|
|
164
363
|
}
|
|
165
364
|
}
|
|
166
365
|
} catch (err) {
|
|
366
|
+
normalizeTerminalErrorName(err);
|
|
167
367
|
logEvent({
|
|
168
368
|
event: "handler_error",
|
|
169
369
|
action: unwrapped.Action,
|
|
370
|
+
input: summarizeEvent(unwrapped),
|
|
170
371
|
message: err instanceof Error ? err.message : String(err),
|
|
171
372
|
name: err instanceof Error ? err.name : void 0
|
|
172
373
|
});
|
|
173
374
|
throw err;
|
|
174
375
|
}
|
|
175
376
|
}
|
|
377
|
+
function validatePlanProtocolShape(event) {
|
|
378
|
+
const raw = event;
|
|
379
|
+
const protocol = raw.PlanProtocol;
|
|
380
|
+
if (protocol !== void 0 && protocol !== "v1" && protocol !== "v2") {
|
|
381
|
+
const error = new Error(
|
|
382
|
+
`[handler] unsupported PlanProtocol ${JSON.stringify(protocol)}; expected "v1", "v2", or absent`
|
|
383
|
+
);
|
|
384
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
if (event.Action === "plan") return;
|
|
388
|
+
const hasV1Locator = typeof raw.PlanGcsUri === "string";
|
|
389
|
+
const hasV2Manifest = typeof raw.PlanV2ManifestGcsUri === "string";
|
|
390
|
+
const hasV2Prefix = typeof raw.PlanV2ArtifactGcsPrefix === "string";
|
|
391
|
+
const valid = protocol === "v2" ? !hasV1Locator && hasV2Manifest && hasV2Prefix : hasV1Locator && !hasV2Manifest && !hasV2Prefix;
|
|
392
|
+
if (!valid) {
|
|
393
|
+
const error = new Error(
|
|
394
|
+
`[handler] ${protocol === "v2" ? "v2" : "v1"} ${event.Action} event has mixed or missing plan locators`
|
|
395
|
+
);
|
|
396
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
397
|
+
throw error;
|
|
398
|
+
}
|
|
399
|
+
if (protocol === "v2" && event.Action === "assemble" && event.AudioGcsUri !== null) {
|
|
400
|
+
const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
|
|
401
|
+
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
402
|
+
throw error;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
function normalizeTerminalErrorName(error) {
|
|
406
|
+
if (!error || typeof error !== "object") return;
|
|
407
|
+
const candidate = error;
|
|
408
|
+
if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE") {
|
|
409
|
+
candidate.name = candidate.code;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
176
412
|
var MAX_ENVELOPE_DEPTH = 4;
|
|
177
413
|
function unwrapEvent(event) {
|
|
178
414
|
let cursor = event;
|
|
@@ -209,18 +445,21 @@ function summarizeEvent(event) {
|
|
|
209
445
|
return {
|
|
210
446
|
projectGcsUri: event.ProjectGcsUri,
|
|
211
447
|
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
448
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
212
449
|
format: event.Config.format,
|
|
213
450
|
fps: event.Config.fps
|
|
214
451
|
};
|
|
215
452
|
case "renderChunk":
|
|
216
453
|
return {
|
|
217
|
-
|
|
454
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
455
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
|
|
218
456
|
chunkIndex: event.ChunkIndex,
|
|
219
457
|
format: event.Format
|
|
220
458
|
};
|
|
221
459
|
case "assemble":
|
|
222
460
|
return {
|
|
223
|
-
|
|
461
|
+
planProtocol: event.PlanProtocol ?? "v1",
|
|
462
|
+
...event.PlanProtocol === "v2" ? { planV2ManifestGcsUri: event.PlanV2ManifestGcsUri } : { planGcsUri: event.PlanGcsUri },
|
|
224
463
|
chunkCount: event.ChunkGcsUris.length,
|
|
225
464
|
hasAudio: event.AudioGcsUri !== null,
|
|
226
465
|
outputGcsUri: event.OutputGcsUri,
|
|
@@ -234,14 +473,17 @@ function primeChrome(deps) {
|
|
|
234
473
|
process.env.PRODUCER_HEADLESS_SHELL_PATH = resolveChromeExecutablePath();
|
|
235
474
|
}
|
|
236
475
|
async function handlePlan(event, deps) {
|
|
476
|
+
if (event.PlanProtocol === "v2") {
|
|
477
|
+
return handlePlanV2(event, deps);
|
|
478
|
+
}
|
|
237
479
|
const started = Date.now();
|
|
238
480
|
const storage = deps?.storage ?? getStorage();
|
|
239
481
|
const primitive = deps?.primitives?.plan ?? plan;
|
|
240
482
|
primeChrome(deps);
|
|
241
|
-
const work =
|
|
242
|
-
const projectArchive =
|
|
243
|
-
const projectDir =
|
|
244
|
-
const planDir =
|
|
483
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-"));
|
|
484
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
485
|
+
const projectDir = join2(work, "project");
|
|
486
|
+
const planDir = join2(work, "plan");
|
|
245
487
|
try {
|
|
246
488
|
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
247
489
|
await untarDirectory(projectArchive, projectDir);
|
|
@@ -249,11 +491,11 @@ async function handlePlan(event, deps) {
|
|
|
249
491
|
...event.Config
|
|
250
492
|
};
|
|
251
493
|
const result = await primitive(projectDir, config, planDir);
|
|
252
|
-
const planTar =
|
|
494
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
253
495
|
await tarDirectory(planDir, planTar);
|
|
254
|
-
const planTarUri = `${
|
|
255
|
-
const audioPath =
|
|
256
|
-
const hasAudio = existsSync3(audioPath) &&
|
|
496
|
+
const planTarUri = `${trimTrailingSlash2(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
|
497
|
+
const audioPath = join2(planDir, "audio.aac");
|
|
498
|
+
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
257
499
|
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
|
258
500
|
return {
|
|
259
501
|
Action: "plan",
|
|
@@ -275,19 +517,99 @@ async function handlePlan(event, deps) {
|
|
|
275
517
|
cleanupDir(work);
|
|
276
518
|
}
|
|
277
519
|
}
|
|
520
|
+
async function handlePlanV2(event, deps) {
|
|
521
|
+
const started = Date.now();
|
|
522
|
+
const storage = deps?.storage ?? getStorage();
|
|
523
|
+
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
524
|
+
primeChrome(deps);
|
|
525
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-plan-v2-"));
|
|
526
|
+
const projectArchive = join2(work, "project.tar.gz");
|
|
527
|
+
const projectDir = join2(work, "project");
|
|
528
|
+
try {
|
|
529
|
+
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
|
530
|
+
await untarDirectory(projectArchive, projectDir);
|
|
531
|
+
const publisher = new GcsPlanV2ArtifactPublisher({
|
|
532
|
+
storage,
|
|
533
|
+
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
|
534
|
+
temporaryRoot: work
|
|
535
|
+
});
|
|
536
|
+
const manifest = await primitive(projectDir, { ...event.Config }, publisher, {
|
|
537
|
+
stagingParentDir: work
|
|
538
|
+
});
|
|
539
|
+
return {
|
|
540
|
+
Action: "plan",
|
|
541
|
+
PlanProtocol: "v2",
|
|
542
|
+
PlanV2ManifestGcsUri: publisher.manifestUri,
|
|
543
|
+
PlanV2ArtifactGcsPrefix: publisher.artifactPrefix,
|
|
544
|
+
PlanHash: manifest.planHash,
|
|
545
|
+
ChunkCount: manifest.chunkCount,
|
|
546
|
+
TotalFrames: manifest.totalFrames,
|
|
547
|
+
Fps: manifest.fps,
|
|
548
|
+
Width: manifest.width,
|
|
549
|
+
Height: manifest.height,
|
|
550
|
+
Format: manifest.format,
|
|
551
|
+
HasAudio: manifest.artifacts.some((artifact) => artifact.path === "audio.aac"),
|
|
552
|
+
AudioGcsUri: null,
|
|
553
|
+
FfmpegVersion: manifest.ffmpegVersion,
|
|
554
|
+
ProducerVersion: manifest.producerVersion,
|
|
555
|
+
DurationMs: Date.now() - started
|
|
556
|
+
};
|
|
557
|
+
} finally {
|
|
558
|
+
cleanupDir(work);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
278
561
|
async function handleRenderChunk(event, deps) {
|
|
562
|
+
if (event.PlanProtocol === "v2") {
|
|
563
|
+
return handleRenderChunkV2(event, deps);
|
|
564
|
+
}
|
|
279
565
|
const started = Date.now();
|
|
280
566
|
const storage = deps?.storage ?? getStorage();
|
|
281
567
|
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
282
568
|
primeChrome(deps);
|
|
283
|
-
const work =
|
|
284
|
-
const planTar =
|
|
285
|
-
const planDir =
|
|
569
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-"));
|
|
570
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
571
|
+
const planDir = join2(work, "plan");
|
|
286
572
|
try {
|
|
287
573
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
288
574
|
await untarDirectory(planTar, planDir);
|
|
289
575
|
verifyPlanHash(planDir, event.PlanHash);
|
|
290
|
-
const chunkOutputBase =
|
|
576
|
+
const chunkOutputBase = join2(
|
|
577
|
+
work,
|
|
578
|
+
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
579
|
+
);
|
|
580
|
+
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
|
581
|
+
const chunkUri = await uploadChunkOutput(
|
|
582
|
+
storage,
|
|
583
|
+
result,
|
|
584
|
+
event.ChunkOutputGcsPrefix,
|
|
585
|
+
event.ChunkIndex
|
|
586
|
+
);
|
|
587
|
+
return {
|
|
588
|
+
Action: "renderChunk",
|
|
589
|
+
ChunkGcsUri: chunkUri,
|
|
590
|
+
ChunkIndex: event.ChunkIndex,
|
|
591
|
+
Sha256: result.sha256,
|
|
592
|
+
FramesEncoded: result.framesEncoded,
|
|
593
|
+
DurationMs: Date.now() - started
|
|
594
|
+
};
|
|
595
|
+
} finally {
|
|
596
|
+
cleanupDir(work);
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
async function handleRenderChunkV2(event, deps) {
|
|
600
|
+
const started = Date.now();
|
|
601
|
+
const storage = deps?.storage ?? getStorage();
|
|
602
|
+
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
603
|
+
primeChrome(deps);
|
|
604
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-chunk-v2-"));
|
|
605
|
+
try {
|
|
606
|
+
const planDir = await downloadAndMaterializePlanV2(
|
|
607
|
+
storage,
|
|
608
|
+
event,
|
|
609
|
+
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
610
|
+
work
|
|
611
|
+
);
|
|
612
|
+
const chunkOutputBase = join2(
|
|
291
613
|
work,
|
|
292
614
|
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
293
615
|
);
|
|
@@ -311,7 +633,7 @@ async function handleRenderChunk(event, deps) {
|
|
|
311
633
|
}
|
|
312
634
|
}
|
|
313
635
|
async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
314
|
-
const trimmed =
|
|
636
|
+
const trimmed = trimTrailingSlash2(prefix);
|
|
315
637
|
if (result.outputKind === "file") {
|
|
316
638
|
const ext = extname(result.outputPath);
|
|
317
639
|
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
@@ -325,25 +647,28 @@ async function uploadChunkOutput(storage, result, prefix, chunkIndex) {
|
|
|
325
647
|
return uri;
|
|
326
648
|
}
|
|
327
649
|
async function handleAssemble(event, deps) {
|
|
650
|
+
if (event.PlanProtocol === "v2") {
|
|
651
|
+
return handleAssembleV2(event, deps);
|
|
652
|
+
}
|
|
328
653
|
const started = Date.now();
|
|
329
654
|
const storage = deps?.storage ?? getStorage();
|
|
330
655
|
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
331
|
-
const work =
|
|
332
|
-
const planTar =
|
|
333
|
-
const planDir =
|
|
656
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-"));
|
|
657
|
+
const planTar = join2(work, "plan.tar.gz");
|
|
658
|
+
const planDir = join2(work, "plan");
|
|
334
659
|
try {
|
|
335
660
|
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
|
336
661
|
await untarDirectory(planTar, planDir);
|
|
337
662
|
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
338
663
|
let audioPath = null;
|
|
339
|
-
const planAudio =
|
|
340
|
-
if (existsSync3(planAudio) &&
|
|
664
|
+
const planAudio = join2(planDir, "audio.aac");
|
|
665
|
+
if (existsSync3(planAudio) && statSync3(planAudio).size > 0) {
|
|
341
666
|
audioPath = planAudio;
|
|
342
667
|
} else if (event.AudioGcsUri) {
|
|
343
668
|
audioPath = planAudio;
|
|
344
669
|
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
|
345
670
|
}
|
|
346
|
-
const finalOutput = event.Format === "png-sequence" ?
|
|
671
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
347
672
|
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
348
673
|
cfr: event.Cfr === true
|
|
349
674
|
});
|
|
@@ -365,9 +690,100 @@ async function handleAssemble(event, deps) {
|
|
|
365
690
|
cleanupDir(work);
|
|
366
691
|
}
|
|
367
692
|
}
|
|
693
|
+
async function handleAssembleV2(event, deps) {
|
|
694
|
+
const started = Date.now();
|
|
695
|
+
const storage = deps?.storage ?? getStorage();
|
|
696
|
+
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
697
|
+
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-cr-assemble-v2-"));
|
|
698
|
+
try {
|
|
699
|
+
const planDir = await downloadAndMaterializePlanV2(storage, event, { role: "assembler" }, work);
|
|
700
|
+
const audioPath = existsSync3(join2(planDir, "audio.aac")) ? join2(planDir, "audio.aac") : null;
|
|
701
|
+
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
|
702
|
+
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
703
|
+
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
704
|
+
cfr: event.Cfr === true
|
|
705
|
+
});
|
|
706
|
+
if (event.Format === "png-sequence") {
|
|
707
|
+
const tarball = `${finalOutput}.tar.gz`;
|
|
708
|
+
await tarDirectory(finalOutput, tarball);
|
|
709
|
+
await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
|
|
710
|
+
} else {
|
|
711
|
+
await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
|
|
712
|
+
}
|
|
713
|
+
return {
|
|
714
|
+
Action: "assemble",
|
|
715
|
+
OutputGcsUri: event.OutputGcsUri,
|
|
716
|
+
FramesEncoded: result.framesEncoded,
|
|
717
|
+
FileSize: result.fileSize,
|
|
718
|
+
DurationMs: Date.now() - started
|
|
719
|
+
};
|
|
720
|
+
} finally {
|
|
721
|
+
cleanupDir(work);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
async function downloadAndMaterializePlanV2(storage, event, target, work) {
|
|
725
|
+
const transportDir = join2(work, "plan-v2");
|
|
726
|
+
mkdirSync3(transportDir, { recursive: true });
|
|
727
|
+
await downloadGcsObjectToFile(
|
|
728
|
+
storage,
|
|
729
|
+
event.PlanV2ManifestGcsUri,
|
|
730
|
+
join2(transportDir, "plan.json")
|
|
731
|
+
);
|
|
732
|
+
const manifest = readPlanV2Manifest(transportDir);
|
|
733
|
+
if (manifest.planHash !== event.PlanHash) {
|
|
734
|
+
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
735
|
+
}
|
|
736
|
+
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
737
|
+
const uniqueArtifacts = [
|
|
738
|
+
...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
739
|
+
];
|
|
740
|
+
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
741
|
+
await downloadPlanV2Artifact(storage, event.PlanV2ArtifactGcsPrefix, transportDir, artifact);
|
|
742
|
+
});
|
|
743
|
+
const planDir = join2(work, "plan");
|
|
744
|
+
materializePlanV2Target(transportDir, target, planDir);
|
|
745
|
+
return planDir;
|
|
746
|
+
}
|
|
747
|
+
async function downloadPlanV2Artifact(storage, artifactPrefix, planV2Dir, artifact) {
|
|
748
|
+
await downloadGcsObjectToFileVerified(
|
|
749
|
+
storage,
|
|
750
|
+
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
751
|
+
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
752
|
+
artifact.sha256
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
function planV2BlobPath(planV2Dir, digest) {
|
|
756
|
+
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
757
|
+
}
|
|
758
|
+
function planV2BlobUri(prefix, digest) {
|
|
759
|
+
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
760
|
+
}
|
|
761
|
+
function throwPlanHashMismatch(expected, actual) {
|
|
762
|
+
const error = new Error(
|
|
763
|
+
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
|
|
764
|
+
);
|
|
765
|
+
error.name = "PLAN_HASH_MISMATCH";
|
|
766
|
+
throw error;
|
|
767
|
+
}
|
|
768
|
+
async function mapConcurrent(values, concurrency, fn) {
|
|
769
|
+
let cursor = 0;
|
|
770
|
+
async function worker() {
|
|
771
|
+
while (cursor < values.length) {
|
|
772
|
+
const index = cursor++;
|
|
773
|
+
await fn(values[index]);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
const results = await Promise.allSettled(
|
|
777
|
+
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
778
|
+
);
|
|
779
|
+
const failure = results.find(
|
|
780
|
+
(result) => result.status === "rejected"
|
|
781
|
+
);
|
|
782
|
+
if (failure) throw failure.reason;
|
|
783
|
+
}
|
|
368
784
|
async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
369
|
-
const chunksDir =
|
|
370
|
-
|
|
785
|
+
const chunksDir = join2(workDir, "chunks");
|
|
786
|
+
mkdirSync3(chunksDir, { recursive: true });
|
|
371
787
|
const local = new Array(uris.length);
|
|
372
788
|
await Promise.all(
|
|
373
789
|
uris.map(async (uri, i) => {
|
|
@@ -375,10 +791,10 @@ async function downloadChunkObjects(storage, uris, workDir, format) {
|
|
|
375
791
|
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
376
792
|
}
|
|
377
793
|
const { key } = parseGcsUri(uri);
|
|
378
|
-
const localPath =
|
|
794
|
+
const localPath = join2(chunksDir, basename(key));
|
|
379
795
|
await downloadGcsObjectToFile(storage, uri, localPath);
|
|
380
796
|
if (format === "png-sequence") {
|
|
381
|
-
const dirPath =
|
|
797
|
+
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
382
798
|
await untarDirectory(localPath, dirPath);
|
|
383
799
|
local[i] = dirPath;
|
|
384
800
|
} else {
|
|
@@ -393,10 +809,10 @@ function getEventGcsUris(event) {
|
|
|
393
809
|
case "plan":
|
|
394
810
|
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
|
|
395
811
|
case "renderChunk":
|
|
396
|
-
return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
|
|
812
|
+
return event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix, event.ChunkOutputGcsPrefix] : [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
|
|
397
813
|
case "assemble":
|
|
398
814
|
return [
|
|
399
|
-
event.PlanGcsUri,
|
|
815
|
+
...event.PlanProtocol === "v2" ? [event.PlanV2ManifestGcsUri, event.PlanV2ArtifactGcsPrefix] : [event.PlanGcsUri],
|
|
400
816
|
...event.ChunkGcsUris,
|
|
401
817
|
event.OutputGcsUri,
|
|
402
818
|
event.AudioGcsUri
|
|
@@ -432,17 +848,17 @@ function validateEventGcsUris(event) {
|
|
|
432
848
|
function pad(n) {
|
|
433
849
|
return n.toString().padStart(4, "0");
|
|
434
850
|
}
|
|
435
|
-
function
|
|
851
|
+
function trimTrailingSlash2(prefix) {
|
|
436
852
|
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
437
853
|
}
|
|
438
854
|
function cleanupDir(dir) {
|
|
439
855
|
try {
|
|
440
|
-
|
|
856
|
+
rmSync3(dir, { recursive: true, force: true });
|
|
441
857
|
} catch {
|
|
442
858
|
}
|
|
443
859
|
}
|
|
444
860
|
function verifyPlanHash(planDir, expected) {
|
|
445
|
-
const planJsonPath =
|
|
861
|
+
const planJsonPath = join2(planDir, "plan.json");
|
|
446
862
|
let parsed;
|
|
447
863
|
try {
|
|
448
864
|
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
@@ -465,12 +881,17 @@ var NON_RETRYABLE_ERROR_NAMES = /* @__PURE__ */ new Set([
|
|
|
465
881
|
// Handler-boundary guards.
|
|
466
882
|
"GCS_URI_NOT_ALLOWED",
|
|
467
883
|
"PLAN_HASH_MISMATCH",
|
|
884
|
+
"PLAN_ARTIFACT_DIGEST_MISMATCH",
|
|
885
|
+
"PLAN_PROTOCOL_UNSUPPORTED",
|
|
886
|
+
"PLAN_V2_INTEGRITY_UNRECOVERABLE",
|
|
468
887
|
// Producer error class names (`.name`) + their string code aliases — the
|
|
469
888
|
// class sets `.name` to the class name but wraps a `code`; cover both so a
|
|
470
889
|
// raw-code throw is caught too. Mirrors the AWS state machine's
|
|
471
890
|
// non-retryable list.
|
|
472
891
|
"FormatNotSupportedInDistributedError",
|
|
473
892
|
"PlanTooLargeError",
|
|
893
|
+
"PlanProtocolUnsupportedError",
|
|
894
|
+
"PlanV2IntegrityError",
|
|
474
895
|
"RenderChunkValidationError",
|
|
475
896
|
"FFMPEG_VERSION_MISMATCH",
|
|
476
897
|
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
|
@@ -513,13 +934,13 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
|
|
513
934
|
}
|
|
514
935
|
|
|
515
936
|
// src/sdk/deploySite.ts
|
|
516
|
-
import { mkdtempSync as
|
|
517
|
-
import { tmpdir as
|
|
518
|
-
import { join as
|
|
937
|
+
import { mkdtempSync as mkdtempSync3, rmSync as rmSync4, statSync as statSync4 } from "node:fs";
|
|
938
|
+
import { tmpdir as tmpdir3 } from "node:os";
|
|
939
|
+
import { join as join3 } from "node:path";
|
|
519
940
|
import { Storage as Storage2 } from "@google-cloud/storage";
|
|
520
941
|
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
|
521
942
|
async function deploySite(opts) {
|
|
522
|
-
if (!
|
|
943
|
+
if (!statSync4(opts.projectDir).isDirectory()) {
|
|
523
944
|
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
|
524
945
|
}
|
|
525
946
|
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
|
@@ -538,11 +959,11 @@ async function deploySite(opts) {
|
|
|
538
959
|
uploaded: false
|
|
539
960
|
};
|
|
540
961
|
}
|
|
541
|
-
const workdir =
|
|
962
|
+
const workdir = mkdtempSync3(join3(tmpdir3(), "hf-deploy-site-"));
|
|
542
963
|
try {
|
|
543
|
-
const tarball =
|
|
964
|
+
const tarball = join3(workdir, "project.tar.gz");
|
|
544
965
|
await tarDirectory(opts.projectDir, tarball);
|
|
545
|
-
const size =
|
|
966
|
+
const size = statSync4(tarball).size;
|
|
546
967
|
await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");
|
|
547
968
|
return {
|
|
548
969
|
siteId,
|
|
@@ -553,7 +974,7 @@ async function deploySite(opts) {
|
|
|
553
974
|
uploaded: true
|
|
554
975
|
};
|
|
555
976
|
} finally {
|
|
556
|
-
|
|
977
|
+
rmSync4(workdir, { recursive: true, force: true });
|
|
557
978
|
}
|
|
558
979
|
}
|
|
559
980
|
async function headObject(file) {
|
|
@@ -640,7 +1061,8 @@ async function renderToCloudRun(opts) {
|
|
|
640
1061
|
PlanOutputGcsPrefix: planOutputGcsPrefix,
|
|
641
1062
|
OutputGcsUri: outputGcsUri,
|
|
642
1063
|
ServiceUrl: opts.serviceUrl,
|
|
643
|
-
Config: opts.config
|
|
1064
|
+
Config: opts.config,
|
|
1065
|
+
PlanProtocol: opts.planProtocol ?? "v1"
|
|
644
1066
|
};
|
|
645
1067
|
validateWorkflowsInputSize(argument);
|
|
646
1068
|
const executions = opts.executions ?? await defaultExecutionsClient();
|
|
@@ -841,22 +1263,26 @@ function getTerraformModuleDir() {
|
|
|
841
1263
|
}
|
|
842
1264
|
export {
|
|
843
1265
|
ChromeBinaryUnavailableError,
|
|
1266
|
+
GcsPlanV2ArtifactPublisher,
|
|
844
1267
|
InvalidConfigError2 as InvalidConfigError,
|
|
845
1268
|
computeRenderCost,
|
|
846
1269
|
createApp,
|
|
847
1270
|
deploySite,
|
|
848
1271
|
dispatch,
|
|
849
1272
|
downloadGcsObjectToFile,
|
|
1273
|
+
downloadGcsObjectToFileVerified,
|
|
850
1274
|
formatGcsUri,
|
|
851
1275
|
getRenderProgress,
|
|
852
1276
|
getTerraformModuleDir,
|
|
853
1277
|
parseGcsUri,
|
|
854
1278
|
renderToCloudRun,
|
|
855
1279
|
resolveChromeExecutablePath,
|
|
1280
|
+
sha256File,
|
|
856
1281
|
startServer,
|
|
857
1282
|
tarDirectory,
|
|
858
1283
|
untarDirectory,
|
|
859
1284
|
unwrapEvent,
|
|
1285
|
+
uploadContentAddressedFileToGcs,
|
|
860
1286
|
uploadFileToGcs,
|
|
861
1287
|
validateDistributedRenderConfig
|
|
862
1288
|
};
|