@hyperframes/aws-lambda 0.8.177 → 0.8.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/_chunks/chunk-AK7J3QQN.js +1006 -0
- package/dist/_chunks/chunk-PK7PO3MH.js +596 -0
- package/dist/_chunks/chunk-YS3KY3C5.js +19 -0
- package/dist/_chunks/chunk-ZY5YIH5W.js +380 -0
- package/dist/cdk/index.js +13 -1
- package/dist/contentStore.d.ts +5 -0
- package/dist/handler.js +5 -1137
- package/dist/index.js +26 -1648
- package/dist/s3PlanV2Publisher.d.ts +1 -0
- package/dist/s3Transport.d.ts +14 -1
- package/dist/sdk/deploySite.d.ts +17 -17
- package/dist/sdk/index.d.ts +3 -2
- package/dist/sdk/index.js +38 -572
- package/dist/sdk/prebuiltHandler.d.ts +17 -0
- package/dist/sdk/renderToLambda.d.ts +8 -2
- package/dist/siteManifest.d.ts +18 -0
- package/package.json +4 -2
- package/prebuilt/handler-zip.json +6 -0
- package/prebuilt/template.yaml +782 -0
- package/scripts/build-zip.ts +3 -3
- package/scripts/pin-handler-zip.test.ts +101 -0
- package/scripts/pin-handler-zip.ts +59 -0
package/dist/handler.js
CHANGED
|
@@ -1,1141 +1,9 @@
|
|
|
1
|
-
// src/handler.ts
|
|
2
|
-
import { existsSync as existsSync3, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, readFileSync, rmSync as rmSync3, statSync as statSync3 } from "node:fs";
|
|
3
|
-
import { tmpdir as tmpdir2 } from "node:os";
|
|
4
|
-
import { basename, join as join2 } from "node:path";
|
|
5
|
-
import { GetAccountSettingsCommand, LambdaClient } from "@aws-sdk/client-lambda";
|
|
6
|
-
import { S3Client } from "@aws-sdk/client-s3";
|
|
7
1
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
plan,
|
|
14
|
-
isPlanAudioArtifactPath,
|
|
15
|
-
PLAN_AUDIO_RELATIVE_PATH,
|
|
16
|
-
resolvePlanAudioPath,
|
|
17
|
-
planV2WithPublisher,
|
|
18
|
-
readPlanV2Manifest,
|
|
19
|
-
renderChunk
|
|
20
|
-
} from "@hyperframes/producer/distributed";
|
|
21
|
-
|
|
22
|
-
// src/chromium.ts
|
|
23
|
-
import { existsSync } from "node:fs";
|
|
24
|
-
var ChromeBinaryUnavailableError = class extends Error {
|
|
25
|
-
// Lambda's runtime serializes the error envelope's `errorType` from
|
|
26
|
-
// `err.name`; this class-field override sets it across the structured
|
|
27
|
-
// clone. Read indirectly; fallow can't follow.
|
|
28
|
-
// fallow-ignore-next-line unused-class-member
|
|
29
|
-
name = "ChromeBinaryUnavailableError";
|
|
30
|
-
source;
|
|
31
|
-
resolvedPath;
|
|
32
|
-
constructor(source, resolvedPath, hint) {
|
|
33
|
-
super(`[chromium] Chrome binary unavailable (source=${source}): ${hint}`);
|
|
34
|
-
this.source = source;
|
|
35
|
-
this.resolvedPath = resolvedPath;
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
var SPARTICUZ_WEDGE_HINT = "@sparticuz/chromium.executablePath() returned a falsy value or a path that doesn't exist on disk. This typically happens after a chunk hits `Sandbox.Timedout` mid-extraction and leaves /tmp in a wedged state \u2014 subsequent invocations land on the same warm instance and never re-extract. Recycle the function (e.g. `aws lambda update-function-configuration ... --environment ...` with a bumped marker var, or redeploy via `hyperframes lambda deploy --skip-build`) to force fresh execution environments. Tracking: investigate the upstream wedge so this auto-recovers.";
|
|
39
|
-
function resolveChromeSource() {
|
|
40
|
-
const raw = process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE?.toLowerCase();
|
|
41
|
-
if (raw === "chrome-headless-shell" || raw === "shell") return "chrome-headless-shell";
|
|
42
|
-
return "sparticuz";
|
|
43
|
-
}
|
|
44
|
-
async function resolveChromeExecutablePath() {
|
|
45
|
-
const source = resolveChromeSource();
|
|
46
|
-
if (source === "sparticuz") {
|
|
47
|
-
const mod = await loadSparticuzChromium();
|
|
48
|
-
const path = await mod.executablePath();
|
|
49
|
-
if (!path || typeof path !== "string") {
|
|
50
|
-
throw new ChromeBinaryUnavailableError(source, null, SPARTICUZ_WEDGE_HINT);
|
|
51
|
-
}
|
|
52
|
-
if (!existsSync(path)) {
|
|
53
|
-
throw new ChromeBinaryUnavailableError(source, path, SPARTICUZ_WEDGE_HINT);
|
|
54
|
-
}
|
|
55
|
-
return path;
|
|
56
|
-
}
|
|
57
|
-
const explicit = process.env.HYPERFRAMES_LAMBDA_CHROME_PATH;
|
|
58
|
-
if (!explicit) {
|
|
59
|
-
throw new ChromeBinaryUnavailableError(
|
|
60
|
-
source,
|
|
61
|
-
null,
|
|
62
|
-
"HYPERFRAMES_LAMBDA_CHROME_SOURCE=chrome-headless-shell requires HYPERFRAMES_LAMBDA_CHROME_PATH to be set to the absolute path of the bundled binary."
|
|
63
|
-
);
|
|
64
|
-
}
|
|
65
|
-
if (!existsSync(explicit)) {
|
|
66
|
-
throw new ChromeBinaryUnavailableError(
|
|
67
|
-
source,
|
|
68
|
-
explicit,
|
|
69
|
-
`HYPERFRAMES_LAMBDA_CHROME_PATH=${JSON.stringify(explicit)} does not exist on disk.`
|
|
70
|
-
);
|
|
71
|
-
}
|
|
72
|
-
return explicit;
|
|
73
|
-
}
|
|
74
|
-
var cachedSparticuz = null;
|
|
75
|
-
async function loadSparticuzChromium() {
|
|
76
|
-
if (cachedSparticuz) return cachedSparticuz;
|
|
77
|
-
const mod = await import("@sparticuz/chromium");
|
|
78
|
-
const resolved = "default" in mod ? mod.default : mod;
|
|
79
|
-
cachedSparticuz = resolved;
|
|
80
|
-
return resolved;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// src/s3Stitch.ts
|
|
84
|
-
import {
|
|
85
|
-
AbortMultipartUploadCommand,
|
|
86
|
-
CompleteMultipartUploadCommand,
|
|
87
|
-
CreateMultipartUploadCommand,
|
|
88
|
-
GetObjectCommand as GetObjectCommand2,
|
|
89
|
-
UploadPartCommand,
|
|
90
|
-
UploadPartCopyCommand
|
|
91
|
-
} from "@aws-sdk/client-s3";
|
|
92
|
-
import { buildStitchedHeader, readMp4Layout } from "@hyperframes/producer/distributed";
|
|
93
|
-
|
|
94
|
-
// src/s3Transport.ts
|
|
95
|
-
import {
|
|
96
|
-
createReadStream,
|
|
97
|
-
createWriteStream,
|
|
98
|
-
existsSync as existsSync2,
|
|
99
|
-
mkdirSync,
|
|
100
|
-
rmSync,
|
|
101
|
-
statSync
|
|
102
|
-
} from "node:fs";
|
|
103
|
-
import { createHash } from "node:crypto";
|
|
104
|
-
import { dirname } from "node:path";
|
|
105
|
-
import { pipeline } from "node:stream/promises";
|
|
106
|
-
import {
|
|
107
|
-
GetObjectCommand,
|
|
108
|
-
HeadObjectCommand,
|
|
109
|
-
PutObjectCommand
|
|
110
|
-
} from "@aws-sdk/client-s3";
|
|
111
|
-
import * as tar from "tar";
|
|
112
|
-
function parseS3Uri(uri) {
|
|
113
|
-
if (!uri.startsWith("s3://")) {
|
|
114
|
-
throw new Error(`[s3Transport] expected s3:// URI, got: ${JSON.stringify(uri)}`);
|
|
115
|
-
}
|
|
116
|
-
const rest = uri.slice("s3://".length);
|
|
117
|
-
const slash = rest.indexOf("/");
|
|
118
|
-
if (slash === -1) {
|
|
119
|
-
throw new Error(`[s3Transport] missing key in s3 URI: ${JSON.stringify(uri)}`);
|
|
120
|
-
}
|
|
121
|
-
const bucket = rest.slice(0, slash);
|
|
122
|
-
const key = rest.slice(slash + 1);
|
|
123
|
-
if (!bucket || !key) {
|
|
124
|
-
throw new Error(`[s3Transport] empty bucket or key in s3 URI: ${JSON.stringify(uri)}`);
|
|
125
|
-
}
|
|
126
|
-
return { bucket, key };
|
|
127
|
-
}
|
|
128
|
-
async function downloadS3ObjectToFile(client, uri, destPath) {
|
|
129
|
-
const { bucket, key } = parseS3Uri(uri);
|
|
130
|
-
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
131
|
-
const body = response.Body;
|
|
132
|
-
if (!body) {
|
|
133
|
-
throw new Error(`[s3Transport] s3 GetObject returned empty body for ${uri}`);
|
|
134
|
-
}
|
|
135
|
-
mkdirSync(dirname(destPath), { recursive: true });
|
|
136
|
-
await pipeline(body, createWriteStream(destPath));
|
|
137
|
-
}
|
|
138
|
-
async function downloadS3ObjectToFileVerified(client, uri, destPath, expectedSha256) {
|
|
139
|
-
assertSha256(expectedSha256);
|
|
140
|
-
await downloadS3ObjectToFile(client, uri, destPath);
|
|
141
|
-
const actual = await sha256File(destPath);
|
|
142
|
-
if (actual !== expectedSha256) {
|
|
143
|
-
rmSync(destPath, { force: true });
|
|
144
|
-
const error = new Error(
|
|
145
|
-
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: ${uri} expected ${expectedSha256}, got ${actual}`
|
|
146
|
-
);
|
|
147
|
-
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
148
|
-
throw error;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
async function uploadFileToS3(client, localPath, uri, contentType) {
|
|
152
|
-
if (!existsSync2(localPath)) {
|
|
153
|
-
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
|
154
|
-
}
|
|
155
|
-
const { bucket, key } = parseS3Uri(uri);
|
|
156
|
-
const size = statSync(localPath).size;
|
|
157
|
-
await client.send(
|
|
158
|
-
new PutObjectCommand({
|
|
159
|
-
Bucket: bucket,
|
|
160
|
-
Key: key,
|
|
161
|
-
Body: createReadStream(localPath),
|
|
162
|
-
ContentType: contentType,
|
|
163
|
-
ContentLength: size
|
|
164
|
-
})
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
async function uploadContentAddressedFileToS3(client, localPath, uri, expectedSha256, contentType) {
|
|
168
|
-
assertSha256(expectedSha256);
|
|
169
|
-
if (!existsSync2(localPath)) {
|
|
170
|
-
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
|
171
|
-
}
|
|
172
|
-
const { bucket, key } = parseS3Uri(uri);
|
|
173
|
-
const size = statSync(localPath).size;
|
|
174
|
-
const existing = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
175
|
-
if (existing === "matching") return "reused";
|
|
176
|
-
if (existing === "conflict") throwImmutableObjectConflict(uri);
|
|
177
|
-
const body = createReadStream(localPath);
|
|
178
|
-
try {
|
|
179
|
-
await client.send(
|
|
180
|
-
new PutObjectCommand({
|
|
181
|
-
Bucket: bucket,
|
|
182
|
-
Key: key,
|
|
183
|
-
Body: body,
|
|
184
|
-
ContentType: contentType,
|
|
185
|
-
ContentLength: size,
|
|
186
|
-
Metadata: { sha256: expectedSha256 },
|
|
187
|
-
ChecksumSHA256: Buffer.from(expectedSha256, "hex").toString("base64"),
|
|
188
|
-
// HEAD followed by an unconditional PUT can overwrite a conflicting
|
|
189
|
-
// object published by a concurrent planner. Conditional create makes
|
|
190
|
-
// immutable CAS and fixed-key manifest publication race-safe.
|
|
191
|
-
IfNoneMatch: "*"
|
|
192
|
-
})
|
|
193
|
-
);
|
|
194
|
-
return "uploaded";
|
|
195
|
-
} catch (error) {
|
|
196
|
-
if (!isS3PreconditionFailed(error)) throw error;
|
|
197
|
-
const raced = await inspectContentAddressedObject(client, bucket, key, size, expectedSha256);
|
|
198
|
-
if (raced === "matching") return "reused";
|
|
199
|
-
if (raced === "conflict") throwImmutableObjectConflict(uri);
|
|
200
|
-
throw error;
|
|
201
|
-
} finally {
|
|
202
|
-
body.destroy();
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
async function sha256File(path) {
|
|
206
|
-
const hash = createHash("sha256");
|
|
207
|
-
for await (const chunk of createReadStream(path)) {
|
|
208
|
-
hash.update(chunk);
|
|
209
|
-
}
|
|
210
|
-
return hash.digest("hex");
|
|
211
|
-
}
|
|
212
|
-
function assertSha256(value) {
|
|
213
|
-
if (!/^[a-f0-9]{64}$/.test(value)) {
|
|
214
|
-
throw new Error(
|
|
215
|
-
`[s3Transport] expected lowercase SHA-256 digest, got ${JSON.stringify(value)}`
|
|
216
|
-
);
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
async function inspectContentAddressedObject(client, bucket, key, expectedSize, expectedSha256) {
|
|
220
|
-
try {
|
|
221
|
-
const existing = await client.send(
|
|
222
|
-
new HeadObjectCommand({ Bucket: bucket, Key: key, ChecksumMode: "ENABLED" })
|
|
223
|
-
);
|
|
224
|
-
return existing.ContentLength === expectedSize && existing.Metadata?.sha256 === expectedSha256 ? "matching" : "conflict";
|
|
225
|
-
} catch (error) {
|
|
226
|
-
if (isS3NotFound(error)) return "missing";
|
|
227
|
-
throw error;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
function throwImmutableObjectConflict(uri) {
|
|
231
|
-
const error = new Error(
|
|
232
|
-
`[s3Transport] PLAN_ARTIFACT_DIGEST_MISMATCH: immutable object ${uri} already exists with different digest metadata or size`
|
|
233
|
-
);
|
|
234
|
-
error.name = "PLAN_ARTIFACT_DIGEST_MISMATCH";
|
|
235
|
-
throw error;
|
|
236
|
-
}
|
|
237
|
-
function isS3NotFound(error) {
|
|
238
|
-
if (!isRecord(error)) return false;
|
|
239
|
-
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
240
|
-
return error.name === "NotFound" || error.name === "NoSuchKey" || metadata?.httpStatusCode === 404;
|
|
241
|
-
}
|
|
242
|
-
function isS3PreconditionFailed(error) {
|
|
243
|
-
if (!isRecord(error)) return false;
|
|
244
|
-
const metadata = isRecord(error.$metadata) ? error.$metadata : void 0;
|
|
245
|
-
return error.name === "PreconditionFailed" || metadata?.httpStatusCode === 412;
|
|
246
|
-
}
|
|
247
|
-
function isRecord(value) {
|
|
248
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
249
|
-
}
|
|
250
|
-
async function tarDirectory(sourceDir, destTarball) {
|
|
251
|
-
if (!existsSync2(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
|
252
|
-
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
|
|
253
|
-
}
|
|
254
|
-
mkdirSync(dirname(destTarball), { recursive: true });
|
|
255
|
-
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
|
|
256
|
-
}
|
|
257
|
-
async function untarDirectory(tarballPath, destDir) {
|
|
258
|
-
if (!existsSync2(tarballPath)) {
|
|
259
|
-
throw new Error(`[s3Transport] tarball missing: ${tarballPath}`);
|
|
260
|
-
}
|
|
261
|
-
if (existsSync2(destDir)) {
|
|
262
|
-
rmSync(destDir, { recursive: true, force: true });
|
|
263
|
-
}
|
|
264
|
-
mkdirSync(destDir, { recursive: true });
|
|
265
|
-
await tar.extract({ file: tarballPath, cwd: destDir });
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
// src/s3Stitch.ts
|
|
269
|
-
var MIN_PART_BYTES = 5 * 1024 * 1024;
|
|
270
|
-
var segmentSize = (s) => "bytes" in s ? s.bytes.length : s.end - s.start;
|
|
271
|
-
function planParts(segments, min = MIN_PART_BYTES) {
|
|
272
|
-
const parts = [];
|
|
273
|
-
let pending = [];
|
|
274
|
-
let pendingSize = 0;
|
|
275
|
-
const flush = () => {
|
|
276
|
-
if (pending.length) parts.push({ upload: pending });
|
|
277
|
-
pending = [];
|
|
278
|
-
pendingSize = 0;
|
|
279
|
-
};
|
|
280
|
-
for (const segment of segments) {
|
|
281
|
-
const borrow = pendingSize ? min - pendingSize : 0;
|
|
282
|
-
if ("uri" in segment && segmentSize(segment) - borrow >= min) {
|
|
283
|
-
if (borrow) pending.push({ ...segment, end: segment.start + borrow });
|
|
284
|
-
flush();
|
|
285
|
-
parts.push({ copy: { ...segment, start: segment.start + borrow } });
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
288
|
-
pending.push(segment);
|
|
289
|
-
pendingSize += segmentSize(segment);
|
|
290
|
-
if (pendingSize >= min) flush();
|
|
291
|
-
}
|
|
292
|
-
flush();
|
|
293
|
-
return parts;
|
|
294
|
-
}
|
|
295
|
-
async function getRange(s3, uri, start, end) {
|
|
296
|
-
const { bucket, key } = parseS3Uri(uri);
|
|
297
|
-
const response = await s3.send(
|
|
298
|
-
new GetObjectCommand2({ Bucket: bucket, Key: key, Range: `bytes=${start}-${end - 1}` })
|
|
299
|
-
);
|
|
300
|
-
if (!response.Body) throw new Error(`[s3Stitch] empty body for ${uri}`);
|
|
301
|
-
return Buffer.from(await response.Body.transformToByteArray());
|
|
302
|
-
}
|
|
303
|
-
async function sendPart(s3, part, partNumber, target) {
|
|
304
|
-
if ("copy" in part) {
|
|
305
|
-
const { uri, start, end } = part.copy;
|
|
306
|
-
const source = parseS3Uri(uri);
|
|
307
|
-
const result2 = await s3.send(
|
|
308
|
-
new UploadPartCopyCommand({
|
|
309
|
-
...target,
|
|
310
|
-
PartNumber: partNumber,
|
|
311
|
-
CopySource: encodeURI(`${source.bucket}/${source.key}`),
|
|
312
|
-
CopySourceRange: `bytes=${start}-${end - 1}`
|
|
313
|
-
})
|
|
314
|
-
);
|
|
315
|
-
return result2.CopyPartResult?.ETag;
|
|
316
|
-
}
|
|
317
|
-
const body = await Promise.all(
|
|
318
|
-
part.upload.map((s) => "bytes" in s ? s.bytes : getRange(s3, s.uri, s.start, s.end))
|
|
319
|
-
);
|
|
320
|
-
const result = await s3.send(
|
|
321
|
-
new UploadPartCommand({ ...target, PartNumber: partNumber, Body: Buffer.concat(body) })
|
|
322
|
-
);
|
|
323
|
-
return result.ETag;
|
|
324
|
-
}
|
|
325
|
-
async function sendParts(s3, parts, target) {
|
|
326
|
-
const settled = await Promise.allSettled(
|
|
327
|
-
parts.map(async (part, index) => {
|
|
328
|
-
const etag = await sendPart(s3, part, index + 1, target);
|
|
329
|
-
if (!etag) throw new Error(`[s3Stitch] part ${index + 1} returned no ETag`);
|
|
330
|
-
return etag;
|
|
331
|
-
})
|
|
332
|
-
);
|
|
333
|
-
const failure = settled.find(
|
|
334
|
-
(result) => result.status === "rejected"
|
|
335
|
-
);
|
|
336
|
-
if (failure) throw failure.reason;
|
|
337
|
-
return settled.map((result) => result.value);
|
|
338
|
-
}
|
|
339
|
-
async function uploadParts(s3, parts, destUri, contentType) {
|
|
340
|
-
const { bucket, key } = parseS3Uri(destUri);
|
|
341
|
-
const created = await s3.send(
|
|
342
|
-
new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: contentType })
|
|
343
|
-
);
|
|
344
|
-
if (!created.UploadId) throw new Error(`[s3Stitch] no upload id for ${destUri}`);
|
|
345
|
-
const target = { Bucket: bucket, Key: key, UploadId: created.UploadId };
|
|
346
|
-
try {
|
|
347
|
-
const etags = await sendParts(s3, parts, target);
|
|
348
|
-
await s3.send(
|
|
349
|
-
new CompleteMultipartUploadCommand({
|
|
350
|
-
...target,
|
|
351
|
-
MultipartUpload: { Parts: etags.map((ETag, index) => ({ ETag, PartNumber: index + 1 })) }
|
|
352
|
-
})
|
|
353
|
-
);
|
|
354
|
-
} catch (error) {
|
|
355
|
-
await s3.send(new AbortMultipartUploadCommand(target)).catch((abortError) => {
|
|
356
|
-
console.error(
|
|
357
|
-
`[s3Stitch] abort failed for ${destUri}; parts remain until a lifecycle rule removes them`,
|
|
358
|
-
abortError
|
|
359
|
-
);
|
|
360
|
-
});
|
|
361
|
-
throw error;
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
async function stitchMp4InS3(s3, chunkUris, destUri, minPartBytes = MIN_PART_BYTES) {
|
|
365
|
-
const layouts = await Promise.all(
|
|
366
|
-
chunkUris.map((uri) => readMp4Layout((start, end) => getRange(s3, uri, start, end)))
|
|
367
|
-
);
|
|
368
|
-
const segments = [
|
|
369
|
-
{ bytes: buildStitchedHeader(layouts) },
|
|
370
|
-
...layouts.map((layout, i) => ({
|
|
371
|
-
uri: chunkUris[i],
|
|
372
|
-
start: layout.payloadStart,
|
|
373
|
-
end: layout.payloadEnd
|
|
374
|
-
}))
|
|
375
|
-
];
|
|
376
|
-
const parts = planParts(segments, minPartBytes);
|
|
377
|
-
await uploadParts(s3, parts, destUri, "video/mp4");
|
|
378
|
-
const uploaded = parts.flatMap((part) => "upload" in part ? part.upload : []);
|
|
379
|
-
return {
|
|
380
|
-
fileSize: segments.reduce((total, s) => total + segmentSize(s), 0),
|
|
381
|
-
parts: parts.length,
|
|
382
|
-
downloadedBytes: uploaded.reduce((total, s) => total + ("uri" in s ? segmentSize(s) : 0), 0)
|
|
383
|
-
};
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// src/formatExtension.ts
|
|
387
|
-
var FORMAT_EXTENSIONS = {
|
|
388
|
-
mp4: ".mp4",
|
|
389
|
-
mov: ".mov",
|
|
390
|
-
webm: ".webm",
|
|
391
|
-
"png-sequence": ""
|
|
392
|
-
};
|
|
393
|
-
function formatExtension(format) {
|
|
394
|
-
return FORMAT_EXTENSIONS[format];
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// src/s3PlanV2Publisher.ts
|
|
398
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
399
|
-
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync as rmSync2, statSync as statSync2, writeFileSync } from "node:fs";
|
|
400
|
-
import { tmpdir } from "node:os";
|
|
401
|
-
import { join } from "node:path";
|
|
402
|
-
import {
|
|
403
|
-
PlanV2IntegrityError
|
|
404
|
-
} from "@hyperframes/producer/distributed";
|
|
405
|
-
function isRecord2(value) {
|
|
406
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
407
|
-
}
|
|
408
|
-
function assertSha2562(value, label) {
|
|
409
|
-
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) {
|
|
410
|
-
throw new PlanV2IntegrityError(`${label} must be a lowercase SHA-256 digest`);
|
|
411
|
-
}
|
|
412
|
-
return value;
|
|
413
|
-
}
|
|
414
|
-
function manifestDigests(manifestBytes) {
|
|
415
|
-
let value;
|
|
416
|
-
try {
|
|
417
|
-
value = JSON.parse(manifestBytes);
|
|
418
|
-
} catch {
|
|
419
|
-
throw new PlanV2IntegrityError("S3 publisher received invalid manifest JSON");
|
|
420
|
-
}
|
|
421
|
-
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
422
|
-
throw new PlanV2IntegrityError("S3 publisher manifest requires an artifacts array");
|
|
423
|
-
}
|
|
424
|
-
return new Set(
|
|
425
|
-
value.artifacts.map((artifact, index) => {
|
|
426
|
-
if (!isRecord2(artifact)) {
|
|
427
|
-
throw new PlanV2IntegrityError(`S3 publisher artifacts[${index}] must be an object`);
|
|
428
|
-
}
|
|
429
|
-
return assertSha2562(artifact.sha256, `S3 publisher artifacts[${index}].sha256`);
|
|
430
|
-
})
|
|
431
|
-
);
|
|
432
|
-
}
|
|
433
|
-
function trimTrailingSlash(value) {
|
|
434
|
-
let end = value.length;
|
|
435
|
-
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
436
|
-
return value.slice(0, end);
|
|
437
|
-
}
|
|
438
|
-
var S3PlanV2ArtifactPublisher = class {
|
|
439
|
-
artifactPrefix;
|
|
440
|
-
manifestUri;
|
|
441
|
-
#s3;
|
|
442
|
-
#temporaryRoot;
|
|
443
|
-
#publishedDigests = /* @__PURE__ */ new Set();
|
|
444
|
-
#state = "open";
|
|
445
|
-
constructor(options) {
|
|
446
|
-
const outputPrefix = `${trimTrailingSlash(options.planOutputS3Prefix)}/v2`;
|
|
447
|
-
parseS3Uri(outputPrefix);
|
|
448
|
-
this.#s3 = options.s3;
|
|
449
|
-
this.artifactPrefix = `${outputPrefix}/artifacts/sha256`;
|
|
450
|
-
this.manifestUri = `${outputPrefix}/manifest.json`;
|
|
451
|
-
this.#temporaryRoot = options.temporaryRoot ?? tmpdir();
|
|
452
|
-
mkdirSync2(this.#temporaryRoot, { recursive: true });
|
|
453
|
-
}
|
|
454
|
-
async putBlob(blob) {
|
|
455
|
-
this.#assertOpen("publish a blob");
|
|
456
|
-
const digest = assertSha2562(blob.sha256, "S3 published blob sha256");
|
|
457
|
-
if (this.#publishedDigests.has(digest)) return;
|
|
458
|
-
const sourceSize = statSync2(blob.sourcePath).size;
|
|
459
|
-
if (sourceSize !== blob.sizeBytes) {
|
|
460
|
-
throw new PlanV2IntegrityError(
|
|
461
|
-
`S3 published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
462
|
-
);
|
|
463
|
-
}
|
|
464
|
-
const uri = `${this.artifactPrefix}/${digest.slice(0, 2)}/${digest}`;
|
|
465
|
-
await uploadContentAddressedFileToS3(this.#s3, blob.sourcePath, uri, digest);
|
|
466
|
-
this.#publishedDigests.add(digest);
|
|
467
|
-
}
|
|
468
|
-
async commitManifest(manifestBytes) {
|
|
469
|
-
this.#assertOpen("commit a manifest");
|
|
470
|
-
for (const digest of manifestDigests(manifestBytes)) {
|
|
471
|
-
if (!this.#publishedDigests.has(digest)) {
|
|
472
|
-
throw new PlanV2IntegrityError(
|
|
473
|
-
`cannot commit S3 manifest before referenced blob is durable: ${digest}`
|
|
474
|
-
);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
const manifestDigest = createHash2("sha256").update(manifestBytes, "utf8").digest("hex");
|
|
478
|
-
const stagingDir = mkdtempSync(join(this.#temporaryRoot, "hf-plan-v2-manifest-"));
|
|
479
|
-
const manifestPath = join(stagingDir, "manifest.json");
|
|
480
|
-
try {
|
|
481
|
-
writeFileSync(manifestPath, manifestBytes, "utf8");
|
|
482
|
-
await uploadContentAddressedFileToS3(
|
|
483
|
-
this.#s3,
|
|
484
|
-
manifestPath,
|
|
485
|
-
this.manifestUri,
|
|
486
|
-
manifestDigest,
|
|
487
|
-
"application/json"
|
|
488
|
-
);
|
|
489
|
-
this.#state = "committed";
|
|
490
|
-
} finally {
|
|
491
|
-
rmSync2(stagingDir, { recursive: true, force: true });
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
async abort() {
|
|
495
|
-
if (this.#state === "open") this.#state = "aborted";
|
|
496
|
-
}
|
|
497
|
-
#assertOpen(operation) {
|
|
498
|
-
if (this.#state !== "open") {
|
|
499
|
-
throw new PlanV2IntegrityError(`cannot ${operation} after publisher is ${this.#state}`);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
};
|
|
503
|
-
|
|
504
|
-
// src/handler.ts
|
|
505
|
-
var cachedS3Client = null;
|
|
506
|
-
function getS3Client() {
|
|
507
|
-
if (cachedS3Client) return cachedS3Client;
|
|
508
|
-
cachedS3Client = new S3Client({});
|
|
509
|
-
return cachedS3Client;
|
|
510
|
-
}
|
|
511
|
-
async function handler(event, deps) {
|
|
512
|
-
const unwrapped = unwrapEvent(event);
|
|
513
|
-
validatePlanProtocolShape(unwrapped);
|
|
514
|
-
validateEventS3Uris(unwrapped);
|
|
515
|
-
primeRuntimeEnv();
|
|
516
|
-
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
|
|
517
|
-
try {
|
|
518
|
-
switch (unwrapped.Action) {
|
|
519
|
-
case "plan":
|
|
520
|
-
return await handlePlan(unwrapped, deps);
|
|
521
|
-
case "renderChunk":
|
|
522
|
-
return await handleRenderChunk(unwrapped, deps);
|
|
523
|
-
case "assemble":
|
|
524
|
-
return await handleAssemble(unwrapped, deps);
|
|
525
|
-
default: {
|
|
526
|
-
const _exhaustive = unwrapped;
|
|
527
|
-
throw new Error(
|
|
528
|
-
`[handler] unknown Action: ${JSON.stringify(
|
|
529
|
-
_exhaustive.Action
|
|
530
|
-
)}. Expected one of "plan", "renderChunk", "assemble".`
|
|
531
|
-
);
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
} catch (err) {
|
|
535
|
-
normalizeTerminalErrorName(err);
|
|
536
|
-
logEvent({
|
|
537
|
-
event: "handler_error",
|
|
538
|
-
action: unwrapped.Action,
|
|
539
|
-
input: summarizeEvent(unwrapped),
|
|
540
|
-
message: err instanceof Error ? err.message : String(err),
|
|
541
|
-
name: err instanceof Error ? err.name : void 0
|
|
542
|
-
});
|
|
543
|
-
throw err;
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
function normalizeTerminalErrorName(error) {
|
|
547
|
-
if (!error || typeof error !== "object") return;
|
|
548
|
-
const candidate = error;
|
|
549
|
-
if (candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" || candidate.code === "PLAN_TOO_LARGE" || candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" || candidate.code === "FONT_FETCH_FAILED" || candidate.code === "FONT_FETCH_UNAVAILABLE" || candidate.code === "NOT_MEDIA_PAYLOAD" || candidate.code === "VIDEO_SOURCE_UNRENDERABLE" || candidate.code === "VIDEO_EXTRACTION_FAILED" || candidate.code === "INVALID_VIDEO_METADATA" || candidate.code === "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED" || candidate.code === "CHUNK_EXCEEDS_WORKER_TIMEOUT") {
|
|
550
|
-
candidate.name = candidate.code;
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
var MAX_ENVELOPE_DEPTH = 4;
|
|
554
|
-
function unwrapEvent(event) {
|
|
555
|
-
let cursor = event;
|
|
556
|
-
for (let i = 0; i < MAX_ENVELOPE_DEPTH; i++) {
|
|
557
|
-
if (cursor && typeof cursor === "object") {
|
|
558
|
-
const obj = cursor;
|
|
559
|
-
if (typeof obj.Action === "string" && isLambdaAction(obj.Action)) {
|
|
560
|
-
return cursor;
|
|
561
|
-
}
|
|
562
|
-
if ("Payload" in obj) {
|
|
563
|
-
cursor = obj.Payload;
|
|
564
|
-
continue;
|
|
565
|
-
}
|
|
566
|
-
if ("Input" in obj) {
|
|
567
|
-
cursor = obj.Input;
|
|
568
|
-
continue;
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
break;
|
|
572
|
-
}
|
|
573
|
-
throw new Error(
|
|
574
|
-
`[handler] event has no recognised Action; unwrapped ${MAX_ENVELOPE_DEPTH} levels of Payload/Input without finding one.`
|
|
575
|
-
);
|
|
576
|
-
}
|
|
577
|
-
function isLambdaAction(value) {
|
|
578
|
-
return value === "plan" || value === "renderChunk" || value === "assemble";
|
|
579
|
-
}
|
|
580
|
-
function validatePlanProtocolShape(event) {
|
|
581
|
-
const raw = event;
|
|
582
|
-
const protocol = raw.PlanProtocol;
|
|
583
|
-
if (protocol !== void 0 && protocol !== "v1" && protocol !== "v2") {
|
|
584
|
-
const error = new Error(
|
|
585
|
-
`[handler] unsupported PlanProtocol ${JSON.stringify(protocol)}; expected "v1", "v2", or absent`
|
|
586
|
-
);
|
|
587
|
-
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
588
|
-
throw error;
|
|
589
|
-
}
|
|
590
|
-
if (event.Action === "plan") return;
|
|
591
|
-
const effectiveProtocol = protocol ?? "v2";
|
|
592
|
-
const hasV1Locator = typeof raw.PlanS3Uri === "string";
|
|
593
|
-
const hasV2Manifest = typeof raw.PlanV2ManifestS3Uri === "string";
|
|
594
|
-
const hasV2Prefix = typeof raw.PlanV2ArtifactS3Prefix === "string";
|
|
595
|
-
const valid = effectiveProtocol === "v2" ? !hasV1Locator && hasV2Manifest && hasV2Prefix : hasV1Locator && !hasV2Manifest && !hasV2Prefix;
|
|
596
|
-
if (!valid) {
|
|
597
|
-
const error = new Error(
|
|
598
|
-
`[handler] ${effectiveProtocol} ${event.Action} event has mixed or missing plan locators`
|
|
599
|
-
);
|
|
600
|
-
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
601
|
-
throw error;
|
|
602
|
-
}
|
|
603
|
-
if (effectiveProtocol === "v2" && event.Action === "assemble" && event.AudioS3Uri !== null) {
|
|
604
|
-
const error = new Error("[handler] v2 assemble audio must be materialized from the manifest");
|
|
605
|
-
error.name = "PLAN_PROTOCOL_UNSUPPORTED";
|
|
606
|
-
throw error;
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
function logEvent(payload) {
|
|
610
|
-
console.log(JSON.stringify(payload));
|
|
611
|
-
}
|
|
612
|
-
function summarizeEvent(event) {
|
|
613
|
-
switch (event.Action) {
|
|
614
|
-
case "plan":
|
|
615
|
-
return {
|
|
616
|
-
projectS3Uri: event.ProjectS3Uri,
|
|
617
|
-
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
618
|
-
planProtocol: event.PlanProtocol ?? "v2",
|
|
619
|
-
format: event.Config.format,
|
|
620
|
-
fps: event.Config.fps
|
|
621
|
-
};
|
|
622
|
-
case "renderChunk":
|
|
623
|
-
return {
|
|
624
|
-
planProtocol: event.PlanProtocol ?? "v2",
|
|
625
|
-
...event.PlanProtocol !== "v1" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
|
|
626
|
-
chunkIndex: event.ChunkIndex,
|
|
627
|
-
format: event.Format
|
|
628
|
-
};
|
|
629
|
-
case "assemble":
|
|
630
|
-
return {
|
|
631
|
-
planProtocol: event.PlanProtocol ?? "v2",
|
|
632
|
-
...event.PlanProtocol !== "v1" ? { planV2ManifestS3Uri: event.PlanV2ManifestS3Uri } : { planS3Uri: event.PlanS3Uri },
|
|
633
|
-
chunkCount: event.ChunkS3Uris.length,
|
|
634
|
-
hasAudio: event.AudioS3Uri !== null,
|
|
635
|
-
outputS3Uri: event.OutputS3Uri,
|
|
636
|
-
format: event.Format
|
|
637
|
-
};
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
var runtimeEnvPrimed = false;
|
|
641
|
-
function primeRuntimeEnv() {
|
|
642
|
-
if (runtimeEnvPrimed) return;
|
|
643
|
-
runtimeEnvPrimed = true;
|
|
644
|
-
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
|
645
|
-
const bin = join2(taskRoot, "bin");
|
|
646
|
-
if (existsSync3(bin)) {
|
|
647
|
-
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
var ACCOUNT_CONCURRENCY_SHARE = 0.9;
|
|
651
|
-
async function resolveWorkerConcurrency(event, deps) {
|
|
652
|
-
if (event.Config.workerConcurrency !== void 0) return event.Config.workerConcurrency;
|
|
653
|
-
try {
|
|
654
|
-
return await functionConcurrency(deps?.lambda);
|
|
655
|
-
} catch (err) {
|
|
656
|
-
logEvent({
|
|
657
|
-
event: "worker_concurrency",
|
|
658
|
-
level: "WARN",
|
|
659
|
-
source: "fallback",
|
|
660
|
-
concurrency: DEFAULT_MAX_PARALLEL_CHUNKS,
|
|
661
|
-
message: `Could not read the account's Lambda concurrency (${String(err)}); planning for ${DEFAULT_MAX_PARALLEL_CHUNKS} chunks at once. Allow lambda:GetAccountSettings on the function role, or set the concurrency explicitly.`
|
|
662
|
-
});
|
|
663
|
-
return DEFAULT_MAX_PARALLEL_CHUNKS;
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
async function functionConcurrency(lambda = new LambdaClient({})) {
|
|
667
|
-
const reserved = Number(process.env.HYPERFRAMES_LAMBDA_RESERVED_CONCURRENCY);
|
|
668
|
-
if (reserved > 0) return reserved;
|
|
669
|
-
const { AccountLimit } = await lambda.send(new GetAccountSettingsCommand({}));
|
|
670
|
-
const unreserved = AccountLimit?.UnreservedConcurrentExecutions;
|
|
671
|
-
if (unreserved === void 0) throw new Error("no UnreservedConcurrentExecutions in the reply");
|
|
672
|
-
const concurrency = Math.max(1, Math.floor(unreserved * ACCOUNT_CONCURRENCY_SHARE));
|
|
673
|
-
logEvent({ event: "worker_concurrency", source: "account", unreserved, concurrency });
|
|
674
|
-
return concurrency;
|
|
675
|
-
}
|
|
676
|
-
async function planConfig(event, deps) {
|
|
677
|
-
const config = {
|
|
678
|
-
...event.Config,
|
|
679
|
-
workerConcurrency: await resolveWorkerConcurrency(event, deps)
|
|
680
|
-
};
|
|
681
|
-
const remainingMs = deps?.getRemainingTimeInMillis?.();
|
|
682
|
-
if (config.workerTimeoutSeconds === void 0 && remainingMs !== void 0) {
|
|
683
|
-
config.workerTimeoutSeconds = Math.ceil(remainingMs / 1e3);
|
|
684
|
-
}
|
|
685
|
-
return config;
|
|
686
|
-
}
|
|
687
|
-
async function handlePlan(event, deps) {
|
|
688
|
-
if (event.PlanProtocol !== "v1") {
|
|
689
|
-
return handlePlanV2(event, deps);
|
|
690
|
-
}
|
|
691
|
-
const started = Date.now();
|
|
692
|
-
const config = await planConfig(event, deps);
|
|
693
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
694
|
-
const primitive = deps?.primitives?.plan ?? plan;
|
|
695
|
-
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
696
|
-
const chromePath = await resolveChromeExecutablePath();
|
|
697
|
-
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
698
|
-
}
|
|
699
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-"));
|
|
700
|
-
const projectArchive = join2(work, "project.tar.gz");
|
|
701
|
-
const projectDir = join2(work, "project");
|
|
702
|
-
const planDir = join2(work, "plan");
|
|
703
|
-
try {
|
|
704
|
-
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
705
|
-
await untarDirectory(projectArchive, projectDir);
|
|
706
|
-
const result = await primitive(projectDir, config, planDir, {
|
|
707
|
-
chunksCarryAudio: true
|
|
708
|
-
});
|
|
709
|
-
const planTar = join2(work, "plan.tar.gz");
|
|
710
|
-
await tarDirectory(planDir, planTar);
|
|
711
|
-
const planTarUri = `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
|
712
|
-
const audioPath = join2(planDir, PLAN_AUDIO_RELATIVE_PATH);
|
|
713
|
-
const hasAudio = existsSync3(audioPath) && statSync3(audioPath).size > 0;
|
|
714
|
-
const audioUri = hasAudio ? `${trimTrailingSlash2(event.PlanOutputS3Prefix)}/${PLAN_AUDIO_RELATIVE_PATH}` : null;
|
|
715
|
-
await Promise.all([
|
|
716
|
-
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
|
717
|
-
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null
|
|
718
|
-
]);
|
|
719
|
-
return {
|
|
720
|
-
Action: "plan",
|
|
721
|
-
PlanS3Uri: planTarUri,
|
|
722
|
-
PlanHash: result.planHash,
|
|
723
|
-
ChunkCount: result.chunkCount,
|
|
724
|
-
MaxChunkConcurrency: Math.min(config.workerConcurrency, result.chunkCount),
|
|
725
|
-
TotalFrames: result.totalFrames,
|
|
726
|
-
Fps: result.fps,
|
|
727
|
-
Width: result.width,
|
|
728
|
-
Height: result.height,
|
|
729
|
-
Format: result.format,
|
|
730
|
-
HasAudio: audioUri !== null,
|
|
731
|
-
AudioS3Uri: audioUri,
|
|
732
|
-
FfmpegVersion: result.ffmpegVersion,
|
|
733
|
-
ProducerVersion: result.producerVersion,
|
|
734
|
-
DurationMs: Date.now() - started
|
|
735
|
-
};
|
|
736
|
-
} finally {
|
|
737
|
-
cleanupDir(work);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
async function handlePlanV2(event, deps) {
|
|
741
|
-
const started = Date.now();
|
|
742
|
-
const config = await planConfig(event, deps);
|
|
743
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
744
|
-
const primitive = deps?.primitives?.planV2WithPublisher ?? planV2WithPublisher;
|
|
745
|
-
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
746
|
-
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
747
|
-
}
|
|
748
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-plan-v2-"));
|
|
749
|
-
const projectArchive = join2(work, "project.tar.gz");
|
|
750
|
-
const projectDir = join2(work, "project");
|
|
751
|
-
try {
|
|
752
|
-
let stageStarted = Date.now();
|
|
753
|
-
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
|
754
|
-
logEvent({ event: "plan_stage", stage: "download", ms: Date.now() - stageStarted });
|
|
755
|
-
stageStarted = Date.now();
|
|
756
|
-
await untarDirectory(projectArchive, projectDir);
|
|
757
|
-
logEvent({ event: "plan_stage", stage: "untar", ms: Date.now() - stageStarted });
|
|
758
|
-
const publisher = new S3PlanV2ArtifactPublisher({
|
|
759
|
-
s3,
|
|
760
|
-
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
|
761
|
-
temporaryRoot: work
|
|
762
|
-
});
|
|
763
|
-
const manifest = await primitive(projectDir, config, publisher, {
|
|
764
|
-
stagingParentDir: work,
|
|
765
|
-
chunksCarryAudio: true
|
|
766
|
-
});
|
|
767
|
-
return {
|
|
768
|
-
Action: "plan",
|
|
769
|
-
PlanProtocol: "v2",
|
|
770
|
-
PlanV2ManifestS3Uri: publisher.manifestUri,
|
|
771
|
-
PlanV2ArtifactS3Prefix: publisher.artifactPrefix,
|
|
772
|
-
PlanHash: manifest.planHash,
|
|
773
|
-
ChunkCount: manifest.chunkCount,
|
|
774
|
-
MaxChunkConcurrency: Math.min(config.workerConcurrency, manifest.chunkCount),
|
|
775
|
-
TotalFrames: manifest.totalFrames,
|
|
776
|
-
Fps: manifest.fps,
|
|
777
|
-
Width: manifest.width,
|
|
778
|
-
Height: manifest.height,
|
|
779
|
-
Format: manifest.format,
|
|
780
|
-
HasAudio: manifest.artifacts.some((artifact) => isPlanAudioArtifactPath(artifact.path)),
|
|
781
|
-
AudioS3Uri: null,
|
|
782
|
-
FfmpegVersion: manifest.ffmpegVersion,
|
|
783
|
-
ProducerVersion: manifest.producerVersion,
|
|
784
|
-
DurationMs: Date.now() - started
|
|
785
|
-
};
|
|
786
|
-
} finally {
|
|
787
|
-
cleanupDir(work);
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
async function handleRenderChunk(event, deps) {
|
|
791
|
-
if (event.PlanProtocol !== "v1") {
|
|
792
|
-
return handleRenderChunkV2(event, deps);
|
|
793
|
-
}
|
|
794
|
-
const started = Date.now();
|
|
795
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
796
|
-
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
797
|
-
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
798
|
-
const chromePath = await resolveChromeExecutablePath();
|
|
799
|
-
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
|
800
|
-
}
|
|
801
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-"));
|
|
802
|
-
const planTar = join2(work, "plan.tar.gz");
|
|
803
|
-
const planDir = join2(work, "plan");
|
|
804
|
-
try {
|
|
805
|
-
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
806
|
-
await untarDirectory(planTar, planDir);
|
|
807
|
-
verifyPlanHash(planDir, event.PlanHash);
|
|
808
|
-
const chunkOutputBase = join2(
|
|
809
|
-
work,
|
|
810
|
-
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
811
|
-
);
|
|
812
|
-
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
|
813
|
-
const chunkUri = await uploadChunkOutput(
|
|
814
|
-
s3,
|
|
815
|
-
result,
|
|
816
|
-
event.ChunkOutputS3Prefix,
|
|
817
|
-
event.ChunkIndex
|
|
818
|
-
);
|
|
819
|
-
return {
|
|
820
|
-
Action: "renderChunk",
|
|
821
|
-
ChunkS3Uri: chunkUri,
|
|
822
|
-
ChunkIndex: event.ChunkIndex,
|
|
823
|
-
Sha256: result.sha256,
|
|
824
|
-
FramesEncoded: result.framesEncoded,
|
|
825
|
-
CaptureMode: result.captureMode,
|
|
826
|
-
DurationMs: Date.now() - started
|
|
827
|
-
};
|
|
828
|
-
} finally {
|
|
829
|
-
cleanupDir(work);
|
|
830
|
-
}
|
|
831
|
-
}
|
|
832
|
-
async function handleRenderChunkV2(event, deps) {
|
|
833
|
-
const started = Date.now();
|
|
834
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
835
|
-
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
|
836
|
-
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
|
837
|
-
process.env.PRODUCER_HEADLESS_SHELL_PATH = await resolveChromeExecutablePath();
|
|
838
|
-
}
|
|
839
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-chunk-v2-"));
|
|
840
|
-
try {
|
|
841
|
-
const { planDir } = await downloadAndMaterializePlanV2(
|
|
842
|
-
s3,
|
|
843
|
-
event,
|
|
844
|
-
{ role: "chunk", chunkIndex: event.ChunkIndex },
|
|
845
|
-
work
|
|
846
|
-
);
|
|
847
|
-
const chunkOutputBase = join2(
|
|
848
|
-
work,
|
|
849
|
-
event.Format === "png-sequence" ? `chunk-${pad(event.ChunkIndex)}` : `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`
|
|
850
|
-
);
|
|
851
|
-
const result = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
|
852
|
-
const chunkUri = await uploadChunkOutput(
|
|
853
|
-
s3,
|
|
854
|
-
result,
|
|
855
|
-
event.ChunkOutputS3Prefix,
|
|
856
|
-
event.ChunkIndex
|
|
857
|
-
);
|
|
858
|
-
return {
|
|
859
|
-
Action: "renderChunk",
|
|
860
|
-
ChunkS3Uri: chunkUri,
|
|
861
|
-
ChunkIndex: event.ChunkIndex,
|
|
862
|
-
Sha256: result.sha256,
|
|
863
|
-
FramesEncoded: result.framesEncoded,
|
|
864
|
-
CaptureMode: result.captureMode,
|
|
865
|
-
DurationMs: Date.now() - started
|
|
866
|
-
};
|
|
867
|
-
} finally {
|
|
868
|
-
cleanupDir(work);
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
async function uploadChunkOutput(s3, result, prefix, chunkIndex) {
|
|
872
|
-
const trimmed = trimTrailingSlash2(prefix);
|
|
873
|
-
if (result.outputKind === "file") {
|
|
874
|
-
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
|
875
|
-
const uri2 = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
|
876
|
-
await uploadFileToS3(s3, result.outputPath, uri2);
|
|
877
|
-
return uri2;
|
|
878
|
-
}
|
|
879
|
-
const tarball = `${result.outputPath}.tar.gz`;
|
|
880
|
-
await tarDirectory(result.outputPath, tarball);
|
|
881
|
-
const uri = `${trimmed}/chunks/${pad(chunkIndex)}.tar.gz`;
|
|
882
|
-
await uploadFileToS3(s3, tarball, uri, "application/gzip");
|
|
883
|
-
return uri;
|
|
884
|
-
}
|
|
885
|
-
async function handleAssemble(event, deps) {
|
|
886
|
-
if (event.PlanProtocol !== "v1") {
|
|
887
|
-
return handleAssembleV2(event, deps);
|
|
888
|
-
}
|
|
889
|
-
const started = Date.now();
|
|
890
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
891
|
-
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
892
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-"));
|
|
893
|
-
const planTar = join2(work, "plan.tar.gz");
|
|
894
|
-
const planDir = join2(work, "plan");
|
|
895
|
-
try {
|
|
896
|
-
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
|
897
|
-
await untarDirectory(planTar, planDir);
|
|
898
|
-
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
899
|
-
let audioPath = null;
|
|
900
|
-
if (event.AudioS3Uri) {
|
|
901
|
-
audioPath = resolvePlanAudioPath(planDir) ?? join2(planDir, PLAN_AUDIO_RELATIVE_PATH);
|
|
902
|
-
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
|
903
|
-
}
|
|
904
|
-
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
905
|
-
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
906
|
-
cfr: event.Cfr === true
|
|
907
|
-
});
|
|
908
|
-
if (event.Format === "png-sequence") {
|
|
909
|
-
const tarball = `${finalOutput}.tar.gz`;
|
|
910
|
-
await tarDirectory(finalOutput, tarball);
|
|
911
|
-
await uploadFileToS3(s3, tarball, event.OutputS3Uri, "application/gzip");
|
|
912
|
-
} else {
|
|
913
|
-
await uploadFileToS3(s3, finalOutput, event.OutputS3Uri);
|
|
914
|
-
}
|
|
915
|
-
return {
|
|
916
|
-
Action: "assemble",
|
|
917
|
-
OutputS3Uri: event.OutputS3Uri,
|
|
918
|
-
FramesEncoded: result.framesEncoded,
|
|
919
|
-
FileSize: result.fileSize,
|
|
920
|
-
DurationMs: Date.now() - started
|
|
921
|
-
};
|
|
922
|
-
} finally {
|
|
923
|
-
cleanupDir(work);
|
|
924
|
-
}
|
|
925
|
-
}
|
|
926
|
-
async function handleAssembleV2(event, deps) {
|
|
927
|
-
const started = Date.now();
|
|
928
|
-
const s3 = deps?.s3 ?? getS3Client();
|
|
929
|
-
const primitive = deps?.primitives?.assemble ?? assemble;
|
|
930
|
-
const work = mkdtempSync2(join2(deps?.tmpRoot ?? tmpdir2(), "hf-lambda-assemble-v2-"));
|
|
931
|
-
try {
|
|
932
|
-
const { planDir, manifest } = await downloadAndMaterializePlanV2(
|
|
933
|
-
s3,
|
|
934
|
-
event,
|
|
935
|
-
{ role: "assembler" },
|
|
936
|
-
work
|
|
937
|
-
);
|
|
938
|
-
if (chunksAreStitchedMp4(planDir, event.Cfr === true)) {
|
|
939
|
-
return await stitchChunksInS3(s3, event, manifest, started);
|
|
940
|
-
}
|
|
941
|
-
const audioPath = resolvePlanAudioPath(planDir);
|
|
942
|
-
const chunksStarted = Date.now();
|
|
943
|
-
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
|
944
|
-
const chunksMs = Date.now() - chunksStarted;
|
|
945
|
-
const finalOutput = event.Format === "png-sequence" ? join2(work, "output-frames") : join2(work, `output${formatExtension(event.Format)}`);
|
|
946
|
-
const result = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
|
947
|
-
cfr: event.Cfr === true
|
|
948
|
-
});
|
|
949
|
-
const uploadStarted = Date.now();
|
|
950
|
-
if (event.Format === "png-sequence") {
|
|
951
|
-
const tarball = `${finalOutput}.tar.gz`;
|
|
952
|
-
await tarDirectory(finalOutput, tarball);
|
|
953
|
-
await uploadFileToS3(s3, tarball, event.OutputS3Uri, "application/gzip");
|
|
954
|
-
} else {
|
|
955
|
-
await uploadFileToS3(s3, finalOutput, event.OutputS3Uri);
|
|
956
|
-
}
|
|
957
|
-
logEvent({
|
|
958
|
-
event: "assemble_stages",
|
|
959
|
-
planMs: chunksStarted - started,
|
|
960
|
-
chunksMs,
|
|
961
|
-
assembleMs: result.durationMs,
|
|
962
|
-
uploadMs: Date.now() - uploadStarted
|
|
963
|
-
});
|
|
964
|
-
return {
|
|
965
|
-
Action: "assemble",
|
|
966
|
-
OutputS3Uri: event.OutputS3Uri,
|
|
967
|
-
FramesEncoded: result.framesEncoded,
|
|
968
|
-
FileSize: result.fileSize,
|
|
969
|
-
DurationMs: Date.now() - started
|
|
970
|
-
};
|
|
971
|
-
} finally {
|
|
972
|
-
cleanupDir(work);
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
async function stitchChunksInS3(s3, event, plan2, started) {
|
|
976
|
-
if (event.ChunkS3Uris.length !== plan2.chunkCount) {
|
|
977
|
-
throw new Error(
|
|
978
|
-
`[handler] ${event.ChunkS3Uris.length} chunk URIs for a plan of ${plan2.chunkCount} chunks`
|
|
979
|
-
);
|
|
980
|
-
}
|
|
981
|
-
const stitchStarted = Date.now();
|
|
982
|
-
const stitched = await stitchMp4InS3(s3, event.ChunkS3Uris, event.OutputS3Uri);
|
|
983
|
-
logEvent({
|
|
984
|
-
event: "assemble_stages",
|
|
985
|
-
mode: "s3-stitch",
|
|
986
|
-
planMs: stitchStarted - started,
|
|
987
|
-
stitchMs: Date.now() - stitchStarted,
|
|
988
|
-
parts: stitched.parts,
|
|
989
|
-
downloadedBytes: stitched.downloadedBytes
|
|
990
|
-
});
|
|
991
|
-
return {
|
|
992
|
-
Action: "assemble",
|
|
993
|
-
OutputS3Uri: event.OutputS3Uri,
|
|
994
|
-
FramesEncoded: plan2.totalFrames,
|
|
995
|
-
FileSize: stitched.fileSize,
|
|
996
|
-
DurationMs: Date.now() - started
|
|
997
|
-
};
|
|
998
|
-
}
|
|
999
|
-
async function downloadAndMaterializePlanV2(s3, event, target, work) {
|
|
1000
|
-
const transportDir = join2(work, "plan-v2");
|
|
1001
|
-
mkdirSync3(transportDir, { recursive: true });
|
|
1002
|
-
await downloadS3ObjectToFile(s3, event.PlanV2ManifestS3Uri, join2(transportDir, "plan.json"));
|
|
1003
|
-
const manifest = readPlanV2Manifest(transportDir);
|
|
1004
|
-
if (manifest.planHash !== event.PlanHash) {
|
|
1005
|
-
throwPlanHashMismatch(event.PlanHash, manifest.planHash);
|
|
1006
|
-
}
|
|
1007
|
-
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
1008
|
-
const uniqueArtifacts = [
|
|
1009
|
-
...new Map(artifacts.map((artifact) => [artifact.sha256, artifact])).values()
|
|
1010
|
-
];
|
|
1011
|
-
await mapConcurrent(uniqueArtifacts, 16, async (artifact) => {
|
|
1012
|
-
await downloadPlanV2Artifact(s3, event.PlanV2ArtifactS3Prefix, transportDir, artifact);
|
|
1013
|
-
});
|
|
1014
|
-
const planDir = join2(work, "plan");
|
|
1015
|
-
materializePlanV2Target(transportDir, target, planDir);
|
|
1016
|
-
return { planDir, manifest };
|
|
1017
|
-
}
|
|
1018
|
-
async function downloadPlanV2Artifact(s3, artifactPrefix, planV2Dir, artifact) {
|
|
1019
|
-
await downloadS3ObjectToFileVerified(
|
|
1020
|
-
s3,
|
|
1021
|
-
planV2BlobUri(artifactPrefix, artifact.sha256),
|
|
1022
|
-
planV2BlobPath(planV2Dir, artifact.sha256),
|
|
1023
|
-
artifact.sha256
|
|
1024
|
-
);
|
|
1025
|
-
}
|
|
1026
|
-
function planV2BlobPath(planV2Dir, digest) {
|
|
1027
|
-
return join2(planV2Dir, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
1028
|
-
}
|
|
1029
|
-
function planV2BlobUri(prefix, digest) {
|
|
1030
|
-
return `${trimTrailingSlash2(prefix)}/${digest.slice(0, 2)}/${digest}`;
|
|
1031
|
-
}
|
|
1032
|
-
function throwPlanHashMismatch(expected, actual) {
|
|
1033
|
-
const error = new Error(
|
|
1034
|
-
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match v2 manifest planHash=${actual}`
|
|
1035
|
-
);
|
|
1036
|
-
error.name = "PLAN_HASH_MISMATCH";
|
|
1037
|
-
throw error;
|
|
1038
|
-
}
|
|
1039
|
-
async function mapConcurrent(values, concurrency, fn) {
|
|
1040
|
-
let cursor = 0;
|
|
1041
|
-
async function worker() {
|
|
1042
|
-
while (cursor < values.length) {
|
|
1043
|
-
const index = cursor++;
|
|
1044
|
-
await fn(values[index]);
|
|
1045
|
-
}
|
|
1046
|
-
}
|
|
1047
|
-
const results = await Promise.allSettled(
|
|
1048
|
-
Array.from({ length: Math.min(concurrency, values.length) }, () => worker())
|
|
1049
|
-
);
|
|
1050
|
-
const failure = results.find(
|
|
1051
|
-
(result) => result.status === "rejected"
|
|
1052
|
-
);
|
|
1053
|
-
if (failure) throw failure.reason;
|
|
1054
|
-
}
|
|
1055
|
-
async function downloadChunkObjects(s3, uris, workDir, format) {
|
|
1056
|
-
const chunksDir = join2(workDir, "chunks");
|
|
1057
|
-
mkdirSync3(chunksDir, { recursive: true });
|
|
1058
|
-
const local = new Array(uris.length);
|
|
1059
|
-
await Promise.all(
|
|
1060
|
-
uris.map(async (uri, i) => {
|
|
1061
|
-
if (!uri) {
|
|
1062
|
-
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
|
1063
|
-
}
|
|
1064
|
-
const { key } = parseS3Uri(uri);
|
|
1065
|
-
const localPath = join2(chunksDir, basename(key));
|
|
1066
|
-
await downloadS3ObjectToFile(s3, uri, localPath);
|
|
1067
|
-
if (format === "png-sequence") {
|
|
1068
|
-
const dirPath = join2(chunksDir, `frames-${pad(i)}`);
|
|
1069
|
-
await untarDirectory(localPath, dirPath);
|
|
1070
|
-
local[i] = dirPath;
|
|
1071
|
-
} else {
|
|
1072
|
-
local[i] = localPath;
|
|
1073
|
-
}
|
|
1074
|
-
})
|
|
1075
|
-
);
|
|
1076
|
-
return local;
|
|
1077
|
-
}
|
|
1078
|
-
function getEventS3Uris(event) {
|
|
1079
|
-
switch (event.Action) {
|
|
1080
|
-
case "plan":
|
|
1081
|
-
return [event.ProjectS3Uri, event.PlanOutputS3Prefix];
|
|
1082
|
-
case "renderChunk":
|
|
1083
|
-
return event.PlanProtocol !== "v1" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix, event.ChunkOutputS3Prefix] : [event.PlanS3Uri, event.ChunkOutputS3Prefix];
|
|
1084
|
-
case "assemble":
|
|
1085
|
-
return [
|
|
1086
|
-
...event.PlanProtocol !== "v1" ? [event.PlanV2ManifestS3Uri, event.PlanV2ArtifactS3Prefix] : [event.PlanS3Uri],
|
|
1087
|
-
...event.ChunkS3Uris,
|
|
1088
|
-
event.OutputS3Uri,
|
|
1089
|
-
event.AudioS3Uri
|
|
1090
|
-
].filter((u) => u != null);
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
function validateEventS3Uris(event) {
|
|
1094
|
-
const allowedBucket = process.env.HYPERFRAMES_RENDER_BUCKET?.trim();
|
|
1095
|
-
if (!allowedBucket) return;
|
|
1096
|
-
for (const uri of getEventS3Uris(event)) {
|
|
1097
|
-
const { bucket } = parseS3Uri(uri);
|
|
1098
|
-
if (bucket !== allowedBucket) {
|
|
1099
|
-
const err = new Error(
|
|
1100
|
-
`[handler] S3_URI_NOT_ALLOWED: URI ${JSON.stringify(uri)} targets bucket "${bucket}" but only "${allowedBucket}" is permitted`
|
|
1101
|
-
);
|
|
1102
|
-
err.name = "S3_URI_NOT_ALLOWED";
|
|
1103
|
-
throw err;
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
}
|
|
1107
|
-
function pad(n) {
|
|
1108
|
-
return n.toString().padStart(4, "0");
|
|
1109
|
-
}
|
|
1110
|
-
function trimTrailingSlash2(prefix) {
|
|
1111
|
-
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
1112
|
-
}
|
|
1113
|
-
function cleanupDir(dir) {
|
|
1114
|
-
try {
|
|
1115
|
-
rmSync3(dir, { recursive: true, force: true });
|
|
1116
|
-
} catch {
|
|
1117
|
-
}
|
|
1118
|
-
}
|
|
1119
|
-
function verifyPlanHash(planDir, expected) {
|
|
1120
|
-
const planJsonPath = join2(planDir, "plan.json");
|
|
1121
|
-
let parsed;
|
|
1122
|
-
try {
|
|
1123
|
-
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8"));
|
|
1124
|
-
} catch (err) {
|
|
1125
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
1126
|
-
const error = new Error(`PLAN_HASH_MISMATCH: failed to read ${planJsonPath}: ${msg}`);
|
|
1127
|
-
error.name = "PLAN_HASH_MISMATCH";
|
|
1128
|
-
throw error;
|
|
1129
|
-
}
|
|
1130
|
-
const actual = parsed.planHash;
|
|
1131
|
-
if (typeof actual !== "string" || actual !== expected) {
|
|
1132
|
-
const error = new Error(
|
|
1133
|
-
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match plan.json planHash=${String(actual)}`
|
|
1134
|
-
);
|
|
1135
|
-
error.name = "PLAN_HASH_MISMATCH";
|
|
1136
|
-
throw error;
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
2
|
+
handler,
|
|
3
|
+
unwrapEvent
|
|
4
|
+
} from "./_chunks/chunk-AK7J3QQN.js";
|
|
5
|
+
import "./_chunks/chunk-ZY5YIH5W.js";
|
|
6
|
+
import "./_chunks/chunk-YS3KY3C5.js";
|
|
1139
7
|
export {
|
|
1140
8
|
handler,
|
|
1141
9
|
unwrapEvent
|