@hyperframes/producer 0.8.178 → 0.8.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distributed.js +2189 -2107
- package/dist/index.js +3450 -3368
- package/dist/public-server.js +11 -0
- package/dist/services/distributed/chunkAudio.d.ts +11 -0
- package/dist/services/distributed/mp4Stitch.d.ts +1 -0
- package/dist/services/distributed/plan.d.ts +2 -0
- package/package.json +7 -7
package/dist/distributed.js
CHANGED
|
@@ -3,14 +3,14 @@ import { audioFailureDiagnostics as audioFailureDiagnostics3 } from "@hyperframe
|
|
|
3
3
|
import {
|
|
4
4
|
copyFileSync as copyFileSync5,
|
|
5
5
|
cpSync as cpSync2,
|
|
6
|
-
existsSync as
|
|
6
|
+
existsSync as existsSync15,
|
|
7
7
|
linkSync,
|
|
8
|
-
mkdirSync as
|
|
9
|
-
renameSync as
|
|
10
|
-
rmSync as
|
|
11
|
-
writeFileSync as
|
|
8
|
+
mkdirSync as mkdirSync10,
|
|
9
|
+
renameSync as renameSync4,
|
|
10
|
+
rmSync as rmSync8,
|
|
11
|
+
writeFileSync as writeFileSync6
|
|
12
12
|
} from "node:fs";
|
|
13
|
-
import { extname as extname4, isAbsolute as isAbsolute3, join as
|
|
13
|
+
import { extname as extname4, isAbsolute as isAbsolute3, join as join22, relative as relative5, sep as sep2 } from "node:path";
|
|
14
14
|
import {
|
|
15
15
|
fpsToNumber as fpsToNumber10,
|
|
16
16
|
HF_COLOR_GRADING_ATTR as HF_COLOR_GRADING_ATTR2
|
|
@@ -7369,2151 +7369,2329 @@ function applyRuntimeEnvSnapshot(snapshot, env = process.env) {
|
|
|
7369
7369
|
};
|
|
7370
7370
|
}
|
|
7371
7371
|
|
|
7372
|
-
// src/services/distributed/
|
|
7373
|
-
import {
|
|
7374
|
-
import {
|
|
7375
|
-
import {
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7379
|
-
|
|
7380
|
-
|
|
7381
|
-
|
|
7382
|
-
|
|
7383
|
-
|
|
7384
|
-
|
|
7385
|
-
|
|
7386
|
-
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
7390
|
-
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
|
|
7394
|
-
|
|
7395
|
-
|
|
7396
|
-
|
|
7397
|
-
|
|
7398
|
-
|
|
7399
|
-
meta: "metadata",
|
|
7400
|
-
"plan.json": "metadata"
|
|
7401
|
-
};
|
|
7402
|
-
function hashComponent(value) {
|
|
7403
|
-
return createHash5("sha256").update(value).digest("hex").slice(0, 12);
|
|
7372
|
+
// src/services/distributed/chunkAudio.ts
|
|
7373
|
+
import { closeSync as closeSync4, existsSync as existsSync14, ftruncateSync, mkdirSync as mkdirSync9, openSync as openSync4, readFileSync as readFileSync9 } from "node:fs";
|
|
7374
|
+
import { writeFileSync as writeFileSync5, writeSync } from "node:fs";
|
|
7375
|
+
import { join as join20 } from "node:path";
|
|
7376
|
+
import { appendRenderProvenanceArgs, describeFfmpegFailure as describeFfmpegFailure2, runFfmpeg as runFfmpeg5 } from "@hyperframes/engine";
|
|
7377
|
+
|
|
7378
|
+
// src/services/distributed/mp4Stitch.ts
|
|
7379
|
+
import {
|
|
7380
|
+
createReadStream as createReadStream4,
|
|
7381
|
+
createWriteStream,
|
|
7382
|
+
mkdirSync as mkdirSync8,
|
|
7383
|
+
mkdtempSync as mkdtempSync3,
|
|
7384
|
+
renameSync as renameSync3,
|
|
7385
|
+
rmSync as rmSync7
|
|
7386
|
+
} from "node:fs";
|
|
7387
|
+
import { open } from "node:fs/promises";
|
|
7388
|
+
import { dirname as dirname9, join as join19 } from "node:path";
|
|
7389
|
+
import { finished, pipeline } from "node:stream/promises";
|
|
7390
|
+
function boxAt(src, off, at, limit) {
|
|
7391
|
+
let size = src.readUInt32BE(off);
|
|
7392
|
+
let bodyStart = at + 8;
|
|
7393
|
+
if (size === 1) {
|
|
7394
|
+
size = Number(src.readBigUInt64BE(off + 8));
|
|
7395
|
+
bodyStart = at + 16;
|
|
7396
|
+
}
|
|
7397
|
+
if (size < 8 || at + size > limit) throw new Error(`[mp4Stitch] malformed box at ${at}`);
|
|
7398
|
+
return { type: src.toString("latin1", off + 4, off + 8), start: at, bodyStart, end: at + size };
|
|
7404
7399
|
}
|
|
7405
|
-
function
|
|
7406
|
-
const
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
const videoKey = segments[1] ?? "unknown";
|
|
7410
|
-
return {
|
|
7411
|
-
category: "video-frames",
|
|
7412
|
-
componentLabel: `video-frames:${hashComponent(videoKey)}`
|
|
7413
|
-
};
|
|
7400
|
+
function childBoxes(buf, parent) {
|
|
7401
|
+
const boxes = [];
|
|
7402
|
+
for (let at = parent.bodyStart; at < parent.end; at = boxes.at(-1).end) {
|
|
7403
|
+
boxes.push(boxAt(buf, at, at, parent.end));
|
|
7414
7404
|
}
|
|
7415
|
-
|
|
7416
|
-
return { category, componentLabel: category };
|
|
7405
|
+
return boxes;
|
|
7417
7406
|
}
|
|
7418
|
-
function
|
|
7419
|
-
return
|
|
7420
|
-
totalBytes: 0,
|
|
7421
|
-
fileCount: 0,
|
|
7422
|
-
compiledBytes: 0,
|
|
7423
|
-
sourceMediaBytes: 0,
|
|
7424
|
-
videoFramesBytes: 0,
|
|
7425
|
-
videoFrameFileCount: 0,
|
|
7426
|
-
audioBytes: 0,
|
|
7427
|
-
metadataBytes: 0,
|
|
7428
|
-
otherBytes: 0,
|
|
7429
|
-
topComponents: []
|
|
7430
|
-
};
|
|
7407
|
+
function findChild(buf, parent, type) {
|
|
7408
|
+
return childBoxes(buf, parent).find((box2) => box2.type === type);
|
|
7431
7409
|
}
|
|
7432
|
-
function
|
|
7433
|
-
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
result.sourceMediaBytes += sizeBytes;
|
|
7437
|
-
}
|
|
7438
|
-
return;
|
|
7439
|
-
}
|
|
7440
|
-
if (category === "video-frames") {
|
|
7441
|
-
result.videoFramesBytes += sizeBytes;
|
|
7442
|
-
result.videoFrameFileCount += 1;
|
|
7443
|
-
return;
|
|
7444
|
-
}
|
|
7445
|
-
if (category === "audio") {
|
|
7446
|
-
result.audioBytes += sizeBytes;
|
|
7447
|
-
return;
|
|
7448
|
-
}
|
|
7449
|
-
if (category === "metadata") {
|
|
7450
|
-
result.metadataBytes += sizeBytes;
|
|
7451
|
-
return;
|
|
7452
|
-
}
|
|
7453
|
-
result.otherBytes += sizeBytes;
|
|
7410
|
+
function child(buf, parent, type) {
|
|
7411
|
+
const found = findChild(buf, parent, type);
|
|
7412
|
+
if (!found) throw new Error(`[mp4Stitch] ${parent.type} has no ${type}`);
|
|
7413
|
+
return found;
|
|
7454
7414
|
}
|
|
7455
|
-
|
|
7456
|
-
|
|
7457
|
-
|
|
7458
|
-
|
|
7459
|
-
|
|
7460
|
-
|
|
7461
|
-
|
|
7462
|
-
return;
|
|
7463
|
-
|
|
7464
|
-
|
|
7465
|
-
const segments = rel.split(/[\\/]+/).filter(Boolean);
|
|
7466
|
-
const classified = classifyFile(segments, rootKind);
|
|
7467
|
-
result.totalBytes += sizeBytes;
|
|
7468
|
-
result.fileCount += 1;
|
|
7469
|
-
addCategoryBytes(result, classified.category, filePath, sizeBytes);
|
|
7470
|
-
const component = components.get(classified.componentLabel) ?? {
|
|
7471
|
-
label: classified.componentLabel,
|
|
7472
|
-
sizeBytes: 0,
|
|
7473
|
-
fileCount: 0
|
|
7415
|
+
var SCAN_BYTES = 64 * 1024;
|
|
7416
|
+
async function readMp4Layout(read) {
|
|
7417
|
+
const found = /* @__PURE__ */ new Map();
|
|
7418
|
+
let payload;
|
|
7419
|
+
let buf = Buffer.alloc(0);
|
|
7420
|
+
let bufAt = 0;
|
|
7421
|
+
const load = async (start, end) => {
|
|
7422
|
+
if (start >= bufAt && end <= bufAt + buf.length) return;
|
|
7423
|
+
buf = await read(start, Math.max(end, start + SCAN_BYTES));
|
|
7424
|
+
bufAt = start;
|
|
7474
7425
|
};
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
7478
|
-
}
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
|
|
7485
|
-
}
|
|
7486
|
-
for (const entry of entries) {
|
|
7487
|
-
const full = join19(dir, entry.name);
|
|
7488
|
-
if (entry.isDirectory()) {
|
|
7489
|
-
walkRegularFiles(full, visit);
|
|
7490
|
-
} else if (entry.isFile()) {
|
|
7491
|
-
visit(full);
|
|
7426
|
+
for (let at = 0; !(found.has("ftyp") && found.has("moov") && payload); ) {
|
|
7427
|
+
await load(at, at + 16);
|
|
7428
|
+
if (at + 8 > bufAt + buf.length) {
|
|
7429
|
+
throw new Error(`[mp4Stitch] chunk needs ftyp, moov and mdat; found ${[...found.keys()]}`);
|
|
7430
|
+
}
|
|
7431
|
+
const box2 = boxAt(buf, at - bufAt, at, Number.MAX_SAFE_INTEGER);
|
|
7432
|
+
if (box2.type === "mdat") payload ??= box2;
|
|
7433
|
+
if (box2.type === "ftyp" || box2.type === "moov") {
|
|
7434
|
+
await load(at, box2.end);
|
|
7435
|
+
found.set(box2.type, buf.subarray(at - bufAt, box2.end - bufAt));
|
|
7492
7436
|
}
|
|
7437
|
+
at = box2.end;
|
|
7493
7438
|
}
|
|
7439
|
+
const [ftyp, moov] = [found.get("ftyp"), found.get("moov")];
|
|
7440
|
+
if (moov.length !== moovSize(moov)) throw new Error("[mp4Stitch] moov is cut off");
|
|
7441
|
+
return { ftyp, moov, payloadStart: payload.bodyStart, payloadEnd: payload.end };
|
|
7494
7442
|
}
|
|
7495
|
-
|
|
7496
|
-
|
|
7497
|
-
const
|
|
7498
|
-
|
|
7499
|
-
rootDir,
|
|
7500
|
-
(filePath) => recordFile(result, components, rootDir, rootKind, filePath)
|
|
7501
|
-
);
|
|
7502
|
-
result.topComponents = [...components.values()].sort((a, b) => b.sizeBytes - a.sizeBytes || a.label.localeCompare(b.label)).slice(0, MAX_TOP_COMPONENTS);
|
|
7503
|
-
return result;
|
|
7504
|
-
}
|
|
7505
|
-
|
|
7506
|
-
// src/services/distributed/plan.ts
|
|
7507
|
-
function logPlanStage(log, stage, startedAt) {
|
|
7508
|
-
log.info("[plan] stage", { stage, ms: Date.now() - startedAt });
|
|
7443
|
+
var moovSize = (moov) => boxAt(moov, 0, 0, Number.MAX_SAFE_INTEGER).end;
|
|
7444
|
+
function readTracks(moov) {
|
|
7445
|
+
const root = boxAt(moov, 0, 0, moov.length);
|
|
7446
|
+
return childBoxes(moov, root).filter((box2) => box2.type === "trak").map((trak) => readTrack(moov, trak));
|
|
7509
7447
|
}
|
|
7510
|
-
function
|
|
7511
|
-
const
|
|
7512
|
-
const
|
|
7513
|
-
|
|
7514
|
-
|
|
7515
|
-
|
|
7516
|
-
{
|
|
7517
|
-
|
|
7518
|
-
|
|
7519
|
-
details: {
|
|
7520
|
-
mediaType: "audio",
|
|
7521
|
-
audioFailures: audioFailureDiagnostics3(audioFailures),
|
|
7522
|
-
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
|
|
7523
|
-
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
|
|
7524
|
-
failureOwner,
|
|
7525
|
-
retryable
|
|
7526
|
-
}
|
|
7527
|
-
}
|
|
7528
|
-
],
|
|
7529
|
-
log
|
|
7448
|
+
function rows(buf, table, columns, countAt = 4) {
|
|
7449
|
+
const count = buf.readUInt32BE(table.bodyStart + countAt);
|
|
7450
|
+
const first = table.bodyStart + countAt + 4;
|
|
7451
|
+
return Array.from(
|
|
7452
|
+
{ length: count },
|
|
7453
|
+
(_, row) => Array.from(
|
|
7454
|
+
{ length: columns },
|
|
7455
|
+
(_2, col) => buf.readUInt32BE(first + (row * columns + col) * 4)
|
|
7456
|
+
)
|
|
7530
7457
|
);
|
|
7531
7458
|
}
|
|
7532
|
-
|
|
7533
|
-
"
|
|
7534
|
-
|
|
7535
|
-
"
|
|
7536
|
-
|
|
7537
|
-
|
|
7538
|
-
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
7542
|
-
|
|
7543
|
-
|
|
7544
|
-
var MIN_CHUNK_SIZE = 10;
|
|
7545
|
-
var MAX_PARALLEL_CHUNKS_CEILING = 256;
|
|
7546
|
-
var ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME = 0.11;
|
|
7547
|
-
var ESTIMATED_CHUNK_OVERHEAD_SECONDS = 60;
|
|
7548
|
-
var WORKER_TIMEOUT_MARGIN = 0.75;
|
|
7549
|
-
function estimateFrameSeconds(megapixels, measuredSecondsPerFrame = 0) {
|
|
7550
|
-
return Math.max(ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels, measuredSecondsPerFrame);
|
|
7551
|
-
}
|
|
7552
|
-
function estimateChunkSeconds(frames, megapixels, measuredSecondsPerFrame) {
|
|
7553
|
-
return ESTIMATED_CHUNK_OVERHEAD_SECONDS + frames * estimateFrameSeconds(megapixels, measuredSecondsPerFrame);
|
|
7459
|
+
function readMediaTime(moov, trak) {
|
|
7460
|
+
const edts = findChild(moov, trak, "edts");
|
|
7461
|
+
if (!edts) return 0;
|
|
7462
|
+
const elst = child(moov, edts, "elst");
|
|
7463
|
+
const wide = moov[elst.bodyStart] === 1;
|
|
7464
|
+
const entries = moov.readUInt32BE(elst.bodyStart + 4);
|
|
7465
|
+
for (let i = 0, at = elst.bodyStart + 8; i < entries; i++, at += wide ? 20 : 12) {
|
|
7466
|
+
const offset = at + (wide ? 8 : 4);
|
|
7467
|
+
const mediaTime = wide ? Number(moov.readBigInt64BE(offset)) : moov.readInt32BE(offset);
|
|
7468
|
+
if (mediaTime !== -1) return mediaTime;
|
|
7469
|
+
}
|
|
7470
|
+
return 0;
|
|
7554
7471
|
}
|
|
7555
|
-
function
|
|
7556
|
-
|
|
7472
|
+
function readSizes(moov, stsz) {
|
|
7473
|
+
const fixedSize = moov.readUInt32BE(stsz.bodyStart + 4);
|
|
7474
|
+
if (!fixedSize) return rows(moov, stsz, 1, 8).map(([size]) => size);
|
|
7475
|
+
return Array(moov.readUInt32BE(stsz.bodyStart + 8)).fill(fixedSize);
|
|
7557
7476
|
}
|
|
7558
|
-
function
|
|
7559
|
-
|
|
7477
|
+
function readSync2(moov, stbl, sampleCount) {
|
|
7478
|
+
const stss = findChild(moov, stbl, "stss");
|
|
7479
|
+
if (!stss) return null;
|
|
7480
|
+
const sync = Array(sampleCount).fill(false);
|
|
7481
|
+
for (const [n] of rows(moov, stss, 1)) sync[n - 1] = true;
|
|
7482
|
+
return sync;
|
|
7560
7483
|
}
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
|
|
7565
|
-
constructor(predictedChunkSeconds, workerTimeoutSeconds) {
|
|
7566
|
-
super(
|
|
7567
|
-
`[plan] each chunk is predicted to take about ${predictedChunkSeconds} s even split ${MAX_PARALLEL_CHUNKS_CEILING} ways, over the ${workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN} s budget (${WORKER_TIMEOUT_MARGIN * 100}% of the ${workerTimeoutSeconds} s worker time limit) (${CHUNK_EXCEEDS_WORKER_TIMEOUT}). Shorten the video or lower its resolution, raise the worker time limit, or pass an explicit chunkSize or maxParallelChunks to override the estimate.`
|
|
7568
|
-
);
|
|
7569
|
-
this.name = "ChunkExceedsWorkerTimeoutError";
|
|
7570
|
-
}
|
|
7571
|
-
};
|
|
7572
|
-
function resolveMaxParallelChunks(config, totalFrames, megapixels, log = defaultLogger, measuredSecondsPerFrame) {
|
|
7573
|
-
if (config.maxParallelChunks !== void 0) return config.maxParallelChunks;
|
|
7574
|
-
const { workerTimeoutSeconds, workerConcurrency } = config;
|
|
7575
|
-
const wave = Math.min(
|
|
7576
|
-
workerConcurrency ?? DEFAULT_MAX_PARALLEL_CHUNKS,
|
|
7577
|
-
MAX_PARALLEL_CHUNKS_CEILING
|
|
7578
|
-
);
|
|
7579
|
-
if (!chunkSizingUsesFrameCost(config)) return wave;
|
|
7580
|
-
const secondsPerFrame = estimateFrameSeconds(megapixels, measuredSecondsPerFrame);
|
|
7581
|
-
const minFramesPerChunk = Math.ceil(ESTIMATED_CHUNK_OVERHEAD_SECONDS / secondsPerFrame);
|
|
7582
|
-
const wide = Math.max(1, Math.min(wave, Math.floor(totalFrames / minFramesPerChunk)));
|
|
7583
|
-
if (workerTimeoutSeconds === void 0) return wide;
|
|
7584
|
-
const maxFramesPerChunk = Math.floor(
|
|
7585
|
-
(workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN - ESTIMATED_CHUNK_OVERHEAD_SECONDS) / secondsPerFrame
|
|
7586
|
-
);
|
|
7587
|
-
const needed = maxFramesPerChunk < 1 ? Infinity : Math.ceil(totalFrames / maxFramesPerChunk);
|
|
7588
|
-
if (needed > MAX_PARALLEL_CHUNKS_CEILING) {
|
|
7589
|
-
const framesAtCeiling = Math.ceil(totalFrames / MAX_PARALLEL_CHUNKS_CEILING);
|
|
7590
|
-
throw new ChunkExceedsWorkerTimeoutError(
|
|
7591
|
-
Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels, measuredSecondsPerFrame)),
|
|
7592
|
-
workerTimeoutSeconds
|
|
7593
|
-
);
|
|
7594
|
-
}
|
|
7595
|
-
if (needed > wave) {
|
|
7596
|
-
log.warn(
|
|
7597
|
-
`[plan] ${needed} chunks are needed to keep each under the ${workerTimeoutSeconds} s worker time limit, but the plan assumes ${wave} workers at once, so the render runs in ${Math.ceil(needed / wave)} waves. Raise --concurrency to ${needed} to render in one wave (Lambda: hyperframes lambda deploy --concurrency ${needed}, or a higher account concurrency quota).`,
|
|
7598
|
-
{ needed, workerConcurrency: wave, workerTimeoutSeconds }
|
|
7599
|
-
);
|
|
7600
|
-
}
|
|
7601
|
-
return Math.max(wide, needed);
|
|
7484
|
+
function readChunkOffsets(moov, stbl) {
|
|
7485
|
+
const stco = findChild(moov, stbl, "stco");
|
|
7486
|
+
if (stco) return rows(moov, stco, 1).map(([offset]) => offset);
|
|
7487
|
+
return rows(moov, child(moov, stbl, "co64"), 2).map(([hi, lo]) => hi * 2 ** 32 + lo);
|
|
7602
7488
|
}
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
observedAt;
|
|
7611
|
-
constructor(sizeBytes, limitBytes, breakdown, observedAt) {
|
|
7612
|
-
const breakdownSuffix = breakdown ? ` Breakdown: video-frames=${formatBytes(breakdown.videoFramesBytes)}, compiled=${formatBytes(breakdown.compiledBytes)} (source-media=${formatBytes(breakdown.sourceMediaBytes)}), audio=${formatBytes(breakdown.audioBytes)}, metadata=${formatBytes(breakdown.metadataBytes)}, other=${formatBytes(breakdown.otherBytes)}, files=${breakdown.fileCount}.` : "";
|
|
7613
|
-
const observationSuffix = observedAt ? ` Observed at ${observedAt}.` : "";
|
|
7614
|
-
super(
|
|
7615
|
-
`[plan] planDir size ${formatBytes(sizeBytes)} exceeds the configured ceiling ${formatBytes(limitBytes)} (PLAN_TOO_LARGE). The default 2 GB cap fits inside AWS Lambda's 10 GB /tmp budget alongside the chunk worker's frame buffer and ffmpeg's working set. To unblock: use the content-addressed \`planV2()\` transport, shorten the composition, lower the framerate, or use the in-process renderer (\`executeRenderJob\`), which has no planDir size cap.` + observationSuffix + breakdownSuffix
|
|
7616
|
-
);
|
|
7617
|
-
this.name = "PlanTooLargeError";
|
|
7618
|
-
this.sizeBytes = sizeBytes;
|
|
7619
|
-
this.limitBytes = limitBytes;
|
|
7620
|
-
this.breakdown = breakdown;
|
|
7621
|
-
this.observedAt = observedAt;
|
|
7622
|
-
}
|
|
7623
|
-
};
|
|
7624
|
-
var FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED = "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED";
|
|
7625
|
-
var FormatNotSupportedInDistributedError = class extends Error {
|
|
7626
|
-
// fallow-ignore-next-line unused-class-member
|
|
7627
|
-
code = FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED;
|
|
7628
|
-
format;
|
|
7629
|
-
reason;
|
|
7630
|
-
constructor(format, reason) {
|
|
7631
|
-
super(
|
|
7632
|
-
`[plan] format ${JSON.stringify(format)} is not supported in distributed mode: ${reason}. Render with the in-process renderer (\`executeRenderJob\`), which has full format support, or pick a distributed-supported format: mp4 SDR, mov ProRes 4444, png-sequence, or webm VP9.`
|
|
7633
|
-
);
|
|
7634
|
-
this.name = "FormatNotSupportedInDistributedError";
|
|
7635
|
-
this.format = format;
|
|
7636
|
-
this.reason = reason;
|
|
7637
|
-
}
|
|
7638
|
-
};
|
|
7639
|
-
function formatBytes(bytes) {
|
|
7640
|
-
if (bytes < 1024) return `${bytes} B`;
|
|
7641
|
-
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
7642
|
-
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
|
|
7643
|
-
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`;
|
|
7489
|
+
function readSamplesPerChunk(moov, stbl, chunkCount) {
|
|
7490
|
+
const stsc = rows(moov, child(moov, stbl, "stsc"), 3);
|
|
7491
|
+
return stsc.flatMap(([firstChunk, perChunk, description], i) => {
|
|
7492
|
+
if (description !== 1) throw new Error("[mp4Stitch] chunk uses more than one sample entry");
|
|
7493
|
+
const nextFirst = stsc[i + 1]?.[0] ?? chunkCount + 1;
|
|
7494
|
+
return Array(nextFirst - firstChunk).fill(perChunk);
|
|
7495
|
+
});
|
|
7644
7496
|
}
|
|
7645
|
-
function
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7649
|
-
|
|
7650
|
-
|
|
7497
|
+
function readTrack(moov, trak) {
|
|
7498
|
+
const mdia = child(moov, trak, "mdia");
|
|
7499
|
+
const hdlr = child(moov, mdia, "hdlr");
|
|
7500
|
+
const stbl = child(moov, child(moov, mdia, "minf"), "stbl");
|
|
7501
|
+
if (findChild(moov, stbl, "ctts")) throw new Error("[mp4Stitch] chunk has ctts (B-frames)");
|
|
7502
|
+
const stts = rows(moov, child(moov, stbl, "stts"), 2);
|
|
7503
|
+
const durations = stts.flatMap(([n, delta]) => Array(n).fill(delta));
|
|
7504
|
+
const sizes = readSizes(moov, child(moov, stbl, "stsz"));
|
|
7505
|
+
const chunkOffsets = readChunkOffsets(moov, stbl);
|
|
7506
|
+
const samplesPerChunk = readSamplesPerChunk(moov, stbl, chunkOffsets.length);
|
|
7507
|
+
if (durations.length !== sizes.length || sum(samplesPerChunk) !== sizes.length) {
|
|
7508
|
+
throw new Error("[mp4Stitch] sample tables disagree on the sample count");
|
|
7651
7509
|
}
|
|
7510
|
+
const stsd = child(moov, stbl, "stsd");
|
|
7511
|
+
return {
|
|
7512
|
+
trak,
|
|
7513
|
+
handler: moov.toString("latin1", hdlr.bodyStart + 8, hdlr.bodyStart + 12),
|
|
7514
|
+
timescale: timescaleOf(moov, child(moov, mdia, "mdhd")),
|
|
7515
|
+
mediaTime: readMediaTime(moov, trak),
|
|
7516
|
+
stsd: moov.subarray(stsd.start, stsd.end),
|
|
7517
|
+
durations,
|
|
7518
|
+
sizes,
|
|
7519
|
+
sync: readSync2(moov, stbl, sizes.length),
|
|
7520
|
+
samplesPerChunk,
|
|
7521
|
+
chunkOffsets
|
|
7522
|
+
};
|
|
7652
7523
|
}
|
|
7653
|
-
function
|
|
7654
|
-
return
|
|
7524
|
+
function sum(values) {
|
|
7525
|
+
return values.reduce((total, value) => total + value, 0);
|
|
7655
7526
|
}
|
|
7656
|
-
function
|
|
7657
|
-
const
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
sizeBytes: breakdown.totalBytes,
|
|
7662
|
-
limitBytes: input.limitBytes,
|
|
7663
|
-
fileCount: breakdown.fileCount,
|
|
7664
|
-
compiledBytes: breakdown.compiledBytes,
|
|
7665
|
-
sourceMediaBytes: breakdown.sourceMediaBytes,
|
|
7666
|
-
videoFramesBytes: breakdown.videoFramesBytes,
|
|
7667
|
-
videoFrameFileCount: breakdown.videoFrameFileCount,
|
|
7668
|
-
audioBytes: breakdown.audioBytes,
|
|
7669
|
-
metadataBytes: breakdown.metadataBytes,
|
|
7670
|
-
otherBytes: breakdown.otherBytes,
|
|
7671
|
-
topComponents: breakdown.topComponents
|
|
7672
|
-
});
|
|
7673
|
-
throw new PlanTooLargeError(breakdown.totalBytes, input.limitBytes, breakdown, input.observedAt);
|
|
7527
|
+
function box(type, ...body) {
|
|
7528
|
+
const header = Buffer.alloc(8);
|
|
7529
|
+
header.writeUInt32BE(8 + sum(body.map((part) => part.length)));
|
|
7530
|
+
header.write(type, 4, "latin1");
|
|
7531
|
+
return Buffer.concat([header, ...body]);
|
|
7674
7532
|
}
|
|
7675
|
-
function
|
|
7676
|
-
|
|
7677
|
-
assertPositiveInteger("maxParallelChunks", maxParallelChunks);
|
|
7678
|
-
if (configChunkSize !== void 0) {
|
|
7679
|
-
assertPositiveInteger("configChunkSize", configChunkSize);
|
|
7680
|
-
}
|
|
7681
|
-
if (targetChunkFrames !== void 0) {
|
|
7682
|
-
assertPositiveInteger("targetChunkFrames", targetChunkFrames);
|
|
7683
|
-
}
|
|
7684
|
-
const autoSizeParallel = targetChunkFrames === void 0 ? maxParallelChunks : Math.min(maxParallelChunks, Math.max(1, Math.ceil(totalFrames / targetChunkFrames)));
|
|
7685
|
-
const resolvedChunkSize = configChunkSize ?? Math.max(MIN_CHUNK_SIZE, Math.ceil(totalFrames / autoSizeParallel));
|
|
7686
|
-
const naiveCount = Math.ceil(totalFrames / resolvedChunkSize);
|
|
7687
|
-
const chunkCount = Math.min(maxParallelChunks, Math.max(1, naiveCount));
|
|
7688
|
-
const effectiveChunkSize = Math.max(resolvedChunkSize, Math.ceil(totalFrames / chunkCount));
|
|
7689
|
-
const tightChunkCount = Math.min(chunkCount, Math.ceil(totalFrames / effectiveChunkSize));
|
|
7690
|
-
return { chunkCount: tightChunkCount, effectiveChunkSize };
|
|
7533
|
+
function fullBox(type, version, body) {
|
|
7534
|
+
return box(type, Buffer.from([version, 0, 0, 0]), body);
|
|
7691
7535
|
}
|
|
7692
|
-
function
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
);
|
|
7697
|
-
}
|
|
7536
|
+
function u32s(values) {
|
|
7537
|
+
const buf = Buffer.alloc(values.length * 4);
|
|
7538
|
+
values.forEach((value, i) => buf.writeUInt32BE(value, i * 4));
|
|
7539
|
+
return buf;
|
|
7698
7540
|
}
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
];
|
|
7705
|
-
function removeFreezeOwnedPlanFiles(planDir) {
|
|
7706
|
-
for (const relativePath of FREEZE_OWNED_PLAN_FILES) {
|
|
7707
|
-
rmSync7(join20(planDir, relativePath), { force: true });
|
|
7541
|
+
function runs(values) {
|
|
7542
|
+
const out = [];
|
|
7543
|
+
for (const value of values) {
|
|
7544
|
+
const last = out.at(-1);
|
|
7545
|
+
if (last && last[1] === value) last[0]++;
|
|
7546
|
+
else out.push([1, value]);
|
|
7708
7547
|
}
|
|
7548
|
+
return out;
|
|
7709
7549
|
}
|
|
7710
|
-
function
|
|
7711
|
-
const
|
|
7712
|
-
|
|
7713
|
-
|
|
7714
|
-
|
|
7715
|
-
slices.push({ index: i, startFrame, endFrame });
|
|
7716
|
-
}
|
|
7717
|
-
return slices;
|
|
7550
|
+
function rebuild(buf, parent, replace) {
|
|
7551
|
+
const parts = childBoxes(buf, parent).map(
|
|
7552
|
+
(box2) => replace[box2.type]?.(box2) ?? buf.subarray(box2.start, box2.end)
|
|
7553
|
+
);
|
|
7554
|
+
return box(parent.type, ...parts);
|
|
7718
7555
|
}
|
|
7719
|
-
|
|
7720
|
-
|
|
7721
|
-
|
|
7722
|
-
|
|
7723
|
-
|
|
7724
|
-
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
const
|
|
7728
|
-
const
|
|
7729
|
-
|
|
7730
|
-
|
|
7731
|
-
|
|
7732
|
-
|
|
7733
|
-
return sha256Hex(canonicalJsonStringify(fingerprint));
|
|
7556
|
+
var DURATION_AT = {
|
|
7557
|
+
mvhd: [16, 24],
|
|
7558
|
+
mdhd: [16, 24],
|
|
7559
|
+
tkhd: [20, 28]
|
|
7560
|
+
};
|
|
7561
|
+
function withDuration(buf, target, duration) {
|
|
7562
|
+
const copy = Buffer.from(buf.subarray(target.start, target.end));
|
|
7563
|
+
const body = target.bodyStart - target.start;
|
|
7564
|
+
const wide = copy[body] === 1;
|
|
7565
|
+
const at = body + DURATION_AT[target.type][wide ? 1 : 0];
|
|
7566
|
+
if (wide) copy.writeBigUInt64BE(BigInt(duration), at);
|
|
7567
|
+
else if (duration <= 4294967295) copy.writeUInt32BE(duration, at);
|
|
7568
|
+
else throw new Error(`[mp4Stitch] ${target.type} duration ${duration} needs a version 1 box`);
|
|
7569
|
+
return copy;
|
|
7734
7570
|
}
|
|
7735
|
-
function
|
|
7736
|
-
|
|
7737
|
-
const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);
|
|
7738
|
-
const locksVp9CpuUsed = encoder === "libvpx-vp9-software" ? { vp9CpuUsed: normalizeVp9CpuUsed(input.engineConfig.vp9CpuUsed) } : {};
|
|
7739
|
-
return {
|
|
7740
|
-
captureMode: forceScreenshot ? "screenshot" : "beginframe",
|
|
7741
|
-
forceScreenshot,
|
|
7742
|
-
deviceScaleFactor,
|
|
7743
|
-
useLayeredHdrComposite: false,
|
|
7744
|
-
browserGpuMode: "software",
|
|
7745
|
-
// Match `LOCKED_WARMUP_TICKS` in `frameCapture.ts` — kept as a literal so
|
|
7746
|
-
// a worker that ships a different value will trip `PLAN_HASH_MISMATCH`
|
|
7747
|
-
// (the locked config flows into planHash via the canonical JSON).
|
|
7748
|
-
warmupTicks: 60,
|
|
7749
|
-
encoder,
|
|
7750
|
-
quality: config.quality ?? "standard",
|
|
7751
|
-
ffmpegVersion,
|
|
7752
|
-
preset,
|
|
7753
|
-
crf: config.crf,
|
|
7754
|
-
bitrate: config.bitrate,
|
|
7755
|
-
...locksVp9CpuUsed,
|
|
7756
|
-
// GOP === chunkSize so every chunk's first frame is an IDR keyframe and
|
|
7757
|
-
// ffmpeg concat-copy round-trips losslessly.
|
|
7758
|
-
gopSize: input.effectiveChunkSize,
|
|
7759
|
-
closedGop: true,
|
|
7760
|
-
forceKeyframes: "n=0",
|
|
7761
|
-
pixelFormat,
|
|
7762
|
-
chunkSize: input.effectiveChunkSize,
|
|
7763
|
-
chunkCount: input.chunkCount,
|
|
7764
|
-
runtimeEnv: input.runtimeEnv,
|
|
7765
|
-
variables: config.variables,
|
|
7766
|
-
...input.chunksCarryAudio ? { chunksCarryAudio: true } : {}
|
|
7767
|
-
};
|
|
7768
|
-
}
|
|
7769
|
-
function carriesChunkAudio(config, options) {
|
|
7770
|
-
return options.chunksCarryAudio === true && config.format === "mp4" && config.cfr !== true;
|
|
7571
|
+
function timescaleOf(buf, header) {
|
|
7572
|
+
return buf.readUInt32BE(header.bodyStart + (buf[header.bodyStart] === 1 ? 20 : 12));
|
|
7771
7573
|
}
|
|
7772
|
-
function
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
);
|
|
7574
|
+
function buildStbl(stsd, track) {
|
|
7575
|
+
const stts = runs(track.durations);
|
|
7576
|
+
const stsc = [];
|
|
7577
|
+
let firstChunk = 1;
|
|
7578
|
+
for (const [count, perChunk] of runs(track.samplesPerChunk)) {
|
|
7579
|
+
stsc.push([firstChunk, perChunk, 1]);
|
|
7580
|
+
firstChunk += count;
|
|
7780
7581
|
}
|
|
7781
|
-
|
|
7582
|
+
const co64 = Buffer.alloc(4 + track.chunkOffsets.length * 8);
|
|
7583
|
+
co64.writeUInt32BE(track.chunkOffsets.length);
|
|
7584
|
+
track.chunkOffsets.forEach((offset, i) => co64.writeBigUInt64BE(BigInt(offset), 4 + i * 8));
|
|
7585
|
+
const syncSamples = track.sync?.flatMap((isSync, i) => isSync ? [i + 1] : []) ?? null;
|
|
7586
|
+
return box(
|
|
7587
|
+
"stbl",
|
|
7588
|
+
stsd,
|
|
7589
|
+
fullBox("stts", 0, u32s([stts.length, ...stts.flat()])),
|
|
7590
|
+
...syncSamples ? [fullBox("stss", 0, u32s([syncSamples.length, ...syncSamples]))] : [],
|
|
7591
|
+
fullBox("stsc", 0, u32s([stsc.length, ...stsc.flat()])),
|
|
7592
|
+
fullBox("stsz", 0, u32s([0, track.sizes.length, ...track.sizes])),
|
|
7593
|
+
fullBox("co64", 0, co64)
|
|
7594
|
+
);
|
|
7782
7595
|
}
|
|
7783
|
-
|
|
7784
|
-
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
|
|
7788
|
-
|
|
7789
|
-
|
|
7790
|
-
|
|
7791
|
-
|
|
7596
|
+
var codecOf = (track) => track.stsd.toString("latin1", 20, 24);
|
|
7597
|
+
var sameTrack = (track, plan2) => track.handler === plan2.handler && track.timescale === plan2.timescale && codecOf(track) === codecOf(plan2);
|
|
7598
|
+
function planTracksOf(tracks, plan2, index) {
|
|
7599
|
+
let next = 0;
|
|
7600
|
+
const matched = plan2.map((planTrack) => {
|
|
7601
|
+
const track = tracks[next];
|
|
7602
|
+
if (track && sameTrack(track, planTrack)) return tracks[next++];
|
|
7603
|
+
if (planTrack.handler === "soun") return null;
|
|
7604
|
+
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
7605
|
+
});
|
|
7606
|
+
if (next !== tracks.length)
|
|
7607
|
+
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
7608
|
+
for (const track of tracks) {
|
|
7609
|
+
if (track.handler === "vide" && track.mediaTime !== 0) {
|
|
7610
|
+
throw new Error(`[mp4Stitch] chunk ${index} video starts at media time ${track.mediaTime}`);
|
|
7611
|
+
}
|
|
7792
7612
|
}
|
|
7793
|
-
return
|
|
7613
|
+
return matched;
|
|
7794
7614
|
}
|
|
7795
|
-
function
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
|
|
7800
|
-
|
|
7801
|
-
|
|
7802
|
-
|
|
7803
|
-
pixelFormat: enginePreset.pixelFormat,
|
|
7804
|
-
preset: enginePreset.preset
|
|
7805
|
-
};
|
|
7615
|
+
function buildStitchedHeader(layouts) {
|
|
7616
|
+
const plan2 = readTracks(layouts[0].moov);
|
|
7617
|
+
const chunks = layouts.map((layout, i) => planTracksOf(readTracks(layout.moov), plan2, i));
|
|
7618
|
+
const payloadBases = [];
|
|
7619
|
+
let payloadBytes = 0;
|
|
7620
|
+
for (const layout of layouts) {
|
|
7621
|
+
payloadBases.push(payloadBytes);
|
|
7622
|
+
payloadBytes += layout.payloadEnd - layout.payloadStart;
|
|
7806
7623
|
}
|
|
7807
|
-
|
|
7624
|
+
const template = layouts[0].moov;
|
|
7625
|
+
const root = boxAt(template, 0, 0, template.length);
|
|
7626
|
+
const movieTimescale = timescaleOf(template, child(template, root, "mvhd"));
|
|
7627
|
+
const buildMoov = (dataStart2) => {
|
|
7628
|
+
const traks = plan2.map((track, t) => {
|
|
7629
|
+
const parts = chunks.flatMap((tracks, chunk) => {
|
|
7630
|
+
const part = tracks[t];
|
|
7631
|
+
return part ? [{ ...part, chunk }] : [];
|
|
7632
|
+
});
|
|
7633
|
+
const merged = {
|
|
7634
|
+
durations: parts.flatMap((part) => part.durations),
|
|
7635
|
+
sizes: parts.flatMap((part) => part.sizes),
|
|
7636
|
+
sync: parts.every((part) => !part.sync) ? null : parts.flatMap((part) => part.sync ?? part.sizes.map(() => true)),
|
|
7637
|
+
samplesPerChunk: parts.flatMap((part) => part.samplesPerChunk),
|
|
7638
|
+
chunkOffsets: parts.flatMap(
|
|
7639
|
+
(part) => part.chunkOffsets.map((offset) => {
|
|
7640
|
+
const i = part.chunk;
|
|
7641
|
+
const { payloadStart, payloadEnd } = layouts[i];
|
|
7642
|
+
if (offset < payloadStart || offset >= payloadEnd) {
|
|
7643
|
+
throw new Error(`[mp4Stitch] chunk ${i} has samples outside its mdat`);
|
|
7644
|
+
}
|
|
7645
|
+
return offset - payloadStart + payloadBases[i] + dataStart2;
|
|
7646
|
+
})
|
|
7647
|
+
)
|
|
7648
|
+
};
|
|
7649
|
+
const mediaDuration = sum(merged.durations);
|
|
7650
|
+
const shown = Math.round(
|
|
7651
|
+
(mediaDuration - track.mediaTime) * movieTimescale / track.timescale
|
|
7652
|
+
);
|
|
7653
|
+
const elst = Buffer.alloc(24);
|
|
7654
|
+
elst.writeUInt32BE(1);
|
|
7655
|
+
elst.writeBigUInt64BE(BigInt(shown), 4);
|
|
7656
|
+
elst.writeBigInt64BE(BigInt(track.mediaTime), 12);
|
|
7657
|
+
elst.writeUInt32BE(65536, 20);
|
|
7658
|
+
const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);
|
|
7659
|
+
const trak = rebuild(template, track.trak, {
|
|
7660
|
+
tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),
|
|
7661
|
+
edts: () => Buffer.alloc(0),
|
|
7662
|
+
mdia: (mdia) => rebuild(template, mdia, {
|
|
7663
|
+
mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),
|
|
7664
|
+
minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) })
|
|
7665
|
+
})
|
|
7666
|
+
});
|
|
7667
|
+
return { trak, shown };
|
|
7668
|
+
});
|
|
7669
|
+
let next = 0;
|
|
7670
|
+
return rebuild(template, root, {
|
|
7671
|
+
mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),
|
|
7672
|
+
trak: () => traks[next++].trak
|
|
7673
|
+
});
|
|
7674
|
+
};
|
|
7675
|
+
const ftyp = layouts[0].ftyp;
|
|
7676
|
+
const dataStart = ftyp.length + buildMoov(0).length + 16;
|
|
7677
|
+
const mdat = Buffer.alloc(16);
|
|
7678
|
+
mdat.writeUInt32BE(1);
|
|
7679
|
+
mdat.write("mdat", 4, "latin1");
|
|
7680
|
+
mdat.writeBigUInt64BE(BigInt(16 + payloadBytes), 8);
|
|
7681
|
+
return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);
|
|
7808
7682
|
}
|
|
7809
|
-
function
|
|
7810
|
-
return {
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
7814
|
-
|
|
7815
|
-
|
|
7816
|
-
|
|
7817
|
-
|
|
7818
|
-
|
|
7683
|
+
function fileRange(path) {
|
|
7684
|
+
return async (start, end) => {
|
|
7685
|
+
const file = await open(path, "r", 384);
|
|
7686
|
+
try {
|
|
7687
|
+
const buf = Buffer.alloc(end - start);
|
|
7688
|
+
const { bytesRead } = await file.read(buf, 0, buf.length, start);
|
|
7689
|
+
return buf.subarray(0, bytesRead);
|
|
7690
|
+
} finally {
|
|
7691
|
+
await file.close();
|
|
7692
|
+
}
|
|
7819
7693
|
};
|
|
7820
7694
|
}
|
|
7821
|
-
function
|
|
7822
|
-
const
|
|
7823
|
-
|
|
7695
|
+
async function copyPayload(path, layout, index, out) {
|
|
7696
|
+
const { payloadStart, payloadEnd } = layout;
|
|
7697
|
+
const payload = createReadStream4(path, { start: payloadStart, end: payloadEnd - 1 });
|
|
7698
|
+
await pipeline(payload, out, { end: false });
|
|
7699
|
+
if (payload.bytesRead !== payloadEnd - payloadStart) {
|
|
7700
|
+
throw new Error(
|
|
7701
|
+
`[mp4Stitch] chunk ${index} mdat holds ${payload.bytesRead} of ${payloadEnd - payloadStart} payload bytes`
|
|
7702
|
+
);
|
|
7703
|
+
}
|
|
7824
7704
|
}
|
|
7825
|
-
function
|
|
7826
|
-
|
|
7827
|
-
|
|
7828
|
-
|
|
7705
|
+
async function writeStitchedMp4(chunkPaths, outputPath) {
|
|
7706
|
+
const layouts = await Promise.all(chunkPaths.map((path) => readMp4Layout(fileRange(path))));
|
|
7707
|
+
const header = buildStitchedHeader(layouts);
|
|
7708
|
+
mkdirSync8(dirname9(outputPath), { recursive: true });
|
|
7709
|
+
const workDir = mkdtempSync3(`${outputPath}.stitch-`);
|
|
7710
|
+
const out = createWriteStream(join19(workDir, "out.mp4"));
|
|
7711
|
+
try {
|
|
7712
|
+
out.write(header);
|
|
7713
|
+
for (const [i, path] of chunkPaths.entries()) await copyPayload(path, layouts[i], i, out);
|
|
7714
|
+
out.end();
|
|
7715
|
+
await finished(out);
|
|
7716
|
+
renameSync3(join19(workDir, "out.mp4"), outputPath);
|
|
7717
|
+
} finally {
|
|
7718
|
+
out.destroy();
|
|
7719
|
+
rmSync7(workDir, { recursive: true, force: true });
|
|
7829
7720
|
}
|
|
7830
|
-
return srcPath;
|
|
7831
7721
|
}
|
|
7832
|
-
|
|
7833
|
-
|
|
7834
|
-
|
|
7835
|
-
|
|
7836
|
-
|
|
7837
|
-
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
|
|
7846
|
-
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
try {
|
|
7852
|
-
linkSync(current, destination);
|
|
7853
|
-
} catch {
|
|
7854
|
-
copyFileSync5(current, destination);
|
|
7855
|
-
}
|
|
7856
|
-
}
|
|
7857
|
-
shipped.set(srcPath, result);
|
|
7858
|
-
return result;
|
|
7859
|
-
};
|
|
7722
|
+
|
|
7723
|
+
// src/services/distributed/chunkAudio.ts
|
|
7724
|
+
function audioCutForChunk(grid, chunks, index, fps) {
|
|
7725
|
+
const firstPacketAt = (frame) => Math.min(
|
|
7726
|
+
grid.packetCount,
|
|
7727
|
+
Math.max(
|
|
7728
|
+
0,
|
|
7729
|
+
Math.ceil(
|
|
7730
|
+
(frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) / (grid.packetSamples * fps.num)
|
|
7731
|
+
)
|
|
7732
|
+
)
|
|
7733
|
+
);
|
|
7734
|
+
const first = index === 0 ? 0 : firstPacketAt(chunks[index].startFrame);
|
|
7735
|
+
const end = index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1].startFrame);
|
|
7736
|
+
if (end <= first) return null;
|
|
7737
|
+
const cutSeconds = (packet) => (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;
|
|
7738
|
+
const seekSeconds = index === 0 ? null : cutSeconds(first);
|
|
7739
|
+
const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);
|
|
7740
|
+
return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
|
|
7860
7741
|
}
|
|
7861
|
-
async function
|
|
7862
|
-
const
|
|
7863
|
-
|
|
7864
|
-
|
|
7865
|
-
|
|
7866
|
-
|
|
7867
|
-
|
|
7868
|
-
|
|
7869
|
-
|
|
7870
|
-
|
|
7871
|
-
|
|
7872
|
-
|
|
7873
|
-
|
|
7874
|
-
|
|
7875
|
-
|
|
7742
|
+
async function probeAacPacketGrid(audioPath) {
|
|
7743
|
+
const probe = await runFfprobeJson([
|
|
7744
|
+
"-v",
|
|
7745
|
+
"error",
|
|
7746
|
+
"-select_streams",
|
|
7747
|
+
"a:0",
|
|
7748
|
+
"-show_entries",
|
|
7749
|
+
"stream=sample_rate,time_base,nb_frames:packet=pts,duration",
|
|
7750
|
+
"-read_intervals",
|
|
7751
|
+
"%+#1",
|
|
7752
|
+
"-of",
|
|
7753
|
+
"json",
|
|
7754
|
+
"--",
|
|
7755
|
+
audioPath
|
|
7756
|
+
]);
|
|
7757
|
+
const stream = probe.streams?.[0] ?? {};
|
|
7758
|
+
const packet = probe.packets?.[0] ?? {};
|
|
7759
|
+
const sampleRate = Number(stream.sample_rate);
|
|
7760
|
+
const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);
|
|
7761
|
+
const toSamples = (ticks) => Number(ticks) * sampleRate * tbNum / tbDen;
|
|
7762
|
+
const grid = {
|
|
7763
|
+
sampleRate,
|
|
7764
|
+
firstStart: toSamples(packet.pts),
|
|
7765
|
+
packetSamples: toSamples(packet.duration),
|
|
7766
|
+
packetCount: Number(stream.nb_frames)
|
|
7876
7767
|
};
|
|
7877
|
-
const
|
|
7878
|
-
|
|
7879
|
-
|
|
7880
|
-
quality: config.quality ?? "standard",
|
|
7881
|
-
format: config.format,
|
|
7882
|
-
crf: config.crf,
|
|
7883
|
-
bitrate: config.bitrate,
|
|
7884
|
-
videoFrameFormat: config.videoFrameFormat,
|
|
7885
|
-
outputResolution: config.outputResolution,
|
|
7886
|
-
outputResolutionAspectAgnostic: config.outputResolutionAspectAgnostic,
|
|
7887
|
-
// HDR is banned in distributed mode. force-sdr keeps the
|
|
7888
|
-
// extract / encoder paths off the HDR branches entirely.
|
|
7889
|
-
hdrMode: config.hdrMode ?? "force-sdr",
|
|
7890
|
-
strictness: config.strictness,
|
|
7891
|
-
entryFile: config.entryFile ?? "index.html",
|
|
7892
|
-
logger: config.logger,
|
|
7893
|
-
producerConfig: cfg,
|
|
7894
|
-
variables: config.variables
|
|
7895
|
-
});
|
|
7896
|
-
const entryFile = config.entryFile ?? "index.html";
|
|
7897
|
-
const htmlPath = join20(projectDir, entryFile);
|
|
7898
|
-
if (!existsSync14(htmlPath)) {
|
|
7899
|
-
throw new Error(`[plan] entry file not found: ${htmlPath}`);
|
|
7768
|
+
const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];
|
|
7769
|
+
if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {
|
|
7770
|
+
throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);
|
|
7900
7771
|
}
|
|
7901
|
-
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
7905
|
-
|
|
7906
|
-
|
|
7907
|
-
|
|
7908
|
-
|
|
7909
|
-
|
|
7910
|
-
|
|
7911
|
-
|
|
7912
|
-
|
|
7913
|
-
|
|
7914
|
-
|
|
7772
|
+
return grid;
|
|
7773
|
+
}
|
|
7774
|
+
var SLICE_DIR = "audio-chunks";
|
|
7775
|
+
var SLICE_FRAME = `${SLICE_DIR}/frame.m4a`;
|
|
7776
|
+
var SLICE_LAYOUT = `${SLICE_DIR}/layout.json`;
|
|
7777
|
+
function chunkAudioSlicePath(index) {
|
|
7778
|
+
return `${SLICE_DIR}/${String(index).padStart(4, "0")}.bin`;
|
|
7779
|
+
}
|
|
7780
|
+
function chunkAudioSliceTarget(path) {
|
|
7781
|
+
if (path === SLICE_FRAME || path === SLICE_LAYOUT) return "all";
|
|
7782
|
+
const match = /^audio-chunks\/(\d{4,})\.bin$/.exec(path);
|
|
7783
|
+
return match ? Number(match[1]) : null;
|
|
7784
|
+
}
|
|
7785
|
+
async function writeChunkAudioSlices(input) {
|
|
7786
|
+
const grid = await probeAacPacketGrid(input.audioPath);
|
|
7787
|
+
const { payloadStart, payloadEnd, moov } = await readMp4Layout(fileRange(input.audioPath));
|
|
7788
|
+
const [track] = readTracks(moov);
|
|
7789
|
+
const offsets = [];
|
|
7790
|
+
track.chunkOffsets.forEach((offset, chunk) => {
|
|
7791
|
+
for (let i = 0; i < track.samplesPerChunk[chunk]; i++) {
|
|
7792
|
+
offsets.push(offset);
|
|
7793
|
+
offset += track.sizes[offsets.length - 1];
|
|
7915
7794
|
}
|
|
7916
7795
|
});
|
|
7917
|
-
|
|
7918
|
-
const
|
|
7919
|
-
|
|
7920
|
-
|
|
7921
|
-
|
|
7922
|
-
|
|
7923
|
-
|
|
7924
|
-
|
|
7925
|
-
|
|
7926
|
-
|
|
7927
|
-
|
|
7928
|
-
stageStarted = Date.now();
|
|
7929
|
-
const compileResult = await runCompileStage({
|
|
7930
|
-
urlByAbsentLocalPath,
|
|
7931
|
-
projectDir,
|
|
7932
|
-
workDir,
|
|
7933
|
-
htmlPath,
|
|
7934
|
-
entryFile,
|
|
7935
|
-
job,
|
|
7936
|
-
cfg,
|
|
7937
|
-
needsAlpha,
|
|
7938
|
-
log,
|
|
7939
|
-
assertNotAborted,
|
|
7940
|
-
abortSignal,
|
|
7941
|
-
// Distributed renders fail closed on font-fetch errors so the planDir
|
|
7942
|
-
// is content-addressed against deterministic fonts only.
|
|
7943
|
-
failClosedFontFetch: config.failClosedFontFetch !== false,
|
|
7944
|
-
// Distributed renders must not capture host-specific system fonts —
|
|
7945
|
-
// the Lambda/worker filesystem won't have the same fonts installed.
|
|
7946
|
-
allowSystemFontCapture: false,
|
|
7947
|
-
variables: config.variables
|
|
7948
|
-
});
|
|
7949
|
-
logPlanStage(log, "compile", stageStarted);
|
|
7950
|
-
let compiled = compileResult.compiled;
|
|
7951
|
-
const composition = compileResult.composition;
|
|
7952
|
-
const { deviceScaleFactor } = compileResult;
|
|
7953
|
-
let forceScreenshot = compileResult.forceScreenshot;
|
|
7954
|
-
if (cfg.lowMemoryMode) forceScreenshot = true;
|
|
7955
|
-
const { width, height } = composition;
|
|
7956
|
-
if (config.rejectOnSystemFonts !== false) {
|
|
7957
|
-
validateNoSystemFonts(compiled.html);
|
|
7958
|
-
}
|
|
7959
|
-
stageStarted = Date.now();
|
|
7960
|
-
const probeResult = await runProbeStage({
|
|
7961
|
-
projectDir,
|
|
7962
|
-
workDir,
|
|
7963
|
-
job,
|
|
7964
|
-
cfg,
|
|
7965
|
-
forceScreenshot,
|
|
7966
|
-
log,
|
|
7967
|
-
assertNotAborted,
|
|
7968
|
-
abortSignal,
|
|
7969
|
-
compiled,
|
|
7970
|
-
composition,
|
|
7971
|
-
width,
|
|
7972
|
-
height,
|
|
7973
|
-
needsAlpha,
|
|
7974
|
-
deviceScaleFactor,
|
|
7975
|
-
subTimelineWaitMemo: {}
|
|
7976
|
-
});
|
|
7977
|
-
logPlanStage(log, "probe", stageStarted);
|
|
7978
|
-
compiled = probeResult.compiled;
|
|
7979
|
-
job.duration = probeResult.duration;
|
|
7980
|
-
job.totalFrames = probeResult.totalFrames;
|
|
7981
|
-
const totalFrames = probeResult.totalFrames;
|
|
7982
|
-
validateDistributedDuration({
|
|
7983
|
-
duration: probeResult.duration,
|
|
7984
|
-
totalFrames,
|
|
7985
|
-
fps: fpsToNumber10(job.config.fps)
|
|
7796
|
+
const bytes = readFileSync9(input.audioPath);
|
|
7797
|
+
const layout = { sizeBytes: bytes.length, payloadStart, payloadEnd, partStarts: {} };
|
|
7798
|
+
mkdirSync9(join20(input.planDir, SLICE_DIR), { recursive: true });
|
|
7799
|
+
input.chunks.forEach((_, index) => {
|
|
7800
|
+
const cut = audioCutForChunk(grid, input.chunks, index, input.fps);
|
|
7801
|
+
if (!cut) return;
|
|
7802
|
+
const last = cut.firstPacket + cut.packetCount - 1;
|
|
7803
|
+
const start = offsets[cut.firstPacket];
|
|
7804
|
+
layout.partStarts[index] = start;
|
|
7805
|
+
const end = offsets[last] + track.sizes[last];
|
|
7806
|
+
writeFileSync5(join20(input.planDir, chunkAudioSlicePath(index)), bytes.subarray(start, end));
|
|
7986
7807
|
});
|
|
7987
|
-
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
7991
|
-
|
|
7992
|
-
|
|
7993
|
-
|
|
7994
|
-
|
|
7995
|
-
|
|
7808
|
+
const frame = [bytes.subarray(0, payloadStart), bytes.subarray(payloadEnd)];
|
|
7809
|
+
writeFileSync5(join20(input.planDir, SLICE_FRAME), Buffer.concat(frame));
|
|
7810
|
+
writeFileSync5(join20(input.planDir, SLICE_LAYOUT), JSON.stringify(layout));
|
|
7811
|
+
}
|
|
7812
|
+
function chunkAudioSource(planDir, chunkIndex, scratchPath) {
|
|
7813
|
+
const layoutPath = join20(planDir, SLICE_LAYOUT);
|
|
7814
|
+
if (!existsSync14(layoutPath)) return resolvePlanAudioPath(planDir);
|
|
7815
|
+
const layout = JSON.parse(readFileSync9(layoutPath, "utf-8"));
|
|
7816
|
+
const frame = readFileSync9(join20(planDir, SLICE_FRAME));
|
|
7817
|
+
const fd = openSync4(scratchPath, "w");
|
|
7818
|
+
try {
|
|
7819
|
+
writeSync(fd, frame, 0, layout.payloadStart, 0);
|
|
7820
|
+
writeSync(
|
|
7821
|
+
fd,
|
|
7822
|
+
frame,
|
|
7823
|
+
layout.payloadStart,
|
|
7824
|
+
frame.length - layout.payloadStart,
|
|
7825
|
+
layout.payloadEnd
|
|
7826
|
+
);
|
|
7827
|
+
const start = layout.partStarts[chunkIndex];
|
|
7828
|
+
if (start !== void 0) {
|
|
7829
|
+
const part = readFileSync9(join20(planDir, chunkAudioSlicePath(chunkIndex)));
|
|
7830
|
+
writeSync(fd, part, 0, part.length, start);
|
|
7996
7831
|
}
|
|
7832
|
+
ftruncateSync(fd, layout.sizeBytes);
|
|
7833
|
+
} finally {
|
|
7834
|
+
closeSync4(fd);
|
|
7997
7835
|
}
|
|
7998
|
-
|
|
7999
|
-
|
|
8000
|
-
|
|
8001
|
-
|
|
8002
|
-
|
|
8003
|
-
|
|
8004
|
-
|
|
8005
|
-
|
|
8006
|
-
|
|
8007
|
-
|
|
8008
|
-
stageStarted = Date.now();
|
|
8009
|
-
const extractResult = await runExtractVideosStage({
|
|
8010
|
-
projectDir,
|
|
8011
|
-
compiledDir,
|
|
8012
|
-
job,
|
|
8013
|
-
cfg,
|
|
8014
|
-
composition,
|
|
8015
|
-
abortSignal,
|
|
8016
|
-
assertNotAborted,
|
|
8017
|
-
materializeSymlinks: true,
|
|
8018
|
-
deferRangeExtraction: options.deferVideoExtraction,
|
|
8019
|
-
urlByAbsentLocalPath
|
|
8020
|
-
});
|
|
8021
|
-
logPlanStage(log, "extract", stageStarted);
|
|
8022
|
-
if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
|
|
8023
|
-
if (extractResult.extractionResult) {
|
|
8024
|
-
assertVideoExtractionSucceeded(extractResult.extractionResult);
|
|
8025
|
-
}
|
|
8026
|
-
const stagedVideoFrames = join20(compiledDir, "__hyperframes_video_frames");
|
|
8027
|
-
const videoFramesDst = join20(planDir, "video-frames");
|
|
8028
|
-
if (existsSync14(videoFramesDst)) rmSync7(videoFramesDst, { recursive: true, force: true });
|
|
8029
|
-
const videoSourcesDir = join20(planDir, "video-sources");
|
|
8030
|
-
rmSync7(videoSourcesDir, { recursive: true, force: true });
|
|
8031
|
-
if (existsSync14(stagedVideoFrames)) {
|
|
8032
|
-
renameSync3(stagedVideoFrames, videoFramesDst);
|
|
8033
|
-
} else {
|
|
8034
|
-
mkdirSync8(videoFramesDst, { recursive: true });
|
|
8035
|
-
}
|
|
8036
|
-
if (existsSync14(finalCompiledDir)) rmSync7(finalCompiledDir, { recursive: true, force: true });
|
|
8037
|
-
renameSync3(compiledDir, finalCompiledDir);
|
|
8038
|
-
const shipVideoSource = createVideoSourceShipper({
|
|
8039
|
-
planDir,
|
|
8040
|
-
projectDir,
|
|
8041
|
-
workCompiledDir: compiledDir,
|
|
8042
|
-
workVideoFramesDir: stagedVideoFrames,
|
|
8043
|
-
isRemote: (srcPath) => urlByAbsentLocalPath.has(srcPath)
|
|
8044
|
-
});
|
|
8045
|
-
const sourceRoots = [
|
|
8046
|
-
[stagedVideoFrames, "video-frames"],
|
|
8047
|
-
[compiledDir, "compiled"],
|
|
8048
|
-
[projectDir, ""]
|
|
7836
|
+
return scratchPath;
|
|
7837
|
+
}
|
|
7838
|
+
function chunkMuxArgs(input, cut) {
|
|
7839
|
+
if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];
|
|
7840
|
+
const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];
|
|
7841
|
+
const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];
|
|
7842
|
+
const exact = seek.length ? ["-copypriorss", "0"] : [];
|
|
7843
|
+
return [
|
|
7844
|
+
...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath],
|
|
7845
|
+
...["-map", "0:v:0", "-map", "1:a:0", ...exact]
|
|
8049
7846
|
];
|
|
8050
|
-
|
|
8051
|
-
|
|
8052
|
-
|
|
8053
|
-
|
|
8054
|
-
|
|
8055
|
-
|
|
8056
|
-
|
|
8057
|
-
|
|
8058
|
-
|
|
8059
|
-
|
|
8060
|
-
|
|
8061
|
-
metadata: ext.metadata,
|
|
8062
|
-
...ext.deferredRange ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(sourcePath) } } : {}
|
|
8063
|
-
};
|
|
8064
|
-
})
|
|
8065
|
-
});
|
|
8066
|
-
mkdirSync8(join20(planDir, "meta"), { recursive: true });
|
|
8067
|
-
writePlanRemoteFiles(
|
|
8068
|
-
planDir,
|
|
8069
|
-
remoteFiles.map(([rel, { sha256, sizeBytes }]) => ({
|
|
8070
|
-
path: `compiled/${rel}`,
|
|
8071
|
-
sha256,
|
|
8072
|
-
sizeBytes
|
|
8073
|
-
}))
|
|
8074
|
-
);
|
|
8075
|
-
writeFileSync5(
|
|
8076
|
-
join20(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),
|
|
8077
|
-
JSON.stringify(planVideosJson, null, 2),
|
|
8078
|
-
"utf-8"
|
|
8079
|
-
);
|
|
8080
|
-
const digests = options.digests ?? /* @__PURE__ */ new Map();
|
|
8081
|
-
const chunksCarryAudio2 = carriesChunkAudio(config, options);
|
|
8082
|
-
const audioSeconds = chunksCarryAudio2 ? totalFrames / config.fps : probeResult.duration;
|
|
8083
|
-
stageStarted = Date.now();
|
|
8084
|
-
const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([
|
|
8085
|
-
runAudioStage({
|
|
8086
|
-
projectDir,
|
|
8087
|
-
workDir,
|
|
8088
|
-
compiledDir: finalCompiledDir,
|
|
8089
|
-
urlByAbsentLocalPath,
|
|
8090
|
-
duration: audioSeconds,
|
|
8091
|
-
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
|
|
8092
|
-
audioGain: cfg.audioGain,
|
|
8093
|
-
audios: composition.audios,
|
|
8094
|
-
masterFxChain: composition.masterFxChain,
|
|
8095
|
-
abortSignal,
|
|
8096
|
-
assertNotAborted
|
|
8097
|
-
}).finally(() => logPlanStage(log, "audio", stageStarted)),
|
|
8098
|
-
(async () => {
|
|
8099
|
-
for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {
|
|
8100
|
-
await prehashPlanFiles(dir, digests);
|
|
8101
|
-
}
|
|
8102
|
-
logPlanStage(log, "prehash", stageStarted);
|
|
8103
|
-
if (options.whileAudioMixes) {
|
|
8104
|
-
const prepublishStarted = Date.now();
|
|
8105
|
-
await options.whileAudioMixes();
|
|
8106
|
-
logPlanStage(log, "prepublish", prepublishStarted);
|
|
8107
|
-
}
|
|
8108
|
-
})(),
|
|
8109
|
-
measureFrameCost ? measurePlanSecondsPerFrame({
|
|
8110
|
-
planDir,
|
|
8111
|
-
workDir,
|
|
8112
|
-
html: compiled.html,
|
|
8113
|
-
planVideos: planVideosJson,
|
|
8114
|
-
totalFrames,
|
|
8115
|
-
fps: config.fps,
|
|
8116
|
-
width,
|
|
8117
|
-
height,
|
|
8118
|
-
format: config.format,
|
|
8119
|
-
quality: config.quality ?? "standard",
|
|
8120
|
-
deviceScaleFactor,
|
|
8121
|
-
variables: config.variables,
|
|
8122
|
-
cfg: { ...cfg, forceScreenshot, forceScreenshotExplicitlyOptedOut: !forceScreenshot },
|
|
8123
|
-
log
|
|
8124
|
-
}).finally(() => logPlanStage(log, "frame-cost", stageStarted)) : void 0
|
|
8125
|
-
]);
|
|
8126
|
-
if (audioOutcome.status === "rejected") throw audioOutcome.reason;
|
|
8127
|
-
if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;
|
|
8128
|
-
if (frameCostOutcome.status === "rejected") throw frameCostOutcome.reason;
|
|
8129
|
-
if (frameCostOutcome.value !== void 0) {
|
|
8130
|
-
log.info("[plan] measured frame cost", { secondsPerFrame: frameCostOutcome.value });
|
|
8131
|
-
maxParallel = resolveMaxParallelChunks(
|
|
8132
|
-
config,
|
|
8133
|
-
totalFrames,
|
|
8134
|
-
megapixels,
|
|
8135
|
-
log,
|
|
8136
|
-
frameCostOutcome.value
|
|
7847
|
+
}
|
|
7848
|
+
async function muxChunkWithAudio(input) {
|
|
7849
|
+
const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;
|
|
7850
|
+
const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;
|
|
7851
|
+
const args = [...chunkMuxArgs(input, cut), "-c", "copy"];
|
|
7852
|
+
appendRenderProvenanceArgs(args, input.outputPath);
|
|
7853
|
+
args.push("-y", input.outputPath);
|
|
7854
|
+
const result = await runFfmpeg5(args, { signal: input.signal, timeout: input.timeoutMs });
|
|
7855
|
+
if (!result.success) {
|
|
7856
|
+
throw new Error(
|
|
7857
|
+
`[chunkAudio] chunk mux failed: ${describeFfmpegFailure2(result, input.timeoutMs)}`
|
|
8137
7858
|
);
|
|
8138
7859
|
}
|
|
8139
|
-
const
|
|
8140
|
-
|
|
8141
|
-
|
|
8142
|
-
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8146
|
-
|
|
8147
|
-
|
|
8148
|
-
|
|
8149
|
-
|
|
8150
|
-
|
|
8151
|
-
|
|
8152
|
-
|
|
8153
|
-
|
|
8154
|
-
|
|
8155
|
-
|
|
8156
|
-
|
|
8157
|
-
|
|
8158
|
-
targetDurationSeconds: audioSeconds,
|
|
8159
|
-
signal: abortSignal,
|
|
8160
|
-
timeoutMs: cfg.ffmpegProcessTimeout
|
|
8161
|
-
});
|
|
8162
|
-
if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
|
|
8163
|
-
logPlanStage(log, "audio-peak", stageStarted);
|
|
7860
|
+
const slice = input.chunks[input.chunkIndex];
|
|
7861
|
+
const expected = { video: slice.endFrame - slice.startFrame };
|
|
7862
|
+
if (cut) expected.audio = cut.packetCount;
|
|
7863
|
+
const probe = await runFfprobeJson([
|
|
7864
|
+
"-v",
|
|
7865
|
+
"error",
|
|
7866
|
+
"-count_packets",
|
|
7867
|
+
"-show_entries",
|
|
7868
|
+
"stream=codec_type,nb_read_packets",
|
|
7869
|
+
"-of",
|
|
7870
|
+
"json",
|
|
7871
|
+
"--",
|
|
7872
|
+
input.outputPath
|
|
7873
|
+
]);
|
|
7874
|
+
for (const [type, count] of Object.entries(expected)) {
|
|
7875
|
+
const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);
|
|
7876
|
+
if (packets !== count) {
|
|
7877
|
+
throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);
|
|
7878
|
+
}
|
|
8164
7879
|
}
|
|
8165
|
-
|
|
8166
|
-
|
|
8167
|
-
|
|
8168
|
-
|
|
8169
|
-
|
|
8170
|
-
|
|
8171
|
-
|
|
8172
|
-
|
|
8173
|
-
|
|
8174
|
-
|
|
8175
|
-
|
|
8176
|
-
|
|
8177
|
-
|
|
8178
|
-
|
|
8179
|
-
|
|
8180
|
-
|
|
8181
|
-
|
|
8182
|
-
|
|
8183
|
-
|
|
8184
|
-
|
|
8185
|
-
|
|
8186
|
-
|
|
8187
|
-
|
|
8188
|
-
|
|
8189
|
-
|
|
8190
|
-
|
|
8191
|
-
|
|
8192
|
-
|
|
8193
|
-
|
|
8194
|
-
|
|
8195
|
-
|
|
8196
|
-
|
|
8197
|
-
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
7880
|
+
}
|
|
7881
|
+
|
|
7882
|
+
// src/services/distributed/planSize.ts
|
|
7883
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
7884
|
+
import { lstatSync, readdirSync as readdirSync6 } from "node:fs";
|
|
7885
|
+
import { extname as extname3, join as join21, relative as relative4 } from "node:path";
|
|
7886
|
+
var MAX_TOP_COMPONENTS = 10;
|
|
7887
|
+
var SOURCE_MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
7888
|
+
".aac",
|
|
7889
|
+
".aiff",
|
|
7890
|
+
".avi",
|
|
7891
|
+
".flac",
|
|
7892
|
+
".m4a",
|
|
7893
|
+
".m4v",
|
|
7894
|
+
".mkv",
|
|
7895
|
+
".mov",
|
|
7896
|
+
".mp3",
|
|
7897
|
+
".mp4",
|
|
7898
|
+
".mpeg",
|
|
7899
|
+
".mpg",
|
|
7900
|
+
".ogg",
|
|
7901
|
+
".ogv",
|
|
7902
|
+
".ts",
|
|
7903
|
+
".wav",
|
|
7904
|
+
".webm"
|
|
7905
|
+
]);
|
|
7906
|
+
var PLAN_ROOT_CATEGORY = {
|
|
7907
|
+
compiled: "compiled",
|
|
7908
|
+
[PLAN_AUDIO_RELATIVE_PATH]: "audio",
|
|
7909
|
+
meta: "metadata",
|
|
7910
|
+
"plan.json": "metadata"
|
|
7911
|
+
};
|
|
7912
|
+
function hashComponent(value) {
|
|
7913
|
+
return createHash5("sha256").update(value).digest("hex").slice(0, 12);
|
|
7914
|
+
}
|
|
7915
|
+
function classifyFile(segments, rootKind) {
|
|
7916
|
+
const rootName = segments[0] ?? "";
|
|
7917
|
+
const extractedRootName = rootKind === "compiled" ? "__hyperframes_video_frames" : "video-frames";
|
|
7918
|
+
if (rootName === extractedRootName) {
|
|
7919
|
+
const videoKey = segments[1] ?? "unknown";
|
|
7920
|
+
return {
|
|
7921
|
+
category: "video-frames",
|
|
7922
|
+
componentLabel: `video-frames:${hashComponent(videoKey)}`
|
|
7923
|
+
};
|
|
8203
7924
|
}
|
|
8204
|
-
|
|
8205
|
-
|
|
8206
|
-
|
|
8207
|
-
|
|
8208
|
-
limitBytes: sizeLimitBytes,
|
|
8209
|
-
observedAt: "pre-freeze",
|
|
8210
|
-
log
|
|
8211
|
-
});
|
|
8212
|
-
stageStarted = Date.now();
|
|
8213
|
-
const freezeResult = await freezePlan({
|
|
8214
|
-
planDir,
|
|
8215
|
-
composition: compositionJson,
|
|
8216
|
-
encoder: lockedConfig,
|
|
8217
|
-
chunks,
|
|
8218
|
-
dimensions,
|
|
8219
|
-
producerVersion,
|
|
8220
|
-
fontSnapshotSha,
|
|
8221
|
-
durationSeconds: job.duration ?? 0,
|
|
8222
|
-
totalFrames,
|
|
8223
|
-
hasAudio: audioResult.hasAudio,
|
|
8224
|
-
digests
|
|
8225
|
-
});
|
|
8226
|
-
logPlanStage(log, "freeze", stageStarted);
|
|
8227
|
-
const planHash = freezeResult.planHash;
|
|
8228
|
-
assertPlanSizeWithinLimit({
|
|
8229
|
-
rootDir: planDir,
|
|
8230
|
-
rootKind: "plan",
|
|
8231
|
-
limitBytes: sizeLimitBytes,
|
|
8232
|
-
observedAt: "post-freeze",
|
|
8233
|
-
log
|
|
8234
|
-
});
|
|
7925
|
+
const category = rootKind === "compiled" ? "compiled" : PLAN_ROOT_CATEGORY[rootName] ?? "other";
|
|
7926
|
+
return { category, componentLabel: category };
|
|
7927
|
+
}
|
|
7928
|
+
function emptyBreakdown() {
|
|
8235
7929
|
return {
|
|
8236
|
-
|
|
8237
|
-
|
|
8238
|
-
|
|
8239
|
-
|
|
8240
|
-
|
|
8241
|
-
|
|
8242
|
-
|
|
8243
|
-
|
|
8244
|
-
|
|
8245
|
-
|
|
7930
|
+
totalBytes: 0,
|
|
7931
|
+
fileCount: 0,
|
|
7932
|
+
compiledBytes: 0,
|
|
7933
|
+
sourceMediaBytes: 0,
|
|
7934
|
+
videoFramesBytes: 0,
|
|
7935
|
+
videoFrameFileCount: 0,
|
|
7936
|
+
audioBytes: 0,
|
|
7937
|
+
metadataBytes: 0,
|
|
7938
|
+
otherBytes: 0,
|
|
7939
|
+
topComponents: []
|
|
8246
7940
|
};
|
|
8247
7941
|
}
|
|
8248
|
-
|
|
8249
|
-
|
|
8250
|
-
|
|
8251
|
-
|
|
8252
|
-
|
|
8253
|
-
|
|
8254
|
-
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
7942
|
+
function addCategoryBytes(result, category, filePath, sizeBytes) {
|
|
7943
|
+
if (category === "compiled") {
|
|
7944
|
+
result.compiledBytes += sizeBytes;
|
|
7945
|
+
if (SOURCE_MEDIA_EXTENSIONS.has(extname3(filePath).toLowerCase())) {
|
|
7946
|
+
result.sourceMediaBytes += sizeBytes;
|
|
7947
|
+
}
|
|
7948
|
+
return;
|
|
7949
|
+
}
|
|
7950
|
+
if (category === "video-frames") {
|
|
7951
|
+
result.videoFramesBytes += sizeBytes;
|
|
7952
|
+
result.videoFrameFileCount += 1;
|
|
7953
|
+
return;
|
|
7954
|
+
}
|
|
7955
|
+
if (category === "audio") {
|
|
7956
|
+
result.audioBytes += sizeBytes;
|
|
7957
|
+
return;
|
|
7958
|
+
}
|
|
7959
|
+
if (category === "metadata") {
|
|
7960
|
+
result.metadataBytes += sizeBytes;
|
|
7961
|
+
return;
|
|
7962
|
+
}
|
|
7963
|
+
result.otherBytes += sizeBytes;
|
|
7964
|
+
}
|
|
7965
|
+
function recordFile(result, components, rootDir, rootKind, filePath) {
|
|
7966
|
+
let sizeBytes;
|
|
7967
|
+
try {
|
|
7968
|
+
const stat = lstatSync(filePath);
|
|
7969
|
+
if (!stat.isFile()) return;
|
|
7970
|
+
sizeBytes = stat.size;
|
|
7971
|
+
} catch {
|
|
7972
|
+
return;
|
|
7973
|
+
}
|
|
7974
|
+
const rel = relative4(rootDir, filePath);
|
|
7975
|
+
const segments = rel.split(/[\\/]+/).filter(Boolean);
|
|
7976
|
+
const classified = classifyFile(segments, rootKind);
|
|
7977
|
+
result.totalBytes += sizeBytes;
|
|
7978
|
+
result.fileCount += 1;
|
|
7979
|
+
addCategoryBytes(result, classified.category, filePath, sizeBytes);
|
|
7980
|
+
const component = components.get(classified.componentLabel) ?? {
|
|
7981
|
+
label: classified.componentLabel,
|
|
7982
|
+
sizeBytes: 0,
|
|
7983
|
+
fileCount: 0
|
|
8262
7984
|
};
|
|
7985
|
+
component.sizeBytes += sizeBytes;
|
|
7986
|
+
component.fileCount += 1;
|
|
7987
|
+
components.set(classified.componentLabel, component);
|
|
8263
7988
|
}
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
mkdirSync as mkdirSync10,
|
|
8271
|
-
mkdtempSync as mkdtempSync4,
|
|
8272
|
-
readFileSync as readFileSync9,
|
|
8273
|
-
readdirSync as readdirSync7,
|
|
8274
|
-
renameSync as renameSync5,
|
|
8275
|
-
rmSync as rmSync9,
|
|
8276
|
-
statSync as statSync6,
|
|
8277
|
-
writeFileSync as writeFileSync7
|
|
8278
|
-
} from "node:fs";
|
|
8279
|
-
import { tmpdir } from "node:os";
|
|
8280
|
-
import { dirname as dirname10, isAbsolute as isAbsolute4, join as join23, relative as relative6, resolve as resolve8, sep as sep3 } from "node:path";
|
|
8281
|
-
import { DISTRIBUTED_FPS, isDistributedFps } from "@hyperframes/core";
|
|
8282
|
-
import { createFrameLookupTable as createFrameLookupTable4 } from "@hyperframes/engine";
|
|
8283
|
-
|
|
8284
|
-
// src/services/distributed/planV2Publisher.ts
|
|
8285
|
-
import {
|
|
8286
|
-
copyFileSync as copyFileSync6,
|
|
8287
|
-
existsSync as existsSync15,
|
|
8288
|
-
linkSync as linkSync2,
|
|
8289
|
-
mkdirSync as mkdirSync9,
|
|
8290
|
-
mkdtempSync as mkdtempSync3,
|
|
8291
|
-
renameSync as renameSync4,
|
|
8292
|
-
rmSync as rmSync8,
|
|
8293
|
-
statSync as statSync5,
|
|
8294
|
-
writeFileSync as writeFileSync6
|
|
8295
|
-
} from "node:fs";
|
|
8296
|
-
import { dirname as dirname9, join as join22 } from "node:path";
|
|
8297
|
-
|
|
8298
|
-
// src/services/distributed/planV2Errors.ts
|
|
8299
|
-
var PLAN_V2_INTEGRITY_UNRECOVERABLE = "PLAN_V2_INTEGRITY_UNRECOVERABLE";
|
|
8300
|
-
var PlanV2IntegrityError = class extends Error {
|
|
8301
|
-
// Public adapters inspect this typed code even though OSS producer does not.
|
|
8302
|
-
// fallow-ignore-next-line unused-class-member
|
|
8303
|
-
code = PLAN_V2_INTEGRITY_UNRECOVERABLE;
|
|
8304
|
-
constructor(message) {
|
|
8305
|
-
super(`[planV2] ${message}`);
|
|
8306
|
-
this.name = "PlanV2IntegrityError";
|
|
7989
|
+
function walkRegularFiles(dir, visit) {
|
|
7990
|
+
let entries;
|
|
7991
|
+
try {
|
|
7992
|
+
entries = readdirSync6(dir, { withFileTypes: true });
|
|
7993
|
+
} catch {
|
|
7994
|
+
return;
|
|
8307
7995
|
}
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
|
|
8312
|
-
|
|
8313
|
-
|
|
8314
|
-
|
|
8315
|
-
throw new PlanV2IntegrityError(`${field} must be a lowercase sha256 digest`);
|
|
7996
|
+
for (const entry of entries) {
|
|
7997
|
+
const full = join21(dir, entry.name);
|
|
7998
|
+
if (entry.isDirectory()) {
|
|
7999
|
+
walkRegularFiles(full, visit);
|
|
8000
|
+
} else if (entry.isFile()) {
|
|
8001
|
+
visit(full);
|
|
8002
|
+
}
|
|
8316
8003
|
}
|
|
8317
|
-
return value;
|
|
8318
8004
|
}
|
|
8319
|
-
function
|
|
8320
|
-
const
|
|
8321
|
-
|
|
8005
|
+
function measurePlanSizeBreakdown(rootDir, rootKind = "plan") {
|
|
8006
|
+
const result = emptyBreakdown();
|
|
8007
|
+
const components = /* @__PURE__ */ new Map();
|
|
8008
|
+
walkRegularFiles(
|
|
8009
|
+
rootDir,
|
|
8010
|
+
(filePath) => recordFile(result, components, rootDir, rootKind, filePath)
|
|
8011
|
+
);
|
|
8012
|
+
result.topComponents = [...components.values()].sort((a, b) => b.sizeBytes - a.sizeBytes || a.label.localeCompare(b.label)).slice(0, MAX_TOP_COMPONENTS);
|
|
8013
|
+
return result;
|
|
8322
8014
|
}
|
|
8323
8015
|
|
|
8324
|
-
// src/services/distributed/
|
|
8325
|
-
function
|
|
8326
|
-
|
|
8327
|
-
const code = error.code;
|
|
8328
|
-
return code === "EXDEV" || code === "EPERM" || code === "EACCES" || code === "ENOTSUP";
|
|
8016
|
+
// src/services/distributed/plan.ts
|
|
8017
|
+
function logPlanStage(log, stage, startedAt) {
|
|
8018
|
+
log.info("[plan] stage", { stage, ms: Date.now() - startedAt });
|
|
8329
8019
|
}
|
|
8330
|
-
function
|
|
8331
|
-
|
|
8020
|
+
function applyDistributedAudioWarningPolicy(job, audioError, audioFailures = [], log = defaultLogger) {
|
|
8021
|
+
const failureOwner = audioFailures.length === 0 ? void 0 : audioFailures.some((failure) => failure.owner === "system") ? "system" : "user";
|
|
8022
|
+
const retryable = audioFailures.length === 0 ? void 0 : audioFailures.every((failure) => failure.retryable);
|
|
8023
|
+
applyRenderWarningPolicy(
|
|
8024
|
+
job,
|
|
8025
|
+
[
|
|
8026
|
+
{
|
|
8027
|
+
code: "audio_processing_failed",
|
|
8028
|
+
message: `Audio mix failed; output would be video-only: ${audioError}`,
|
|
8029
|
+
details: {
|
|
8030
|
+
mediaType: "audio",
|
|
8031
|
+
audioFailures: audioFailureDiagnostics3(audioFailures),
|
|
8032
|
+
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
|
|
8033
|
+
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
|
|
8034
|
+
failureOwner,
|
|
8035
|
+
retryable
|
|
8036
|
+
}
|
|
8037
|
+
}
|
|
8038
|
+
],
|
|
8039
|
+
log
|
|
8040
|
+
);
|
|
8332
8041
|
}
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8337
|
-
|
|
8338
|
-
|
|
8042
|
+
var PLAN_PROJECT_DIR_SKIP_SEGMENTS = /* @__PURE__ */ new Set([
|
|
8043
|
+
"node_modules",
|
|
8044
|
+
".git",
|
|
8045
|
+
".cache",
|
|
8046
|
+
"output",
|
|
8047
|
+
"failures",
|
|
8048
|
+
"dist",
|
|
8049
|
+
".next",
|
|
8050
|
+
".turbo"
|
|
8051
|
+
]);
|
|
8052
|
+
var DEFAULT_CHUNK_SIZE = 240;
|
|
8053
|
+
var DEFAULT_MAX_PARALLEL_CHUNKS = 16;
|
|
8054
|
+
var MIN_CHUNK_SIZE = 10;
|
|
8055
|
+
var MAX_PARALLEL_CHUNKS_CEILING = 256;
|
|
8056
|
+
var ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME = 0.11;
|
|
8057
|
+
var ESTIMATED_CHUNK_OVERHEAD_SECONDS = 60;
|
|
8058
|
+
var WORKER_TIMEOUT_MARGIN = 0.75;
|
|
8059
|
+
function estimateFrameSeconds(megapixels, measuredSecondsPerFrame = 0) {
|
|
8060
|
+
return Math.max(ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels, measuredSecondsPerFrame);
|
|
8061
|
+
}
|
|
8062
|
+
function estimateChunkSeconds(frames, megapixels, measuredSecondsPerFrame) {
|
|
8063
|
+
return ESTIMATED_CHUNK_OVERHEAD_SECONDS + frames * estimateFrameSeconds(megapixels, measuredSecondsPerFrame);
|
|
8064
|
+
}
|
|
8065
|
+
function planNeedsFrameCostMeasurement(html) {
|
|
8066
|
+
return new RegExp(`\\b${HF_COLOR_GRADING_ATTR2}=`, "i").test(html);
|
|
8067
|
+
}
|
|
8068
|
+
function chunkSizingUsesFrameCost(config) {
|
|
8069
|
+
return config.maxParallelChunks === void 0 && config.chunkSize === void 0 && config.targetChunkFrames === void 0 && (config.workerTimeoutSeconds !== void 0 || config.workerConcurrency !== void 0);
|
|
8070
|
+
}
|
|
8071
|
+
var CHUNK_EXCEEDS_WORKER_TIMEOUT = "CHUNK_EXCEEDS_WORKER_TIMEOUT";
|
|
8072
|
+
var ChunkExceedsWorkerTimeoutError = class extends Error {
|
|
8073
|
+
// fallow-ignore-next-line unused-class-member
|
|
8074
|
+
code = CHUNK_EXCEEDS_WORKER_TIMEOUT;
|
|
8075
|
+
constructor(predictedChunkSeconds, workerTimeoutSeconds) {
|
|
8076
|
+
super(
|
|
8077
|
+
`[plan] each chunk is predicted to take about ${predictedChunkSeconds} s even split ${MAX_PARALLEL_CHUNKS_CEILING} ways, over the ${workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN} s budget (${WORKER_TIMEOUT_MARGIN * 100}% of the ${workerTimeoutSeconds} s worker time limit) (${CHUNK_EXCEEDS_WORKER_TIMEOUT}). Shorten the video or lower its resolution, raise the worker time limit, or pass an explicit chunkSize or maxParallelChunks to override the estimate.`
|
|
8078
|
+
);
|
|
8079
|
+
this.name = "ChunkExceedsWorkerTimeoutError";
|
|
8339
8080
|
}
|
|
8340
|
-
|
|
8341
|
-
|
|
8081
|
+
};
|
|
8082
|
+
function resolveMaxParallelChunks(config, totalFrames, megapixels, log = defaultLogger, measuredSecondsPerFrame) {
|
|
8083
|
+
if (config.maxParallelChunks !== void 0) return config.maxParallelChunks;
|
|
8084
|
+
const { workerTimeoutSeconds, workerConcurrency } = config;
|
|
8085
|
+
const wave = Math.min(
|
|
8086
|
+
workerConcurrency ?? DEFAULT_MAX_PARALLEL_CHUNKS,
|
|
8087
|
+
MAX_PARALLEL_CHUNKS_CEILING
|
|
8088
|
+
);
|
|
8089
|
+
if (!chunkSizingUsesFrameCost(config)) return wave;
|
|
8090
|
+
const secondsPerFrame = estimateFrameSeconds(megapixels, measuredSecondsPerFrame);
|
|
8091
|
+
const minFramesPerChunk = Math.ceil(ESTIMATED_CHUNK_OVERHEAD_SECONDS / secondsPerFrame);
|
|
8092
|
+
const wide = Math.max(1, Math.min(wave, Math.floor(totalFrames / minFramesPerChunk)));
|
|
8093
|
+
if (workerTimeoutSeconds === void 0) return wide;
|
|
8094
|
+
const maxFramesPerChunk = Math.floor(
|
|
8095
|
+
(workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN - ESTIMATED_CHUNK_OVERHEAD_SECONDS) / secondsPerFrame
|
|
8096
|
+
);
|
|
8097
|
+
const needed = maxFramesPerChunk < 1 ? Infinity : Math.ceil(totalFrames / maxFramesPerChunk);
|
|
8098
|
+
if (needed > MAX_PARALLEL_CHUNKS_CEILING) {
|
|
8099
|
+
const framesAtCeiling = Math.ceil(totalFrames / MAX_PARALLEL_CHUNKS_CEILING);
|
|
8100
|
+
throw new ChunkExceedsWorkerTimeoutError(
|
|
8101
|
+
Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels, measuredSecondsPerFrame)),
|
|
8102
|
+
workerTimeoutSeconds
|
|
8103
|
+
);
|
|
8104
|
+
}
|
|
8105
|
+
if (needed > wave) {
|
|
8106
|
+
log.warn(
|
|
8107
|
+
`[plan] ${needed} chunks are needed to keep each under the ${workerTimeoutSeconds} s worker time limit, but the plan assumes ${wave} workers at once, so the render runs in ${Math.ceil(needed / wave)} waves. Raise --concurrency to ${needed} to render in one wave (Lambda: hyperframes lambda deploy --concurrency ${needed}, or a higher account concurrency quota).`,
|
|
8108
|
+
{ needed, workerConcurrency: wave, workerTimeoutSeconds }
|
|
8109
|
+
);
|
|
8110
|
+
}
|
|
8111
|
+
return Math.max(wide, needed);
|
|
8112
|
+
}
|
|
8113
|
+
var PLAN_DIR_SIZE_LIMIT_BYTES = 2 * 1024 * 1024 * 1024;
|
|
8114
|
+
var PLAN_TOO_LARGE = "PLAN_TOO_LARGE";
|
|
8115
|
+
var PlanTooLargeError = class extends Error {
|
|
8116
|
+
code = PLAN_TOO_LARGE;
|
|
8117
|
+
sizeBytes;
|
|
8118
|
+
limitBytes;
|
|
8119
|
+
breakdown;
|
|
8120
|
+
observedAt;
|
|
8121
|
+
constructor(sizeBytes, limitBytes, breakdown, observedAt) {
|
|
8122
|
+
const breakdownSuffix = breakdown ? ` Breakdown: video-frames=${formatBytes(breakdown.videoFramesBytes)}, compiled=${formatBytes(breakdown.compiledBytes)} (source-media=${formatBytes(breakdown.sourceMediaBytes)}), audio=${formatBytes(breakdown.audioBytes)}, metadata=${formatBytes(breakdown.metadataBytes)}, other=${formatBytes(breakdown.otherBytes)}, files=${breakdown.fileCount}.` : "";
|
|
8123
|
+
const observationSuffix = observedAt ? ` Observed at ${observedAt}.` : "";
|
|
8124
|
+
super(
|
|
8125
|
+
`[plan] planDir size ${formatBytes(sizeBytes)} exceeds the configured ceiling ${formatBytes(limitBytes)} (PLAN_TOO_LARGE). The default 2 GB cap fits inside AWS Lambda's 10 GB /tmp budget alongside the chunk worker's frame buffer and ffmpeg's working set. To unblock: use the content-addressed \`planV2()\` transport, shorten the composition, lower the framerate, or use the in-process renderer (\`executeRenderJob\`), which has no planDir size cap.` + observationSuffix + breakdownSuffix
|
|
8126
|
+
);
|
|
8127
|
+
this.name = "PlanTooLargeError";
|
|
8128
|
+
this.sizeBytes = sizeBytes;
|
|
8129
|
+
this.limitBytes = limitBytes;
|
|
8130
|
+
this.breakdown = breakdown;
|
|
8131
|
+
this.observedAt = observedAt;
|
|
8132
|
+
}
|
|
8133
|
+
};
|
|
8134
|
+
var FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED = "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED";
|
|
8135
|
+
var FormatNotSupportedInDistributedError = class extends Error {
|
|
8136
|
+
// fallow-ignore-next-line unused-class-member
|
|
8137
|
+
code = FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED;
|
|
8138
|
+
format;
|
|
8139
|
+
reason;
|
|
8140
|
+
constructor(format, reason) {
|
|
8141
|
+
super(
|
|
8142
|
+
`[plan] format ${JSON.stringify(format)} is not supported in distributed mode: ${reason}. Render with the in-process renderer (\`executeRenderJob\`), which has full format support, or pick a distributed-supported format: mp4 SDR, mov ProRes 4444, png-sequence, or webm VP9.`
|
|
8143
|
+
);
|
|
8144
|
+
this.name = "FormatNotSupportedInDistributedError";
|
|
8145
|
+
this.format = format;
|
|
8146
|
+
this.reason = reason;
|
|
8147
|
+
}
|
|
8148
|
+
};
|
|
8149
|
+
function formatBytes(bytes) {
|
|
8150
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
8151
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
8152
|
+
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
|
|
8153
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`;
|
|
8154
|
+
}
|
|
8155
|
+
function rejectUnsupportedDistributedFormat(config) {
|
|
8156
|
+
if (config.hdrMode === "force-hdr") {
|
|
8157
|
+
throw new FormatNotSupportedInDistributedError(
|
|
8158
|
+
"mp4-hdr",
|
|
8159
|
+
"HDR (PQ / HLG) requires per-source HDR pre-extract + HDR signaling re-apply on the assembled file; neither is implemented for the distributed pipeline"
|
|
8160
|
+
);
|
|
8342
8161
|
}
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
8346
|
-
|
|
8347
|
-
|
|
8162
|
+
}
|
|
8163
|
+
function measurePlanDirBytes(planDir) {
|
|
8164
|
+
return measurePlanSizeBreakdown(planDir).totalBytes;
|
|
8165
|
+
}
|
|
8166
|
+
function assertPlanSizeWithinLimit(input) {
|
|
8167
|
+
const breakdown = measurePlanSizeBreakdown(input.rootDir, input.rootKind);
|
|
8168
|
+
if (breakdown.totalBytes <= input.limitBytes) return breakdown;
|
|
8169
|
+
input.log.warn("[plan] size budget exceeded", {
|
|
8170
|
+
observedAt: input.observedAt,
|
|
8171
|
+
sizeBytes: breakdown.totalBytes,
|
|
8172
|
+
limitBytes: input.limitBytes,
|
|
8173
|
+
fileCount: breakdown.fileCount,
|
|
8174
|
+
compiledBytes: breakdown.compiledBytes,
|
|
8175
|
+
sourceMediaBytes: breakdown.sourceMediaBytes,
|
|
8176
|
+
videoFramesBytes: breakdown.videoFramesBytes,
|
|
8177
|
+
videoFrameFileCount: breakdown.videoFrameFileCount,
|
|
8178
|
+
audioBytes: breakdown.audioBytes,
|
|
8179
|
+
metadataBytes: breakdown.metadataBytes,
|
|
8180
|
+
otherBytes: breakdown.otherBytes,
|
|
8181
|
+
topComponents: breakdown.topComponents
|
|
8348
8182
|
});
|
|
8183
|
+
throw new PlanTooLargeError(breakdown.totalBytes, input.limitBytes, breakdown, input.observedAt);
|
|
8349
8184
|
}
|
|
8350
|
-
|
|
8351
|
-
|
|
8352
|
-
|
|
8353
|
-
|
|
8354
|
-
|
|
8355
|
-
constructor(destinationDir, options = {}) {
|
|
8356
|
-
if (existsSync15(destinationDir)) {
|
|
8357
|
-
throw new PlanV2IntegrityError(`output directory already exists: ${destinationDir}`);
|
|
8358
|
-
}
|
|
8359
|
-
this.destinationDir = destinationDir;
|
|
8360
|
-
this.#linkFile = options.linkFile ?? linkSync2;
|
|
8361
|
-
mkdirSync9(dirname9(destinationDir), { recursive: true });
|
|
8362
|
-
this.temporaryDir = mkdtempSync3(join22(dirname9(destinationDir), ".plan-v2-publish-"));
|
|
8363
|
-
}
|
|
8364
|
-
async putBlob(blob) {
|
|
8365
|
-
const digest = assertPlanV2Sha256(blob.sha256, "published blob sha256");
|
|
8366
|
-
const sourceSize = statSync5(blob.sourcePath).size;
|
|
8367
|
-
if (sourceSize !== blob.sizeBytes) {
|
|
8368
|
-
throw new PlanV2IntegrityError(
|
|
8369
|
-
`published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
8370
|
-
);
|
|
8371
|
-
}
|
|
8372
|
-
const destinationPath = planV2BlobPath(this.temporaryDir, digest);
|
|
8373
|
-
if (existsSync15(destinationPath)) return;
|
|
8374
|
-
mkdirSync9(dirname9(destinationPath), { recursive: true });
|
|
8375
|
-
const stagingDir = mkdtempSync3(join22(dirname9(destinationPath), ".plan-v2-blob-"));
|
|
8376
|
-
const temporaryPath = join22(stagingDir, "blob");
|
|
8377
|
-
try {
|
|
8378
|
-
try {
|
|
8379
|
-
this.#linkFile(blob.sourcePath, temporaryPath);
|
|
8380
|
-
} catch (error) {
|
|
8381
|
-
if (!canFallbackToCopy(error)) throw error;
|
|
8382
|
-
copyFileSync6(blob.sourcePath, temporaryPath);
|
|
8383
|
-
}
|
|
8384
|
-
renameSync4(temporaryPath, destinationPath);
|
|
8385
|
-
} finally {
|
|
8386
|
-
rmSync8(stagingDir, { recursive: true, force: true });
|
|
8387
|
-
}
|
|
8388
|
-
}
|
|
8389
|
-
async commitManifest(manifestBytes) {
|
|
8390
|
-
for (const digest of new Set(manifestDigests(manifestBytes))) {
|
|
8391
|
-
if (!existsSync15(planV2BlobPath(this.temporaryDir, digest))) {
|
|
8392
|
-
throw new PlanV2IntegrityError(
|
|
8393
|
-
`cannot commit manifest before referenced blob is durable: ${digest}`
|
|
8394
|
-
);
|
|
8395
|
-
}
|
|
8396
|
-
}
|
|
8397
|
-
writeFileSync6(join22(this.temporaryDir, "plan.json"), manifestBytes, "utf-8");
|
|
8398
|
-
renameSync4(this.temporaryDir, this.destinationDir);
|
|
8399
|
-
this.#committed = true;
|
|
8185
|
+
function resolveChunkPlan(totalFrames, configChunkSize, maxParallelChunks, targetChunkFrames) {
|
|
8186
|
+
assertPositiveInteger("totalFrames", totalFrames);
|
|
8187
|
+
assertPositiveInteger("maxParallelChunks", maxParallelChunks);
|
|
8188
|
+
if (configChunkSize !== void 0) {
|
|
8189
|
+
assertPositiveInteger("configChunkSize", configChunkSize);
|
|
8400
8190
|
}
|
|
8401
|
-
|
|
8402
|
-
|
|
8403
|
-
rmSync8(this.temporaryDir, { recursive: true, force: true });
|
|
8404
|
-
}
|
|
8191
|
+
if (targetChunkFrames !== void 0) {
|
|
8192
|
+
assertPositiveInteger("targetChunkFrames", targetChunkFrames);
|
|
8405
8193
|
}
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
|
|
8410
|
-
|
|
8411
|
-
|
|
8412
|
-
|
|
8413
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
8194
|
+
const autoSizeParallel = targetChunkFrames === void 0 ? maxParallelChunks : Math.min(maxParallelChunks, Math.max(1, Math.ceil(totalFrames / targetChunkFrames)));
|
|
8195
|
+
const resolvedChunkSize = configChunkSize ?? Math.max(MIN_CHUNK_SIZE, Math.ceil(totalFrames / autoSizeParallel));
|
|
8196
|
+
const naiveCount = Math.ceil(totalFrames / resolvedChunkSize);
|
|
8197
|
+
const chunkCount = Math.min(maxParallelChunks, Math.max(1, naiveCount));
|
|
8198
|
+
const effectiveChunkSize = Math.max(resolvedChunkSize, Math.ceil(totalFrames / chunkCount));
|
|
8199
|
+
const tightChunkCount = Math.min(chunkCount, Math.ceil(totalFrames / effectiveChunkSize));
|
|
8200
|
+
return { chunkCount: tightChunkCount, effectiveChunkSize };
|
|
8414
8201
|
}
|
|
8415
|
-
function
|
|
8416
|
-
|
|
8202
|
+
function assertPositiveInteger(name, value) {
|
|
8203
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
8204
|
+
throw new Error(
|
|
8205
|
+
`[plan] resolveChunkPlan: ${name} must be a positive integer (received ${String(value)})`
|
|
8206
|
+
);
|
|
8207
|
+
}
|
|
8417
8208
|
}
|
|
8418
|
-
|
|
8419
|
-
|
|
8209
|
+
var FREEZE_OWNED_PLAN_FILES = [
|
|
8210
|
+
"plan.json",
|
|
8211
|
+
join22("meta", "composition.json"),
|
|
8212
|
+
join22("meta", "encoder.json"),
|
|
8213
|
+
join22("meta", "chunks.json")
|
|
8214
|
+
];
|
|
8215
|
+
function removeFreezeOwnedPlanFiles(planDir) {
|
|
8216
|
+
for (const relativePath of FREEZE_OWNED_PLAN_FILES) {
|
|
8217
|
+
rmSync8(join22(planDir, relativePath), { force: true });
|
|
8218
|
+
}
|
|
8420
8219
|
}
|
|
8421
|
-
function
|
|
8422
|
-
const
|
|
8423
|
-
|
|
8424
|
-
const
|
|
8425
|
-
|
|
8426
|
-
|
|
8427
|
-
const detail = error instanceof Error ? error.message : String(error);
|
|
8428
|
-
throw new PlanV2IntegrityError(`${label} is not readable JSON: ${detail}`);
|
|
8220
|
+
function buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize) {
|
|
8221
|
+
const slices = [];
|
|
8222
|
+
for (let i = 0; i < chunkCount; i++) {
|
|
8223
|
+
const startFrame = i * effectiveChunkSize;
|
|
8224
|
+
const endFrame = i === chunkCount - 1 ? totalFrames : Math.min(totalFrames, startFrame + effectiveChunkSize);
|
|
8225
|
+
slices.push({ index: i, startFrame, endFrame });
|
|
8429
8226
|
}
|
|
8227
|
+
return slices;
|
|
8430
8228
|
}
|
|
8431
|
-
function
|
|
8432
|
-
const
|
|
8433
|
-
|
|
8434
|
-
|
|
8229
|
+
async function readFontSnapshotSha() {
|
|
8230
|
+
const module = await import("@hyperframes/core/fonts/embedded-data");
|
|
8231
|
+
const data = module.EMBEDDED_FONT_DATA;
|
|
8232
|
+
if (!data || typeof data !== "object") {
|
|
8233
|
+
throw new Error(
|
|
8234
|
+
"[plan] EMBEDDED_FONT_DATA missing from @hyperframes/core/fonts/embedded-data. Was `bun run build:fonts` run?"
|
|
8235
|
+
);
|
|
8236
|
+
}
|
|
8237
|
+
const dataObj = data;
|
|
8238
|
+
const fingerprint = {};
|
|
8239
|
+
for (const key of Object.keys(dataObj).sort()) {
|
|
8240
|
+
const value = dataObj[key];
|
|
8241
|
+
fingerprint[key] = typeof value === "string" ? value.length : JSON.stringify(value ?? null).length;
|
|
8435
8242
|
}
|
|
8243
|
+
return sha256Hex(canonicalJsonStringify(fingerprint));
|
|
8436
8244
|
}
|
|
8437
|
-
function
|
|
8438
|
-
const
|
|
8439
|
-
const
|
|
8440
|
-
|
|
8441
|
-
|
|
8442
|
-
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8448
|
-
|
|
8449
|
-
|
|
8450
|
-
|
|
8451
|
-
|
|
8452
|
-
|
|
8453
|
-
|
|
8454
|
-
|
|
8455
|
-
|
|
8245
|
+
function buildLockedRenderConfig(input) {
|
|
8246
|
+
const { config, forceScreenshot, deviceScaleFactor, ffmpegVersion } = input;
|
|
8247
|
+
const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);
|
|
8248
|
+
const locksVp9CpuUsed = encoder === "libvpx-vp9-software" ? { vp9CpuUsed: normalizeVp9CpuUsed(input.engineConfig.vp9CpuUsed) } : {};
|
|
8249
|
+
return {
|
|
8250
|
+
captureMode: forceScreenshot ? "screenshot" : "beginframe",
|
|
8251
|
+
forceScreenshot,
|
|
8252
|
+
deviceScaleFactor,
|
|
8253
|
+
useLayeredHdrComposite: false,
|
|
8254
|
+
browserGpuMode: "software",
|
|
8255
|
+
// Match `LOCKED_WARMUP_TICKS` in `frameCapture.ts` — kept as a literal so
|
|
8256
|
+
// a worker that ships a different value will trip `PLAN_HASH_MISMATCH`
|
|
8257
|
+
// (the locked config flows into planHash via the canonical JSON).
|
|
8258
|
+
warmupTicks: 60,
|
|
8259
|
+
encoder,
|
|
8260
|
+
quality: config.quality ?? "standard",
|
|
8261
|
+
ffmpegVersion,
|
|
8262
|
+
preset,
|
|
8263
|
+
crf: config.crf,
|
|
8264
|
+
bitrate: config.bitrate,
|
|
8265
|
+
...locksVp9CpuUsed,
|
|
8266
|
+
// GOP === chunkSize so every chunk's first frame is an IDR keyframe and
|
|
8267
|
+
// ffmpeg concat-copy round-trips losslessly.
|
|
8268
|
+
gopSize: input.effectiveChunkSize,
|
|
8269
|
+
closedGop: true,
|
|
8270
|
+
forceKeyframes: "n=0",
|
|
8271
|
+
pixelFormat,
|
|
8272
|
+
chunkSize: input.effectiveChunkSize,
|
|
8273
|
+
chunkCount: input.chunkCount,
|
|
8274
|
+
runtimeEnv: input.runtimeEnv,
|
|
8275
|
+
variables: config.variables,
|
|
8276
|
+
...input.chunksCarryAudio ? { chunksCarryAudio: true } : {}
|
|
8277
|
+
};
|
|
8278
|
+
}
|
|
8279
|
+
function carriesChunkAudio(config, options) {
|
|
8280
|
+
return options.chunksCarryAudio === true && config.format === "mp4" && config.cfr !== true;
|
|
8281
|
+
}
|
|
8282
|
+
function resolveEncoderTriple(config) {
|
|
8283
|
+
if (config.format === "mp4") {
|
|
8284
|
+
return resolveMp4EncoderTriple(config.codec);
|
|
8456
8285
|
}
|
|
8457
|
-
|
|
8458
|
-
|
|
8286
|
+
if (config.codec !== void 0) {
|
|
8287
|
+
throw new Error(
|
|
8288
|
+
`[plan] DistributedRenderConfig.codec is only valid for format="mp4"; received codec=${JSON.stringify(config.codec)} with format=${JSON.stringify(config.format)}. Omit codec for non-mp4 formats: mov is always ProRes 4444, webm is always libvpx-vp9, and png-sequence has no encoder.`
|
|
8289
|
+
);
|
|
8290
|
+
}
|
|
8291
|
+
return resolveNonMp4EncoderTriple(config.format, config.quality ?? "standard");
|
|
8459
8292
|
}
|
|
8460
|
-
function
|
|
8461
|
-
const
|
|
8462
|
-
if (
|
|
8463
|
-
throw new
|
|
8464
|
-
|
|
8293
|
+
function resolveMp4EncoderTriple(codec) {
|
|
8294
|
+
const c = codec ?? "h264";
|
|
8295
|
+
if (c !== "h264" && c !== "h265") {
|
|
8296
|
+
throw new Error(
|
|
8297
|
+
`[plan] DistributedRenderConfig.codec must be "h264" or "h265" for format="mp4"; received ${JSON.stringify(c)}. Omit codec to default to h264.`
|
|
8465
8298
|
);
|
|
8466
8299
|
}
|
|
8300
|
+
if (c === "h265") {
|
|
8301
|
+
return { encoder: "libx265-software", pixelFormat: "yuv420p", preset: "medium" };
|
|
8302
|
+
}
|
|
8303
|
+
return { encoder: "libx264-software", pixelFormat: "yuv420p", preset: "medium" };
|
|
8467
8304
|
}
|
|
8468
|
-
function
|
|
8469
|
-
|
|
8470
|
-
|
|
8471
|
-
|
|
8472
|
-
|
|
8473
|
-
const
|
|
8474
|
-
|
|
8475
|
-
|
|
8476
|
-
|
|
8477
|
-
|
|
8305
|
+
function resolveNonMp4EncoderTriple(format, quality) {
|
|
8306
|
+
if (format === "mov") {
|
|
8307
|
+
return { encoder: "prores-software", pixelFormat: "yuva444p10le", preset: "4444" };
|
|
8308
|
+
}
|
|
8309
|
+
if (format === "webm") {
|
|
8310
|
+
const enginePreset = getEncoderPreset4(quality, "webm");
|
|
8311
|
+
return {
|
|
8312
|
+
encoder: "libvpx-vp9-software",
|
|
8313
|
+
pixelFormat: enginePreset.pixelFormat,
|
|
8314
|
+
preset: enginePreset.preset
|
|
8315
|
+
};
|
|
8478
8316
|
}
|
|
8317
|
+
return { encoder: "png-sequence", pixelFormat: "rgba", preset: "lossless" };
|
|
8479
8318
|
}
|
|
8480
|
-
function
|
|
8481
|
-
|
|
8482
|
-
|
|
8319
|
+
function resolveDistributedEngineConfig(config) {
|
|
8320
|
+
return {
|
|
8321
|
+
...config.producerConfig ?? config.engineConfig ?? resolveConfig3(),
|
|
8322
|
+
browserGpuMode: "software",
|
|
8323
|
+
forceScreenshot: false,
|
|
8324
|
+
// Distributed rendering deliberately opts into deterministic BeginFrame
|
|
8325
|
+
// on Linux SwiftShader. Preserve the provenance bit consumed by the
|
|
8326
|
+
// engine's software-GPU screenshot clamp; assigning the boolean alone
|
|
8327
|
+
// loses the distinction between a default false and this explicit opt-out.
|
|
8328
|
+
forceScreenshotExplicitlyOptedOut: true
|
|
8329
|
+
};
|
|
8483
8330
|
}
|
|
8484
|
-
function
|
|
8485
|
-
const
|
|
8486
|
-
|
|
8487
|
-
if (outputDir === videoRoot || !outputDir.startsWith(`${videoRoot}${sep3}`)) {
|
|
8488
|
-
throw new PlanV2IntegrityError(`unsafe extracted video id: ${JSON.stringify(videoId)}`);
|
|
8489
|
-
}
|
|
8490
|
-
return outputDir;
|
|
8331
|
+
function pathInside(root, path) {
|
|
8332
|
+
const rel = relative5(root, path);
|
|
8333
|
+
return rel === "" || rel.startsWith("..") || isAbsolute3(rel) ? null : rel;
|
|
8491
8334
|
}
|
|
8492
|
-
function
|
|
8493
|
-
|
|
8494
|
-
|
|
8495
|
-
|
|
8496
|
-
if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {
|
|
8497
|
-
return { chunks: "all", assembler: true };
|
|
8498
|
-
}
|
|
8499
|
-
if (path.startsWith("video-frames/") || path.startsWith("video-sources/") || path.startsWith("compiled/") && videoDependencies?.has(path)) {
|
|
8500
|
-
return {
|
|
8501
|
-
chunks: videoDependencies === null ? "all" : videoDependencies.get(path) ?? [],
|
|
8502
|
-
assembler: false
|
|
8503
|
-
};
|
|
8504
|
-
}
|
|
8505
|
-
if (path === "meta/composition.json" || path === "meta/videos.json" || path.startsWith("compiled/")) {
|
|
8506
|
-
return { chunks: "all", assembler: false };
|
|
8335
|
+
function stableSourcePath(roots, srcPath) {
|
|
8336
|
+
for (const [root, prefix] of roots) {
|
|
8337
|
+
const rel = pathInside(root, srcPath);
|
|
8338
|
+
if (rel !== null) return [prefix, ...rel.split(sep2)].filter(Boolean).join("/");
|
|
8507
8339
|
}
|
|
8508
|
-
return
|
|
8509
|
-
}
|
|
8510
|
-
function isPlanV2DeferredSourceArtifact(artifact) {
|
|
8511
|
-
return artifact.path.startsWith("compiled/") && artifact.chunks !== "all";
|
|
8340
|
+
return srcPath;
|
|
8512
8341
|
}
|
|
8513
|
-
function
|
|
8514
|
-
|
|
8515
|
-
|
|
8516
|
-
|
|
8517
|
-
|
|
8518
|
-
|
|
8519
|
-
|
|
8342
|
+
function createVideoSourceShipper(input) {
|
|
8343
|
+
const shipped = /* @__PURE__ */ new Map();
|
|
8344
|
+
return (srcPath) => {
|
|
8345
|
+
const known = shipped.get(srcPath);
|
|
8346
|
+
if (known) return known;
|
|
8347
|
+
const stagedRel = pathInside(input.workVideoFramesDir, srcPath);
|
|
8348
|
+
const current = stagedRel === null ? srcPath : join22(input.planDir, "video-frames", stagedRel);
|
|
8349
|
+
let result = null;
|
|
8350
|
+
for (const root of stagedRel === null ? [input.workCompiledDir, input.projectDir] : []) {
|
|
8351
|
+
const rel = pathInside(root, srcPath);
|
|
8352
|
+
if (rel !== null && (existsSync15(join22(input.planDir, "compiled", rel)) || input.isRemote(srcPath))) {
|
|
8353
|
+
result = ["compiled", ...rel.split(sep2)].join("/");
|
|
8354
|
+
break;
|
|
8520
8355
|
}
|
|
8521
|
-
return { ...video, outputDir, framePaths: framePaths2, ownedByLookup: false };
|
|
8522
8356
|
}
|
|
8523
|
-
|
|
8524
|
-
|
|
8525
|
-
|
|
8526
|
-
|
|
8527
|
-
|
|
8528
|
-
|
|
8529
|
-
}
|
|
8530
|
-
|
|
8531
|
-
if (!match) {
|
|
8532
|
-
throw new PlanV2IntegrityError(`cannot derive extracted frame index from ${frameName}`);
|
|
8533
|
-
}
|
|
8534
|
-
const oneBasedFrameNumber = Number(match[1]);
|
|
8535
|
-
if (!Number.isSafeInteger(oneBasedFrameNumber) || oneBasedFrameNumber < 1) {
|
|
8536
|
-
throw new PlanV2IntegrityError(
|
|
8537
|
-
`extracted frame filename must use a 1-based safe integer: ${frameName}`
|
|
8538
|
-
);
|
|
8539
|
-
}
|
|
8540
|
-
const frameIndex = oneBasedFrameNumber - 1;
|
|
8541
|
-
if (framePaths.has(frameIndex)) {
|
|
8542
|
-
throw new PlanV2IntegrityError(
|
|
8543
|
-
`duplicate extracted frame index ${frameIndex} in ${outputDir}`
|
|
8544
|
-
);
|
|
8357
|
+
if (result === null) {
|
|
8358
|
+
result = `video-sources/${shipped.size}${extname4(current)}`;
|
|
8359
|
+
const destination = join22(input.planDir, ...result.split("/"));
|
|
8360
|
+
mkdirSync10(join22(input.planDir, "video-sources"), { recursive: true });
|
|
8361
|
+
try {
|
|
8362
|
+
linkSync(current, destination);
|
|
8363
|
+
} catch {
|
|
8364
|
+
copyFileSync5(current, destination);
|
|
8545
8365
|
}
|
|
8546
|
-
framePaths.set(frameIndex, join23(outputDir, frameName));
|
|
8547
8366
|
}
|
|
8548
|
-
|
|
8549
|
-
|
|
8550
|
-
|
|
8551
|
-
framePaths,
|
|
8552
|
-
ownedByLookup: false
|
|
8553
|
-
};
|
|
8554
|
-
});
|
|
8367
|
+
shipped.set(srcPath, result);
|
|
8368
|
+
return result;
|
|
8369
|
+
};
|
|
8555
8370
|
}
|
|
8556
|
-
function
|
|
8557
|
-
|
|
8558
|
-
|
|
8559
|
-
|
|
8560
|
-
|
|
8561
|
-
|
|
8371
|
+
async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, options = {}) {
|
|
8372
|
+
const planDir = executionPlanDir;
|
|
8373
|
+
rejectUnsupportedDistributedFormat(config);
|
|
8374
|
+
validateNoGpuEncode({
|
|
8375
|
+
useGpu: false,
|
|
8376
|
+
browserGpuMode: "software"
|
|
8377
|
+
});
|
|
8378
|
+
if (!existsSync15(planDir)) mkdirSync10(planDir, { recursive: true });
|
|
8379
|
+
const log = config.logger ?? defaultLogger;
|
|
8380
|
+
const sizeLimitBytes = options.executionPlanSizeLimitBytes ?? config.planDirSizeLimitBytes ?? PLAN_DIR_SIZE_LIMIT_BYTES;
|
|
8381
|
+
const abortSignal = config.abortSignal;
|
|
8382
|
+
const assertNotAborted = () => {
|
|
8383
|
+
if (abortSignal?.aborted) {
|
|
8384
|
+
throw new Error("[plan] render_cancelled");
|
|
8562
8385
|
}
|
|
8563
|
-
|
|
8564
|
-
|
|
8565
|
-
|
|
8566
|
-
|
|
8567
|
-
|
|
8568
|
-
|
|
8569
|
-
|
|
8570
|
-
|
|
8571
|
-
|
|
8572
|
-
|
|
8573
|
-
|
|
8574
|
-
|
|
8575
|
-
|
|
8576
|
-
|
|
8386
|
+
};
|
|
8387
|
+
const cfg = resolveDistributedEngineConfig(config);
|
|
8388
|
+
const job = buildSyntheticRenderJob({
|
|
8389
|
+
fps: { num: config.fps, den: 1 },
|
|
8390
|
+
quality: config.quality ?? "standard",
|
|
8391
|
+
format: config.format,
|
|
8392
|
+
crf: config.crf,
|
|
8393
|
+
bitrate: config.bitrate,
|
|
8394
|
+
videoFrameFormat: config.videoFrameFormat,
|
|
8395
|
+
outputResolution: config.outputResolution,
|
|
8396
|
+
outputResolutionAspectAgnostic: config.outputResolutionAspectAgnostic,
|
|
8397
|
+
// HDR is banned in distributed mode. force-sdr keeps the
|
|
8398
|
+
// extract / encoder paths off the HDR branches entirely.
|
|
8399
|
+
hdrMode: config.hdrMode ?? "force-sdr",
|
|
8400
|
+
strictness: config.strictness,
|
|
8401
|
+
entryFile: config.entryFile ?? "index.html",
|
|
8402
|
+
logger: config.logger,
|
|
8403
|
+
producerConfig: cfg,
|
|
8404
|
+
variables: config.variables
|
|
8405
|
+
});
|
|
8406
|
+
const entryFile = config.entryFile ?? "index.html";
|
|
8407
|
+
const htmlPath = join22(projectDir, entryFile);
|
|
8408
|
+
if (!existsSync15(htmlPath)) {
|
|
8409
|
+
throw new Error(`[plan] entry file not found: ${htmlPath}`);
|
|
8577
8410
|
}
|
|
8578
|
-
const
|
|
8579
|
-
|
|
8580
|
-
|
|
8581
|
-
|
|
8582
|
-
|
|
8583
|
-
|
|
8584
|
-
|
|
8585
|
-
|
|
8586
|
-
|
|
8411
|
+
const workDir = join22(planDir, ".plan-work");
|
|
8412
|
+
rmSync8(workDir, { recursive: true, force: true });
|
|
8413
|
+
mkdirSync10(workDir, { recursive: true });
|
|
8414
|
+
const compiledDir = join22(workDir, "compiled");
|
|
8415
|
+
mkdirSync10(compiledDir, { recursive: true });
|
|
8416
|
+
let stageStarted = Date.now();
|
|
8417
|
+
cpSync2(projectDir, compiledDir, {
|
|
8418
|
+
recursive: true,
|
|
8419
|
+
dereference: true,
|
|
8420
|
+
filter: (src) => {
|
|
8421
|
+
const rel = relative5(projectDir, src);
|
|
8422
|
+
if (rel === "" || rel.startsWith("..")) return true;
|
|
8423
|
+
const firstSegment = rel.split(sep2, 1)[0];
|
|
8424
|
+
return firstSegment === void 0 || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
|
|
8587
8425
|
}
|
|
8588
|
-
|
|
8589
|
-
|
|
8426
|
+
});
|
|
8427
|
+
logPlanStage(log, "copy", stageStarted);
|
|
8428
|
+
const finalCompiledDir = join22(planDir, "compiled");
|
|
8429
|
+
const needsAlpha = outputNeedsAlpha(config.format);
|
|
8430
|
+
const remoteFiles = Object.entries(options.remoteFiles ?? {});
|
|
8431
|
+
const remoteLocalPath = (rel) => join22(projectDir, ...rel.split("/"));
|
|
8432
|
+
const urlByAbsentLocalPath = new Map(
|
|
8433
|
+
remoteFiles.map(([rel, file]) => [remoteLocalPath(rel), file.url])
|
|
8434
|
+
);
|
|
8435
|
+
const localPathByUrl = new Map(
|
|
8436
|
+
remoteFiles.map(([rel, file]) => [file.url, remoteLocalPath(rel)])
|
|
8437
|
+
);
|
|
8438
|
+
stageStarted = Date.now();
|
|
8439
|
+
const compileResult = await runCompileStage({
|
|
8440
|
+
urlByAbsentLocalPath,
|
|
8441
|
+
projectDir,
|
|
8442
|
+
workDir,
|
|
8443
|
+
htmlPath,
|
|
8444
|
+
entryFile,
|
|
8445
|
+
job,
|
|
8446
|
+
cfg,
|
|
8447
|
+
needsAlpha,
|
|
8448
|
+
log,
|
|
8449
|
+
assertNotAborted,
|
|
8450
|
+
abortSignal,
|
|
8451
|
+
// Distributed renders fail closed on font-fetch errors so the planDir
|
|
8452
|
+
// is content-addressed against deterministic fonts only.
|
|
8453
|
+
failClosedFontFetch: config.failClosedFontFetch !== false,
|
|
8454
|
+
// Distributed renders must not capture host-specific system fonts —
|
|
8455
|
+
// the Lambda/worker filesystem won't have the same fonts installed.
|
|
8456
|
+
allowSystemFontCapture: false,
|
|
8457
|
+
variables: config.variables
|
|
8458
|
+
});
|
|
8459
|
+
logPlanStage(log, "compile", stageStarted);
|
|
8460
|
+
let compiled = compileResult.compiled;
|
|
8461
|
+
const composition = compileResult.composition;
|
|
8462
|
+
const { deviceScaleFactor } = compileResult;
|
|
8463
|
+
let forceScreenshot = compileResult.forceScreenshot;
|
|
8464
|
+
if (cfg.lowMemoryMode) forceScreenshot = true;
|
|
8465
|
+
const { width, height } = composition;
|
|
8466
|
+
if (config.rejectOnSystemFonts !== false) {
|
|
8467
|
+
validateNoSystemFonts(compiled.html);
|
|
8468
|
+
}
|
|
8469
|
+
stageStarted = Date.now();
|
|
8470
|
+
const probeResult = await runProbeStage({
|
|
8471
|
+
projectDir,
|
|
8472
|
+
workDir,
|
|
8473
|
+
job,
|
|
8474
|
+
cfg,
|
|
8475
|
+
forceScreenshot,
|
|
8476
|
+
log,
|
|
8477
|
+
assertNotAborted,
|
|
8478
|
+
abortSignal,
|
|
8479
|
+
compiled,
|
|
8480
|
+
composition,
|
|
8481
|
+
width,
|
|
8482
|
+
height,
|
|
8483
|
+
needsAlpha,
|
|
8484
|
+
deviceScaleFactor,
|
|
8485
|
+
subTimelineWaitMemo: {}
|
|
8486
|
+
});
|
|
8487
|
+
logPlanStage(log, "probe", stageStarted);
|
|
8488
|
+
compiled = probeResult.compiled;
|
|
8489
|
+
job.duration = probeResult.duration;
|
|
8490
|
+
job.totalFrames = probeResult.totalFrames;
|
|
8491
|
+
const totalFrames = probeResult.totalFrames;
|
|
8492
|
+
validateDistributedDuration({
|
|
8493
|
+
duration: probeResult.duration,
|
|
8494
|
+
totalFrames,
|
|
8495
|
+
fps: fpsToNumber10(job.config.fps)
|
|
8496
|
+
});
|
|
8497
|
+
if (probeResult.fileServer) closeFileServerSafely(probeResult.fileServer, "plan", log);
|
|
8498
|
+
if (probeResult.probeSession) {
|
|
8499
|
+
try {
|
|
8500
|
+
const { closeCaptureSession: closeCaptureSession11 } = await import("@hyperframes/engine");
|
|
8501
|
+
await closeCaptureSession11(probeResult.probeSession);
|
|
8502
|
+
} catch (err) {
|
|
8503
|
+
log.warn("[plan] probe session close failed", {
|
|
8504
|
+
error: err instanceof Error ? err.message : String(err)
|
|
8505
|
+
});
|
|
8590
8506
|
}
|
|
8591
|
-
|
|
8592
|
-
|
|
8507
|
+
}
|
|
8508
|
+
const megapixels = compileResult.outputWidth * compileResult.outputHeight / 1e6;
|
|
8509
|
+
let maxParallel = resolveMaxParallelChunks(config, totalFrames, megapixels, log);
|
|
8510
|
+
const measureFrameCost = chunkSizingUsesFrameCost(config) && planNeedsFrameCostMeasurement(compiled.html);
|
|
8511
|
+
assertPlanSizeWithinLimit({
|
|
8512
|
+
rootDir: compiledDir,
|
|
8513
|
+
rootKind: "compiled",
|
|
8514
|
+
limitBytes: sizeLimitBytes,
|
|
8515
|
+
observedAt: "pre-extract",
|
|
8516
|
+
log
|
|
8593
8517
|
});
|
|
8594
|
-
|
|
8595
|
-
|
|
8596
|
-
|
|
8597
|
-
|
|
8598
|
-
|
|
8599
|
-
|
|
8600
|
-
|
|
8518
|
+
stageStarted = Date.now();
|
|
8519
|
+
const extractResult = await runExtractVideosStage({
|
|
8520
|
+
projectDir,
|
|
8521
|
+
compiledDir,
|
|
8522
|
+
job,
|
|
8523
|
+
cfg,
|
|
8524
|
+
composition,
|
|
8525
|
+
abortSignal,
|
|
8526
|
+
assertNotAborted,
|
|
8527
|
+
materializeSymlinks: true,
|
|
8528
|
+
deferRangeExtraction: options.deferVideoExtraction,
|
|
8529
|
+
urlByAbsentLocalPath
|
|
8530
|
+
});
|
|
8531
|
+
logPlanStage(log, "extract", stageStarted);
|
|
8532
|
+
if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
|
|
8533
|
+
if (extractResult.extractionResult) {
|
|
8534
|
+
assertVideoExtractionSucceeded(extractResult.extractionResult);
|
|
8601
8535
|
}
|
|
8602
|
-
const
|
|
8603
|
-
const
|
|
8604
|
-
|
|
8605
|
-
const
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
8612
|
-
|
|
8613
|
-
|
|
8536
|
+
const stagedVideoFrames = join22(compiledDir, "__hyperframes_video_frames");
|
|
8537
|
+
const videoFramesDst = join22(planDir, "video-frames");
|
|
8538
|
+
if (existsSync15(videoFramesDst)) rmSync8(videoFramesDst, { recursive: true, force: true });
|
|
8539
|
+
const videoSourcesDir = join22(planDir, "video-sources");
|
|
8540
|
+
rmSync8(videoSourcesDir, { recursive: true, force: true });
|
|
8541
|
+
if (existsSync15(stagedVideoFrames)) {
|
|
8542
|
+
renameSync4(stagedVideoFrames, videoFramesDst);
|
|
8543
|
+
} else {
|
|
8544
|
+
mkdirSync10(videoFramesDst, { recursive: true });
|
|
8545
|
+
}
|
|
8546
|
+
if (existsSync15(finalCompiledDir)) rmSync8(finalCompiledDir, { recursive: true, force: true });
|
|
8547
|
+
renameSync4(compiledDir, finalCompiledDir);
|
|
8548
|
+
const shipVideoSource = createVideoSourceShipper({
|
|
8549
|
+
planDir,
|
|
8550
|
+
projectDir,
|
|
8551
|
+
workCompiledDir: compiledDir,
|
|
8552
|
+
workVideoFramesDir: stagedVideoFrames,
|
|
8553
|
+
isRemote: (srcPath) => urlByAbsentLocalPath.has(srcPath)
|
|
8554
|
+
});
|
|
8555
|
+
const sourceRoots = [
|
|
8556
|
+
[stagedVideoFrames, "video-frames"],
|
|
8557
|
+
[compiledDir, "compiled"],
|
|
8558
|
+
[projectDir, ""]
|
|
8559
|
+
];
|
|
8560
|
+
const planVideosJson = buildPlanVideosJson({
|
|
8561
|
+
videos: composition.videos,
|
|
8562
|
+
compositionEnd: job.duration ?? Number.NaN,
|
|
8563
|
+
extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => {
|
|
8564
|
+
const sourcePath = localPathByUrl.get(ext.srcPath) ?? ext.srcPath;
|
|
8565
|
+
return {
|
|
8566
|
+
videoId: ext.videoId,
|
|
8567
|
+
srcPath: stableSourcePath(sourceRoots, sourcePath),
|
|
8568
|
+
framePattern: ext.framePattern,
|
|
8569
|
+
fps: ext.fps,
|
|
8570
|
+
totalFrames: ext.totalFrames,
|
|
8571
|
+
metadata: ext.metadata,
|
|
8572
|
+
...ext.deferredRange ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(sourcePath) } } : {}
|
|
8573
|
+
};
|
|
8574
|
+
})
|
|
8575
|
+
});
|
|
8576
|
+
mkdirSync10(join22(planDir, "meta"), { recursive: true });
|
|
8577
|
+
writePlanRemoteFiles(
|
|
8578
|
+
planDir,
|
|
8579
|
+
remoteFiles.map(([rel, { sha256, sizeBytes }]) => ({
|
|
8580
|
+
path: `compiled/${rel}`,
|
|
8581
|
+
sha256,
|
|
8582
|
+
sizeBytes
|
|
8583
|
+
}))
|
|
8614
8584
|
);
|
|
8615
|
-
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
|
|
8585
|
+
writeFileSync6(
|
|
8586
|
+
join22(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),
|
|
8587
|
+
JSON.stringify(planVideosJson, null, 2),
|
|
8588
|
+
"utf-8"
|
|
8619
8589
|
);
|
|
8620
|
-
const
|
|
8621
|
-
|
|
8622
|
-
|
|
8623
|
-
|
|
8624
|
-
|
|
8625
|
-
|
|
8626
|
-
|
|
8627
|
-
|
|
8628
|
-
|
|
8629
|
-
|
|
8630
|
-
|
|
8631
|
-
|
|
8632
|
-
|
|
8633
|
-
|
|
8634
|
-
|
|
8635
|
-
|
|
8636
|
-
|
|
8637
|
-
|
|
8638
|
-
|
|
8639
|
-
|
|
8640
|
-
|
|
8641
|
-
const indexes = mutable.get(path) ?? /* @__PURE__ */ new Set();
|
|
8642
|
-
indexes.add(chunk.index);
|
|
8643
|
-
mutable.set(path, indexes);
|
|
8590
|
+
const digests = options.digests ?? /* @__PURE__ */ new Map();
|
|
8591
|
+
const chunksCarryAudio2 = carriesChunkAudio(config, options);
|
|
8592
|
+
const audioSeconds = chunksCarryAudio2 ? totalFrames / config.fps : probeResult.duration;
|
|
8593
|
+
stageStarted = Date.now();
|
|
8594
|
+
const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([
|
|
8595
|
+
runAudioStage({
|
|
8596
|
+
projectDir,
|
|
8597
|
+
workDir,
|
|
8598
|
+
compiledDir: finalCompiledDir,
|
|
8599
|
+
urlByAbsentLocalPath,
|
|
8600
|
+
duration: audioSeconds,
|
|
8601
|
+
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
|
|
8602
|
+
audioGain: cfg.audioGain,
|
|
8603
|
+
audios: composition.audios,
|
|
8604
|
+
masterFxChain: composition.masterFxChain,
|
|
8605
|
+
abortSignal,
|
|
8606
|
+
assertNotAborted
|
|
8607
|
+
}).finally(() => logPlanStage(log, "audio", stageStarted)),
|
|
8608
|
+
(async () => {
|
|
8609
|
+
for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {
|
|
8610
|
+
await prehashPlanFiles(dir, digests);
|
|
8644
8611
|
}
|
|
8645
|
-
|
|
8646
|
-
|
|
8647
|
-
|
|
8648
|
-
|
|
8649
|
-
|
|
8650
|
-
return manifest;
|
|
8651
|
-
}
|
|
8652
|
-
function computeManifestHash(manifest) {
|
|
8653
|
-
return sha256Hex(`${PLAN_V2_HASH_PREFIX}${canonicalJsonStringify(manifestPayload(manifest))}`);
|
|
8654
|
-
}
|
|
8655
|
-
function writeBlob(sourcePath, destinationPath) {
|
|
8656
|
-
if (existsSync16(destinationPath)) return;
|
|
8657
|
-
mkdirSync10(dirname10(destinationPath), { recursive: true });
|
|
8658
|
-
const temporaryDir = mkdtempSync4(join23(dirname10(destinationPath), ".plan-v2-blob-"));
|
|
8659
|
-
const temporaryPath = join23(temporaryDir, "blob");
|
|
8660
|
-
try {
|
|
8661
|
-
copyFileSync7(sourcePath, temporaryPath);
|
|
8662
|
-
renameSync5(temporaryPath, destinationPath);
|
|
8663
|
-
} finally {
|
|
8664
|
-
rmSync9(temporaryDir, { recursive: true, force: true });
|
|
8665
|
-
}
|
|
8666
|
-
}
|
|
8667
|
-
function publishBlobFor(absolutePath, digests) {
|
|
8668
|
-
return {
|
|
8669
|
-
sourcePath: absolutePath,
|
|
8670
|
-
sha256: planFileSha256(absolutePath, digests),
|
|
8671
|
-
sizeBytes: statSync6(absolutePath).size
|
|
8672
|
-
};
|
|
8673
|
-
}
|
|
8674
|
-
async function putBlobs(publisher, blobs) {
|
|
8675
|
-
await putTasks(blobs.map((blob) => () => publisher.putBlob(blob)));
|
|
8676
|
-
}
|
|
8677
|
-
async function putTasks(tasks) {
|
|
8678
|
-
const uploadSlots = 32;
|
|
8679
|
-
let next = 0;
|
|
8680
|
-
let failure;
|
|
8681
|
-
const upload = async () => {
|
|
8682
|
-
while (!failure && next < tasks.length) {
|
|
8683
|
-
const task = tasks[next++];
|
|
8684
|
-
try {
|
|
8685
|
-
await task();
|
|
8686
|
-
} catch (reason) {
|
|
8687
|
-
failure ??= { reason };
|
|
8612
|
+
logPlanStage(log, "prehash", stageStarted);
|
|
8613
|
+
if (options.whileAudioMixes) {
|
|
8614
|
+
const prepublishStarted = Date.now();
|
|
8615
|
+
await options.whileAudioMixes();
|
|
8616
|
+
logPlanStage(log, "prepublish", prepublishStarted);
|
|
8688
8617
|
}
|
|
8689
|
-
}
|
|
8690
|
-
|
|
8691
|
-
|
|
8692
|
-
|
|
8693
|
-
|
|
8694
|
-
|
|
8695
|
-
|
|
8696
|
-
|
|
8697
|
-
|
|
8698
|
-
|
|
8699
|
-
|
|
8700
|
-
|
|
8701
|
-
|
|
8702
|
-
|
|
8703
|
-
|
|
8704
|
-
|
|
8705
|
-
|
|
8706
|
-
|
|
8707
|
-
|
|
8708
|
-
|
|
8709
|
-
|
|
8710
|
-
if (
|
|
8711
|
-
|
|
8712
|
-
|
|
8618
|
+
})(),
|
|
8619
|
+
measureFrameCost ? measurePlanSecondsPerFrame({
|
|
8620
|
+
planDir,
|
|
8621
|
+
workDir,
|
|
8622
|
+
html: compiled.html,
|
|
8623
|
+
planVideos: planVideosJson,
|
|
8624
|
+
totalFrames,
|
|
8625
|
+
fps: config.fps,
|
|
8626
|
+
width,
|
|
8627
|
+
height,
|
|
8628
|
+
format: config.format,
|
|
8629
|
+
quality: config.quality ?? "standard",
|
|
8630
|
+
deviceScaleFactor,
|
|
8631
|
+
variables: config.variables,
|
|
8632
|
+
cfg: { ...cfg, forceScreenshot, forceScreenshotExplicitlyOptedOut: !forceScreenshot },
|
|
8633
|
+
log
|
|
8634
|
+
}).finally(() => logPlanStage(log, "frame-cost", stageStarted)) : void 0
|
|
8635
|
+
]);
|
|
8636
|
+
if (audioOutcome.status === "rejected") throw audioOutcome.reason;
|
|
8637
|
+
if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;
|
|
8638
|
+
if (frameCostOutcome.status === "rejected") throw frameCostOutcome.reason;
|
|
8639
|
+
if (frameCostOutcome.value !== void 0) {
|
|
8640
|
+
log.info("[plan] measured frame cost", { secondsPerFrame: frameCostOutcome.value });
|
|
8641
|
+
maxParallel = resolveMaxParallelChunks(
|
|
8642
|
+
config,
|
|
8643
|
+
totalFrames,
|
|
8644
|
+
megapixels,
|
|
8645
|
+
log,
|
|
8646
|
+
frameCostOutcome.value
|
|
8713
8647
|
);
|
|
8714
8648
|
}
|
|
8715
|
-
const
|
|
8716
|
-
|
|
8717
|
-
|
|
8718
|
-
|
|
8719
|
-
|
|
8649
|
+
const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
|
|
8650
|
+
totalFrames,
|
|
8651
|
+
config.chunkSize,
|
|
8652
|
+
maxParallel,
|
|
8653
|
+
config.targetChunkFrames
|
|
8654
|
+
);
|
|
8655
|
+
const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
|
|
8656
|
+
const audioResult = audioOutcome.value;
|
|
8657
|
+
if (audioResult.audioError) {
|
|
8658
|
+
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
|
|
8720
8659
|
}
|
|
8721
|
-
|
|
8722
|
-
|
|
8723
|
-
|
|
8724
|
-
|
|
8725
|
-
|
|
8726
|
-
|
|
8727
|
-
const
|
|
8728
|
-
|
|
8729
|
-
|
|
8730
|
-
|
|
8731
|
-
|
|
8732
|
-
if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
|
|
8733
|
-
artifacts.push({
|
|
8734
|
-
path: file.path,
|
|
8735
|
-
sha256: blob.sha256,
|
|
8736
|
-
sizeBytes: blob.sizeBytes,
|
|
8737
|
-
...targets
|
|
8660
|
+
const planAudioPath = join22(planDir, PLAN_AUDIO_RELATIVE_PATH);
|
|
8661
|
+
if (audioResult.hasAudio && existsSync15(audioResult.audioOutputPath)) {
|
|
8662
|
+
renameSync4(audioResult.audioOutputPath, planAudioPath);
|
|
8663
|
+
}
|
|
8664
|
+
if (chunksCarryAudio2 && existsSync15(planAudioPath)) {
|
|
8665
|
+
stageStarted = Date.now();
|
|
8666
|
+
const peak = await constrainAacTruePeak({
|
|
8667
|
+
audioPath: planAudioPath,
|
|
8668
|
+
targetDurationSeconds: audioSeconds,
|
|
8669
|
+
signal: abortSignal,
|
|
8670
|
+
timeoutMs: cfg.ffmpegProcessTimeout
|
|
8738
8671
|
});
|
|
8672
|
+
if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
|
|
8673
|
+
logPlanStage(log, "audio-peak", stageStarted);
|
|
8674
|
+
if (options.sliceChunkAudio) {
|
|
8675
|
+
stageStarted = Date.now();
|
|
8676
|
+
await writeChunkAudioSlices({
|
|
8677
|
+
audioPath: planAudioPath,
|
|
8678
|
+
planDir,
|
|
8679
|
+
chunks,
|
|
8680
|
+
fps: { num: config.fps, den: 1 }
|
|
8681
|
+
});
|
|
8682
|
+
logPlanStage(log, "audio-slices", stageStarted);
|
|
8683
|
+
}
|
|
8739
8684
|
}
|
|
8740
|
-
const
|
|
8741
|
-
|
|
8742
|
-
|
|
8743
|
-
|
|
8744
|
-
|
|
8745
|
-
|
|
8746
|
-
|
|
8685
|
+
const ffmpegVersion = await readFfmpegVersion();
|
|
8686
|
+
const producerVersion = readProducerVersion();
|
|
8687
|
+
const fontSnapshotSha = await readFontSnapshotSha();
|
|
8688
|
+
const runtimeEnv = snapshotRuntimeEnv();
|
|
8689
|
+
const lockedConfig = buildLockedRenderConfig({
|
|
8690
|
+
config,
|
|
8691
|
+
forceScreenshot,
|
|
8692
|
+
deviceScaleFactor,
|
|
8693
|
+
ffmpegVersion,
|
|
8694
|
+
engineConfig: cfg,
|
|
8695
|
+
effectiveChunkSize,
|
|
8696
|
+
chunkCount,
|
|
8697
|
+
runtimeEnv,
|
|
8698
|
+
chunksCarryAudio: chunksCarryAudio2
|
|
8699
|
+
});
|
|
8700
|
+
const compositionJson = {
|
|
8701
|
+
durationSeconds: job.duration ?? 0,
|
|
8702
|
+
width,
|
|
8703
|
+
height,
|
|
8704
|
+
fps: job.config.fps,
|
|
8705
|
+
videoCount: composition.videos.length,
|
|
8706
|
+
audioCount: composition.audios.length,
|
|
8707
|
+
imageCount: composition.images.length
|
|
8708
|
+
};
|
|
8709
|
+
const dimensions = {
|
|
8710
|
+
fpsNum: config.fps,
|
|
8711
|
+
fpsDen: 1,
|
|
8712
|
+
width,
|
|
8713
|
+
height,
|
|
8714
|
+
format: config.format
|
|
8715
|
+
};
|
|
8716
|
+
try {
|
|
8717
|
+
rmSync8(workDir, { recursive: true, force: true });
|
|
8718
|
+
} catch (err) {
|
|
8719
|
+
log.warn("[plan] failed to remove temp work dir", {
|
|
8720
|
+
workDir,
|
|
8721
|
+
error: err instanceof Error ? err.message : String(err)
|
|
8747
8722
|
});
|
|
8748
8723
|
}
|
|
8749
|
-
|
|
8750
|
-
|
|
8751
|
-
|
|
8752
|
-
|
|
8753
|
-
|
|
8754
|
-
|
|
8755
|
-
|
|
8756
|
-
|
|
8757
|
-
|
|
8758
|
-
|
|
8759
|
-
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
|
|
8763
|
-
|
|
8724
|
+
removeFreezeOwnedPlanFiles(planDir);
|
|
8725
|
+
assertPlanSizeWithinLimit({
|
|
8726
|
+
rootDir: planDir,
|
|
8727
|
+
rootKind: "plan",
|
|
8728
|
+
limitBytes: sizeLimitBytes,
|
|
8729
|
+
observedAt: "pre-freeze",
|
|
8730
|
+
log
|
|
8731
|
+
});
|
|
8732
|
+
stageStarted = Date.now();
|
|
8733
|
+
const freezeResult = await freezePlan({
|
|
8734
|
+
planDir,
|
|
8735
|
+
composition: compositionJson,
|
|
8736
|
+
encoder: lockedConfig,
|
|
8737
|
+
chunks,
|
|
8738
|
+
dimensions,
|
|
8739
|
+
producerVersion,
|
|
8740
|
+
fontSnapshotSha,
|
|
8741
|
+
durationSeconds: job.duration ?? 0,
|
|
8742
|
+
totalFrames,
|
|
8743
|
+
hasAudio: audioResult.hasAudio,
|
|
8744
|
+
digests
|
|
8745
|
+
});
|
|
8746
|
+
logPlanStage(log, "freeze", stageStarted);
|
|
8747
|
+
const planHash = freezeResult.planHash;
|
|
8748
|
+
assertPlanSizeWithinLimit({
|
|
8749
|
+
rootDir: planDir,
|
|
8750
|
+
rootKind: "plan",
|
|
8751
|
+
limitBytes: sizeLimitBytes,
|
|
8752
|
+
observedAt: "post-freeze",
|
|
8753
|
+
log
|
|
8754
|
+
});
|
|
8755
|
+
return {
|
|
8756
|
+
executionPlanDir: planDir,
|
|
8757
|
+
executionPlanHash: planHash,
|
|
8758
|
+
chunkCount,
|
|
8759
|
+
totalFrames,
|
|
8760
|
+
fps: config.fps,
|
|
8761
|
+
width,
|
|
8762
|
+
height,
|
|
8763
|
+
format: config.format,
|
|
8764
|
+
ffmpegVersion,
|
|
8765
|
+
producerVersion
|
|
8764
8766
|
};
|
|
8767
|
+
}
|
|
8768
|
+
async function plan(projectDir, config, planDir, options = {}) {
|
|
8769
|
+
const executionPlan = await buildLocalExecutionPlan(projectDir, config, planDir, options);
|
|
8765
8770
|
return {
|
|
8766
|
-
|
|
8767
|
-
|
|
8768
|
-
|
|
8769
|
-
|
|
8770
|
-
|
|
8771
|
-
|
|
8771
|
+
planDir: executionPlan.executionPlanDir,
|
|
8772
|
+
planProtocol: PLAN_PROTOCOL_V1,
|
|
8773
|
+
planHash: executionPlan.executionPlanHash,
|
|
8774
|
+
chunkCount: executionPlan.chunkCount,
|
|
8775
|
+
totalFrames: executionPlan.totalFrames,
|
|
8776
|
+
fps: executionPlan.fps,
|
|
8777
|
+
width: executionPlan.width,
|
|
8778
|
+
height: executionPlan.height,
|
|
8779
|
+
format: executionPlan.format,
|
|
8780
|
+
ffmpegVersion: executionPlan.ffmpegVersion,
|
|
8781
|
+
producerVersion: executionPlan.producerVersion
|
|
8772
8782
|
};
|
|
8773
8783
|
}
|
|
8774
|
-
|
|
8775
|
-
|
|
8776
|
-
|
|
8777
|
-
|
|
8778
|
-
|
|
8779
|
-
|
|
8780
|
-
|
|
8784
|
+
|
|
8785
|
+
// src/services/distributed/planV2.ts
|
|
8786
|
+
import {
|
|
8787
|
+
copyFileSync as copyFileSync7,
|
|
8788
|
+
existsSync as existsSync17,
|
|
8789
|
+
lstatSync as lstatSync2,
|
|
8790
|
+
mkdirSync as mkdirSync12,
|
|
8791
|
+
mkdtempSync as mkdtempSync5,
|
|
8792
|
+
readFileSync as readFileSync10,
|
|
8793
|
+
readdirSync as readdirSync7,
|
|
8794
|
+
renameSync as renameSync6,
|
|
8795
|
+
rmSync as rmSync10,
|
|
8796
|
+
statSync as statSync6,
|
|
8797
|
+
writeFileSync as writeFileSync8
|
|
8798
|
+
} from "node:fs";
|
|
8799
|
+
import { tmpdir } from "node:os";
|
|
8800
|
+
import { dirname as dirname11, isAbsolute as isAbsolute4, join as join25, relative as relative6, resolve as resolve8, sep as sep3 } from "node:path";
|
|
8801
|
+
import { DISTRIBUTED_FPS, isDistributedFps } from "@hyperframes/core";
|
|
8802
|
+
import { createFrameLookupTable as createFrameLookupTable4 } from "@hyperframes/engine";
|
|
8803
|
+
|
|
8804
|
+
// src/services/distributed/planV2Publisher.ts
|
|
8805
|
+
import {
|
|
8806
|
+
copyFileSync as copyFileSync6,
|
|
8807
|
+
existsSync as existsSync16,
|
|
8808
|
+
linkSync as linkSync2,
|
|
8809
|
+
mkdirSync as mkdirSync11,
|
|
8810
|
+
mkdtempSync as mkdtempSync4,
|
|
8811
|
+
renameSync as renameSync5,
|
|
8812
|
+
rmSync as rmSync9,
|
|
8813
|
+
statSync as statSync5,
|
|
8814
|
+
writeFileSync as writeFileSync7
|
|
8815
|
+
} from "node:fs";
|
|
8816
|
+
import { dirname as dirname10, join as join24 } from "node:path";
|
|
8817
|
+
|
|
8818
|
+
// src/services/distributed/planV2Errors.ts
|
|
8819
|
+
var PLAN_V2_INTEGRITY_UNRECOVERABLE = "PLAN_V2_INTEGRITY_UNRECOVERABLE";
|
|
8820
|
+
var PlanV2IntegrityError = class extends Error {
|
|
8821
|
+
// Public adapters inspect this typed code even though OSS producer does not.
|
|
8822
|
+
// fallow-ignore-next-line unused-class-member
|
|
8823
|
+
code = PLAN_V2_INTEGRITY_UNRECOVERABLE;
|
|
8824
|
+
constructor(message) {
|
|
8825
|
+
super(`[planV2] ${message}`);
|
|
8826
|
+
this.name = "PlanV2IntegrityError";
|
|
8781
8827
|
}
|
|
8782
|
-
|
|
8783
|
-
|
|
8784
|
-
|
|
8785
|
-
|
|
8786
|
-
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
8790
|
-
canonicalJsonStringify(publication.manifest),
|
|
8791
|
-
"utf-8"
|
|
8792
|
-
);
|
|
8793
|
-
renameSync5(tempDir, planV2Dir);
|
|
8794
|
-
return resultFromManifest(planV2Dir, publication.manifest);
|
|
8795
|
-
} catch (error) {
|
|
8796
|
-
rmSync9(tempDir, { recursive: true, force: true });
|
|
8797
|
-
throw error;
|
|
8828
|
+
};
|
|
8829
|
+
|
|
8830
|
+
// src/services/distributed/planV2Layout.ts
|
|
8831
|
+
import { join as join23 } from "node:path";
|
|
8832
|
+
var SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
|
8833
|
+
function assertPlanV2Sha256(value, field) {
|
|
8834
|
+
if (typeof value !== "string" || !SHA256_PATTERN.test(value)) {
|
|
8835
|
+
throw new PlanV2IntegrityError(`${field} must be a lowercase sha256 digest`);
|
|
8798
8836
|
}
|
|
8837
|
+
return value;
|
|
8799
8838
|
}
|
|
8800
|
-
function
|
|
8801
|
-
|
|
8839
|
+
function planV2BlobPath(root, sha256) {
|
|
8840
|
+
const digest = assertPlanV2Sha256(sha256, "artifact sha256");
|
|
8841
|
+
return join23(root, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
8802
8842
|
}
|
|
8803
|
-
|
|
8804
|
-
|
|
8843
|
+
|
|
8844
|
+
// src/services/distributed/planV2Publisher.ts
|
|
8845
|
+
function canFallbackToCopy(error) {
|
|
8846
|
+
if (!(error instanceof Error) || !("code" in error)) return false;
|
|
8847
|
+
const code = error.code;
|
|
8848
|
+
return code === "EXDEV" || code === "EPERM" || code === "EACCES" || code === "ENOTSUP";
|
|
8805
8849
|
}
|
|
8806
|
-
|
|
8850
|
+
function isRecord2(value) {
|
|
8851
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
8852
|
+
}
|
|
8853
|
+
function manifestDigests(manifestBytes) {
|
|
8854
|
+
let value;
|
|
8807
8855
|
try {
|
|
8808
|
-
|
|
8809
|
-
|
|
8810
|
-
|
|
8811
|
-
|
|
8812
|
-
|
|
8856
|
+
value = JSON.parse(manifestBytes);
|
|
8857
|
+
} catch {
|
|
8858
|
+
throw new PlanV2IntegrityError("manifest publisher received invalid JSON");
|
|
8859
|
+
}
|
|
8860
|
+
if (!isRecord2(value) || !Array.isArray(value.artifacts)) {
|
|
8861
|
+
throw new PlanV2IntegrityError("manifest publisher requires an artifacts array");
|
|
8862
|
+
}
|
|
8863
|
+
return value.artifacts.map((artifact, index) => {
|
|
8864
|
+
if (!isRecord2(artifact)) {
|
|
8865
|
+
throw new PlanV2IntegrityError(`manifest artifacts[${index}] must be an object`);
|
|
8813
8866
|
}
|
|
8814
|
-
|
|
8815
|
-
|
|
8816
|
-
|
|
8817
|
-
|
|
8818
|
-
|
|
8819
|
-
|
|
8820
|
-
|
|
8821
|
-
|
|
8822
|
-
|
|
8823
|
-
|
|
8824
|
-
|
|
8825
|
-
try {
|
|
8826
|
-
await publisher.abort();
|
|
8827
|
-
} catch {
|
|
8867
|
+
return assertPlanV2Sha256(artifact.sha256, `manifest artifacts[${index}].sha256`);
|
|
8868
|
+
});
|
|
8869
|
+
}
|
|
8870
|
+
var LocalPlanV2ArtifactPublisher = class {
|
|
8871
|
+
destinationDir;
|
|
8872
|
+
temporaryDir;
|
|
8873
|
+
#linkFile;
|
|
8874
|
+
#committed = false;
|
|
8875
|
+
constructor(destinationDir, options = {}) {
|
|
8876
|
+
if (existsSync16(destinationDir)) {
|
|
8877
|
+
throw new PlanV2IntegrityError(`output directory already exists: ${destinationDir}`);
|
|
8828
8878
|
}
|
|
8829
|
-
|
|
8879
|
+
this.destinationDir = destinationDir;
|
|
8880
|
+
this.#linkFile = options.linkFile ?? linkSync2;
|
|
8881
|
+
mkdirSync11(dirname10(destinationDir), { recursive: true });
|
|
8882
|
+
this.temporaryDir = mkdtempSync4(join24(dirname10(destinationDir), ".plan-v2-publish-"));
|
|
8830
8883
|
}
|
|
8831
|
-
|
|
8832
|
-
|
|
8833
|
-
|
|
8834
|
-
|
|
8835
|
-
|
|
8836
|
-
|
|
8837
|
-
|
|
8838
|
-
|
|
8839
|
-
|
|
8840
|
-
|
|
8841
|
-
|
|
8842
|
-
|
|
8843
|
-
|
|
8844
|
-
// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
|
|
8845
|
-
whileAudioMixes: () => putBlobs(
|
|
8846
|
-
publisher,
|
|
8847
|
-
listFiles(join23(stagingRoot, "compiled")).map(
|
|
8848
|
-
(file) => publishBlobFor(file.absolutePath, digests)
|
|
8849
|
-
)
|
|
8850
|
-
),
|
|
8851
|
-
chunksCarryAudio: options.chunksCarryAudio
|
|
8852
|
-
});
|
|
8853
|
-
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
|
|
8854
|
-
} catch (error) {
|
|
8884
|
+
async putBlob(blob) {
|
|
8885
|
+
const digest = assertPlanV2Sha256(blob.sha256, "published blob sha256");
|
|
8886
|
+
const sourceSize = statSync5(blob.sourcePath).size;
|
|
8887
|
+
if (sourceSize !== blob.sizeBytes) {
|
|
8888
|
+
throw new PlanV2IntegrityError(
|
|
8889
|
+
`published blob size changed for ${digest}: expected ${blob.sizeBytes}, got ${sourceSize}`
|
|
8890
|
+
);
|
|
8891
|
+
}
|
|
8892
|
+
const destinationPath = planV2BlobPath(this.temporaryDir, digest);
|
|
8893
|
+
if (existsSync16(destinationPath)) return;
|
|
8894
|
+
mkdirSync11(dirname10(destinationPath), { recursive: true });
|
|
8895
|
+
const stagingDir = mkdtempSync4(join24(dirname10(destinationPath), ".plan-v2-blob-"));
|
|
8896
|
+
const temporaryPath = join24(stagingDir, "blob");
|
|
8855
8897
|
try {
|
|
8856
|
-
|
|
8857
|
-
|
|
8898
|
+
try {
|
|
8899
|
+
this.#linkFile(blob.sourcePath, temporaryPath);
|
|
8900
|
+
} catch (error) {
|
|
8901
|
+
if (!canFallbackToCopy(error)) throw error;
|
|
8902
|
+
copyFileSync6(blob.sourcePath, temporaryPath);
|
|
8903
|
+
}
|
|
8904
|
+
renameSync5(temporaryPath, destinationPath);
|
|
8905
|
+
} finally {
|
|
8906
|
+
rmSync9(stagingDir, { recursive: true, force: true });
|
|
8858
8907
|
}
|
|
8859
|
-
throw error;
|
|
8860
|
-
} finally {
|
|
8861
|
-
rmSync9(stagingRoot, { recursive: true, force: true });
|
|
8862
8908
|
}
|
|
8863
|
-
|
|
8864
|
-
|
|
8865
|
-
|
|
8866
|
-
|
|
8909
|
+
async commitManifest(manifestBytes) {
|
|
8910
|
+
for (const digest of new Set(manifestDigests(manifestBytes))) {
|
|
8911
|
+
if (!existsSync16(planV2BlobPath(this.temporaryDir, digest))) {
|
|
8912
|
+
throw new PlanV2IntegrityError(
|
|
8913
|
+
`cannot commit manifest before referenced blob is durable: ${digest}`
|
|
8914
|
+
);
|
|
8915
|
+
}
|
|
8916
|
+
}
|
|
8917
|
+
writeFileSync7(join24(this.temporaryDir, "plan.json"), manifestBytes, "utf-8");
|
|
8918
|
+
renameSync5(this.temporaryDir, this.destinationDir);
|
|
8919
|
+
this.#committed = true;
|
|
8867
8920
|
}
|
|
8868
|
-
|
|
8869
|
-
|
|
8870
|
-
|
|
8871
|
-
|
|
8872
|
-
|
|
8921
|
+
async abort() {
|
|
8922
|
+
if (!this.#committed) {
|
|
8923
|
+
rmSync9(this.temporaryDir, { recursive: true, force: true });
|
|
8924
|
+
}
|
|
8925
|
+
}
|
|
8926
|
+
};
|
|
8927
|
+
|
|
8928
|
+
// src/services/distributed/planV2.ts
|
|
8929
|
+
var PLAN_V2_HASH_PREFIX = "hyperframes-plan-manifest-hash-v2\0";
|
|
8930
|
+
var EXTRACTION_CACHE_COMPLETE_SENTINEL = ".hf-complete";
|
|
8931
|
+
var PLAN_V2_MATERIALIZATION_MARKER = ".hyperframes-plan-v2.json";
|
|
8932
|
+
function isRecord3(value) {
|
|
8933
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
8873
8934
|
}
|
|
8874
|
-
function
|
|
8875
|
-
return {
|
|
8876
|
-
planDir: planV2Dir,
|
|
8877
|
-
manifestPath: join23(planV2Dir, "plan.json"),
|
|
8878
|
-
planProtocol: PLAN_PROTOCOL_V2,
|
|
8879
|
-
planHash: manifest.planHash,
|
|
8880
|
-
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
8881
|
-
chunkCount: manifest.chunkCount,
|
|
8882
|
-
totalFrames: manifest.totalFrames,
|
|
8883
|
-
fps: manifest.fps,
|
|
8884
|
-
width: manifest.width,
|
|
8885
|
-
height: manifest.height,
|
|
8886
|
-
format: manifest.format,
|
|
8887
|
-
ffmpegVersion: manifest.ffmpegVersion,
|
|
8888
|
-
producerVersion: manifest.producerVersion,
|
|
8889
|
-
limitations: manifest.limitations
|
|
8890
|
-
};
|
|
8935
|
+
function isSha256(value) {
|
|
8936
|
+
return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
|
|
8891
8937
|
}
|
|
8892
|
-
function
|
|
8893
|
-
return
|
|
8938
|
+
function isNonNegativeIntegerArray(value) {
|
|
8939
|
+
return Array.isArray(value) && value.every((item) => typeof item === "number" && Number.isInteger(item) && item >= 0);
|
|
8894
8940
|
}
|
|
8895
|
-
function
|
|
8896
|
-
|
|
8897
|
-
|
|
8941
|
+
function readJsonFile(path, label) {
|
|
8942
|
+
const contents = readFileSync10(path, "utf-8");
|
|
8943
|
+
try {
|
|
8944
|
+
const parsed = JSON.parse(contents);
|
|
8945
|
+
return parsed;
|
|
8946
|
+
} catch (error) {
|
|
8947
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
8948
|
+
throw new PlanV2IntegrityError(`${label} is not readable JSON: ${detail}`);
|
|
8898
8949
|
}
|
|
8899
|
-
return value;
|
|
8900
8950
|
}
|
|
8901
|
-
function
|
|
8902
|
-
|
|
8903
|
-
|
|
8951
|
+
function assertSafeRelativePath(path) {
|
|
8952
|
+
const normalized = path.split(/[\\/]+/);
|
|
8953
|
+
if (path.length === 0 || isAbsolute4(path) || normalized.some((part) => part === "" || part === "." || part === "..")) {
|
|
8954
|
+
throw new PlanV2IntegrityError(`unsafe artifact path: ${JSON.stringify(path)}`);
|
|
8904
8955
|
}
|
|
8905
|
-
return value;
|
|
8906
8956
|
}
|
|
8907
|
-
function
|
|
8908
|
-
|
|
8909
|
-
|
|
8957
|
+
function listFiles(root) {
|
|
8958
|
+
const files = [];
|
|
8959
|
+
const rootResolved = resolve8(root);
|
|
8960
|
+
function walk(dir) {
|
|
8961
|
+
for (const entry of readdirSync7(dir, { withFileTypes: true })) {
|
|
8962
|
+
const absolutePath = join25(dir, entry.name);
|
|
8963
|
+
if (entry.isDirectory()) {
|
|
8964
|
+
walk(absolutePath);
|
|
8965
|
+
} else if (entry.isFile()) {
|
|
8966
|
+
files.push({
|
|
8967
|
+
path: relative6(rootResolved, absolutePath).split(sep3).join("/"),
|
|
8968
|
+
absolutePath
|
|
8969
|
+
});
|
|
8970
|
+
} else {
|
|
8971
|
+
throw new PlanV2IntegrityError(
|
|
8972
|
+
`unsupported filesystem entry (symlinks and special files are not allowed): ${absolutePath}`
|
|
8973
|
+
);
|
|
8974
|
+
}
|
|
8975
|
+
}
|
|
8910
8976
|
}
|
|
8911
|
-
|
|
8977
|
+
walk(rootResolved);
|
|
8978
|
+
return files.sort((a, b) => a.path.localeCompare(b.path));
|
|
8912
8979
|
}
|
|
8913
|
-
function
|
|
8914
|
-
|
|
8980
|
+
function assertValidExtractionCacheCompleteSentinel(path) {
|
|
8981
|
+
const sentinel = lstatSync2(path);
|
|
8982
|
+
if (!sentinel.isFile() || sentinel.size !== 0) {
|
|
8915
8983
|
throw new PlanV2IntegrityError(
|
|
8916
|
-
|
|
8984
|
+
`${EXTRACTION_CACHE_COMPLETE_SENTINEL} must be a zero-byte regular file`
|
|
8917
8985
|
);
|
|
8918
8986
|
}
|
|
8919
|
-
return value;
|
|
8920
8987
|
}
|
|
8921
|
-
function
|
|
8922
|
-
const
|
|
8923
|
-
if (
|
|
8924
|
-
|
|
8988
|
+
function validateExtractionCacheCompleteSentinels(executionPlanDir) {
|
|
8989
|
+
const videoRoot = join25(executionPlanDir, "video-frames");
|
|
8990
|
+
if (!existsSync17(videoRoot)) return;
|
|
8991
|
+
for (const videoEntry of readdirSync7(videoRoot, { withFileTypes: true })) {
|
|
8992
|
+
if (!videoEntry.isDirectory()) continue;
|
|
8993
|
+
const videoDir = join25(videoRoot, videoEntry.name);
|
|
8994
|
+
if (readdirSync7(videoDir).includes(EXTRACTION_CACHE_COMPLETE_SENTINEL)) {
|
|
8995
|
+
const sentinelPath = join25(videoDir, EXTRACTION_CACHE_COMPLETE_SENTINEL);
|
|
8996
|
+
assertValidExtractionCacheCompleteSentinel(sentinelPath);
|
|
8997
|
+
}
|
|
8925
8998
|
}
|
|
8926
|
-
return readSupportedFps(dimensions.fpsNum);
|
|
8927
8999
|
}
|
|
8928
|
-
function
|
|
8929
|
-
|
|
8930
|
-
|
|
8931
|
-
}
|
|
8932
|
-
return value;
|
|
9000
|
+
function isExtractionCacheCompleteSentinelPath(path) {
|
|
9001
|
+
const segments = path.split("/");
|
|
9002
|
+
return segments.length === 3 && segments[0] === "video-frames" && segments[1] !== "" && segments[2] === EXTRACTION_CACHE_COMPLETE_SENTINEL;
|
|
8933
9003
|
}
|
|
8934
|
-
function
|
|
8935
|
-
|
|
8936
|
-
const
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].sha256 must be a lowercase sha256`);
|
|
8940
|
-
}
|
|
8941
|
-
const sizeBytes = value.sizeBytes;
|
|
8942
|
-
if (typeof sizeBytes !== "number" || !Number.isInteger(sizeBytes) || sizeBytes < 0) {
|
|
8943
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].sizeBytes must be a non-negative integer`);
|
|
8944
|
-
}
|
|
8945
|
-
let chunks;
|
|
8946
|
-
if (value.chunks === "all") {
|
|
8947
|
-
chunks = "all";
|
|
8948
|
-
} else if (isNonNegativeIntegerArray(value.chunks)) {
|
|
8949
|
-
chunks = value.chunks;
|
|
8950
|
-
} else {
|
|
8951
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].chunks is invalid`);
|
|
8952
|
-
}
|
|
8953
|
-
if (typeof value.assembler !== "boolean") {
|
|
8954
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].assembler must be boolean`);
|
|
9004
|
+
function resolveExtractedVideoOutputDir(planDir, videoId) {
|
|
9005
|
+
const videoRoot = resolve8(planDir, "video-frames");
|
|
9006
|
+
const outputDir = resolve8(videoRoot, videoId);
|
|
9007
|
+
if (outputDir === videoRoot || !outputDir.startsWith(`${videoRoot}${sep3}`)) {
|
|
9008
|
+
throw new PlanV2IntegrityError(`unsafe extracted video id: ${JSON.stringify(videoId)}`);
|
|
8955
9009
|
}
|
|
8956
|
-
return
|
|
8957
|
-
path,
|
|
8958
|
-
sha256: value.sha256,
|
|
8959
|
-
sizeBytes,
|
|
8960
|
-
chunks,
|
|
8961
|
-
assembler: value.assembler
|
|
8962
|
-
};
|
|
9010
|
+
return outputDir;
|
|
8963
9011
|
}
|
|
8964
|
-
function
|
|
8965
|
-
if (
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
}
|
|
8969
|
-
if (!isSha256(value.planHash) || !isSha256(value.sourcePlanV1Hash)) {
|
|
8970
|
-
throw new PlanV2IntegrityError("manifest hashes must be lowercase sha256 digests");
|
|
9012
|
+
function artifactTargets(path, videoDependencies, chunkAudio) {
|
|
9013
|
+
if (isPlanAudioArtifactPath(path)) {
|
|
9014
|
+
if (chunkAudio === "none") return { chunks: [], assembler: true };
|
|
9015
|
+
return { chunks: chunkAudio === "whole" ? "all" : [], assembler: false };
|
|
8971
9016
|
}
|
|
8972
|
-
|
|
8973
|
-
|
|
8974
|
-
|
|
8975
|
-
|
|
8976
|
-
if (paths.has(artifact.path))
|
|
8977
|
-
throw new PlanV2IntegrityError(`duplicate artifact path: ${artifact.path}`);
|
|
8978
|
-
paths.add(artifact.path);
|
|
9017
|
+
const slice = chunkAudioSliceTarget(path);
|
|
9018
|
+
if (slice !== null) return { chunks: slice === "all" ? "all" : [slice], assembler: false };
|
|
9019
|
+
if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {
|
|
9020
|
+
return { chunks: "all", assembler: true };
|
|
8979
9021
|
}
|
|
8980
|
-
if (
|
|
8981
|
-
|
|
8982
|
-
|
|
8983
|
-
|
|
9022
|
+
if (path.startsWith("video-frames/") || path.startsWith("video-sources/") || path.startsWith("compiled/") && videoDependencies?.has(path)) {
|
|
9023
|
+
return {
|
|
9024
|
+
chunks: videoDependencies === null ? "all" : videoDependencies.get(path) ?? [],
|
|
9025
|
+
assembler: false
|
|
9026
|
+
};
|
|
8984
9027
|
}
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
planHash: value.planHash,
|
|
8988
|
-
sourcePlanV1Hash: value.sourcePlanV1Hash,
|
|
8989
|
-
chunkCount: readPositiveInteger2(value.chunkCount, "chunkCount"),
|
|
8990
|
-
totalFrames: readPositiveInteger2(value.totalFrames, "totalFrames"),
|
|
8991
|
-
fps: readSupportedFps(value.fps),
|
|
8992
|
-
width: readPositiveInteger2(value.width, "width"),
|
|
8993
|
-
height: readPositiveInteger2(value.height, "height"),
|
|
8994
|
-
format: readDistributedFormat(value.format),
|
|
8995
|
-
ffmpegVersion: readString2(value.ffmpegVersion, "ffmpegVersion"),
|
|
8996
|
-
producerVersion: readString2(value.producerVersion, "producerVersion"),
|
|
8997
|
-
limitations: { videoDependencyMode: value.limitations.videoDependencyMode },
|
|
8998
|
-
artifacts
|
|
8999
|
-
};
|
|
9000
|
-
const { planHash: _planHash, ...payload } = parsed;
|
|
9001
|
-
const expectedHash = computeManifestHash(payload);
|
|
9002
|
-
if (expectedHash !== parsed.planHash) {
|
|
9003
|
-
throw new PlanV2IntegrityError("manifest integrity hash mismatch");
|
|
9028
|
+
if (path === "meta/composition.json" || path === "meta/videos.json" || path.startsWith("compiled/")) {
|
|
9029
|
+
return { chunks: "all", assembler: false };
|
|
9004
9030
|
}
|
|
9005
|
-
|
|
9006
|
-
|
|
9007
|
-
|
|
9031
|
+
return { chunks: "all", assembler: true };
|
|
9032
|
+
}
|
|
9033
|
+
function isPlanV2DeferredSourceArtifact(artifact) {
|
|
9034
|
+
return artifact.path.startsWith("compiled/") && artifact.chunks !== "all";
|
|
9035
|
+
}
|
|
9036
|
+
function listVideoFramePaths(executionPlanDir, videos) {
|
|
9037
|
+
return videos.extracted.map((video) => {
|
|
9038
|
+
const outputDir = resolveExtractedVideoOutputDir(executionPlanDir, video.videoId);
|
|
9039
|
+
if (video.deferredRange) {
|
|
9040
|
+
const framePaths2 = /* @__PURE__ */ new Map();
|
|
9041
|
+
for (let index = 0; index < video.totalFrames; index++) {
|
|
9042
|
+
framePaths2.set(index, join25(outputDir, String(index)));
|
|
9043
|
+
}
|
|
9044
|
+
return { ...video, outputDir, framePaths: framePaths2, ownedByLookup: false };
|
|
9045
|
+
}
|
|
9046
|
+
const frameNames = readdirSync7(outputDir).sort();
|
|
9047
|
+
const framePaths = /* @__PURE__ */ new Map();
|
|
9048
|
+
for (const frameName of frameNames) {
|
|
9049
|
+
if (frameName === EXTRACTION_CACHE_COMPLETE_SENTINEL) {
|
|
9050
|
+
assertValidExtractionCacheCompleteSentinel(join25(outputDir, frameName));
|
|
9051
|
+
continue;
|
|
9052
|
+
}
|
|
9053
|
+
const match = /(\d+)(?=\.[^.]+$)/.exec(frameName);
|
|
9054
|
+
if (!match) {
|
|
9055
|
+
throw new PlanV2IntegrityError(`cannot derive extracted frame index from ${frameName}`);
|
|
9056
|
+
}
|
|
9057
|
+
const oneBasedFrameNumber = Number(match[1]);
|
|
9058
|
+
if (!Number.isSafeInteger(oneBasedFrameNumber) || oneBasedFrameNumber < 1) {
|
|
9059
|
+
throw new PlanV2IntegrityError(
|
|
9060
|
+
`extracted frame filename must use a 1-based safe integer: ${frameName}`
|
|
9061
|
+
);
|
|
9062
|
+
}
|
|
9063
|
+
const frameIndex = oneBasedFrameNumber - 1;
|
|
9064
|
+
if (framePaths.has(frameIndex)) {
|
|
9065
|
+
throw new PlanV2IntegrityError(
|
|
9066
|
+
`duplicate extracted frame index ${frameIndex} in ${outputDir}`
|
|
9067
|
+
);
|
|
9068
|
+
}
|
|
9069
|
+
framePaths.set(frameIndex, join25(outputDir, frameName));
|
|
9070
|
+
}
|
|
9071
|
+
return {
|
|
9072
|
+
...video,
|
|
9073
|
+
outputDir,
|
|
9074
|
+
framePaths,
|
|
9075
|
+
ownedByLookup: false
|
|
9076
|
+
};
|
|
9077
|
+
});
|
|
9078
|
+
}
|
|
9079
|
+
function parsePlanVideosJson2(value) {
|
|
9080
|
+
try {
|
|
9081
|
+
return parsePlanVideosJson(value);
|
|
9082
|
+
} catch (err) {
|
|
9083
|
+
if (err instanceof PlanVideosMetadataError) {
|
|
9084
|
+
throw new PlanV2IntegrityError(err.message);
|
|
9008
9085
|
}
|
|
9086
|
+
throw err;
|
|
9009
9087
|
}
|
|
9010
|
-
return parsed;
|
|
9011
9088
|
}
|
|
9012
|
-
function
|
|
9013
|
-
const
|
|
9014
|
-
if (!
|
|
9015
|
-
|
|
9089
|
+
function materializeExtractedVideoDirectories(planDir) {
|
|
9090
|
+
const videosPath = join25(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9091
|
+
if (!existsSync17(videosPath)) return;
|
|
9092
|
+
const videos = parsePlanVideosJson2(readJsonFile(videosPath, PLAN_VIDEOS_META_RELATIVE_PATH));
|
|
9093
|
+
for (const video of videos.extracted) {
|
|
9094
|
+
mkdirSync12(resolveExtractedVideoOutputDir(planDir, video.videoId), { recursive: true });
|
|
9016
9095
|
}
|
|
9017
|
-
const value = readJsonFile(manifestPath, "v2 manifest");
|
|
9018
|
-
return parsePlanV2Manifest(value);
|
|
9019
9096
|
}
|
|
9020
|
-
function
|
|
9021
|
-
if (
|
|
9022
|
-
|
|
9023
|
-
throw new PlanV2IntegrityError(`chunkIndex ${String(target.chunkIndex)} is out of range`);
|
|
9024
|
-
}
|
|
9025
|
-
return manifest.artifacts.filter(
|
|
9026
|
-
(artifact) => artifact.chunks === "all" || artifact.chunks.some((chunkIndex) => chunkIndex === target.chunkIndex)
|
|
9027
|
-
);
|
|
9097
|
+
function parseChunkSlices(value) {
|
|
9098
|
+
if (!Array.isArray(value)) {
|
|
9099
|
+
throw new PlanV2IntegrityError("meta/chunks.json must be an array");
|
|
9028
9100
|
}
|
|
9029
|
-
|
|
9101
|
+
const indexes = /* @__PURE__ */ new Set();
|
|
9102
|
+
return value.map((chunk, position) => {
|
|
9103
|
+
const field = `meta/chunks.json[${position}]`;
|
|
9104
|
+
if (!isRecord3(chunk)) throw new PlanV2IntegrityError(`${field} must be an object`);
|
|
9105
|
+
const index = readNonNegativeInteger2(chunk.index, `${field}.index`);
|
|
9106
|
+
const startFrame = readNonNegativeInteger2(chunk.startFrame, `${field}.startFrame`);
|
|
9107
|
+
const endFrame = readPositiveInteger2(chunk.endFrame, `${field}.endFrame`);
|
|
9108
|
+
if (endFrame <= startFrame) {
|
|
9109
|
+
throw new PlanV2IntegrityError(`${field}.endFrame must be greater than startFrame`);
|
|
9110
|
+
}
|
|
9111
|
+
if (indexes.has(index)) {
|
|
9112
|
+
throw new PlanV2IntegrityError(`meta/chunks.json contains duplicate chunk index ${index}`);
|
|
9113
|
+
}
|
|
9114
|
+
indexes.add(index);
|
|
9115
|
+
return { index, startFrame, endFrame };
|
|
9116
|
+
});
|
|
9030
9117
|
}
|
|
9031
|
-
function
|
|
9032
|
-
const
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
if (!stats.isFile() || stats.size !== artifact.sizeBytes) {
|
|
9038
|
-
throw new PlanV2IntegrityError(`artifact size mismatch for ${artifact.path}`);
|
|
9039
|
-
}
|
|
9040
|
-
if (sha256File(sourcePath) !== artifact.sha256) {
|
|
9041
|
-
throw new PlanV2IntegrityError(`artifact hash mismatch for ${artifact.path}`);
|
|
9118
|
+
function buildVideoChunkDependencies(executionPlanDir, dimensions) {
|
|
9119
|
+
const videoRoot = join25(executionPlanDir, "video-frames");
|
|
9120
|
+
const hasExtractedFrames = existsSync17(videoRoot) && listFiles(videoRoot).length > 0;
|
|
9121
|
+
const videosPath = join25(executionPlanDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9122
|
+
if (!existsSync17(videosPath)) {
|
|
9123
|
+
return hasExtractedFrames ? { mode: "full-source-pack", dependencies: null } : { mode: "exact-rendered-frames", dependencies: /* @__PURE__ */ new Map() };
|
|
9042
9124
|
}
|
|
9043
|
-
|
|
9125
|
+
const chunksPath = join25(executionPlanDir, "meta", "chunks.json");
|
|
9126
|
+
const videos = readJsonFile(videosPath, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9127
|
+
const chunks = readJsonFile(chunksPath, "meta/chunks.json");
|
|
9128
|
+
const parsedVideos = parsePlanVideosJson2(videos);
|
|
9129
|
+
const parsedChunks = parseChunkSlices(chunks);
|
|
9130
|
+
const extracted = listVideoFramePaths(executionPlanDir, parsedVideos);
|
|
9131
|
+
const fpsNum = readPositiveInteger2(dimensions.fpsNum, "dimensions.fpsNum");
|
|
9132
|
+
const fpsDen = readPositiveInteger2(dimensions.fpsDen, "dimensions.fpsDen");
|
|
9133
|
+
const table = createFrameLookupTable4(
|
|
9134
|
+
parsedVideos.videos,
|
|
9135
|
+
extracted,
|
|
9136
|
+
resolveRenderFpsConfig({ num: fpsNum, den: fpsDen }).value
|
|
9137
|
+
);
|
|
9138
|
+
const deferredSources = new Map(
|
|
9139
|
+
parsedVideos.extracted.flatMap(
|
|
9140
|
+
(video) => video.deferredRange ? [[video.videoId, video.deferredRange.sourcePath]] : []
|
|
9141
|
+
)
|
|
9142
|
+
);
|
|
9143
|
+
const dependencies = collectChunkVideoDependencies({
|
|
9144
|
+
executionPlanDir,
|
|
9145
|
+
table,
|
|
9146
|
+
chunks: parsedChunks,
|
|
9147
|
+
fpsNum,
|
|
9148
|
+
fpsDen,
|
|
9149
|
+
deferredSources
|
|
9150
|
+
});
|
|
9151
|
+
return {
|
|
9152
|
+
mode: deferredSources.size > 0 ? "source-extract" : "exact-rendered-frames",
|
|
9153
|
+
dependencies
|
|
9154
|
+
};
|
|
9044
9155
|
}
|
|
9045
|
-
function
|
|
9046
|
-
|
|
9047
|
-
|
|
9156
|
+
function collectChunkVideoDependencies(input) {
|
|
9157
|
+
const mutable = /* @__PURE__ */ new Map();
|
|
9158
|
+
for (const chunk of input.chunks) {
|
|
9159
|
+
for (let frame = chunk.startFrame; frame < chunk.endFrame; frame++) {
|
|
9160
|
+
const globalTime = frame * input.fpsDen / input.fpsNum;
|
|
9161
|
+
for (const [videoId, payload] of input.table.getActiveFramePayloads(globalTime)) {
|
|
9162
|
+
const path = input.deferredSources.get(videoId) ?? relative6(resolve8(input.executionPlanDir), payload.framePath).split(sep3).join("/");
|
|
9163
|
+
assertSafeRelativePath(path);
|
|
9164
|
+
const indexes = mutable.get(path) ?? /* @__PURE__ */ new Set();
|
|
9165
|
+
indexes.add(chunk.index);
|
|
9166
|
+
mutable.set(path, indexes);
|
|
9167
|
+
}
|
|
9168
|
+
}
|
|
9048
9169
|
}
|
|
9049
|
-
|
|
9050
|
-
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
|
|
9170
|
+
return new Map([...mutable].map(([path, indexes]) => [path, [...indexes].sort((a, b) => a - b)]));
|
|
9171
|
+
}
|
|
9172
|
+
function manifestPayload(manifest) {
|
|
9173
|
+
return manifest;
|
|
9174
|
+
}
|
|
9175
|
+
function computeManifestHash(manifest) {
|
|
9176
|
+
return sha256Hex(`${PLAN_V2_HASH_PREFIX}${canonicalJsonStringify(manifestPayload(manifest))}`);
|
|
9177
|
+
}
|
|
9178
|
+
function writeBlob(sourcePath, destinationPath) {
|
|
9179
|
+
if (existsSync17(destinationPath)) return;
|
|
9180
|
+
mkdirSync12(dirname11(destinationPath), { recursive: true });
|
|
9181
|
+
const temporaryDir = mkdtempSync5(join25(dirname11(destinationPath), ".plan-v2-blob-"));
|
|
9182
|
+
const temporaryPath = join25(temporaryDir, "blob");
|
|
9060
9183
|
try {
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
}
|
|
9066
|
-
if (target.role === "chunk") materializeExtractedVideoDirectories(tempDir);
|
|
9067
|
-
writeFileSync7(
|
|
9068
|
-
join23(tempDir, PLAN_V2_MATERIALIZATION_MARKER),
|
|
9069
|
-
canonicalJsonStringify({
|
|
9070
|
-
manifest,
|
|
9071
|
-
target,
|
|
9072
|
-
...Object.keys(remoteSources).length > 0 ? { remoteSources } : {}
|
|
9073
|
-
}),
|
|
9074
|
-
"utf-8"
|
|
9075
|
-
);
|
|
9076
|
-
renameSync5(tempDir, destinationDir);
|
|
9077
|
-
} catch (error) {
|
|
9078
|
-
rmSync9(tempDir, { recursive: true, force: true });
|
|
9079
|
-
throw error;
|
|
9184
|
+
copyFileSync7(sourcePath, temporaryPath);
|
|
9185
|
+
renameSync6(temporaryPath, destinationPath);
|
|
9186
|
+
} finally {
|
|
9187
|
+
rmSync10(temporaryDir, { recursive: true, force: true });
|
|
9080
9188
|
}
|
|
9189
|
+
}
|
|
9190
|
+
function publishBlobFor(absolutePath, digests) {
|
|
9081
9191
|
return {
|
|
9082
|
-
|
|
9083
|
-
|
|
9084
|
-
|
|
9085
|
-
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
9086
|
-
artifactCount: artifacts.length,
|
|
9087
|
-
sizeBytes: artifacts.reduce((sum2, artifact) => sum2 + artifact.sizeBytes, 0),
|
|
9088
|
-
// Join the artifact's OWN name, not the current constant: a plan written
|
|
9089
|
-
// before the container change carries the legacy name, and materializing it
|
|
9090
|
-
// under the new one would point at a file that was never written.
|
|
9091
|
-
audioPath: target.role === "assembler" ? audioArtifact ? join23(destinationDir, audioArtifact.path) : null : null
|
|
9192
|
+
sourcePath: absolutePath,
|
|
9193
|
+
sha256: planFileSha256(absolutePath, digests),
|
|
9194
|
+
sizeBytes: statSync6(absolutePath).size
|
|
9092
9195
|
};
|
|
9093
9196
|
}
|
|
9094
|
-
function
|
|
9095
|
-
|
|
9096
|
-
for (const [path, url] of Object.entries(remoteSources)) {
|
|
9097
|
-
const artifact = artifacts.find((candidate) => candidate.path === path);
|
|
9098
|
-
if (!artifact || !isPlanV2DeferredSourceArtifact(artifact)) {
|
|
9099
|
-
throw new PlanV2IntegrityError(`remote source is not a selected deferred video: ${path}`);
|
|
9100
|
-
}
|
|
9101
|
-
if (typeof url !== "string" || !/^https?:\/\//.test(url)) {
|
|
9102
|
-
throw new PlanV2IntegrityError(`remote source for ${path} must be an http(s) URL`);
|
|
9103
|
-
}
|
|
9104
|
-
}
|
|
9197
|
+
async function putBlobs(publisher, blobs) {
|
|
9198
|
+
await putTasks(blobs.map((blob) => () => publisher.putBlob(blob)));
|
|
9105
9199
|
}
|
|
9106
|
-
function
|
|
9107
|
-
|
|
9200
|
+
async function putTasks(tasks) {
|
|
9201
|
+
const uploadSlots = 32;
|
|
9202
|
+
let next = 0;
|
|
9203
|
+
let failure;
|
|
9204
|
+
const upload = async () => {
|
|
9205
|
+
while (!failure && next < tasks.length) {
|
|
9206
|
+
const task = tasks[next++];
|
|
9207
|
+
try {
|
|
9208
|
+
await task();
|
|
9209
|
+
} catch (reason) {
|
|
9210
|
+
failure ??= { reason };
|
|
9211
|
+
}
|
|
9212
|
+
}
|
|
9213
|
+
};
|
|
9214
|
+
await Promise.all(Array.from({ length: uploadSlots }, upload));
|
|
9215
|
+
if (failure) throw failure.reason;
|
|
9108
9216
|
}
|
|
9109
|
-
function
|
|
9110
|
-
const
|
|
9111
|
-
if (!
|
|
9112
|
-
|
|
9113
|
-
if (!isRecord3(marker) || !isRecord3(marker.manifest) || !isRecord3(marker.target)) {
|
|
9114
|
-
throw new PlanV2IntegrityError("malformed materialization marker");
|
|
9217
|
+
function buildPlanV2Publication(executionPlanDir, digests) {
|
|
9218
|
+
const executionPlanPath = join25(executionPlanDir, "plan.json");
|
|
9219
|
+
if (!existsSync17(executionPlanPath)) {
|
|
9220
|
+
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
|
|
9115
9221
|
}
|
|
9116
|
-
const
|
|
9117
|
-
if (
|
|
9118
|
-
throw new PlanV2IntegrityError(
|
|
9119
|
-
"materialization target does not match requested execution role"
|
|
9120
|
-
);
|
|
9222
|
+
const executionPlanValue = readJsonFile(executionPlanPath, "execution plan.json");
|
|
9223
|
+
if (!isRecord3(executionPlanValue)) {
|
|
9224
|
+
throw new PlanV2IntegrityError("execution plan.json must be an object");
|
|
9121
9225
|
}
|
|
9122
|
-
const
|
|
9123
|
-
|
|
9124
|
-
const
|
|
9125
|
-
|
|
9126
|
-
|
|
9127
|
-
if (Object.hasOwn(remoteSources, artifact.path)) continue;
|
|
9128
|
-
const path = join23(planDir, ...artifact.path.split("/"));
|
|
9129
|
-
if (!existsSync16(path) || statSync6(path).size !== artifact.sizeBytes) {
|
|
9130
|
-
throw new PlanV2IntegrityError(
|
|
9131
|
-
`materialized artifact missing or truncated: ${artifact.path}`
|
|
9132
|
-
);
|
|
9133
|
-
}
|
|
9134
|
-
if (sha256File(path) !== artifact.sha256) {
|
|
9135
|
-
throw new PlanV2IntegrityError(`materialized artifact hash mismatch: ${artifact.path}`);
|
|
9136
|
-
}
|
|
9226
|
+
const executionPlan = executionPlanValue;
|
|
9227
|
+
readPlanProtocolV1(executionPlan);
|
|
9228
|
+
const executionPlanHash = executionPlan.planHash;
|
|
9229
|
+
if (!isSha256(executionPlanHash)) {
|
|
9230
|
+
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
|
|
9137
9231
|
}
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
import { tmpdir as tmpdir2 } from "node:os";
|
|
9144
|
-
import { join as join27 } from "node:path";
|
|
9145
|
-
|
|
9146
|
-
// src/services/distributed/assemble.ts
|
|
9147
|
-
import {
|
|
9148
|
-
cpSync as cpSync3,
|
|
9149
|
-
existsSync as existsSync17,
|
|
9150
|
-
mkdirSync as mkdirSync12,
|
|
9151
|
-
mkdtempSync as mkdtempSync6,
|
|
9152
|
-
readFileSync as readFileSync10,
|
|
9153
|
-
readdirSync as readdirSync8,
|
|
9154
|
-
renameSync as renameSync7,
|
|
9155
|
-
rmSync as rmSync11,
|
|
9156
|
-
statSync as statSync7,
|
|
9157
|
-
writeFileSync as writeFileSync8
|
|
9158
|
-
} from "node:fs";
|
|
9159
|
-
import { dirname as dirname12, join as join25 } from "node:path";
|
|
9160
|
-
import {
|
|
9161
|
-
appendRenderProvenanceArgs,
|
|
9162
|
-
describeFfmpegFailure as describeFfmpegFailure2,
|
|
9163
|
-
getEncoderPreset as getEncoderPreset5,
|
|
9164
|
-
MIXED_AUDIO_FILENAME as MIXED_AUDIO_FILENAME4,
|
|
9165
|
-
muxVideoWithAudio as muxVideoWithAudio2,
|
|
9166
|
-
resolveConfig as resolveConfig4,
|
|
9167
|
-
runFfmpeg as runFfmpeg5
|
|
9168
|
-
} from "@hyperframes/engine";
|
|
9169
|
-
import { fpsToFfmpegArg as fpsToFfmpegArg3 } from "@hyperframes/core";
|
|
9170
|
-
|
|
9171
|
-
// src/services/distributed/mp4Stitch.ts
|
|
9172
|
-
import {
|
|
9173
|
-
createReadStream as createReadStream4,
|
|
9174
|
-
createWriteStream,
|
|
9175
|
-
mkdirSync as mkdirSync11,
|
|
9176
|
-
mkdtempSync as mkdtempSync5,
|
|
9177
|
-
renameSync as renameSync6,
|
|
9178
|
-
rmSync as rmSync10
|
|
9179
|
-
} from "node:fs";
|
|
9180
|
-
import { open } from "node:fs/promises";
|
|
9181
|
-
import { dirname as dirname11, join as join24 } from "node:path";
|
|
9182
|
-
import { finished, pipeline } from "node:stream/promises";
|
|
9183
|
-
function boxAt(src, off, at, limit) {
|
|
9184
|
-
let size = src.readUInt32BE(off);
|
|
9185
|
-
let bodyStart = at + 8;
|
|
9186
|
-
if (size === 1) {
|
|
9187
|
-
size = Number(src.readBigUInt64BE(off + 8));
|
|
9188
|
-
bodyStart = at + 16;
|
|
9232
|
+
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);
|
|
9233
|
+
if (recomputedExecutionPlanHash !== executionPlanHash) {
|
|
9234
|
+
throw new PlanV2IntegrityError(
|
|
9235
|
+
`execution plan content fingerprint does not match plan.json.planHash: expected ${executionPlanHash}, recomputed ${recomputedExecutionPlanHash}`
|
|
9236
|
+
);
|
|
9189
9237
|
}
|
|
9190
|
-
|
|
9191
|
-
|
|
9238
|
+
const artifacts = [];
|
|
9239
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
9240
|
+
const dimensions = executionPlan.dimensions;
|
|
9241
|
+
if (!isRecord3(dimensions)) {
|
|
9242
|
+
throw new PlanV2IntegrityError("execution plan.json.dimensions must be an object");
|
|
9243
|
+
}
|
|
9244
|
+
validateExtractionCacheCompleteSentinels(executionPlanDir);
|
|
9245
|
+
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
|
|
9246
|
+
const encoder = readJsonFile(join25(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");
|
|
9247
|
+
const files = listFiles(executionPlanDir);
|
|
9248
|
+
const chunkAudio = !isRecord3(encoder) || encoder.chunksCarryAudio !== true ? "none" : files.some((file) => chunkAudioSliceTarget(file.path) !== null) ? "sliced" : "whole";
|
|
9249
|
+
for (const file of files) {
|
|
9250
|
+
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
|
|
9251
|
+
const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunkAudio);
|
|
9252
|
+
if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) && targets.chunks !== "all" && targets.chunks.length === 0 && !targets.assembler) {
|
|
9253
|
+
continue;
|
|
9254
|
+
}
|
|
9255
|
+
const blob = publishBlobFor(file.absolutePath, digests);
|
|
9256
|
+
if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
|
|
9257
|
+
artifacts.push({
|
|
9258
|
+
path: file.path,
|
|
9259
|
+
sha256: blob.sha256,
|
|
9260
|
+
sizeBytes: blob.sizeBytes,
|
|
9261
|
+
...targets
|
|
9262
|
+
});
|
|
9263
|
+
}
|
|
9264
|
+
const storedBlobs = /* @__PURE__ */ new Map();
|
|
9265
|
+
for (const remote of readPlanRemoteFiles(executionPlanDir)) {
|
|
9266
|
+
const { sha256, sizeBytes } = remote;
|
|
9267
|
+
if (!blobs.has(sha256)) storedBlobs.set(sha256, { sha256, sizeBytes });
|
|
9268
|
+
artifacts.push({
|
|
9269
|
+
...remote,
|
|
9270
|
+
...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunkAudio)
|
|
9271
|
+
});
|
|
9272
|
+
}
|
|
9273
|
+
const base = {
|
|
9274
|
+
protocol: PLAN_PROTOCOL_V2,
|
|
9275
|
+
// Retain the established manifest key byte-for-byte. It now acts as the
|
|
9276
|
+
// wire alias for the neutral local execution-plan hash.
|
|
9277
|
+
sourcePlanV1Hash: executionPlanHash,
|
|
9278
|
+
chunkCount: readPositiveInteger2(executionPlan.chunkCount, "chunkCount"),
|
|
9279
|
+
totalFrames: readPositiveInteger2(executionPlan.totalFrames, "totalFrames"),
|
|
9280
|
+
fps: readExecutionPlanFps(dimensions),
|
|
9281
|
+
width: readPositiveInteger2(dimensions.width, "dimensions.width"),
|
|
9282
|
+
height: readPositiveInteger2(dimensions.height, "dimensions.height"),
|
|
9283
|
+
format: readDistributedFormat(dimensions.format),
|
|
9284
|
+
ffmpegVersion: readString2(executionPlan.ffmpegVersion, "ffmpegVersion"),
|
|
9285
|
+
producerVersion: readString2(executionPlan.producerVersion, "producerVersion"),
|
|
9286
|
+
limitations: { videoDependencyMode: videoDependencyPlan.mode },
|
|
9287
|
+
artifacts
|
|
9288
|
+
};
|
|
9289
|
+
return {
|
|
9290
|
+
manifest: {
|
|
9291
|
+
...base,
|
|
9292
|
+
planHash: computeManifestHash(base)
|
|
9293
|
+
},
|
|
9294
|
+
blobs: [...blobs.values()],
|
|
9295
|
+
storedBlobs: [...storedBlobs.values()]
|
|
9296
|
+
};
|
|
9192
9297
|
}
|
|
9193
|
-
function
|
|
9194
|
-
|
|
9195
|
-
|
|
9196
|
-
|
|
9298
|
+
function createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir) {
|
|
9299
|
+
if (existsSync17(planV2Dir)) {
|
|
9300
|
+
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
9301
|
+
}
|
|
9302
|
+
const publication = buildPlanV2Publication(executionPlanDir);
|
|
9303
|
+
if (publication.storedBlobs.length > 0) {
|
|
9304
|
+
throw new PlanV2IntegrityError("a local plan v2 directory cannot reference remote files");
|
|
9305
|
+
}
|
|
9306
|
+
mkdirSync12(dirname11(planV2Dir), { recursive: true });
|
|
9307
|
+
const tempDir = mkdtempSync5(join25(dirname11(planV2Dir), ".plan-v2-build-"));
|
|
9308
|
+
try {
|
|
9309
|
+
for (const blob of publication.blobs) {
|
|
9310
|
+
writeBlob(blob.sourcePath, planV2BlobPath(tempDir, blob.sha256));
|
|
9311
|
+
}
|
|
9312
|
+
writeFileSync8(
|
|
9313
|
+
join25(tempDir, "plan.json"),
|
|
9314
|
+
canonicalJsonStringify(publication.manifest),
|
|
9315
|
+
"utf-8"
|
|
9316
|
+
);
|
|
9317
|
+
renameSync6(tempDir, planV2Dir);
|
|
9318
|
+
return resultFromManifest(planV2Dir, publication.manifest);
|
|
9319
|
+
} catch (error) {
|
|
9320
|
+
rmSync10(tempDir, { recursive: true, force: true });
|
|
9321
|
+
throw error;
|
|
9197
9322
|
}
|
|
9198
|
-
return boxes;
|
|
9199
9323
|
}
|
|
9200
|
-
function
|
|
9201
|
-
return
|
|
9324
|
+
function createPlanV2FromV1(executionPlanDir, planV2Dir) {
|
|
9325
|
+
return createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir);
|
|
9202
9326
|
}
|
|
9203
|
-
function
|
|
9204
|
-
|
|
9205
|
-
if (!found) throw new Error(`[mp4Stitch] ${parent.type} has no ${type}`);
|
|
9206
|
-
return found;
|
|
9327
|
+
async function publishPlanV2FromExecutionPlan(executionPlanDir, publisher) {
|
|
9328
|
+
return publishExecutionPlan(executionPlanDir, publisher, {});
|
|
9207
9329
|
}
|
|
9208
|
-
|
|
9209
|
-
|
|
9210
|
-
|
|
9211
|
-
|
|
9212
|
-
|
|
9213
|
-
|
|
9214
|
-
|
|
9215
|
-
if (start >= bufAt && end <= bufAt + buf.length) return;
|
|
9216
|
-
buf = await read(start, Math.max(end, start + SCAN_BYTES));
|
|
9217
|
-
bufAt = start;
|
|
9218
|
-
};
|
|
9219
|
-
for (let at = 0; !(found.has("ftyp") && found.has("moov") && payload); ) {
|
|
9220
|
-
await load(at, at + 16);
|
|
9221
|
-
if (at + 8 > bufAt + buf.length) {
|
|
9222
|
-
throw new Error(`[mp4Stitch] chunk needs ftyp, moov and mdat; found ${[...found.keys()]}`);
|
|
9330
|
+
async function publishExecutionPlan(executionPlanDir, publisher, { digests, log = defaultLogger }) {
|
|
9331
|
+
try {
|
|
9332
|
+
let stageStarted = Date.now();
|
|
9333
|
+
const publication = buildPlanV2Publication(executionPlanDir, digests);
|
|
9334
|
+
logPlanStage(log, "publication", stageStarted);
|
|
9335
|
+
if (publication.storedBlobs.length > 0 && !publisher.referenceStoredBlob) {
|
|
9336
|
+
throw new PlanV2IntegrityError("this publisher cannot reference remote files");
|
|
9223
9337
|
}
|
|
9224
|
-
|
|
9225
|
-
|
|
9226
|
-
|
|
9227
|
-
|
|
9228
|
-
|
|
9338
|
+
stageStarted = Date.now();
|
|
9339
|
+
await putTasks([
|
|
9340
|
+
...publication.blobs.map((blob) => () => publisher.putBlob(blob)),
|
|
9341
|
+
...publication.storedBlobs.map((blob) => () => publisher.referenceStoredBlob(blob))
|
|
9342
|
+
]);
|
|
9343
|
+
logPlanStage(log, "upload", stageStarted);
|
|
9344
|
+
stageStarted = Date.now();
|
|
9345
|
+
await publisher.commitManifest(canonicalJsonStringify(publication.manifest));
|
|
9346
|
+
logPlanStage(log, "manifest", stageStarted);
|
|
9347
|
+
return publication.manifest;
|
|
9348
|
+
} catch (error) {
|
|
9349
|
+
try {
|
|
9350
|
+
await publisher.abort();
|
|
9351
|
+
} catch {
|
|
9229
9352
|
}
|
|
9230
|
-
|
|
9353
|
+
throw error;
|
|
9231
9354
|
}
|
|
9232
|
-
const [ftyp, moov] = [found.get("ftyp"), found.get("moov")];
|
|
9233
|
-
if (moov.length !== moovSize(moov)) throw new Error("[mp4Stitch] moov is cut off");
|
|
9234
|
-
return { ftyp, moov, payloadStart: payload.bodyStart, payloadEnd: payload.end };
|
|
9235
|
-
}
|
|
9236
|
-
var moovSize = (moov) => boxAt(moov, 0, 0, Number.MAX_SAFE_INTEGER).end;
|
|
9237
|
-
function readTracks(moov) {
|
|
9238
|
-
const root = boxAt(moov, 0, 0, moov.length);
|
|
9239
|
-
return childBoxes(moov, root).filter((box2) => box2.type === "trak").map((trak) => readTrack(moov, trak));
|
|
9240
9355
|
}
|
|
9241
|
-
function
|
|
9242
|
-
|
|
9243
|
-
const first = table.bodyStart + countAt + 4;
|
|
9244
|
-
return Array.from(
|
|
9245
|
-
{ length: count },
|
|
9246
|
-
(_, row) => Array.from(
|
|
9247
|
-
{ length: columns },
|
|
9248
|
-
(_2, col) => buf.readUInt32BE(first + (row * columns + col) * 4)
|
|
9249
|
-
)
|
|
9250
|
-
);
|
|
9356
|
+
async function publishPlanV2FromV1(executionPlanDir, publisher) {
|
|
9357
|
+
return publishPlanV2FromExecutionPlan(executionPlanDir, publisher);
|
|
9251
9358
|
}
|
|
9252
|
-
function
|
|
9253
|
-
const
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
|
|
9257
|
-
|
|
9258
|
-
|
|
9259
|
-
|
|
9260
|
-
|
|
9261
|
-
|
|
9359
|
+
async function planV2WithPublisher(projectDir, config, publisher, options = {}) {
|
|
9360
|
+
const stagingRoot = mkdtempSync5(join25(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
|
|
9361
|
+
const digests = /* @__PURE__ */ new Map();
|
|
9362
|
+
try {
|
|
9363
|
+
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
|
|
9364
|
+
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
|
|
9365
|
+
deferVideoExtraction: true,
|
|
9366
|
+
remoteFiles: options.remoteFiles,
|
|
9367
|
+
digests,
|
|
9368
|
+
// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
|
|
9369
|
+
whileAudioMixes: () => putBlobs(
|
|
9370
|
+
publisher,
|
|
9371
|
+
listFiles(join25(stagingRoot, "compiled")).map(
|
|
9372
|
+
(file) => publishBlobFor(file.absolutePath, digests)
|
|
9373
|
+
)
|
|
9374
|
+
),
|
|
9375
|
+
chunksCarryAudio: options.chunksCarryAudio,
|
|
9376
|
+
sliceChunkAudio: true
|
|
9377
|
+
});
|
|
9378
|
+
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
|
|
9379
|
+
} catch (error) {
|
|
9380
|
+
try {
|
|
9381
|
+
await publisher.abort();
|
|
9382
|
+
} catch {
|
|
9383
|
+
}
|
|
9384
|
+
throw error;
|
|
9385
|
+
} finally {
|
|
9386
|
+
rmSync10(stagingRoot, { recursive: true, force: true });
|
|
9262
9387
|
}
|
|
9263
|
-
return 0;
|
|
9264
|
-
}
|
|
9265
|
-
function readSizes(moov, stsz) {
|
|
9266
|
-
const fixedSize = moov.readUInt32BE(stsz.bodyStart + 4);
|
|
9267
|
-
if (!fixedSize) return rows(moov, stsz, 1, 8).map(([size]) => size);
|
|
9268
|
-
return Array(moov.readUInt32BE(stsz.bodyStart + 8)).fill(fixedSize);
|
|
9269
|
-
}
|
|
9270
|
-
function readSync2(moov, stbl, sampleCount) {
|
|
9271
|
-
const stss = findChild(moov, stbl, "stss");
|
|
9272
|
-
if (!stss) return null;
|
|
9273
|
-
const sync = Array(sampleCount).fill(false);
|
|
9274
|
-
for (const [n] of rows(moov, stss, 1)) sync[n - 1] = true;
|
|
9275
|
-
return sync;
|
|
9276
|
-
}
|
|
9277
|
-
function readChunkOffsets(moov, stbl) {
|
|
9278
|
-
const stco = findChild(moov, stbl, "stco");
|
|
9279
|
-
if (stco) return rows(moov, stco, 1).map(([offset]) => offset);
|
|
9280
|
-
return rows(moov, child(moov, stbl, "co64"), 2).map(([hi, lo]) => hi * 2 ** 32 + lo);
|
|
9281
9388
|
}
|
|
9282
|
-
function
|
|
9283
|
-
|
|
9284
|
-
|
|
9285
|
-
|
|
9286
|
-
|
|
9287
|
-
|
|
9389
|
+
async function planV2(projectDir, config, planV2Dir) {
|
|
9390
|
+
if (existsSync17(planV2Dir)) {
|
|
9391
|
+
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
9392
|
+
}
|
|
9393
|
+
const publisher = new LocalPlanV2ArtifactPublisher(planV2Dir);
|
|
9394
|
+
const manifest = await planV2WithPublisher(projectDir, config, publisher, {
|
|
9395
|
+
stagingParentDir: dirname11(planV2Dir)
|
|
9288
9396
|
});
|
|
9397
|
+
return resultFromManifest(planV2Dir, manifest);
|
|
9289
9398
|
}
|
|
9290
|
-
function
|
|
9291
|
-
const mdia = child(moov, trak, "mdia");
|
|
9292
|
-
const hdlr = child(moov, mdia, "hdlr");
|
|
9293
|
-
const stbl = child(moov, child(moov, mdia, "minf"), "stbl");
|
|
9294
|
-
if (findChild(moov, stbl, "ctts")) throw new Error("[mp4Stitch] chunk has ctts (B-frames)");
|
|
9295
|
-
const stts = rows(moov, child(moov, stbl, "stts"), 2);
|
|
9296
|
-
const durations = stts.flatMap(([n, delta]) => Array(n).fill(delta));
|
|
9297
|
-
const sizes = readSizes(moov, child(moov, stbl, "stsz"));
|
|
9298
|
-
const chunkOffsets = readChunkOffsets(moov, stbl);
|
|
9299
|
-
const samplesPerChunk = readSamplesPerChunk(moov, stbl, chunkOffsets.length);
|
|
9300
|
-
if (durations.length !== sizes.length || sum(samplesPerChunk) !== sizes.length) {
|
|
9301
|
-
throw new Error("[mp4Stitch] sample tables disagree on the sample count");
|
|
9302
|
-
}
|
|
9303
|
-
const stsd = child(moov, stbl, "stsd");
|
|
9399
|
+
function resultFromManifest(planV2Dir, manifest) {
|
|
9304
9400
|
return {
|
|
9305
|
-
|
|
9306
|
-
|
|
9307
|
-
|
|
9308
|
-
|
|
9309
|
-
|
|
9310
|
-
|
|
9311
|
-
|
|
9312
|
-
|
|
9313
|
-
|
|
9314
|
-
|
|
9401
|
+
planDir: planV2Dir,
|
|
9402
|
+
manifestPath: join25(planV2Dir, "plan.json"),
|
|
9403
|
+
planProtocol: PLAN_PROTOCOL_V2,
|
|
9404
|
+
planHash: manifest.planHash,
|
|
9405
|
+
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
9406
|
+
chunkCount: manifest.chunkCount,
|
|
9407
|
+
totalFrames: manifest.totalFrames,
|
|
9408
|
+
fps: manifest.fps,
|
|
9409
|
+
width: manifest.width,
|
|
9410
|
+
height: manifest.height,
|
|
9411
|
+
format: manifest.format,
|
|
9412
|
+
ffmpegVersion: manifest.ffmpegVersion,
|
|
9413
|
+
producerVersion: manifest.producerVersion,
|
|
9414
|
+
limitations: manifest.limitations
|
|
9315
9415
|
};
|
|
9316
9416
|
}
|
|
9317
|
-
function
|
|
9318
|
-
return
|
|
9319
|
-
}
|
|
9320
|
-
function box(type, ...body) {
|
|
9321
|
-
const header = Buffer.alloc(8);
|
|
9322
|
-
header.writeUInt32BE(8 + sum(body.map((part) => part.length)));
|
|
9323
|
-
header.write(type, 4, "latin1");
|
|
9324
|
-
return Buffer.concat([header, ...body]);
|
|
9325
|
-
}
|
|
9326
|
-
function fullBox(type, version, body) {
|
|
9327
|
-
return box(type, Buffer.from([version, 0, 0, 0]), body);
|
|
9328
|
-
}
|
|
9329
|
-
function u32s(values) {
|
|
9330
|
-
const buf = Buffer.alloc(values.length * 4);
|
|
9331
|
-
values.forEach((value, i) => buf.writeUInt32BE(value, i * 4));
|
|
9332
|
-
return buf;
|
|
9417
|
+
function getPlanV2ExecutionPlanHash(plan2) {
|
|
9418
|
+
return plan2.sourcePlanV1Hash;
|
|
9333
9419
|
}
|
|
9334
|
-
function
|
|
9335
|
-
|
|
9336
|
-
|
|
9337
|
-
const last = out.at(-1);
|
|
9338
|
-
if (last && last[1] === value) last[0]++;
|
|
9339
|
-
else out.push([1, value]);
|
|
9420
|
+
function readString2(value, field) {
|
|
9421
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
9422
|
+
throw new PlanV2IntegrityError(`${field} must be a non-empty string`);
|
|
9340
9423
|
}
|
|
9341
|
-
return
|
|
9424
|
+
return value;
|
|
9342
9425
|
}
|
|
9343
|
-
function
|
|
9344
|
-
|
|
9345
|
-
|
|
9346
|
-
|
|
9347
|
-
return
|
|
9426
|
+
function readPositiveInteger2(value, field) {
|
|
9427
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
9428
|
+
throw new PlanV2IntegrityError(`${field} must be a positive integer`);
|
|
9429
|
+
}
|
|
9430
|
+
return value;
|
|
9348
9431
|
}
|
|
9349
|
-
|
|
9350
|
-
|
|
9351
|
-
|
|
9352
|
-
|
|
9353
|
-
|
|
9354
|
-
function withDuration(buf, target, duration) {
|
|
9355
|
-
const copy = Buffer.from(buf.subarray(target.start, target.end));
|
|
9356
|
-
const body = target.bodyStart - target.start;
|
|
9357
|
-
const wide = copy[body] === 1;
|
|
9358
|
-
const at = body + DURATION_AT[target.type][wide ? 1 : 0];
|
|
9359
|
-
if (wide) copy.writeBigUInt64BE(BigInt(duration), at);
|
|
9360
|
-
else if (duration <= 4294967295) copy.writeUInt32BE(duration, at);
|
|
9361
|
-
else throw new Error(`[mp4Stitch] ${target.type} duration ${duration} needs a version 1 box`);
|
|
9362
|
-
return copy;
|
|
9432
|
+
function readNonNegativeInteger2(value, field) {
|
|
9433
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
9434
|
+
throw new PlanV2IntegrityError(`${field} must be a non-negative integer`);
|
|
9435
|
+
}
|
|
9436
|
+
return value;
|
|
9363
9437
|
}
|
|
9364
|
-
function
|
|
9365
|
-
|
|
9438
|
+
function readSupportedFps(value) {
|
|
9439
|
+
if (!isDistributedFps(value)) {
|
|
9440
|
+
throw new PlanV2IntegrityError(
|
|
9441
|
+
`dimensions.fpsNum must be one of ${DISTRIBUTED_FPS.join(", ")}`
|
|
9442
|
+
);
|
|
9443
|
+
}
|
|
9444
|
+
return value;
|
|
9366
9445
|
}
|
|
9367
|
-
function
|
|
9368
|
-
const
|
|
9369
|
-
|
|
9370
|
-
|
|
9371
|
-
for (const [count, perChunk] of runs(track.samplesPerChunk)) {
|
|
9372
|
-
stsc.push([firstChunk, perChunk, 1]);
|
|
9373
|
-
firstChunk += count;
|
|
9446
|
+
function readExecutionPlanFps(dimensions) {
|
|
9447
|
+
const fpsDen = readPositiveInteger2(dimensions.fpsDen, "dimensions.fpsDen");
|
|
9448
|
+
if (fpsDen !== 1) {
|
|
9449
|
+
throw new PlanV2IntegrityError("dimensions.fpsDen must be 1 for plan v2");
|
|
9374
9450
|
}
|
|
9375
|
-
|
|
9376
|
-
co64.writeUInt32BE(track.chunkOffsets.length);
|
|
9377
|
-
track.chunkOffsets.forEach((offset, i) => co64.writeBigUInt64BE(BigInt(offset), 4 + i * 8));
|
|
9378
|
-
const syncSamples = track.sync?.flatMap((isSync, i) => isSync ? [i + 1] : []) ?? null;
|
|
9379
|
-
return box(
|
|
9380
|
-
"stbl",
|
|
9381
|
-
stsd,
|
|
9382
|
-
fullBox("stts", 0, u32s([stts.length, ...stts.flat()])),
|
|
9383
|
-
...syncSamples ? [fullBox("stss", 0, u32s([syncSamples.length, ...syncSamples]))] : [],
|
|
9384
|
-
fullBox("stsc", 0, u32s([stsc.length, ...stsc.flat()])),
|
|
9385
|
-
fullBox("stsz", 0, u32s([0, track.sizes.length, ...track.sizes])),
|
|
9386
|
-
fullBox("co64", 0, co64)
|
|
9387
|
-
);
|
|
9451
|
+
return readSupportedFps(dimensions.fpsNum);
|
|
9388
9452
|
}
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9392
|
-
let next = 0;
|
|
9393
|
-
const matched = plan2.map((planTrack) => {
|
|
9394
|
-
const track = tracks[next];
|
|
9395
|
-
if (track && sameTrack(track, planTrack)) return tracks[next++];
|
|
9396
|
-
if (planTrack.handler === "soun") return null;
|
|
9397
|
-
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
9398
|
-
});
|
|
9399
|
-
if (next !== tracks.length)
|
|
9400
|
-
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
9401
|
-
for (const track of tracks) {
|
|
9402
|
-
if (track.handler === "vide" && track.mediaTime !== 0) {
|
|
9403
|
-
throw new Error(`[mp4Stitch] chunk ${index} video starts at media time ${track.mediaTime}`);
|
|
9404
|
-
}
|
|
9453
|
+
function readDistributedFormat(value) {
|
|
9454
|
+
if (value !== "mp4" && value !== "mov" && value !== "webm" && value !== "png-sequence") {
|
|
9455
|
+
throw new PlanV2IntegrityError("dimensions.format is unsupported");
|
|
9405
9456
|
}
|
|
9406
|
-
return
|
|
9457
|
+
return value;
|
|
9407
9458
|
}
|
|
9408
|
-
function
|
|
9409
|
-
|
|
9410
|
-
const
|
|
9411
|
-
|
|
9412
|
-
|
|
9413
|
-
|
|
9414
|
-
payloadBases.push(payloadBytes);
|
|
9415
|
-
payloadBytes += layout.payloadEnd - layout.payloadStart;
|
|
9459
|
+
function parseArtifact(value, index) {
|
|
9460
|
+
if (!isRecord3(value)) throw new PlanV2IntegrityError(`artifacts[${index}] must be an object`);
|
|
9461
|
+
const path = readString2(value.path, `artifacts[${index}].path`);
|
|
9462
|
+
assertSafeRelativePath(path);
|
|
9463
|
+
if (!isSha256(value.sha256)) {
|
|
9464
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].sha256 must be a lowercase sha256`);
|
|
9416
9465
|
}
|
|
9417
|
-
const
|
|
9418
|
-
|
|
9419
|
-
|
|
9420
|
-
|
|
9421
|
-
|
|
9422
|
-
|
|
9423
|
-
|
|
9424
|
-
|
|
9425
|
-
|
|
9426
|
-
|
|
9427
|
-
|
|
9428
|
-
|
|
9429
|
-
|
|
9430
|
-
|
|
9431
|
-
|
|
9432
|
-
|
|
9433
|
-
|
|
9434
|
-
|
|
9435
|
-
|
|
9436
|
-
|
|
9437
|
-
|
|
9438
|
-
return offset - payloadStart + payloadBases[i] + dataStart2;
|
|
9439
|
-
})
|
|
9440
|
-
)
|
|
9441
|
-
};
|
|
9442
|
-
const mediaDuration = sum(merged.durations);
|
|
9443
|
-
const shown = Math.round(
|
|
9444
|
-
(mediaDuration - track.mediaTime) * movieTimescale / track.timescale
|
|
9445
|
-
);
|
|
9446
|
-
const elst = Buffer.alloc(24);
|
|
9447
|
-
elst.writeUInt32BE(1);
|
|
9448
|
-
elst.writeBigUInt64BE(BigInt(shown), 4);
|
|
9449
|
-
elst.writeBigInt64BE(BigInt(track.mediaTime), 12);
|
|
9450
|
-
elst.writeUInt32BE(65536, 20);
|
|
9451
|
-
const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);
|
|
9452
|
-
const trak = rebuild(template, track.trak, {
|
|
9453
|
-
tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),
|
|
9454
|
-
edts: () => Buffer.alloc(0),
|
|
9455
|
-
mdia: (mdia) => rebuild(template, mdia, {
|
|
9456
|
-
mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),
|
|
9457
|
-
minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) })
|
|
9458
|
-
})
|
|
9459
|
-
});
|
|
9460
|
-
return { trak, shown };
|
|
9461
|
-
});
|
|
9462
|
-
let next = 0;
|
|
9463
|
-
return rebuild(template, root, {
|
|
9464
|
-
mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),
|
|
9465
|
-
trak: () => traks[next++].trak
|
|
9466
|
-
});
|
|
9466
|
+
const sizeBytes = value.sizeBytes;
|
|
9467
|
+
if (typeof sizeBytes !== "number" || !Number.isInteger(sizeBytes) || sizeBytes < 0) {
|
|
9468
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].sizeBytes must be a non-negative integer`);
|
|
9469
|
+
}
|
|
9470
|
+
let chunks;
|
|
9471
|
+
if (value.chunks === "all") {
|
|
9472
|
+
chunks = "all";
|
|
9473
|
+
} else if (isNonNegativeIntegerArray(value.chunks)) {
|
|
9474
|
+
chunks = value.chunks;
|
|
9475
|
+
} else {
|
|
9476
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].chunks is invalid`);
|
|
9477
|
+
}
|
|
9478
|
+
if (typeof value.assembler !== "boolean") {
|
|
9479
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].assembler must be boolean`);
|
|
9480
|
+
}
|
|
9481
|
+
return {
|
|
9482
|
+
path,
|
|
9483
|
+
sha256: value.sha256,
|
|
9484
|
+
sizeBytes,
|
|
9485
|
+
chunks,
|
|
9486
|
+
assembler: value.assembler
|
|
9467
9487
|
};
|
|
9468
|
-
const ftyp = layouts[0].ftyp;
|
|
9469
|
-
const dataStart = ftyp.length + buildMoov(0).length + 16;
|
|
9470
|
-
const mdat = Buffer.alloc(16);
|
|
9471
|
-
mdat.writeUInt32BE(1);
|
|
9472
|
-
mdat.write("mdat", 4, "latin1");
|
|
9473
|
-
mdat.writeBigUInt64BE(BigInt(16 + payloadBytes), 8);
|
|
9474
|
-
return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);
|
|
9475
9488
|
}
|
|
9476
|
-
function
|
|
9477
|
-
|
|
9478
|
-
|
|
9479
|
-
|
|
9480
|
-
|
|
9481
|
-
|
|
9482
|
-
|
|
9483
|
-
|
|
9484
|
-
|
|
9489
|
+
function parsePlanV2Manifest(value) {
|
|
9490
|
+
if (!isRecord3(value)) throw new PlanV2IntegrityError("manifest must be an object");
|
|
9491
|
+
if (!isRecord3(value.protocol) || value.protocol.schemaVersion !== PLAN_PROTOCOL_V2.schemaVersion || value.protocol.artifactLayout !== PLAN_PROTOCOL_V2.artifactLayout || value.protocol.hashSchema !== PLAN_PROTOCOL_V2.hashSchema) {
|
|
9492
|
+
throw new PlanV2IntegrityError("manifest protocol is not the supported v2 descriptor");
|
|
9493
|
+
}
|
|
9494
|
+
if (!isSha256(value.planHash) || !isSha256(value.sourcePlanV1Hash)) {
|
|
9495
|
+
throw new PlanV2IntegrityError("manifest hashes must be lowercase sha256 digests");
|
|
9496
|
+
}
|
|
9497
|
+
if (!Array.isArray(value.artifacts)) throw new PlanV2IntegrityError("artifacts must be an array");
|
|
9498
|
+
const artifacts = value.artifacts.map(parseArtifact);
|
|
9499
|
+
const paths = /* @__PURE__ */ new Set();
|
|
9500
|
+
for (const artifact of artifacts) {
|
|
9501
|
+
if (paths.has(artifact.path))
|
|
9502
|
+
throw new PlanV2IntegrityError(`duplicate artifact path: ${artifact.path}`);
|
|
9503
|
+
paths.add(artifact.path);
|
|
9504
|
+
}
|
|
9505
|
+
if (!paths.has("plan.json"))
|
|
9506
|
+
throw new PlanV2IntegrityError("manifest must include the local execution plan.json artifact");
|
|
9507
|
+
if (!isRecord3(value.limitations) || value.limitations.videoDependencyMode !== "exact-rendered-frames" && value.limitations.videoDependencyMode !== "full-source-pack" && value.limitations.videoDependencyMode !== "source-extract") {
|
|
9508
|
+
throw new PlanV2IntegrityError("unsupported video dependency mode");
|
|
9509
|
+
}
|
|
9510
|
+
const parsed = {
|
|
9511
|
+
protocol: PLAN_PROTOCOL_V2,
|
|
9512
|
+
planHash: value.planHash,
|
|
9513
|
+
sourcePlanV1Hash: value.sourcePlanV1Hash,
|
|
9514
|
+
chunkCount: readPositiveInteger2(value.chunkCount, "chunkCount"),
|
|
9515
|
+
totalFrames: readPositiveInteger2(value.totalFrames, "totalFrames"),
|
|
9516
|
+
fps: readSupportedFps(value.fps),
|
|
9517
|
+
width: readPositiveInteger2(value.width, "width"),
|
|
9518
|
+
height: readPositiveInteger2(value.height, "height"),
|
|
9519
|
+
format: readDistributedFormat(value.format),
|
|
9520
|
+
ffmpegVersion: readString2(value.ffmpegVersion, "ffmpegVersion"),
|
|
9521
|
+
producerVersion: readString2(value.producerVersion, "producerVersion"),
|
|
9522
|
+
limitations: { videoDependencyMode: value.limitations.videoDependencyMode },
|
|
9523
|
+
artifacts
|
|
9524
|
+
};
|
|
9525
|
+
const { planHash: _planHash, ...payload } = parsed;
|
|
9526
|
+
const expectedHash = computeManifestHash(payload);
|
|
9527
|
+
if (expectedHash !== parsed.planHash) {
|
|
9528
|
+
throw new PlanV2IntegrityError("manifest integrity hash mismatch");
|
|
9529
|
+
}
|
|
9530
|
+
for (const [index, artifact] of parsed.artifacts.entries()) {
|
|
9531
|
+
if (artifact.chunks !== "all" && artifact.chunks.some((chunkIndex) => chunkIndex >= parsed.chunkCount)) {
|
|
9532
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].chunks contains an out-of-range index`);
|
|
9485
9533
|
}
|
|
9486
|
-
}
|
|
9534
|
+
}
|
|
9535
|
+
return parsed;
|
|
9487
9536
|
}
|
|
9488
|
-
|
|
9489
|
-
const
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
9493
|
-
|
|
9494
|
-
|
|
9537
|
+
function readPlanV2Manifest(planV2Dir) {
|
|
9538
|
+
const manifestPath = join25(planV2Dir, "plan.json");
|
|
9539
|
+
if (!existsSync17(manifestPath)) {
|
|
9540
|
+
throw new PlanV2IntegrityError(`missing v2 manifest: ${manifestPath}`);
|
|
9541
|
+
}
|
|
9542
|
+
const value = readJsonFile(manifestPath, "v2 manifest");
|
|
9543
|
+
return parsePlanV2Manifest(value);
|
|
9544
|
+
}
|
|
9545
|
+
function listPlanV2ArtifactsForTarget(manifest, target) {
|
|
9546
|
+
if (target.role === "chunk") {
|
|
9547
|
+
if (!Number.isInteger(target.chunkIndex) || target.chunkIndex < 0 || target.chunkIndex >= manifest.chunkCount) {
|
|
9548
|
+
throw new PlanV2IntegrityError(`chunkIndex ${String(target.chunkIndex)} is out of range`);
|
|
9549
|
+
}
|
|
9550
|
+
return manifest.artifacts.filter(
|
|
9551
|
+
(artifact) => artifact.chunks === "all" || artifact.chunks.some((chunkIndex) => chunkIndex === target.chunkIndex)
|
|
9495
9552
|
);
|
|
9496
9553
|
}
|
|
9554
|
+
return manifest.artifacts.filter((artifact) => artifact.assembler);
|
|
9497
9555
|
}
|
|
9498
|
-
|
|
9499
|
-
const
|
|
9500
|
-
|
|
9501
|
-
|
|
9502
|
-
|
|
9503
|
-
const
|
|
9556
|
+
function verifyBlob(planV2Dir, artifact) {
|
|
9557
|
+
const sourcePath = planV2BlobPath(planV2Dir, artifact.sha256);
|
|
9558
|
+
if (!existsSync17(sourcePath)) {
|
|
9559
|
+
throw new PlanV2IntegrityError(`missing content-addressed artifact ${artifact.sha256}`);
|
|
9560
|
+
}
|
|
9561
|
+
const stats = statSync6(sourcePath);
|
|
9562
|
+
if (!stats.isFile() || stats.size !== artifact.sizeBytes) {
|
|
9563
|
+
throw new PlanV2IntegrityError(`artifact size mismatch for ${artifact.path}`);
|
|
9564
|
+
}
|
|
9565
|
+
if (sha256File(sourcePath) !== artifact.sha256) {
|
|
9566
|
+
throw new PlanV2IntegrityError(`artifact hash mismatch for ${artifact.path}`);
|
|
9567
|
+
}
|
|
9568
|
+
return sourcePath;
|
|
9569
|
+
}
|
|
9570
|
+
function materializePlanV2Target(planV2Dir, target, destinationDir, options = {}) {
|
|
9571
|
+
if (existsSync17(destinationDir)) {
|
|
9572
|
+
throw new PlanV2IntegrityError(`materialization destination already exists: ${destinationDir}`);
|
|
9573
|
+
}
|
|
9574
|
+
const manifest = readPlanV2Manifest(planV2Dir);
|
|
9575
|
+
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
9576
|
+
const remoteSources = options.remoteSources ?? {};
|
|
9577
|
+
assertRemoteSources(remoteSources, artifacts);
|
|
9578
|
+
const audioArtifact = artifacts.find((artifact) => isPlanAudioArtifactPath(artifact.path));
|
|
9579
|
+
const verified = artifacts.filter((artifact) => !Object.hasOwn(remoteSources, artifact.path)).map((artifact) => ({
|
|
9580
|
+
artifact,
|
|
9581
|
+
sourcePath: verifyBlob(planV2Dir, artifact)
|
|
9582
|
+
}));
|
|
9583
|
+
mkdirSync12(dirname11(destinationDir), { recursive: true });
|
|
9584
|
+
const tempDir = mkdtempSync5(join25(dirname11(destinationDir), ".plan-v2-materialize-"));
|
|
9504
9585
|
try {
|
|
9505
|
-
|
|
9506
|
-
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9586
|
+
for (const { artifact, sourcePath } of verified) {
|
|
9587
|
+
const destinationPath = join25(tempDir, ...artifact.path.split("/"));
|
|
9588
|
+
mkdirSync12(dirname11(destinationPath), { recursive: true });
|
|
9589
|
+
copyFileSync7(sourcePath, destinationPath);
|
|
9590
|
+
}
|
|
9591
|
+
if (target.role === "chunk") materializeExtractedVideoDirectories(tempDir);
|
|
9592
|
+
writeFileSync8(
|
|
9593
|
+
join25(tempDir, PLAN_V2_MATERIALIZATION_MARKER),
|
|
9594
|
+
canonicalJsonStringify({
|
|
9595
|
+
manifest,
|
|
9596
|
+
target,
|
|
9597
|
+
...Object.keys(remoteSources).length > 0 ? { remoteSources } : {}
|
|
9598
|
+
}),
|
|
9599
|
+
"utf-8"
|
|
9600
|
+
);
|
|
9601
|
+
renameSync6(tempDir, destinationDir);
|
|
9602
|
+
} catch (error) {
|
|
9603
|
+
rmSync10(tempDir, { recursive: true, force: true });
|
|
9604
|
+
throw error;
|
|
9605
|
+
}
|
|
9606
|
+
return {
|
|
9607
|
+
planDir: destinationDir,
|
|
9608
|
+
target,
|
|
9609
|
+
planHash: manifest.planHash,
|
|
9610
|
+
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
9611
|
+
artifactCount: artifacts.length,
|
|
9612
|
+
sizeBytes: artifacts.reduce((sum2, artifact) => sum2 + artifact.sizeBytes, 0),
|
|
9613
|
+
// Join the artifact's OWN name, not the current constant: a plan written
|
|
9614
|
+
// before the container change carries the legacy name, and materializing it
|
|
9615
|
+
// under the new one would point at a file that was never written.
|
|
9616
|
+
audioPath: target.role === "assembler" ? audioArtifact ? join25(destinationDir, audioArtifact.path) : null : null
|
|
9617
|
+
};
|
|
9618
|
+
}
|
|
9619
|
+
function assertRemoteSources(remoteSources, artifacts) {
|
|
9620
|
+
if (!isRecord3(remoteSources)) throw new PlanV2IntegrityError("remoteSources must be an object");
|
|
9621
|
+
for (const [path, url] of Object.entries(remoteSources)) {
|
|
9622
|
+
const artifact = artifacts.find((candidate) => candidate.path === path);
|
|
9623
|
+
if (!artifact || !isPlanV2DeferredSourceArtifact(artifact)) {
|
|
9624
|
+
throw new PlanV2IntegrityError(`remote source is not a selected deferred video: ${path}`);
|
|
9625
|
+
}
|
|
9626
|
+
if (typeof url !== "string" || !/^https?:\/\//.test(url)) {
|
|
9627
|
+
throw new PlanV2IntegrityError(`remote source for ${path} must be an http(s) URL`);
|
|
9628
|
+
}
|
|
9629
|
+
}
|
|
9630
|
+
}
|
|
9631
|
+
function validatePlanV2MaterializedTarget(planDir, expectedTarget) {
|
|
9632
|
+
return readValidatedPlanV2Materialization(planDir, expectedTarget)?.manifest ?? null;
|
|
9633
|
+
}
|
|
9634
|
+
function readValidatedPlanV2Materialization(planDir, expectedTarget) {
|
|
9635
|
+
const markerPath = join25(planDir, PLAN_V2_MATERIALIZATION_MARKER);
|
|
9636
|
+
if (!existsSync17(markerPath)) return null;
|
|
9637
|
+
const marker = readJsonFile(markerPath, "materialization marker");
|
|
9638
|
+
if (!isRecord3(marker) || !isRecord3(marker.manifest) || !isRecord3(marker.target)) {
|
|
9639
|
+
throw new PlanV2IntegrityError("malformed materialization marker");
|
|
9640
|
+
}
|
|
9641
|
+
const target = marker.target;
|
|
9642
|
+
if (target.role !== expectedTarget.role || expectedTarget.role === "chunk" && target.chunkIndex !== expectedTarget.chunkIndex) {
|
|
9643
|
+
throw new PlanV2IntegrityError(
|
|
9644
|
+
"materialization target does not match requested execution role"
|
|
9645
|
+
);
|
|
9513
9646
|
}
|
|
9647
|
+
const manifest = parsePlanV2Manifest(marker.manifest);
|
|
9648
|
+
const required = listPlanV2ArtifactsForTarget(manifest, expectedTarget);
|
|
9649
|
+
const remoteSources = marker.remoteSources ?? {};
|
|
9650
|
+
assertRemoteSources(remoteSources, required);
|
|
9651
|
+
for (const artifact of required) {
|
|
9652
|
+
if (Object.hasOwn(remoteSources, artifact.path)) continue;
|
|
9653
|
+
const path = join25(planDir, ...artifact.path.split("/"));
|
|
9654
|
+
if (!existsSync17(path) || statSync6(path).size !== artifact.sizeBytes) {
|
|
9655
|
+
throw new PlanV2IntegrityError(
|
|
9656
|
+
`materialized artifact missing or truncated: ${artifact.path}`
|
|
9657
|
+
);
|
|
9658
|
+
}
|
|
9659
|
+
if (sha256File(path) !== artifact.sha256) {
|
|
9660
|
+
throw new PlanV2IntegrityError(`materialized artifact hash mismatch: ${artifact.path}`);
|
|
9661
|
+
}
|
|
9662
|
+
}
|
|
9663
|
+
return { manifest, remoteSources };
|
|
9514
9664
|
}
|
|
9515
9665
|
|
|
9666
|
+
// src/services/distributed/planV2Execution.ts
|
|
9667
|
+
import { mkdtempSync as mkdtempSync7, rmSync as rmSync13 } from "node:fs";
|
|
9668
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
9669
|
+
import { join as join28 } from "node:path";
|
|
9670
|
+
|
|
9516
9671
|
// src/services/distributed/assemble.ts
|
|
9672
|
+
import {
|
|
9673
|
+
cpSync as cpSync3,
|
|
9674
|
+
existsSync as existsSync18,
|
|
9675
|
+
mkdirSync as mkdirSync13,
|
|
9676
|
+
mkdtempSync as mkdtempSync6,
|
|
9677
|
+
readFileSync as readFileSync11,
|
|
9678
|
+
readdirSync as readdirSync8,
|
|
9679
|
+
renameSync as renameSync7,
|
|
9680
|
+
rmSync as rmSync11,
|
|
9681
|
+
statSync as statSync7,
|
|
9682
|
+
writeFileSync as writeFileSync9
|
|
9683
|
+
} from "node:fs";
|
|
9684
|
+
import { dirname as dirname12, join as join26 } from "node:path";
|
|
9685
|
+
import {
|
|
9686
|
+
appendRenderProvenanceArgs as appendRenderProvenanceArgs2,
|
|
9687
|
+
describeFfmpegFailure as describeFfmpegFailure3,
|
|
9688
|
+
getEncoderPreset as getEncoderPreset5,
|
|
9689
|
+
MIXED_AUDIO_FILENAME as MIXED_AUDIO_FILENAME4,
|
|
9690
|
+
muxVideoWithAudio as muxVideoWithAudio2,
|
|
9691
|
+
resolveConfig as resolveConfig4,
|
|
9692
|
+
runFfmpeg as runFfmpeg6
|
|
9693
|
+
} from "@hyperframes/engine";
|
|
9694
|
+
import { fpsToFfmpegArg as fpsToFfmpegArg3 } from "@hyperframes/core";
|
|
9517
9695
|
function cfrRateControlArgs(locked) {
|
|
9518
9696
|
const preset = getEncoderPreset5(locked.quality ?? "standard", "mp4");
|
|
9519
9697
|
if (locked.crf == null && locked.bitrate) {
|
|
@@ -9540,8 +9718,8 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9540
9718
|
}
|
|
9541
9719
|
const format = plan2.dimensions.format;
|
|
9542
9720
|
const cfrEncoder = cfr ? readCfrEncoder(planDir, format) : null;
|
|
9543
|
-
if (!
|
|
9544
|
-
|
|
9721
|
+
if (!existsSync18(dirname12(outputPath))) {
|
|
9722
|
+
mkdirSync13(dirname12(outputPath), { recursive: true });
|
|
9545
9723
|
}
|
|
9546
9724
|
const workDir = mkdtempSync6(`${outputPath}.assemble-work-`);
|
|
9547
9725
|
const fps = { num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen };
|
|
@@ -9563,8 +9741,8 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9563
9741
|
};
|
|
9564
9742
|
try {
|
|
9565
9743
|
const videoSources = cfrEncoder ? [await reencodeCfr(ctx, chunkPaths, cfrEncoder)] : chunkPaths;
|
|
9566
|
-
const finalPath =
|
|
9567
|
-
const audio = audioPath !== null &&
|
|
9744
|
+
const finalPath = join26(workDir, `output.${format}`);
|
|
9745
|
+
const audio = audioPath !== null && existsSync18(audioPath) ? audioPath : null;
|
|
9568
9746
|
await writeJoined(ctx, videoSources, audio, finalPath);
|
|
9569
9747
|
renameSync7(finalPath, outputPath);
|
|
9570
9748
|
log.info("[assemble] stage timings", {
|
|
@@ -9582,7 +9760,7 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9582
9760
|
});
|
|
9583
9761
|
}
|
|
9584
9762
|
}
|
|
9585
|
-
const fileSize =
|
|
9763
|
+
const fileSize = existsSync18(outputPath) ? statSync7(outputPath).size : 0;
|
|
9586
9764
|
return {
|
|
9587
9765
|
outputPath,
|
|
9588
9766
|
durationMs: Date.now() - start,
|
|
@@ -9591,25 +9769,25 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9591
9769
|
};
|
|
9592
9770
|
}
|
|
9593
9771
|
function readAssemblePlan(planDir, chunkPaths) {
|
|
9594
|
-
const planJsonPath =
|
|
9595
|
-
const chunksJsonPath =
|
|
9596
|
-
if (!
|
|
9772
|
+
const planJsonPath = join26(planDir, "plan.json");
|
|
9773
|
+
const chunksJsonPath = join26(planDir, "meta", "chunks.json");
|
|
9774
|
+
if (!existsSync18(planJsonPath)) {
|
|
9597
9775
|
throw new Error(`[assemble] planDir missing plan.json: ${planJsonPath}`);
|
|
9598
9776
|
}
|
|
9599
|
-
const plan2 = JSON.parse(
|
|
9777
|
+
const plan2 = JSON.parse(readFileSync11(planJsonPath, "utf-8"));
|
|
9600
9778
|
readPlanProtocolV1(plan2, DISTRIBUTED_RENDER_CAPABILITIES.roles.assembler);
|
|
9601
9779
|
validatePlanV2MaterializedTarget(planDir, { role: "assembler" });
|
|
9602
|
-
if (!
|
|
9780
|
+
if (!existsSync18(chunksJsonPath)) {
|
|
9603
9781
|
throw new Error(`[assemble] planDir missing meta/chunks.json: ${chunksJsonPath}`);
|
|
9604
9782
|
}
|
|
9605
|
-
const chunks = JSON.parse(
|
|
9783
|
+
const chunks = JSON.parse(readFileSync11(chunksJsonPath, "utf-8"));
|
|
9606
9784
|
if (chunkPaths.length !== chunks.length) {
|
|
9607
9785
|
throw new Error(
|
|
9608
9786
|
`[assemble] chunkPaths length (${chunkPaths.length}) does not match chunks.json length (${chunks.length}). Adapters must pass one path per chunk, ordered by index.`
|
|
9609
9787
|
);
|
|
9610
9788
|
}
|
|
9611
9789
|
for (const path of chunkPaths) {
|
|
9612
|
-
if (!
|
|
9790
|
+
if (!existsSync18(path)) {
|
|
9613
9791
|
throw new Error(`[assemble] chunk path does not exist: ${path}`);
|
|
9614
9792
|
}
|
|
9615
9793
|
}
|
|
@@ -9619,7 +9797,7 @@ function videoInputArgs(ctx, videoSources) {
|
|
|
9619
9797
|
return videoSources.length === 1 ? ["-i", videoSources[0]] : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);
|
|
9620
9798
|
}
|
|
9621
9799
|
async function reencodeCfr(ctx, chunkPaths, encoder) {
|
|
9622
|
-
const cfrOutputPath =
|
|
9800
|
+
const cfrOutputPath = join26(ctx.workDir, `cfr.${ctx.format}`);
|
|
9623
9801
|
const cfrArgs = [
|
|
9624
9802
|
...videoInputArgs(ctx, chunkPaths),
|
|
9625
9803
|
"-c:v",
|
|
@@ -9632,7 +9810,7 @@ async function reencodeCfr(ctx, chunkPaths, encoder) {
|
|
|
9632
9810
|
"-r",
|
|
9633
9811
|
ctx.fpsArg
|
|
9634
9812
|
];
|
|
9635
|
-
|
|
9813
|
+
appendRenderProvenanceArgs2(cfrArgs, cfrOutputPath);
|
|
9636
9814
|
cfrArgs.push("-y", cfrOutputPath);
|
|
9637
9815
|
await ctx.timed(
|
|
9638
9816
|
"cfrMs",
|
|
@@ -9649,7 +9827,7 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9649
9827
|
if (audioPath === null) {
|
|
9650
9828
|
const joinArgs = [...videoInputArgs(ctx, videoSources), "-c", "copy"];
|
|
9651
9829
|
if (ctx.format === "mp4") joinArgs.push("-movflags", "+faststart");
|
|
9652
|
-
|
|
9830
|
+
appendRenderProvenanceArgs2(joinArgs, finalPath);
|
|
9653
9831
|
joinArgs.push("-r", ctx.fpsArg, "-y", finalPath);
|
|
9654
9832
|
await ctx.timed(
|
|
9655
9833
|
"joinMs",
|
|
@@ -9657,7 +9835,7 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9657
9835
|
);
|
|
9658
9836
|
return;
|
|
9659
9837
|
}
|
|
9660
|
-
const paddedAudioPath =
|
|
9838
|
+
const paddedAudioPath = join26(ctx.workDir, "audio-padded.m4a");
|
|
9661
9839
|
const padTrimResult = await ctx.timed(
|
|
9662
9840
|
"audioMs",
|
|
9663
9841
|
() => padOrTrimAudioToVideoFrameCount({
|
|
@@ -9697,9 +9875,9 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9697
9875
|
}
|
|
9698
9876
|
}
|
|
9699
9877
|
function concatDemuxerInput(chunkPaths, workDir, fpsArg) {
|
|
9700
|
-
const listPath =
|
|
9878
|
+
const listPath = join26(workDir, "concat-list.txt");
|
|
9701
9879
|
const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");
|
|
9702
|
-
|
|
9880
|
+
writeFileSync9(listPath, `${body}
|
|
9703
9881
|
`, "utf-8");
|
|
9704
9882
|
return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath];
|
|
9705
9883
|
}
|
|
@@ -9714,10 +9892,10 @@ async function probeJoinedFrameInfo(videoPaths, fps, signal) {
|
|
|
9714
9892
|
};
|
|
9715
9893
|
}
|
|
9716
9894
|
async function runAssembleFfmpeg(args, step, signal, timeout) {
|
|
9717
|
-
const result = await
|
|
9895
|
+
const result = await runFfmpeg6(args, { signal, timeout });
|
|
9718
9896
|
if (!result.success) {
|
|
9719
9897
|
throw encoderFailureError(`[assemble] ffmpeg ${step} failed`, {
|
|
9720
|
-
error:
|
|
9898
|
+
error: describeFfmpegFailure3(result, timeout),
|
|
9721
9899
|
failureReason: result.failureReason
|
|
9722
9900
|
});
|
|
9723
9901
|
}
|
|
@@ -9730,9 +9908,9 @@ function chunksAreStitchedMp4(planDir, cfr) {
|
|
|
9730
9908
|
return true;
|
|
9731
9909
|
}
|
|
9732
9910
|
function chunksCarryAudio(planDir) {
|
|
9733
|
-
const encoderJsonPath =
|
|
9734
|
-
if (!
|
|
9735
|
-
return JSON.parse(
|
|
9911
|
+
const encoderJsonPath = join26(planDir, "meta", "encoder.json");
|
|
9912
|
+
if (!existsSync18(encoderJsonPath)) return false;
|
|
9913
|
+
return JSON.parse(readFileSync11(encoderJsonPath, "utf-8")).chunksCarryAudio === true;
|
|
9736
9914
|
}
|
|
9737
9915
|
function readCfrEncoder(planDir, format) {
|
|
9738
9916
|
if (format !== "mp4") {
|
|
@@ -9740,11 +9918,11 @@ function readCfrEncoder(planDir, format) {
|
|
|
9740
9918
|
`[assemble] cfr=true is only supported for format="mp4" (got "${format}"). Stream-copy paths for webm and mov already produce exact avg_frame_rate; cfr re-encode is not needed.`
|
|
9741
9919
|
);
|
|
9742
9920
|
}
|
|
9743
|
-
const encoderJsonPath =
|
|
9744
|
-
if (!
|
|
9921
|
+
const encoderJsonPath = join26(planDir, "meta", "encoder.json");
|
|
9922
|
+
if (!existsSync18(encoderJsonPath)) {
|
|
9745
9923
|
throw new Error(`[assemble] planDir missing meta/encoder.json: ${encoderJsonPath}`);
|
|
9746
9924
|
}
|
|
9747
|
-
const encoderJson = JSON.parse(
|
|
9925
|
+
const encoderJson = JSON.parse(readFileSync11(encoderJsonPath, "utf-8"));
|
|
9748
9926
|
if (encoderJson.encoder === "libx265-software") {
|
|
9749
9927
|
throw new Error(
|
|
9750
9928
|
`[assemble] cfr=true is not yet supported with codec: "h265". The cfr re-encode pass uses libx264 and would silently transcode the h265 chunks. Either disable cfr or render with codec: "h264".`
|
|
@@ -9753,8 +9931,8 @@ function readCfrEncoder(planDir, format) {
|
|
|
9753
9931
|
return encoderJson;
|
|
9754
9932
|
}
|
|
9755
9933
|
function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, startTimeMs) {
|
|
9756
|
-
if (
|
|
9757
|
-
|
|
9934
|
+
if (existsSync18(outputPath)) rmSync11(outputPath, { recursive: true, force: true });
|
|
9935
|
+
mkdirSync13(outputPath, { recursive: true });
|
|
9758
9936
|
let globalIdx = 0;
|
|
9759
9937
|
for (const chunkDir of chunkPaths) {
|
|
9760
9938
|
if (!statSync7(chunkDir).isDirectory()) {
|
|
@@ -9767,8 +9945,8 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9767
9945
|
throw new Error(`[assemble] png-sequence chunk has no frames: ${chunkDir}`);
|
|
9768
9946
|
}
|
|
9769
9947
|
for (const frame of frames) {
|
|
9770
|
-
const dst =
|
|
9771
|
-
cpSync3(
|
|
9948
|
+
const dst = join26(outputPath, formatExportFrameName(globalIdx, "png"));
|
|
9949
|
+
cpSync3(join26(chunkDir, frame), dst);
|
|
9772
9950
|
globalIdx += 1;
|
|
9773
9951
|
}
|
|
9774
9952
|
}
|
|
@@ -9777,14 +9955,14 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9777
9955
|
`[assemble] png-sequence frame count mismatch: merged ${globalIdx} frames vs plan.totalFrames=${totalFrames}. Using on-disk count.`
|
|
9778
9956
|
);
|
|
9779
9957
|
}
|
|
9780
|
-
if (audioPath !== null &&
|
|
9781
|
-
const sidecar =
|
|
9958
|
+
if (audioPath !== null && existsSync18(audioPath)) {
|
|
9959
|
+
const sidecar = join26(outputPath, MIXED_AUDIO_FILENAME4);
|
|
9782
9960
|
cpSync3(audioPath, sidecar);
|
|
9783
9961
|
}
|
|
9784
9962
|
let fileSize = 0;
|
|
9785
9963
|
for (const name of readdirSync8(outputPath)) {
|
|
9786
9964
|
try {
|
|
9787
|
-
fileSize += statSync7(
|
|
9965
|
+
fileSize += statSync7(join26(outputPath, name)).size;
|
|
9788
9966
|
} catch {
|
|
9789
9967
|
}
|
|
9790
9968
|
}
|
|
@@ -9798,8 +9976,8 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9798
9976
|
|
|
9799
9977
|
// src/services/distributed/renderChunk.ts
|
|
9800
9978
|
import { randomBytes } from "node:crypto";
|
|
9801
|
-
import { existsSync as
|
|
9802
|
-
import { join as
|
|
9979
|
+
import { existsSync as existsSync19, mkdirSync as mkdirSync14, readFileSync as readFileSync12, readdirSync as readdirSync9, rmSync as rmSync12, writeFileSync as writeFileSync10 } from "node:fs";
|
|
9980
|
+
import { join as join27 } from "node:path";
|
|
9803
9981
|
import {
|
|
9804
9982
|
assertSwiftShader,
|
|
9805
9983
|
BROWSER_GPU_NOT_SOFTWARE as BROWSER_GPU_NOT_SOFTWARE4,
|
|
@@ -9815,104 +9993,6 @@ import {
|
|
|
9815
9993
|
readWebGlVendorInfoFromCanvas,
|
|
9816
9994
|
resolveConfig as resolveConfig5
|
|
9817
9995
|
} from "@hyperframes/engine";
|
|
9818
|
-
|
|
9819
|
-
// src/services/distributed/chunkAudio.ts
|
|
9820
|
-
import { appendRenderProvenanceArgs as appendRenderProvenanceArgs2, describeFfmpegFailure as describeFfmpegFailure3, runFfmpeg as runFfmpeg6 } from "@hyperframes/engine";
|
|
9821
|
-
function audioCutForChunk(grid, chunks, index, fps) {
|
|
9822
|
-
const firstPacketAt = (frame) => Math.min(
|
|
9823
|
-
grid.packetCount,
|
|
9824
|
-
Math.max(
|
|
9825
|
-
0,
|
|
9826
|
-
Math.ceil(
|
|
9827
|
-
(frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) / (grid.packetSamples * fps.num)
|
|
9828
|
-
)
|
|
9829
|
-
)
|
|
9830
|
-
);
|
|
9831
|
-
const first = index === 0 ? 0 : firstPacketAt(chunks[index].startFrame);
|
|
9832
|
-
const end = index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1].startFrame);
|
|
9833
|
-
if (end <= first) return null;
|
|
9834
|
-
const cutSeconds = (packet) => (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;
|
|
9835
|
-
const seekSeconds = index === 0 ? null : cutSeconds(first);
|
|
9836
|
-
const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);
|
|
9837
|
-
return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
|
|
9838
|
-
}
|
|
9839
|
-
async function probeAacPacketGrid(audioPath) {
|
|
9840
|
-
const probe = await runFfprobeJson([
|
|
9841
|
-
"-v",
|
|
9842
|
-
"error",
|
|
9843
|
-
"-select_streams",
|
|
9844
|
-
"a:0",
|
|
9845
|
-
"-show_entries",
|
|
9846
|
-
"stream=sample_rate,time_base,nb_frames:packet=pts,duration",
|
|
9847
|
-
"-read_intervals",
|
|
9848
|
-
"%+#1",
|
|
9849
|
-
"-of",
|
|
9850
|
-
"json",
|
|
9851
|
-
"--",
|
|
9852
|
-
audioPath
|
|
9853
|
-
]);
|
|
9854
|
-
const stream = probe.streams?.[0] ?? {};
|
|
9855
|
-
const packet = probe.packets?.[0] ?? {};
|
|
9856
|
-
const sampleRate = Number(stream.sample_rate);
|
|
9857
|
-
const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);
|
|
9858
|
-
const toSamples = (ticks) => Number(ticks) * sampleRate * tbNum / tbDen;
|
|
9859
|
-
const grid = {
|
|
9860
|
-
sampleRate,
|
|
9861
|
-
firstStart: toSamples(packet.pts),
|
|
9862
|
-
packetSamples: toSamples(packet.duration),
|
|
9863
|
-
packetCount: Number(stream.nb_frames)
|
|
9864
|
-
};
|
|
9865
|
-
const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];
|
|
9866
|
-
if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {
|
|
9867
|
-
throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);
|
|
9868
|
-
}
|
|
9869
|
-
return grid;
|
|
9870
|
-
}
|
|
9871
|
-
function chunkMuxArgs(input, cut) {
|
|
9872
|
-
if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];
|
|
9873
|
-
const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];
|
|
9874
|
-
const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];
|
|
9875
|
-
const exact = seek.length ? ["-copypriorss", "0"] : [];
|
|
9876
|
-
return [
|
|
9877
|
-
...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath],
|
|
9878
|
-
...["-map", "0:v:0", "-map", "1:a:0", ...exact]
|
|
9879
|
-
];
|
|
9880
|
-
}
|
|
9881
|
-
async function muxChunkWithAudio(input) {
|
|
9882
|
-
const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;
|
|
9883
|
-
const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;
|
|
9884
|
-
const args = [...chunkMuxArgs(input, cut), "-c", "copy"];
|
|
9885
|
-
appendRenderProvenanceArgs2(args, input.outputPath);
|
|
9886
|
-
args.push("-y", input.outputPath);
|
|
9887
|
-
const result = await runFfmpeg6(args, { signal: input.signal, timeout: input.timeoutMs });
|
|
9888
|
-
if (!result.success) {
|
|
9889
|
-
throw new Error(
|
|
9890
|
-
`[chunkAudio] chunk mux failed: ${describeFfmpegFailure3(result, input.timeoutMs)}`
|
|
9891
|
-
);
|
|
9892
|
-
}
|
|
9893
|
-
const slice = input.chunks[input.chunkIndex];
|
|
9894
|
-
const expected = { video: slice.endFrame - slice.startFrame };
|
|
9895
|
-
if (cut) expected.audio = cut.packetCount;
|
|
9896
|
-
const probe = await runFfprobeJson([
|
|
9897
|
-
"-v",
|
|
9898
|
-
"error",
|
|
9899
|
-
"-count_packets",
|
|
9900
|
-
"-show_entries",
|
|
9901
|
-
"stream=codec_type,nb_read_packets",
|
|
9902
|
-
"-of",
|
|
9903
|
-
"json",
|
|
9904
|
-
"--",
|
|
9905
|
-
input.outputPath
|
|
9906
|
-
]);
|
|
9907
|
-
for (const [type, count] of Object.entries(expected)) {
|
|
9908
|
-
const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);
|
|
9909
|
-
if (packets !== count) {
|
|
9910
|
-
throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);
|
|
9911
|
-
}
|
|
9912
|
-
}
|
|
9913
|
-
}
|
|
9914
|
-
|
|
9915
|
-
// src/services/distributed/renderChunk.ts
|
|
9916
9996
|
import { readWebGlVendorInfoFromCanvas as readWebGlVendorInfoFromCanvas2 } from "@hyperframes/engine";
|
|
9917
9997
|
var LEGACY_DISTRIBUTED_VP9_CPU_USED = 2;
|
|
9918
9998
|
function validatePlanVideosForChunk(value) {
|
|
@@ -9978,10 +10058,10 @@ async function beginFrameSessionNeedsScreenshotFallback(session, probe = probeBe
|
|
|
9978
10058
|
return !await probe(session.page, timeoutMs, probeTick, session.beginFrameIntervalMs);
|
|
9979
10059
|
}
|
|
9980
10060
|
function hashChunkOutput(outputPath, kind) {
|
|
9981
|
-
if (kind === "file") return sha256Hex(
|
|
10061
|
+
if (kind === "file") return sha256Hex(readFileSync12(outputPath));
|
|
9982
10062
|
const entries = readdirSync9(outputPath).filter((name) => /\.(png|jpg|jpeg)$/i.test(name)).sort();
|
|
9983
10063
|
const lines = entries.map(
|
|
9984
|
-
(name) => `${name}\0${sha256Hex(
|
|
10064
|
+
(name) => `${name}\0${sha256Hex(readFileSync12(join27(outputPath, name)))}`
|
|
9985
10065
|
);
|
|
9986
10066
|
return sha256Hex(lines.join("\0"));
|
|
9987
10067
|
}
|
|
@@ -9998,16 +10078,16 @@ function resolveLockedVp9CpuUsed(lockedEncoder) {
|
|
|
9998
10078
|
async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
9999
10079
|
const start = Date.now();
|
|
10000
10080
|
const log = defaultLogger;
|
|
10001
|
-
const planJsonPath =
|
|
10002
|
-
const encoderJsonPath =
|
|
10003
|
-
const chunksJsonPath =
|
|
10004
|
-
if (!
|
|
10081
|
+
const planJsonPath = join27(planDir, "plan.json");
|
|
10082
|
+
const encoderJsonPath = join27(planDir, "meta", "encoder.json");
|
|
10083
|
+
const chunksJsonPath = join27(planDir, "meta", "chunks.json");
|
|
10084
|
+
if (!existsSync19(planJsonPath)) {
|
|
10005
10085
|
throw new RenderChunkValidationError(
|
|
10006
10086
|
MISSING_PLAN_ARTIFACT,
|
|
10007
10087
|
`[renderChunk] planDir is missing required artifact: ${planJsonPath}`
|
|
10008
10088
|
);
|
|
10009
10089
|
}
|
|
10010
|
-
const plan2 = JSON.parse(
|
|
10090
|
+
const plan2 = JSON.parse(readFileSync12(planJsonPath, "utf-8"));
|
|
10011
10091
|
readPlanProtocolV1(plan2, DISTRIBUTED_RENDER_CAPABILITIES.roles.chunk);
|
|
10012
10092
|
const planHashStarted = Date.now();
|
|
10013
10093
|
const v2Materialization = readValidatedPlanV2Materialization(planDir, {
|
|
@@ -10017,20 +10097,20 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10017
10097
|
const v2Manifest = v2Materialization?.manifest ?? null;
|
|
10018
10098
|
const remoteSources = v2Materialization?.remoteSources ?? {};
|
|
10019
10099
|
for (const required of [encoderJsonPath, chunksJsonPath]) {
|
|
10020
|
-
if (!
|
|
10100
|
+
if (!existsSync19(required)) {
|
|
10021
10101
|
throw new RenderChunkValidationError(
|
|
10022
10102
|
MISSING_PLAN_ARTIFACT,
|
|
10023
10103
|
`[renderChunk] planDir is missing required artifact: ${required}`
|
|
10024
10104
|
);
|
|
10025
10105
|
}
|
|
10026
10106
|
}
|
|
10027
|
-
const encoder = JSON.parse(
|
|
10028
|
-
const chunks = JSON.parse(
|
|
10029
|
-
const videosJsonPath =
|
|
10107
|
+
const encoder = JSON.parse(readFileSync12(encoderJsonPath, "utf-8"));
|
|
10108
|
+
const chunks = JSON.parse(readFileSync12(chunksJsonPath, "utf-8"));
|
|
10109
|
+
const videosJsonPath = join27(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
10030
10110
|
let planVideos = null;
|
|
10031
|
-
if (
|
|
10111
|
+
if (existsSync19(videosJsonPath)) {
|
|
10032
10112
|
try {
|
|
10033
|
-
planVideos = validatePlanVideosForChunk(JSON.parse(
|
|
10113
|
+
planVideos = validatePlanVideosForChunk(JSON.parse(readFileSync12(videosJsonPath, "utf-8")));
|
|
10034
10114
|
} catch (err) {
|
|
10035
10115
|
if (err instanceof RenderChunkValidationError) throw err;
|
|
10036
10116
|
throw new RenderChunkValidationError(
|
|
@@ -10056,8 +10136,8 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10056
10136
|
`[renderChunk] chunk ${chunkIndex} has non-positive frame count: ${framesInChunk}`
|
|
10057
10137
|
);
|
|
10058
10138
|
}
|
|
10059
|
-
const compiledDir =
|
|
10060
|
-
if (!
|
|
10139
|
+
const compiledDir = join27(planDir, "compiled");
|
|
10140
|
+
if (!existsSync19(compiledDir)) {
|
|
10061
10141
|
throw new RenderChunkValidationError(
|
|
10062
10142
|
MISSING_PLAN_ARTIFACT,
|
|
10063
10143
|
`[renderChunk] planDir missing compiled/ directory: ${compiledDir}`
|
|
@@ -10113,9 +10193,9 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10113
10193
|
forceScreenshotExplicitlyOptedOut: !encoder.forceScreenshot
|
|
10114
10194
|
};
|
|
10115
10195
|
const workDir = `${outputChunkPath}.work.${process.pid}.${randomBytes(4).toString("hex")}`;
|
|
10116
|
-
|
|
10117
|
-
const framesDir =
|
|
10118
|
-
|
|
10196
|
+
mkdirSync14(workDir, { recursive: true });
|
|
10197
|
+
const framesDir = join27(workDir, "captured-frames");
|
|
10198
|
+
mkdirSync14(framesDir, { recursive: true });
|
|
10119
10199
|
const videoFramesDir = `${workDir}.video-frames`;
|
|
10120
10200
|
const deferredVideoFrames = planVideos ? await extractDeferredVideoFramesForChunk({
|
|
10121
10201
|
planDir,
|
|
@@ -10161,7 +10241,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10161
10241
|
dimensions: plan2.dimensions,
|
|
10162
10242
|
encoder,
|
|
10163
10243
|
videoCount: planVideos?.videos.length ?? 0,
|
|
10164
|
-
html:
|
|
10244
|
+
html: readFileSync12(join27(compiledDir, "index.html"), "utf-8"),
|
|
10165
10245
|
// The plan ships a deferred source only to chunks that show it; don't wait on the rest.
|
|
10166
10246
|
skipReadinessVideoIds: deferredVideoFrames.filter((video) => video.framePaths.size === 0 || remoteVideoIds.has(video.videoId)).map((video) => video.videoId),
|
|
10167
10247
|
videoMetadataHints: collectVideoMetadataHints(remoteVideos)
|
|
@@ -10295,7 +10375,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10295
10375
|
},
|
|
10296
10376
|
resetForScreenshotRetry: () => {
|
|
10297
10377
|
rmSync12(workDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
10298
|
-
|
|
10378
|
+
mkdirSync14(framesDir, { recursive: true });
|
|
10299
10379
|
capturePerfs.length = 0;
|
|
10300
10380
|
captureAttempts.length = 0;
|
|
10301
10381
|
job.framesRendered = 0;
|
|
@@ -10326,10 +10406,10 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10326
10406
|
const effectiveBitrate = encoder.crf != null ? void 0 : encoder.bitrate;
|
|
10327
10407
|
const videoOnlyPath = encoder.chunksCarryAudio ? `${outputChunkPath}.video.mp4` : outputChunkPath;
|
|
10328
10408
|
if (isPngSequence) {
|
|
10329
|
-
if (!
|
|
10409
|
+
if (!existsSync19(outputChunkPath)) mkdirSync14(outputChunkPath, { recursive: true });
|
|
10330
10410
|
} else {
|
|
10331
|
-
const outDir =
|
|
10332
|
-
if (!
|
|
10411
|
+
const outDir = join27(outputChunkPath, "..");
|
|
10412
|
+
if (!existsSync19(outDir)) mkdirSync14(outDir, { recursive: true });
|
|
10333
10413
|
}
|
|
10334
10414
|
const encodeStarted = Date.now();
|
|
10335
10415
|
await runEncodeStage({
|
|
@@ -10372,9 +10452,10 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10372
10452
|
gopSize: framesInChunk
|
|
10373
10453
|
});
|
|
10374
10454
|
if (encoder.chunksCarryAudio) {
|
|
10455
|
+
const audioScratchPath = `${outputChunkPath}.audio.m4a`;
|
|
10375
10456
|
await muxChunkWithAudio({
|
|
10376
10457
|
videoPath: videoOnlyPath,
|
|
10377
|
-
audioPath:
|
|
10458
|
+
audioPath: chunkAudioSource(planDir, chunkIndex, audioScratchPath),
|
|
10378
10459
|
outputPath: outputChunkPath,
|
|
10379
10460
|
chunks,
|
|
10380
10461
|
chunkIndex,
|
|
@@ -10382,6 +10463,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10382
10463
|
timeoutMs: cfg.ffmpegProcessTimeout
|
|
10383
10464
|
});
|
|
10384
10465
|
rmSync12(videoOnlyPath, { force: true });
|
|
10466
|
+
rmSync12(audioScratchPath, { force: true });
|
|
10385
10467
|
}
|
|
10386
10468
|
encodeStageMs = Date.now() - encodeStarted;
|
|
10387
10469
|
} finally {
|
|
@@ -10420,7 +10502,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10420
10502
|
producerVersion: plan2.producerVersion,
|
|
10421
10503
|
ffmpegVersion
|
|
10422
10504
|
};
|
|
10423
|
-
|
|
10505
|
+
writeFileSync10(perfPath, `${JSON.stringify(perfPayload, null, 2)}
|
|
10424
10506
|
`, "utf-8");
|
|
10425
10507
|
try {
|
|
10426
10508
|
rmSync12(workDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
@@ -10452,8 +10534,8 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10452
10534
|
|
|
10453
10535
|
// src/services/distributed/planV2Execution.ts
|
|
10454
10536
|
async function renderChunkV2(planV2Dir, chunkIndex, outputChunkPath) {
|
|
10455
|
-
const workRoot = mkdtempSync7(
|
|
10456
|
-
const materializedPlanDir =
|
|
10537
|
+
const workRoot = mkdtempSync7(join28(tmpdir2(), "hf-plan-v2-chunk-"));
|
|
10538
|
+
const materializedPlanDir = join28(workRoot, "plan");
|
|
10457
10539
|
try {
|
|
10458
10540
|
materializePlanV2Target(planV2Dir, { role: "chunk", chunkIndex }, materializedPlanDir);
|
|
10459
10541
|
return await renderChunk(materializedPlanDir, chunkIndex, outputChunkPath);
|
|
@@ -10462,8 +10544,8 @@ async function renderChunkV2(planV2Dir, chunkIndex, outputChunkPath) {
|
|
|
10462
10544
|
}
|
|
10463
10545
|
}
|
|
10464
10546
|
async function assembleV2(planV2Dir, chunkPaths, outputPath, options) {
|
|
10465
|
-
const workRoot = mkdtempSync7(
|
|
10466
|
-
const materializedPlanDir =
|
|
10547
|
+
const workRoot = mkdtempSync7(join28(tmpdir2(), "hf-plan-v2-assembler-"));
|
|
10548
|
+
const materializedPlanDir = join28(workRoot, "plan");
|
|
10467
10549
|
try {
|
|
10468
10550
|
const materialized = materializePlanV2Target(
|
|
10469
10551
|
planV2Dir,
|
|
@@ -10725,9 +10807,9 @@ function validateIntDimension(field, value) {
|
|
|
10725
10807
|
}
|
|
10726
10808
|
|
|
10727
10809
|
// src/services/distributed/projectHash.ts
|
|
10728
|
-
import { readdirSync as readdirSync10, readFileSync as
|
|
10810
|
+
import { readdirSync as readdirSync10, readFileSync as readFileSync13, realpathSync as realpathSync2, statSync as statSync8 } from "node:fs";
|
|
10729
10811
|
import { createHash as createHash6 } from "node:crypto";
|
|
10730
|
-
import { join as
|
|
10812
|
+
import { join as join29 } from "node:path";
|
|
10731
10813
|
var SITE_ID_SCHEME = "site-v2\0";
|
|
10732
10814
|
function listSiteFiles(projectDir) {
|
|
10733
10815
|
const files = [];
|
|
@@ -10735,7 +10817,7 @@ function listSiteFiles(projectDir) {
|
|
|
10735
10817
|
function walk(dir, rel) {
|
|
10736
10818
|
for (const name of readdirSync10(dir).sort()) {
|
|
10737
10819
|
if (rel === "" && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(name)) continue;
|
|
10738
|
-
const full =
|
|
10820
|
+
const full = join29(dir, name);
|
|
10739
10821
|
const relPath = rel === "" ? name : `${rel}/${name}`;
|
|
10740
10822
|
let stat;
|
|
10741
10823
|
try {
|
|
@@ -10764,7 +10846,7 @@ function hashProjectDir(projectDir, files = listSiteFiles(projectDir)) {
|
|
|
10764
10846
|
for (const rel of files) {
|
|
10765
10847
|
hash.update(rel);
|
|
10766
10848
|
hash.update("\0");
|
|
10767
|
-
hash.update(
|
|
10849
|
+
hash.update(readFileSync13(join29(projectDir, rel)));
|
|
10768
10850
|
}
|
|
10769
10851
|
return hash.digest("hex").slice(0, 16);
|
|
10770
10852
|
}
|