@hyperframes/producer 0.8.177 → 0.8.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distributed.d.ts +3 -3
- package/dist/distributed.js +1472 -1239
- package/dist/hyperframe.manifest.json +1 -1
- package/dist/hyperframe.runtime.iife.js +57 -55
- package/dist/index.js +1364 -1154
- package/dist/public-server.js +40 -13
- package/dist/services/distributed/chunkAudio.d.ts +11 -0
- package/dist/services/distributed/chunkFrames.d.ts +4 -0
- package/dist/services/distributed/mp4Stitch.d.ts +1 -0
- package/dist/services/distributed/plan.d.ts +10 -4
- package/dist/services/distributed/planV2.d.ts +15 -2
- package/dist/services/distributed/planV2Publisher.d.ts +1 -0
- package/dist/services/distributed/projectHash.d.ts +13 -8
- package/dist/services/htmlCompiler.d.ts +2 -0
- package/dist/services/render/stages/audioStage.d.ts +1 -0
- package/dist/services/render/stages/compileStage.d.ts +2 -0
- package/dist/services/render/stages/extractVideosStage.d.ts +1 -0
- package/dist/services/render/stages/freezePlan.d.ts +8 -0
- package/package.json +7 -7
- package/dist/services/audioExtractor.d.ts +0 -38
package/dist/distributed.js
CHANGED
|
@@ -3,14 +3,14 @@ import { audioFailureDiagnostics as audioFailureDiagnostics3 } from "@hyperframe
|
|
|
3
3
|
import {
|
|
4
4
|
copyFileSync as copyFileSync5,
|
|
5
5
|
cpSync as cpSync2,
|
|
6
|
-
existsSync as
|
|
6
|
+
existsSync as existsSync15,
|
|
7
7
|
linkSync,
|
|
8
|
-
mkdirSync as
|
|
9
|
-
renameSync as
|
|
10
|
-
rmSync as
|
|
11
|
-
writeFileSync as
|
|
8
|
+
mkdirSync as mkdirSync10,
|
|
9
|
+
renameSync as renameSync4,
|
|
10
|
+
rmSync as rmSync8,
|
|
11
|
+
writeFileSync as writeFileSync6
|
|
12
12
|
} from "node:fs";
|
|
13
|
-
import { extname as extname4, isAbsolute as isAbsolute3, join as
|
|
13
|
+
import { extname as extname4, isAbsolute as isAbsolute3, join as join22, relative as relative5, sep as sep2 } from "node:path";
|
|
14
14
|
import {
|
|
15
15
|
fpsToNumber as fpsToNumber10,
|
|
16
16
|
HF_COLOR_GRADING_ATTR as HF_COLOR_GRADING_ATTR2
|
|
@@ -1148,6 +1148,9 @@ import {
|
|
|
1148
1148
|
extractAudioMetadata
|
|
1149
1149
|
} from "@hyperframes/engine";
|
|
1150
1150
|
|
|
1151
|
+
// src/services/render/renderDirOwner.ts
|
|
1152
|
+
import { RENDER_TEMP_DIR } from "@hyperframes/core/project-folders";
|
|
1153
|
+
|
|
1151
1154
|
// src/services/render/capturePlan.ts
|
|
1152
1155
|
function assertWorkerCount(workerCount) {
|
|
1153
1156
|
if (!Number.isInteger(workerCount) || workerCount < 1) {
|
|
@@ -2337,7 +2340,7 @@ function detectShaderTransitionUsage(html) {
|
|
|
2337
2340
|
}
|
|
2338
2341
|
return false;
|
|
2339
2342
|
}
|
|
2340
|
-
async function resolveMediaDuration(src, mediaStart, playbackRate, baseDir, downloadDir, tagName, elementIdentity, log) {
|
|
2343
|
+
async function resolveMediaDuration(src, mediaStart, playbackRate, baseDir, downloadDir, tagName, elementIdentity, log, urlByAbsentLocalPath) {
|
|
2341
2344
|
let filePath = src;
|
|
2342
2345
|
if (isHttpUrl(src)) {
|
|
2343
2346
|
if (!existsSync6(downloadDir)) mkdirSync4(downloadDir, { recursive: true });
|
|
@@ -2351,8 +2354,9 @@ async function resolveMediaDuration(src, mediaStart, playbackRate, baseDir, down
|
|
|
2351
2354
|
} else {
|
|
2352
2355
|
filePath = decodedUrlPath(src);
|
|
2353
2356
|
if (!filePath.startsWith("/")) filePath = join5(baseDir, filePath);
|
|
2357
|
+
filePath = urlByAbsentLocalPath?.get(filePath) ?? filePath;
|
|
2354
2358
|
}
|
|
2355
|
-
if (!existsSync6(filePath)) {
|
|
2359
|
+
if (!isHttpUrl(filePath) && !existsSync6(filePath)) {
|
|
2356
2360
|
return { duration: null, resolvedPath: filePath };
|
|
2357
2361
|
}
|
|
2358
2362
|
const withSrcContext = (error) => {
|
|
@@ -2415,7 +2419,7 @@ function markInferredVariableMediaDurations(html, unresolvedMedia) {
|
|
|
2415
2419
|
}
|
|
2416
2420
|
return changed ? document2.toString() : html;
|
|
2417
2421
|
}
|
|
2418
|
-
async function compileHtmlFile(html, baseDir, downloadDir, log) {
|
|
2422
|
+
async function compileHtmlFile(html, baseDir, downloadDir, log, urlByAbsentLocalPath) {
|
|
2419
2423
|
const { html: staticCompiled, unresolved } = compileTimingAttrs(html);
|
|
2420
2424
|
const mediaUnresolved = unresolved.filter(
|
|
2421
2425
|
(el) => (el.tagName === "video" || el.tagName === "audio") && el.src
|
|
@@ -2431,7 +2435,8 @@ async function compileHtmlFile(html, baseDir, downloadDir, log) {
|
|
|
2431
2435
|
downloadDir,
|
|
2432
2436
|
el.tagName,
|
|
2433
2437
|
el.id,
|
|
2434
|
-
log
|
|
2438
|
+
log,
|
|
2439
|
+
urlByAbsentLocalPath
|
|
2435
2440
|
).then(({ duration }) => ({ id: el.id, duration }))
|
|
2436
2441
|
)
|
|
2437
2442
|
);
|
|
@@ -2473,7 +2478,7 @@ async function compileHtmlFile(html, baseDir, downloadDir, log) {
|
|
|
2473
2478
|
compiledHtml = compiledHtml.replace(/(<audio\b[^>]*)\s+crossorigin(?:=["'][^"']*["'])?/gi, "$1");
|
|
2474
2479
|
return { html: compiledHtml, unresolvedCompositions };
|
|
2475
2480
|
}
|
|
2476
|
-
async function parseSubCompositions(html, projectDir, downloadDir, visited = /* @__PURE__ */ new Set()) {
|
|
2481
|
+
async function parseSubCompositions(html, projectDir, downloadDir, visited = /* @__PURE__ */ new Set(), urlByAbsentLocalPath) {
|
|
2477
2482
|
const subCompositions = /* @__PURE__ */ new Map();
|
|
2478
2483
|
const { document: document2 } = parseHTML4(html);
|
|
2479
2484
|
const compEls = document2.querySelectorAll("[data-composition-src]");
|
|
@@ -2499,13 +2504,16 @@ async function parseSubCompositions(html, projectDir, downloadDir, visited = /*
|
|
|
2499
2504
|
const { html: compiledSub } = await compileHtmlFile(
|
|
2500
2505
|
item.rawSubHtml,
|
|
2501
2506
|
dirname4(item.filePath),
|
|
2502
|
-
downloadDir
|
|
2507
|
+
downloadDir,
|
|
2508
|
+
void 0,
|
|
2509
|
+
urlByAbsentLocalPath
|
|
2503
2510
|
);
|
|
2504
2511
|
const nested = await parseSubCompositions(
|
|
2505
2512
|
compiledSub,
|
|
2506
2513
|
projectDir,
|
|
2507
2514
|
downloadDir,
|
|
2508
|
-
item.nestedVisited
|
|
2515
|
+
item.nestedVisited,
|
|
2516
|
+
urlByAbsentLocalPath
|
|
2509
2517
|
);
|
|
2510
2518
|
return {
|
|
2511
2519
|
srcPath: item.srcPath,
|
|
@@ -3263,9 +3271,16 @@ async function compileForRender(projectDir, htmlPath, downloadDir, options = {})
|
|
|
3263
3271
|
rawHtml,
|
|
3264
3272
|
projectDir,
|
|
3265
3273
|
downloadDir,
|
|
3266
|
-
options.log
|
|
3274
|
+
options.log,
|
|
3275
|
+
options.urlByAbsentLocalPath
|
|
3276
|
+
);
|
|
3277
|
+
const { subCompositions } = await parseSubCompositions(
|
|
3278
|
+
compiledHtml,
|
|
3279
|
+
projectDir,
|
|
3280
|
+
downloadDir,
|
|
3281
|
+
void 0,
|
|
3282
|
+
options.urlByAbsentLocalPath
|
|
3267
3283
|
);
|
|
3268
|
-
const { subCompositions } = await parseSubCompositions(compiledHtml, projectDir, downloadDir);
|
|
3269
3284
|
const fullHtml = ensureFullDocument(compiledHtml);
|
|
3270
3285
|
const inlinedHtml = inlineSubCompositions(
|
|
3271
3286
|
fullHtml,
|
|
@@ -3761,6 +3776,7 @@ async function runCompileStage(input) {
|
|
|
3761
3776
|
allowSystemFontCapture,
|
|
3762
3777
|
abortSignal,
|
|
3763
3778
|
variables: input.variables,
|
|
3779
|
+
urlByAbsentLocalPath: input.urlByAbsentLocalPath,
|
|
3764
3780
|
animatedGifCacheDir: cfg.extractCacheDir ? join6(cfg.extractCacheDir, "animated-gif") : void 0,
|
|
3765
3781
|
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout
|
|
3766
3782
|
});
|
|
@@ -4759,7 +4775,8 @@ async function runExtractVideosStage(input) {
|
|
|
4759
4775
|
abortSignal,
|
|
4760
4776
|
assertNotAborted,
|
|
4761
4777
|
materializeSymlinks,
|
|
4762
|
-
deferRangeExtraction
|
|
4778
|
+
deferRangeExtraction,
|
|
4779
|
+
urlByAbsentLocalPath
|
|
4763
4780
|
} = input;
|
|
4764
4781
|
const stage2Start = Date.now();
|
|
4765
4782
|
const extractionPolicy = resolveVideoExtractionPolicy();
|
|
@@ -4853,7 +4870,8 @@ async function runExtractVideosStage(input) {
|
|
|
4853
4870
|
maxTransientRetries: extractionPolicy.maxTransientRetries,
|
|
4854
4871
|
collectProbeFailures: extractionPolicy.failureMode === "enforce",
|
|
4855
4872
|
deferRangeExtraction,
|
|
4856
|
-
contentKeyedDirs: [join8(compiledDir, REMOTE_MEDIA_SUBDIR)]
|
|
4873
|
+
contentKeyedDirs: [join8(compiledDir, REMOTE_MEDIA_SUBDIR)],
|
|
4874
|
+
urlByAbsentLocalPath
|
|
4857
4875
|
},
|
|
4858
4876
|
abortSignal,
|
|
4859
4877
|
cfg,
|
|
@@ -4936,6 +4954,7 @@ async function runAudioStage(input) {
|
|
|
4936
4954
|
audioGain,
|
|
4937
4955
|
audios,
|
|
4938
4956
|
masterFxChain,
|
|
4957
|
+
urlByAbsentLocalPath,
|
|
4939
4958
|
abortSignal,
|
|
4940
4959
|
assertNotAborted,
|
|
4941
4960
|
log
|
|
@@ -4955,7 +4974,7 @@ async function runAudioStage(input) {
|
|
|
4955
4974
|
audioOutputPath,
|
|
4956
4975
|
duration,
|
|
4957
4976
|
abortSignal,
|
|
4958
|
-
{ ffmpegProcessTimeout, audioGain, masterFxChain },
|
|
4977
|
+
{ ffmpegProcessTimeout, audioGain, masterFxChain, urlByAbsentLocalPath },
|
|
4959
4978
|
compiledDir
|
|
4960
4979
|
);
|
|
4961
4980
|
} catch (err) {
|
|
@@ -6833,6 +6852,8 @@ function buildChunkCaptureOptions(input) {
|
|
|
6833
6852
|
// The controller's snapshot, so the first capture sees the in-process renderer's `__hfVariables`.
|
|
6834
6853
|
variables: encoder.variables,
|
|
6835
6854
|
subTimelineWaitMemo: {},
|
|
6855
|
+
skipReadinessVideoIds: input.skipReadinessVideoIds,
|
|
6856
|
+
videoMetadataHints: input.videoMetadataHints,
|
|
6836
6857
|
...videoCaptureBeyondViewport !== void 0 ? { captureBeyondViewport: videoCaptureBeyondViewport } : {},
|
|
6837
6858
|
// A fixed warmup count keeps `beginFrameTimeTicks` host-independent.
|
|
6838
6859
|
lockWarmupTicks: true,
|
|
@@ -6841,6 +6862,19 @@ function buildChunkCaptureOptions(input) {
|
|
|
6841
6862
|
}
|
|
6842
6863
|
async function extractDeferredVideoFramesForChunk(input) {
|
|
6843
6864
|
const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
|
|
6865
|
+
const remoteSources = input.remoteSources ?? {};
|
|
6866
|
+
for (const path of Object.keys(remoteSources)) {
|
|
6867
|
+
if (!deferred.some((video) => video.deferredRange.sourcePath === path)) {
|
|
6868
|
+
throw new RenderChunkValidationError(
|
|
6869
|
+
MISSING_PLAN_ARTIFACT,
|
|
6870
|
+
`[renderChunk] ${path} is read by URL but no deferred video uses it`
|
|
6871
|
+
);
|
|
6872
|
+
}
|
|
6873
|
+
defaultLogger.info("[renderChunk] reading video by URL", {
|
|
6874
|
+
path,
|
|
6875
|
+
url: remoteSources[path].split("?")[0]
|
|
6876
|
+
});
|
|
6877
|
+
}
|
|
6844
6878
|
if (deferred.length === 0) return [];
|
|
6845
6879
|
const table = createFrameLookupTable2(
|
|
6846
6880
|
input.planVideos.videos,
|
|
@@ -6868,8 +6902,9 @@ async function extractDeferredVideoFramesForChunk(input) {
|
|
|
6868
6902
|
return { ...video, outputDir, framePaths: /* @__PURE__ */ new Map(), ownedByLookup: false };
|
|
6869
6903
|
}
|
|
6870
6904
|
const range = video.deferredRange;
|
|
6871
|
-
const
|
|
6872
|
-
|
|
6905
|
+
const remoteUrl = remoteSources[range.sourcePath];
|
|
6906
|
+
const sourcePath = remoteUrl ?? join16(input.planDir, ...range.sourcePath.split("/"));
|
|
6907
|
+
if (remoteUrl === void 0 && !existsSync12(sourcePath)) {
|
|
6873
6908
|
throw new RenderChunkValidationError(
|
|
6874
6909
|
MISSING_PLAN_ARTIFACT,
|
|
6875
6910
|
`[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`
|
|
@@ -7209,6 +7244,16 @@ function collectPlanAssetShas(planDir, digests) {
|
|
|
7209
7244
|
}
|
|
7210
7245
|
return { compositionHtml, assets };
|
|
7211
7246
|
}
|
|
7247
|
+
var PLAN_REMOTE_FILES_RELATIVE_PATH = "meta/remote-files.json";
|
|
7248
|
+
function writePlanRemoteFiles(planDir, files) {
|
|
7249
|
+
if (files.length === 0) return;
|
|
7250
|
+
const sorted = [...files].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
7251
|
+
writeFileSync4(join18(planDir, PLAN_REMOTE_FILES_RELATIVE_PATH), JSON.stringify(sorted), "utf-8");
|
|
7252
|
+
}
|
|
7253
|
+
function readPlanRemoteFiles(planDir) {
|
|
7254
|
+
const path = join18(planDir, PLAN_REMOTE_FILES_RELATIVE_PATH);
|
|
7255
|
+
return existsSync13(path) ? JSON.parse(readFileSync8(path, "utf-8")) : [];
|
|
7256
|
+
}
|
|
7212
7257
|
function recomputePlanHashFromPlanDir(planDir, digests) {
|
|
7213
7258
|
const planJsonPath = join18(planDir, "plan.json");
|
|
7214
7259
|
const encoderJsonPath = join18(planDir, "meta", "encoder.json");
|
|
@@ -7324,167 +7369,677 @@ function applyRuntimeEnvSnapshot(snapshot, env = process.env) {
|
|
|
7324
7369
|
};
|
|
7325
7370
|
}
|
|
7326
7371
|
|
|
7327
|
-
// src/services/distributed/
|
|
7328
|
-
import {
|
|
7329
|
-
import {
|
|
7330
|
-
import {
|
|
7331
|
-
|
|
7332
|
-
|
|
7333
|
-
|
|
7334
|
-
|
|
7335
|
-
|
|
7336
|
-
|
|
7337
|
-
|
|
7338
|
-
|
|
7339
|
-
|
|
7340
|
-
|
|
7341
|
-
|
|
7342
|
-
|
|
7343
|
-
|
|
7344
|
-
|
|
7345
|
-
|
|
7346
|
-
|
|
7347
|
-
|
|
7348
|
-
|
|
7349
|
-
|
|
7350
|
-
|
|
7351
|
-
|
|
7352
|
-
|
|
7353
|
-
|
|
7354
|
-
meta: "metadata",
|
|
7355
|
-
"plan.json": "metadata"
|
|
7356
|
-
};
|
|
7357
|
-
function hashComponent(value) {
|
|
7358
|
-
return createHash5("sha256").update(value).digest("hex").slice(0, 12);
|
|
7372
|
+
// src/services/distributed/chunkAudio.ts
|
|
7373
|
+
import { closeSync as closeSync4, existsSync as existsSync14, ftruncateSync, mkdirSync as mkdirSync9, openSync as openSync4, readFileSync as readFileSync9 } from "node:fs";
|
|
7374
|
+
import { writeFileSync as writeFileSync5, writeSync } from "node:fs";
|
|
7375
|
+
import { join as join20 } from "node:path";
|
|
7376
|
+
import { appendRenderProvenanceArgs, describeFfmpegFailure as describeFfmpegFailure2, runFfmpeg as runFfmpeg5 } from "@hyperframes/engine";
|
|
7377
|
+
|
|
7378
|
+
// src/services/distributed/mp4Stitch.ts
|
|
7379
|
+
import {
|
|
7380
|
+
createReadStream as createReadStream4,
|
|
7381
|
+
createWriteStream,
|
|
7382
|
+
mkdirSync as mkdirSync8,
|
|
7383
|
+
mkdtempSync as mkdtempSync3,
|
|
7384
|
+
renameSync as renameSync3,
|
|
7385
|
+
rmSync as rmSync7
|
|
7386
|
+
} from "node:fs";
|
|
7387
|
+
import { open } from "node:fs/promises";
|
|
7388
|
+
import { dirname as dirname9, join as join19 } from "node:path";
|
|
7389
|
+
import { finished, pipeline } from "node:stream/promises";
|
|
7390
|
+
function boxAt(src, off, at, limit) {
|
|
7391
|
+
let size = src.readUInt32BE(off);
|
|
7392
|
+
let bodyStart = at + 8;
|
|
7393
|
+
if (size === 1) {
|
|
7394
|
+
size = Number(src.readBigUInt64BE(off + 8));
|
|
7395
|
+
bodyStart = at + 16;
|
|
7396
|
+
}
|
|
7397
|
+
if (size < 8 || at + size > limit) throw new Error(`[mp4Stitch] malformed box at ${at}`);
|
|
7398
|
+
return { type: src.toString("latin1", off + 4, off + 8), start: at, bodyStart, end: at + size };
|
|
7359
7399
|
}
|
|
7360
|
-
function
|
|
7361
|
-
const
|
|
7362
|
-
|
|
7363
|
-
|
|
7364
|
-
const videoKey = segments[1] ?? "unknown";
|
|
7365
|
-
return {
|
|
7366
|
-
category: "video-frames",
|
|
7367
|
-
componentLabel: `video-frames:${hashComponent(videoKey)}`
|
|
7368
|
-
};
|
|
7400
|
+
function childBoxes(buf, parent) {
|
|
7401
|
+
const boxes = [];
|
|
7402
|
+
for (let at = parent.bodyStart; at < parent.end; at = boxes.at(-1).end) {
|
|
7403
|
+
boxes.push(boxAt(buf, at, at, parent.end));
|
|
7369
7404
|
}
|
|
7370
|
-
|
|
7371
|
-
return { category, componentLabel: category };
|
|
7405
|
+
return boxes;
|
|
7372
7406
|
}
|
|
7373
|
-
function
|
|
7374
|
-
return
|
|
7375
|
-
totalBytes: 0,
|
|
7376
|
-
fileCount: 0,
|
|
7377
|
-
compiledBytes: 0,
|
|
7378
|
-
sourceMediaBytes: 0,
|
|
7379
|
-
videoFramesBytes: 0,
|
|
7380
|
-
videoFrameFileCount: 0,
|
|
7381
|
-
audioBytes: 0,
|
|
7382
|
-
metadataBytes: 0,
|
|
7383
|
-
otherBytes: 0,
|
|
7384
|
-
topComponents: []
|
|
7385
|
-
};
|
|
7407
|
+
function findChild(buf, parent, type) {
|
|
7408
|
+
return childBoxes(buf, parent).find((box2) => box2.type === type);
|
|
7386
7409
|
}
|
|
7387
|
-
function
|
|
7388
|
-
|
|
7389
|
-
|
|
7390
|
-
|
|
7391
|
-
result.sourceMediaBytes += sizeBytes;
|
|
7392
|
-
}
|
|
7393
|
-
return;
|
|
7394
|
-
}
|
|
7395
|
-
if (category === "video-frames") {
|
|
7396
|
-
result.videoFramesBytes += sizeBytes;
|
|
7397
|
-
result.videoFrameFileCount += 1;
|
|
7398
|
-
return;
|
|
7399
|
-
}
|
|
7400
|
-
if (category === "audio") {
|
|
7401
|
-
result.audioBytes += sizeBytes;
|
|
7402
|
-
return;
|
|
7403
|
-
}
|
|
7404
|
-
if (category === "metadata") {
|
|
7405
|
-
result.metadataBytes += sizeBytes;
|
|
7406
|
-
return;
|
|
7407
|
-
}
|
|
7408
|
-
result.otherBytes += sizeBytes;
|
|
7410
|
+
function child(buf, parent, type) {
|
|
7411
|
+
const found = findChild(buf, parent, type);
|
|
7412
|
+
if (!found) throw new Error(`[mp4Stitch] ${parent.type} has no ${type}`);
|
|
7413
|
+
return found;
|
|
7409
7414
|
}
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
return;
|
|
7418
|
-
|
|
7419
|
-
|
|
7420
|
-
const segments = rel.split(/[\\/]+/).filter(Boolean);
|
|
7421
|
-
const classified = classifyFile(segments, rootKind);
|
|
7422
|
-
result.totalBytes += sizeBytes;
|
|
7423
|
-
result.fileCount += 1;
|
|
7424
|
-
addCategoryBytes(result, classified.category, filePath, sizeBytes);
|
|
7425
|
-
const component = components.get(classified.componentLabel) ?? {
|
|
7426
|
-
label: classified.componentLabel,
|
|
7427
|
-
sizeBytes: 0,
|
|
7428
|
-
fileCount: 0
|
|
7415
|
+
var SCAN_BYTES = 64 * 1024;
|
|
7416
|
+
async function readMp4Layout(read) {
|
|
7417
|
+
const found = /* @__PURE__ */ new Map();
|
|
7418
|
+
let payload;
|
|
7419
|
+
let buf = Buffer.alloc(0);
|
|
7420
|
+
let bufAt = 0;
|
|
7421
|
+
const load = async (start, end) => {
|
|
7422
|
+
if (start >= bufAt && end <= bufAt + buf.length) return;
|
|
7423
|
+
buf = await read(start, Math.max(end, start + SCAN_BYTES));
|
|
7424
|
+
bufAt = start;
|
|
7429
7425
|
};
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
}
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
|
|
7437
|
-
|
|
7438
|
-
|
|
7439
|
-
|
|
7440
|
-
}
|
|
7441
|
-
for (const entry of entries) {
|
|
7442
|
-
const full = join19(dir, entry.name);
|
|
7443
|
-
if (entry.isDirectory()) {
|
|
7444
|
-
walkRegularFiles(full, visit);
|
|
7445
|
-
} else if (entry.isFile()) {
|
|
7446
|
-
visit(full);
|
|
7426
|
+
for (let at = 0; !(found.has("ftyp") && found.has("moov") && payload); ) {
|
|
7427
|
+
await load(at, at + 16);
|
|
7428
|
+
if (at + 8 > bufAt + buf.length) {
|
|
7429
|
+
throw new Error(`[mp4Stitch] chunk needs ftyp, moov and mdat; found ${[...found.keys()]}`);
|
|
7430
|
+
}
|
|
7431
|
+
const box2 = boxAt(buf, at - bufAt, at, Number.MAX_SAFE_INTEGER);
|
|
7432
|
+
if (box2.type === "mdat") payload ??= box2;
|
|
7433
|
+
if (box2.type === "ftyp" || box2.type === "moov") {
|
|
7434
|
+
await load(at, box2.end);
|
|
7435
|
+
found.set(box2.type, buf.subarray(at - bufAt, box2.end - bufAt));
|
|
7447
7436
|
}
|
|
7437
|
+
at = box2.end;
|
|
7448
7438
|
}
|
|
7439
|
+
const [ftyp, moov] = [found.get("ftyp"), found.get("moov")];
|
|
7440
|
+
if (moov.length !== moovSize(moov)) throw new Error("[mp4Stitch] moov is cut off");
|
|
7441
|
+
return { ftyp, moov, payloadStart: payload.bodyStart, payloadEnd: payload.end };
|
|
7449
7442
|
}
|
|
7450
|
-
|
|
7451
|
-
|
|
7452
|
-
const
|
|
7453
|
-
|
|
7454
|
-
|
|
7455
|
-
|
|
7443
|
+
var moovSize = (moov) => boxAt(moov, 0, 0, Number.MAX_SAFE_INTEGER).end;
|
|
7444
|
+
function readTracks(moov) {
|
|
7445
|
+
const root = boxAt(moov, 0, 0, moov.length);
|
|
7446
|
+
return childBoxes(moov, root).filter((box2) => box2.type === "trak").map((trak) => readTrack(moov, trak));
|
|
7447
|
+
}
|
|
7448
|
+
function rows(buf, table, columns, countAt = 4) {
|
|
7449
|
+
const count = buf.readUInt32BE(table.bodyStart + countAt);
|
|
7450
|
+
const first = table.bodyStart + countAt + 4;
|
|
7451
|
+
return Array.from(
|
|
7452
|
+
{ length: count },
|
|
7453
|
+
(_, row) => Array.from(
|
|
7454
|
+
{ length: columns },
|
|
7455
|
+
(_2, col) => buf.readUInt32BE(first + (row * columns + col) * 4)
|
|
7456
|
+
)
|
|
7456
7457
|
);
|
|
7457
|
-
result.topComponents = [...components.values()].sort((a, b) => b.sizeBytes - a.sizeBytes || a.label.localeCompare(b.label)).slice(0, MAX_TOP_COMPONENTS);
|
|
7458
|
-
return result;
|
|
7459
7458
|
}
|
|
7460
|
-
|
|
7461
|
-
|
|
7462
|
-
|
|
7463
|
-
|
|
7459
|
+
function readMediaTime(moov, trak) {
|
|
7460
|
+
const edts = findChild(moov, trak, "edts");
|
|
7461
|
+
if (!edts) return 0;
|
|
7462
|
+
const elst = child(moov, edts, "elst");
|
|
7463
|
+
const wide = moov[elst.bodyStart] === 1;
|
|
7464
|
+
const entries = moov.readUInt32BE(elst.bodyStart + 4);
|
|
7465
|
+
for (let i = 0, at = elst.bodyStart + 8; i < entries; i++, at += wide ? 20 : 12) {
|
|
7466
|
+
const offset = at + (wide ? 8 : 4);
|
|
7467
|
+
const mediaTime = wide ? Number(moov.readBigInt64BE(offset)) : moov.readInt32BE(offset);
|
|
7468
|
+
if (mediaTime !== -1) return mediaTime;
|
|
7469
|
+
}
|
|
7470
|
+
return 0;
|
|
7464
7471
|
}
|
|
7465
|
-
function
|
|
7466
|
-
const
|
|
7467
|
-
|
|
7468
|
-
|
|
7469
|
-
job,
|
|
7470
|
-
[
|
|
7471
|
-
{
|
|
7472
|
-
code: "audio_processing_failed",
|
|
7473
|
-
message: `Audio mix failed; output would be video-only: ${audioError}`,
|
|
7474
|
-
details: {
|
|
7475
|
-
mediaType: "audio",
|
|
7476
|
-
audioFailures: audioFailureDiagnostics3(audioFailures),
|
|
7477
|
-
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
|
|
7478
|
-
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
|
|
7479
|
-
failureOwner,
|
|
7480
|
-
retryable
|
|
7481
|
-
}
|
|
7482
|
-
}
|
|
7483
|
-
],
|
|
7484
|
-
log
|
|
7485
|
-
);
|
|
7472
|
+
function readSizes(moov, stsz) {
|
|
7473
|
+
const fixedSize = moov.readUInt32BE(stsz.bodyStart + 4);
|
|
7474
|
+
if (!fixedSize) return rows(moov, stsz, 1, 8).map(([size]) => size);
|
|
7475
|
+
return Array(moov.readUInt32BE(stsz.bodyStart + 8)).fill(fixedSize);
|
|
7486
7476
|
}
|
|
7487
|
-
|
|
7477
|
+
function readSync2(moov, stbl, sampleCount) {
|
|
7478
|
+
const stss = findChild(moov, stbl, "stss");
|
|
7479
|
+
if (!stss) return null;
|
|
7480
|
+
const sync = Array(sampleCount).fill(false);
|
|
7481
|
+
for (const [n] of rows(moov, stss, 1)) sync[n - 1] = true;
|
|
7482
|
+
return sync;
|
|
7483
|
+
}
|
|
7484
|
+
function readChunkOffsets(moov, stbl) {
|
|
7485
|
+
const stco = findChild(moov, stbl, "stco");
|
|
7486
|
+
if (stco) return rows(moov, stco, 1).map(([offset]) => offset);
|
|
7487
|
+
return rows(moov, child(moov, stbl, "co64"), 2).map(([hi, lo]) => hi * 2 ** 32 + lo);
|
|
7488
|
+
}
|
|
7489
|
+
function readSamplesPerChunk(moov, stbl, chunkCount) {
|
|
7490
|
+
const stsc = rows(moov, child(moov, stbl, "stsc"), 3);
|
|
7491
|
+
return stsc.flatMap(([firstChunk, perChunk, description], i) => {
|
|
7492
|
+
if (description !== 1) throw new Error("[mp4Stitch] chunk uses more than one sample entry");
|
|
7493
|
+
const nextFirst = stsc[i + 1]?.[0] ?? chunkCount + 1;
|
|
7494
|
+
return Array(nextFirst - firstChunk).fill(perChunk);
|
|
7495
|
+
});
|
|
7496
|
+
}
|
|
7497
|
+
function readTrack(moov, trak) {
|
|
7498
|
+
const mdia = child(moov, trak, "mdia");
|
|
7499
|
+
const hdlr = child(moov, mdia, "hdlr");
|
|
7500
|
+
const stbl = child(moov, child(moov, mdia, "minf"), "stbl");
|
|
7501
|
+
if (findChild(moov, stbl, "ctts")) throw new Error("[mp4Stitch] chunk has ctts (B-frames)");
|
|
7502
|
+
const stts = rows(moov, child(moov, stbl, "stts"), 2);
|
|
7503
|
+
const durations = stts.flatMap(([n, delta]) => Array(n).fill(delta));
|
|
7504
|
+
const sizes = readSizes(moov, child(moov, stbl, "stsz"));
|
|
7505
|
+
const chunkOffsets = readChunkOffsets(moov, stbl);
|
|
7506
|
+
const samplesPerChunk = readSamplesPerChunk(moov, stbl, chunkOffsets.length);
|
|
7507
|
+
if (durations.length !== sizes.length || sum(samplesPerChunk) !== sizes.length) {
|
|
7508
|
+
throw new Error("[mp4Stitch] sample tables disagree on the sample count");
|
|
7509
|
+
}
|
|
7510
|
+
const stsd = child(moov, stbl, "stsd");
|
|
7511
|
+
return {
|
|
7512
|
+
trak,
|
|
7513
|
+
handler: moov.toString("latin1", hdlr.bodyStart + 8, hdlr.bodyStart + 12),
|
|
7514
|
+
timescale: timescaleOf(moov, child(moov, mdia, "mdhd")),
|
|
7515
|
+
mediaTime: readMediaTime(moov, trak),
|
|
7516
|
+
stsd: moov.subarray(stsd.start, stsd.end),
|
|
7517
|
+
durations,
|
|
7518
|
+
sizes,
|
|
7519
|
+
sync: readSync2(moov, stbl, sizes.length),
|
|
7520
|
+
samplesPerChunk,
|
|
7521
|
+
chunkOffsets
|
|
7522
|
+
};
|
|
7523
|
+
}
|
|
7524
|
+
function sum(values) {
|
|
7525
|
+
return values.reduce((total, value) => total + value, 0);
|
|
7526
|
+
}
|
|
7527
|
+
function box(type, ...body) {
|
|
7528
|
+
const header = Buffer.alloc(8);
|
|
7529
|
+
header.writeUInt32BE(8 + sum(body.map((part) => part.length)));
|
|
7530
|
+
header.write(type, 4, "latin1");
|
|
7531
|
+
return Buffer.concat([header, ...body]);
|
|
7532
|
+
}
|
|
7533
|
+
function fullBox(type, version, body) {
|
|
7534
|
+
return box(type, Buffer.from([version, 0, 0, 0]), body);
|
|
7535
|
+
}
|
|
7536
|
+
function u32s(values) {
|
|
7537
|
+
const buf = Buffer.alloc(values.length * 4);
|
|
7538
|
+
values.forEach((value, i) => buf.writeUInt32BE(value, i * 4));
|
|
7539
|
+
return buf;
|
|
7540
|
+
}
|
|
7541
|
+
function runs(values) {
|
|
7542
|
+
const out = [];
|
|
7543
|
+
for (const value of values) {
|
|
7544
|
+
const last = out.at(-1);
|
|
7545
|
+
if (last && last[1] === value) last[0]++;
|
|
7546
|
+
else out.push([1, value]);
|
|
7547
|
+
}
|
|
7548
|
+
return out;
|
|
7549
|
+
}
|
|
7550
|
+
function rebuild(buf, parent, replace) {
|
|
7551
|
+
const parts = childBoxes(buf, parent).map(
|
|
7552
|
+
(box2) => replace[box2.type]?.(box2) ?? buf.subarray(box2.start, box2.end)
|
|
7553
|
+
);
|
|
7554
|
+
return box(parent.type, ...parts);
|
|
7555
|
+
}
|
|
7556
|
+
var DURATION_AT = {
|
|
7557
|
+
mvhd: [16, 24],
|
|
7558
|
+
mdhd: [16, 24],
|
|
7559
|
+
tkhd: [20, 28]
|
|
7560
|
+
};
|
|
7561
|
+
function withDuration(buf, target, duration) {
|
|
7562
|
+
const copy = Buffer.from(buf.subarray(target.start, target.end));
|
|
7563
|
+
const body = target.bodyStart - target.start;
|
|
7564
|
+
const wide = copy[body] === 1;
|
|
7565
|
+
const at = body + DURATION_AT[target.type][wide ? 1 : 0];
|
|
7566
|
+
if (wide) copy.writeBigUInt64BE(BigInt(duration), at);
|
|
7567
|
+
else if (duration <= 4294967295) copy.writeUInt32BE(duration, at);
|
|
7568
|
+
else throw new Error(`[mp4Stitch] ${target.type} duration ${duration} needs a version 1 box`);
|
|
7569
|
+
return copy;
|
|
7570
|
+
}
|
|
7571
|
+
function timescaleOf(buf, header) {
|
|
7572
|
+
return buf.readUInt32BE(header.bodyStart + (buf[header.bodyStart] === 1 ? 20 : 12));
|
|
7573
|
+
}
|
|
7574
|
+
function buildStbl(stsd, track) {
|
|
7575
|
+
const stts = runs(track.durations);
|
|
7576
|
+
const stsc = [];
|
|
7577
|
+
let firstChunk = 1;
|
|
7578
|
+
for (const [count, perChunk] of runs(track.samplesPerChunk)) {
|
|
7579
|
+
stsc.push([firstChunk, perChunk, 1]);
|
|
7580
|
+
firstChunk += count;
|
|
7581
|
+
}
|
|
7582
|
+
const co64 = Buffer.alloc(4 + track.chunkOffsets.length * 8);
|
|
7583
|
+
co64.writeUInt32BE(track.chunkOffsets.length);
|
|
7584
|
+
track.chunkOffsets.forEach((offset, i) => co64.writeBigUInt64BE(BigInt(offset), 4 + i * 8));
|
|
7585
|
+
const syncSamples = track.sync?.flatMap((isSync, i) => isSync ? [i + 1] : []) ?? null;
|
|
7586
|
+
return box(
|
|
7587
|
+
"stbl",
|
|
7588
|
+
stsd,
|
|
7589
|
+
fullBox("stts", 0, u32s([stts.length, ...stts.flat()])),
|
|
7590
|
+
...syncSamples ? [fullBox("stss", 0, u32s([syncSamples.length, ...syncSamples]))] : [],
|
|
7591
|
+
fullBox("stsc", 0, u32s([stsc.length, ...stsc.flat()])),
|
|
7592
|
+
fullBox("stsz", 0, u32s([0, track.sizes.length, ...track.sizes])),
|
|
7593
|
+
fullBox("co64", 0, co64)
|
|
7594
|
+
);
|
|
7595
|
+
}
|
|
7596
|
+
var codecOf = (track) => track.stsd.toString("latin1", 20, 24);
|
|
7597
|
+
var sameTrack = (track, plan2) => track.handler === plan2.handler && track.timescale === plan2.timescale && codecOf(track) === codecOf(plan2);
|
|
7598
|
+
function planTracksOf(tracks, plan2, index) {
|
|
7599
|
+
let next = 0;
|
|
7600
|
+
const matched = plan2.map((planTrack) => {
|
|
7601
|
+
const track = tracks[next];
|
|
7602
|
+
if (track && sameTrack(track, planTrack)) return tracks[next++];
|
|
7603
|
+
if (planTrack.handler === "soun") return null;
|
|
7604
|
+
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
7605
|
+
});
|
|
7606
|
+
if (next !== tracks.length)
|
|
7607
|
+
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
7608
|
+
for (const track of tracks) {
|
|
7609
|
+
if (track.handler === "vide" && track.mediaTime !== 0) {
|
|
7610
|
+
throw new Error(`[mp4Stitch] chunk ${index} video starts at media time ${track.mediaTime}`);
|
|
7611
|
+
}
|
|
7612
|
+
}
|
|
7613
|
+
return matched;
|
|
7614
|
+
}
|
|
7615
|
+
function buildStitchedHeader(layouts) {
|
|
7616
|
+
const plan2 = readTracks(layouts[0].moov);
|
|
7617
|
+
const chunks = layouts.map((layout, i) => planTracksOf(readTracks(layout.moov), plan2, i));
|
|
7618
|
+
const payloadBases = [];
|
|
7619
|
+
let payloadBytes = 0;
|
|
7620
|
+
for (const layout of layouts) {
|
|
7621
|
+
payloadBases.push(payloadBytes);
|
|
7622
|
+
payloadBytes += layout.payloadEnd - layout.payloadStart;
|
|
7623
|
+
}
|
|
7624
|
+
const template = layouts[0].moov;
|
|
7625
|
+
const root = boxAt(template, 0, 0, template.length);
|
|
7626
|
+
const movieTimescale = timescaleOf(template, child(template, root, "mvhd"));
|
|
7627
|
+
const buildMoov = (dataStart2) => {
|
|
7628
|
+
const traks = plan2.map((track, t) => {
|
|
7629
|
+
const parts = chunks.flatMap((tracks, chunk) => {
|
|
7630
|
+
const part = tracks[t];
|
|
7631
|
+
return part ? [{ ...part, chunk }] : [];
|
|
7632
|
+
});
|
|
7633
|
+
const merged = {
|
|
7634
|
+
durations: parts.flatMap((part) => part.durations),
|
|
7635
|
+
sizes: parts.flatMap((part) => part.sizes),
|
|
7636
|
+
sync: parts.every((part) => !part.sync) ? null : parts.flatMap((part) => part.sync ?? part.sizes.map(() => true)),
|
|
7637
|
+
samplesPerChunk: parts.flatMap((part) => part.samplesPerChunk),
|
|
7638
|
+
chunkOffsets: parts.flatMap(
|
|
7639
|
+
(part) => part.chunkOffsets.map((offset) => {
|
|
7640
|
+
const i = part.chunk;
|
|
7641
|
+
const { payloadStart, payloadEnd } = layouts[i];
|
|
7642
|
+
if (offset < payloadStart || offset >= payloadEnd) {
|
|
7643
|
+
throw new Error(`[mp4Stitch] chunk ${i} has samples outside its mdat`);
|
|
7644
|
+
}
|
|
7645
|
+
return offset - payloadStart + payloadBases[i] + dataStart2;
|
|
7646
|
+
})
|
|
7647
|
+
)
|
|
7648
|
+
};
|
|
7649
|
+
const mediaDuration = sum(merged.durations);
|
|
7650
|
+
const shown = Math.round(
|
|
7651
|
+
(mediaDuration - track.mediaTime) * movieTimescale / track.timescale
|
|
7652
|
+
);
|
|
7653
|
+
const elst = Buffer.alloc(24);
|
|
7654
|
+
elst.writeUInt32BE(1);
|
|
7655
|
+
elst.writeBigUInt64BE(BigInt(shown), 4);
|
|
7656
|
+
elst.writeBigInt64BE(BigInt(track.mediaTime), 12);
|
|
7657
|
+
elst.writeUInt32BE(65536, 20);
|
|
7658
|
+
const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);
|
|
7659
|
+
const trak = rebuild(template, track.trak, {
|
|
7660
|
+
tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),
|
|
7661
|
+
edts: () => Buffer.alloc(0),
|
|
7662
|
+
mdia: (mdia) => rebuild(template, mdia, {
|
|
7663
|
+
mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),
|
|
7664
|
+
minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) })
|
|
7665
|
+
})
|
|
7666
|
+
});
|
|
7667
|
+
return { trak, shown };
|
|
7668
|
+
});
|
|
7669
|
+
let next = 0;
|
|
7670
|
+
return rebuild(template, root, {
|
|
7671
|
+
mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),
|
|
7672
|
+
trak: () => traks[next++].trak
|
|
7673
|
+
});
|
|
7674
|
+
};
|
|
7675
|
+
const ftyp = layouts[0].ftyp;
|
|
7676
|
+
const dataStart = ftyp.length + buildMoov(0).length + 16;
|
|
7677
|
+
const mdat = Buffer.alloc(16);
|
|
7678
|
+
mdat.writeUInt32BE(1);
|
|
7679
|
+
mdat.write("mdat", 4, "latin1");
|
|
7680
|
+
mdat.writeBigUInt64BE(BigInt(16 + payloadBytes), 8);
|
|
7681
|
+
return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);
|
|
7682
|
+
}
|
|
7683
|
+
function fileRange(path) {
|
|
7684
|
+
return async (start, end) => {
|
|
7685
|
+
const file = await open(path, "r", 384);
|
|
7686
|
+
try {
|
|
7687
|
+
const buf = Buffer.alloc(end - start);
|
|
7688
|
+
const { bytesRead } = await file.read(buf, 0, buf.length, start);
|
|
7689
|
+
return buf.subarray(0, bytesRead);
|
|
7690
|
+
} finally {
|
|
7691
|
+
await file.close();
|
|
7692
|
+
}
|
|
7693
|
+
};
|
|
7694
|
+
}
|
|
7695
|
+
async function copyPayload(path, layout, index, out) {
|
|
7696
|
+
const { payloadStart, payloadEnd } = layout;
|
|
7697
|
+
const payload = createReadStream4(path, { start: payloadStart, end: payloadEnd - 1 });
|
|
7698
|
+
await pipeline(payload, out, { end: false });
|
|
7699
|
+
if (payload.bytesRead !== payloadEnd - payloadStart) {
|
|
7700
|
+
throw new Error(
|
|
7701
|
+
`[mp4Stitch] chunk ${index} mdat holds ${payload.bytesRead} of ${payloadEnd - payloadStart} payload bytes`
|
|
7702
|
+
);
|
|
7703
|
+
}
|
|
7704
|
+
}
|
|
7705
|
+
async function writeStitchedMp4(chunkPaths, outputPath) {
|
|
7706
|
+
const layouts = await Promise.all(chunkPaths.map((path) => readMp4Layout(fileRange(path))));
|
|
7707
|
+
const header = buildStitchedHeader(layouts);
|
|
7708
|
+
mkdirSync8(dirname9(outputPath), { recursive: true });
|
|
7709
|
+
const workDir = mkdtempSync3(`${outputPath}.stitch-`);
|
|
7710
|
+
const out = createWriteStream(join19(workDir, "out.mp4"));
|
|
7711
|
+
try {
|
|
7712
|
+
out.write(header);
|
|
7713
|
+
for (const [i, path] of chunkPaths.entries()) await copyPayload(path, layouts[i], i, out);
|
|
7714
|
+
out.end();
|
|
7715
|
+
await finished(out);
|
|
7716
|
+
renameSync3(join19(workDir, "out.mp4"), outputPath);
|
|
7717
|
+
} finally {
|
|
7718
|
+
out.destroy();
|
|
7719
|
+
rmSync7(workDir, { recursive: true, force: true });
|
|
7720
|
+
}
|
|
7721
|
+
}
|
|
7722
|
+
|
|
7723
|
+
// src/services/distributed/chunkAudio.ts
|
|
7724
|
+
function audioCutForChunk(grid, chunks, index, fps) {
|
|
7725
|
+
const firstPacketAt = (frame) => Math.min(
|
|
7726
|
+
grid.packetCount,
|
|
7727
|
+
Math.max(
|
|
7728
|
+
0,
|
|
7729
|
+
Math.ceil(
|
|
7730
|
+
(frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) / (grid.packetSamples * fps.num)
|
|
7731
|
+
)
|
|
7732
|
+
)
|
|
7733
|
+
);
|
|
7734
|
+
const first = index === 0 ? 0 : firstPacketAt(chunks[index].startFrame);
|
|
7735
|
+
const end = index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1].startFrame);
|
|
7736
|
+
if (end <= first) return null;
|
|
7737
|
+
const cutSeconds = (packet) => (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;
|
|
7738
|
+
const seekSeconds = index === 0 ? null : cutSeconds(first);
|
|
7739
|
+
const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);
|
|
7740
|
+
return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
|
|
7741
|
+
}
|
|
7742
|
+
async function probeAacPacketGrid(audioPath) {
|
|
7743
|
+
const probe = await runFfprobeJson([
|
|
7744
|
+
"-v",
|
|
7745
|
+
"error",
|
|
7746
|
+
"-select_streams",
|
|
7747
|
+
"a:0",
|
|
7748
|
+
"-show_entries",
|
|
7749
|
+
"stream=sample_rate,time_base,nb_frames:packet=pts,duration",
|
|
7750
|
+
"-read_intervals",
|
|
7751
|
+
"%+#1",
|
|
7752
|
+
"-of",
|
|
7753
|
+
"json",
|
|
7754
|
+
"--",
|
|
7755
|
+
audioPath
|
|
7756
|
+
]);
|
|
7757
|
+
const stream = probe.streams?.[0] ?? {};
|
|
7758
|
+
const packet = probe.packets?.[0] ?? {};
|
|
7759
|
+
const sampleRate = Number(stream.sample_rate);
|
|
7760
|
+
const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);
|
|
7761
|
+
const toSamples = (ticks) => Number(ticks) * sampleRate * tbNum / tbDen;
|
|
7762
|
+
const grid = {
|
|
7763
|
+
sampleRate,
|
|
7764
|
+
firstStart: toSamples(packet.pts),
|
|
7765
|
+
packetSamples: toSamples(packet.duration),
|
|
7766
|
+
packetCount: Number(stream.nb_frames)
|
|
7767
|
+
};
|
|
7768
|
+
const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];
|
|
7769
|
+
if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {
|
|
7770
|
+
throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);
|
|
7771
|
+
}
|
|
7772
|
+
return grid;
|
|
7773
|
+
}
|
|
7774
|
+
var SLICE_DIR = "audio-chunks";
|
|
7775
|
+
var SLICE_FRAME = `${SLICE_DIR}/frame.m4a`;
|
|
7776
|
+
var SLICE_LAYOUT = `${SLICE_DIR}/layout.json`;
|
|
7777
|
+
function chunkAudioSlicePath(index) {
|
|
7778
|
+
return `${SLICE_DIR}/${String(index).padStart(4, "0")}.bin`;
|
|
7779
|
+
}
|
|
7780
|
+
function chunkAudioSliceTarget(path) {
|
|
7781
|
+
if (path === SLICE_FRAME || path === SLICE_LAYOUT) return "all";
|
|
7782
|
+
const match = /^audio-chunks\/(\d{4,})\.bin$/.exec(path);
|
|
7783
|
+
return match ? Number(match[1]) : null;
|
|
7784
|
+
}
|
|
7785
|
+
async function writeChunkAudioSlices(input) {
|
|
7786
|
+
const grid = await probeAacPacketGrid(input.audioPath);
|
|
7787
|
+
const { payloadStart, payloadEnd, moov } = await readMp4Layout(fileRange(input.audioPath));
|
|
7788
|
+
const [track] = readTracks(moov);
|
|
7789
|
+
const offsets = [];
|
|
7790
|
+
track.chunkOffsets.forEach((offset, chunk) => {
|
|
7791
|
+
for (let i = 0; i < track.samplesPerChunk[chunk]; i++) {
|
|
7792
|
+
offsets.push(offset);
|
|
7793
|
+
offset += track.sizes[offsets.length - 1];
|
|
7794
|
+
}
|
|
7795
|
+
});
|
|
7796
|
+
const bytes = readFileSync9(input.audioPath);
|
|
7797
|
+
const layout = { sizeBytes: bytes.length, payloadStart, payloadEnd, partStarts: {} };
|
|
7798
|
+
mkdirSync9(join20(input.planDir, SLICE_DIR), { recursive: true });
|
|
7799
|
+
input.chunks.forEach((_, index) => {
|
|
7800
|
+
const cut = audioCutForChunk(grid, input.chunks, index, input.fps);
|
|
7801
|
+
if (!cut) return;
|
|
7802
|
+
const last = cut.firstPacket + cut.packetCount - 1;
|
|
7803
|
+
const start = offsets[cut.firstPacket];
|
|
7804
|
+
layout.partStarts[index] = start;
|
|
7805
|
+
const end = offsets[last] + track.sizes[last];
|
|
7806
|
+
writeFileSync5(join20(input.planDir, chunkAudioSlicePath(index)), bytes.subarray(start, end));
|
|
7807
|
+
});
|
|
7808
|
+
const frame = [bytes.subarray(0, payloadStart), bytes.subarray(payloadEnd)];
|
|
7809
|
+
writeFileSync5(join20(input.planDir, SLICE_FRAME), Buffer.concat(frame));
|
|
7810
|
+
writeFileSync5(join20(input.planDir, SLICE_LAYOUT), JSON.stringify(layout));
|
|
7811
|
+
}
|
|
7812
|
+
function chunkAudioSource(planDir, chunkIndex, scratchPath) {
|
|
7813
|
+
const layoutPath = join20(planDir, SLICE_LAYOUT);
|
|
7814
|
+
if (!existsSync14(layoutPath)) return resolvePlanAudioPath(planDir);
|
|
7815
|
+
const layout = JSON.parse(readFileSync9(layoutPath, "utf-8"));
|
|
7816
|
+
const frame = readFileSync9(join20(planDir, SLICE_FRAME));
|
|
7817
|
+
const fd = openSync4(scratchPath, "w");
|
|
7818
|
+
try {
|
|
7819
|
+
writeSync(fd, frame, 0, layout.payloadStart, 0);
|
|
7820
|
+
writeSync(
|
|
7821
|
+
fd,
|
|
7822
|
+
frame,
|
|
7823
|
+
layout.payloadStart,
|
|
7824
|
+
frame.length - layout.payloadStart,
|
|
7825
|
+
layout.payloadEnd
|
|
7826
|
+
);
|
|
7827
|
+
const start = layout.partStarts[chunkIndex];
|
|
7828
|
+
if (start !== void 0) {
|
|
7829
|
+
const part = readFileSync9(join20(planDir, chunkAudioSlicePath(chunkIndex)));
|
|
7830
|
+
writeSync(fd, part, 0, part.length, start);
|
|
7831
|
+
}
|
|
7832
|
+
ftruncateSync(fd, layout.sizeBytes);
|
|
7833
|
+
} finally {
|
|
7834
|
+
closeSync4(fd);
|
|
7835
|
+
}
|
|
7836
|
+
return scratchPath;
|
|
7837
|
+
}
|
|
7838
|
+
function chunkMuxArgs(input, cut) {
|
|
7839
|
+
if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];
|
|
7840
|
+
const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];
|
|
7841
|
+
const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];
|
|
7842
|
+
const exact = seek.length ? ["-copypriorss", "0"] : [];
|
|
7843
|
+
return [
|
|
7844
|
+
...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath],
|
|
7845
|
+
...["-map", "0:v:0", "-map", "1:a:0", ...exact]
|
|
7846
|
+
];
|
|
7847
|
+
}
|
|
7848
|
+
async function muxChunkWithAudio(input) {
|
|
7849
|
+
const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;
|
|
7850
|
+
const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;
|
|
7851
|
+
const args = [...chunkMuxArgs(input, cut), "-c", "copy"];
|
|
7852
|
+
appendRenderProvenanceArgs(args, input.outputPath);
|
|
7853
|
+
args.push("-y", input.outputPath);
|
|
7854
|
+
const result = await runFfmpeg5(args, { signal: input.signal, timeout: input.timeoutMs });
|
|
7855
|
+
if (!result.success) {
|
|
7856
|
+
throw new Error(
|
|
7857
|
+
`[chunkAudio] chunk mux failed: ${describeFfmpegFailure2(result, input.timeoutMs)}`
|
|
7858
|
+
);
|
|
7859
|
+
}
|
|
7860
|
+
const slice = input.chunks[input.chunkIndex];
|
|
7861
|
+
const expected = { video: slice.endFrame - slice.startFrame };
|
|
7862
|
+
if (cut) expected.audio = cut.packetCount;
|
|
7863
|
+
const probe = await runFfprobeJson([
|
|
7864
|
+
"-v",
|
|
7865
|
+
"error",
|
|
7866
|
+
"-count_packets",
|
|
7867
|
+
"-show_entries",
|
|
7868
|
+
"stream=codec_type,nb_read_packets",
|
|
7869
|
+
"-of",
|
|
7870
|
+
"json",
|
|
7871
|
+
"--",
|
|
7872
|
+
input.outputPath
|
|
7873
|
+
]);
|
|
7874
|
+
for (const [type, count] of Object.entries(expected)) {
|
|
7875
|
+
const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);
|
|
7876
|
+
if (packets !== count) {
|
|
7877
|
+
throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);
|
|
7878
|
+
}
|
|
7879
|
+
}
|
|
7880
|
+
}
|
|
7881
|
+
|
|
7882
|
+
// src/services/distributed/planSize.ts
|
|
7883
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
7884
|
+
import { lstatSync, readdirSync as readdirSync6 } from "node:fs";
|
|
7885
|
+
import { extname as extname3, join as join21, relative as relative4 } from "node:path";
|
|
7886
|
+
var MAX_TOP_COMPONENTS = 10;
|
|
7887
|
+
var SOURCE_MEDIA_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
7888
|
+
".aac",
|
|
7889
|
+
".aiff",
|
|
7890
|
+
".avi",
|
|
7891
|
+
".flac",
|
|
7892
|
+
".m4a",
|
|
7893
|
+
".m4v",
|
|
7894
|
+
".mkv",
|
|
7895
|
+
".mov",
|
|
7896
|
+
".mp3",
|
|
7897
|
+
".mp4",
|
|
7898
|
+
".mpeg",
|
|
7899
|
+
".mpg",
|
|
7900
|
+
".ogg",
|
|
7901
|
+
".ogv",
|
|
7902
|
+
".ts",
|
|
7903
|
+
".wav",
|
|
7904
|
+
".webm"
|
|
7905
|
+
]);
|
|
7906
|
+
var PLAN_ROOT_CATEGORY = {
|
|
7907
|
+
compiled: "compiled",
|
|
7908
|
+
[PLAN_AUDIO_RELATIVE_PATH]: "audio",
|
|
7909
|
+
meta: "metadata",
|
|
7910
|
+
"plan.json": "metadata"
|
|
7911
|
+
};
|
|
7912
|
+
function hashComponent(value) {
|
|
7913
|
+
return createHash5("sha256").update(value).digest("hex").slice(0, 12);
|
|
7914
|
+
}
|
|
7915
|
+
function classifyFile(segments, rootKind) {
|
|
7916
|
+
const rootName = segments[0] ?? "";
|
|
7917
|
+
const extractedRootName = rootKind === "compiled" ? "__hyperframes_video_frames" : "video-frames";
|
|
7918
|
+
if (rootName === extractedRootName) {
|
|
7919
|
+
const videoKey = segments[1] ?? "unknown";
|
|
7920
|
+
return {
|
|
7921
|
+
category: "video-frames",
|
|
7922
|
+
componentLabel: `video-frames:${hashComponent(videoKey)}`
|
|
7923
|
+
};
|
|
7924
|
+
}
|
|
7925
|
+
const category = rootKind === "compiled" ? "compiled" : PLAN_ROOT_CATEGORY[rootName] ?? "other";
|
|
7926
|
+
return { category, componentLabel: category };
|
|
7927
|
+
}
|
|
7928
|
+
function emptyBreakdown() {
|
|
7929
|
+
return {
|
|
7930
|
+
totalBytes: 0,
|
|
7931
|
+
fileCount: 0,
|
|
7932
|
+
compiledBytes: 0,
|
|
7933
|
+
sourceMediaBytes: 0,
|
|
7934
|
+
videoFramesBytes: 0,
|
|
7935
|
+
videoFrameFileCount: 0,
|
|
7936
|
+
audioBytes: 0,
|
|
7937
|
+
metadataBytes: 0,
|
|
7938
|
+
otherBytes: 0,
|
|
7939
|
+
topComponents: []
|
|
7940
|
+
};
|
|
7941
|
+
}
|
|
7942
|
+
function addCategoryBytes(result, category, filePath, sizeBytes) {
|
|
7943
|
+
if (category === "compiled") {
|
|
7944
|
+
result.compiledBytes += sizeBytes;
|
|
7945
|
+
if (SOURCE_MEDIA_EXTENSIONS.has(extname3(filePath).toLowerCase())) {
|
|
7946
|
+
result.sourceMediaBytes += sizeBytes;
|
|
7947
|
+
}
|
|
7948
|
+
return;
|
|
7949
|
+
}
|
|
7950
|
+
if (category === "video-frames") {
|
|
7951
|
+
result.videoFramesBytes += sizeBytes;
|
|
7952
|
+
result.videoFrameFileCount += 1;
|
|
7953
|
+
return;
|
|
7954
|
+
}
|
|
7955
|
+
if (category === "audio") {
|
|
7956
|
+
result.audioBytes += sizeBytes;
|
|
7957
|
+
return;
|
|
7958
|
+
}
|
|
7959
|
+
if (category === "metadata") {
|
|
7960
|
+
result.metadataBytes += sizeBytes;
|
|
7961
|
+
return;
|
|
7962
|
+
}
|
|
7963
|
+
result.otherBytes += sizeBytes;
|
|
7964
|
+
}
|
|
7965
|
+
function recordFile(result, components, rootDir, rootKind, filePath) {
|
|
7966
|
+
let sizeBytes;
|
|
7967
|
+
try {
|
|
7968
|
+
const stat = lstatSync(filePath);
|
|
7969
|
+
if (!stat.isFile()) return;
|
|
7970
|
+
sizeBytes = stat.size;
|
|
7971
|
+
} catch {
|
|
7972
|
+
return;
|
|
7973
|
+
}
|
|
7974
|
+
const rel = relative4(rootDir, filePath);
|
|
7975
|
+
const segments = rel.split(/[\\/]+/).filter(Boolean);
|
|
7976
|
+
const classified = classifyFile(segments, rootKind);
|
|
7977
|
+
result.totalBytes += sizeBytes;
|
|
7978
|
+
result.fileCount += 1;
|
|
7979
|
+
addCategoryBytes(result, classified.category, filePath, sizeBytes);
|
|
7980
|
+
const component = components.get(classified.componentLabel) ?? {
|
|
7981
|
+
label: classified.componentLabel,
|
|
7982
|
+
sizeBytes: 0,
|
|
7983
|
+
fileCount: 0
|
|
7984
|
+
};
|
|
7985
|
+
component.sizeBytes += sizeBytes;
|
|
7986
|
+
component.fileCount += 1;
|
|
7987
|
+
components.set(classified.componentLabel, component);
|
|
7988
|
+
}
|
|
7989
|
+
function walkRegularFiles(dir, visit) {
|
|
7990
|
+
let entries;
|
|
7991
|
+
try {
|
|
7992
|
+
entries = readdirSync6(dir, { withFileTypes: true });
|
|
7993
|
+
} catch {
|
|
7994
|
+
return;
|
|
7995
|
+
}
|
|
7996
|
+
for (const entry of entries) {
|
|
7997
|
+
const full = join21(dir, entry.name);
|
|
7998
|
+
if (entry.isDirectory()) {
|
|
7999
|
+
walkRegularFiles(full, visit);
|
|
8000
|
+
} else if (entry.isFile()) {
|
|
8001
|
+
visit(full);
|
|
8002
|
+
}
|
|
8003
|
+
}
|
|
8004
|
+
}
|
|
8005
|
+
function measurePlanSizeBreakdown(rootDir, rootKind = "plan") {
|
|
8006
|
+
const result = emptyBreakdown();
|
|
8007
|
+
const components = /* @__PURE__ */ new Map();
|
|
8008
|
+
walkRegularFiles(
|
|
8009
|
+
rootDir,
|
|
8010
|
+
(filePath) => recordFile(result, components, rootDir, rootKind, filePath)
|
|
8011
|
+
);
|
|
8012
|
+
result.topComponents = [...components.values()].sort((a, b) => b.sizeBytes - a.sizeBytes || a.label.localeCompare(b.label)).slice(0, MAX_TOP_COMPONENTS);
|
|
8013
|
+
return result;
|
|
8014
|
+
}
|
|
8015
|
+
|
|
8016
|
+
// src/services/distributed/plan.ts
|
|
8017
|
+
function logPlanStage(log, stage, startedAt) {
|
|
8018
|
+
log.info("[plan] stage", { stage, ms: Date.now() - startedAt });
|
|
8019
|
+
}
|
|
8020
|
+
function applyDistributedAudioWarningPolicy(job, audioError, audioFailures = [], log = defaultLogger) {
|
|
8021
|
+
const failureOwner = audioFailures.length === 0 ? void 0 : audioFailures.some((failure) => failure.owner === "system") ? "system" : "user";
|
|
8022
|
+
const retryable = audioFailures.length === 0 ? void 0 : audioFailures.every((failure) => failure.retryable);
|
|
8023
|
+
applyRenderWarningPolicy(
|
|
8024
|
+
job,
|
|
8025
|
+
[
|
|
8026
|
+
{
|
|
8027
|
+
code: "audio_processing_failed",
|
|
8028
|
+
message: `Audio mix failed; output would be video-only: ${audioError}`,
|
|
8029
|
+
details: {
|
|
8030
|
+
mediaType: "audio",
|
|
8031
|
+
audioFailures: audioFailureDiagnostics3(audioFailures),
|
|
8032
|
+
failureReasons: [...new Set(audioFailures.map((failure) => failure.reason))],
|
|
8033
|
+
failureStages: [...new Set(audioFailures.map((failure) => failure.stage))],
|
|
8034
|
+
failureOwner,
|
|
8035
|
+
retryable
|
|
8036
|
+
}
|
|
8037
|
+
}
|
|
8038
|
+
],
|
|
8039
|
+
log
|
|
8040
|
+
);
|
|
8041
|
+
}
|
|
8042
|
+
var PLAN_PROJECT_DIR_SKIP_SEGMENTS = /* @__PURE__ */ new Set([
|
|
7488
8043
|
"node_modules",
|
|
7489
8044
|
".git",
|
|
7490
8045
|
".cache",
|
|
@@ -7653,13 +8208,13 @@ function assertPositiveInteger(name, value) {
|
|
|
7653
8208
|
}
|
|
7654
8209
|
var FREEZE_OWNED_PLAN_FILES = [
|
|
7655
8210
|
"plan.json",
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
8211
|
+
join22("meta", "composition.json"),
|
|
8212
|
+
join22("meta", "encoder.json"),
|
|
8213
|
+
join22("meta", "chunks.json")
|
|
7659
8214
|
];
|
|
7660
8215
|
function removeFreezeOwnedPlanFiles(planDir) {
|
|
7661
8216
|
for (const relativePath of FREEZE_OWNED_PLAN_FILES) {
|
|
7662
|
-
|
|
8217
|
+
rmSync8(join22(planDir, relativePath), { force: true });
|
|
7663
8218
|
}
|
|
7664
8219
|
}
|
|
7665
8220
|
function buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize) {
|
|
@@ -7790,19 +8345,19 @@ function createVideoSourceShipper(input) {
|
|
|
7790
8345
|
const known = shipped.get(srcPath);
|
|
7791
8346
|
if (known) return known;
|
|
7792
8347
|
const stagedRel = pathInside(input.workVideoFramesDir, srcPath);
|
|
7793
|
-
const current = stagedRel === null ? srcPath :
|
|
8348
|
+
const current = stagedRel === null ? srcPath : join22(input.planDir, "video-frames", stagedRel);
|
|
7794
8349
|
let result = null;
|
|
7795
8350
|
for (const root of stagedRel === null ? [input.workCompiledDir, input.projectDir] : []) {
|
|
7796
8351
|
const rel = pathInside(root, srcPath);
|
|
7797
|
-
if (rel !== null &&
|
|
8352
|
+
if (rel !== null && (existsSync15(join22(input.planDir, "compiled", rel)) || input.isRemote(srcPath))) {
|
|
7798
8353
|
result = ["compiled", ...rel.split(sep2)].join("/");
|
|
7799
8354
|
break;
|
|
7800
8355
|
}
|
|
7801
8356
|
}
|
|
7802
8357
|
if (result === null) {
|
|
7803
8358
|
result = `video-sources/${shipped.size}${extname4(current)}`;
|
|
7804
|
-
const destination =
|
|
7805
|
-
|
|
8359
|
+
const destination = join22(input.planDir, ...result.split("/"));
|
|
8360
|
+
mkdirSync10(join22(input.planDir, "video-sources"), { recursive: true });
|
|
7806
8361
|
try {
|
|
7807
8362
|
linkSync(current, destination);
|
|
7808
8363
|
} catch {
|
|
@@ -7820,7 +8375,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7820
8375
|
useGpu: false,
|
|
7821
8376
|
browserGpuMode: "software"
|
|
7822
8377
|
});
|
|
7823
|
-
if (!
|
|
8378
|
+
if (!existsSync15(planDir)) mkdirSync10(planDir, { recursive: true });
|
|
7824
8379
|
const log = config.logger ?? defaultLogger;
|
|
7825
8380
|
const sizeLimitBytes = options.executionPlanSizeLimitBytes ?? config.planDirSizeLimitBytes ?? PLAN_DIR_SIZE_LIMIT_BYTES;
|
|
7826
8381
|
const abortSignal = config.abortSignal;
|
|
@@ -7849,15 +8404,15 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7849
8404
|
variables: config.variables
|
|
7850
8405
|
});
|
|
7851
8406
|
const entryFile = config.entryFile ?? "index.html";
|
|
7852
|
-
const htmlPath =
|
|
7853
|
-
if (!
|
|
8407
|
+
const htmlPath = join22(projectDir, entryFile);
|
|
8408
|
+
if (!existsSync15(htmlPath)) {
|
|
7854
8409
|
throw new Error(`[plan] entry file not found: ${htmlPath}`);
|
|
7855
8410
|
}
|
|
7856
|
-
const workDir =
|
|
7857
|
-
|
|
7858
|
-
|
|
7859
|
-
const compiledDir =
|
|
7860
|
-
|
|
8411
|
+
const workDir = join22(planDir, ".plan-work");
|
|
8412
|
+
rmSync8(workDir, { recursive: true, force: true });
|
|
8413
|
+
mkdirSync10(workDir, { recursive: true });
|
|
8414
|
+
const compiledDir = join22(workDir, "compiled");
|
|
8415
|
+
mkdirSync10(compiledDir, { recursive: true });
|
|
7861
8416
|
let stageStarted = Date.now();
|
|
7862
8417
|
cpSync2(projectDir, compiledDir, {
|
|
7863
8418
|
recursive: true,
|
|
@@ -7870,10 +8425,19 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7870
8425
|
}
|
|
7871
8426
|
});
|
|
7872
8427
|
logPlanStage(log, "copy", stageStarted);
|
|
7873
|
-
const finalCompiledDir =
|
|
8428
|
+
const finalCompiledDir = join22(planDir, "compiled");
|
|
7874
8429
|
const needsAlpha = outputNeedsAlpha(config.format);
|
|
8430
|
+
const remoteFiles = Object.entries(options.remoteFiles ?? {});
|
|
8431
|
+
const remoteLocalPath = (rel) => join22(projectDir, ...rel.split("/"));
|
|
8432
|
+
const urlByAbsentLocalPath = new Map(
|
|
8433
|
+
remoteFiles.map(([rel, file]) => [remoteLocalPath(rel), file.url])
|
|
8434
|
+
);
|
|
8435
|
+
const localPathByUrl = new Map(
|
|
8436
|
+
remoteFiles.map(([rel, file]) => [file.url, remoteLocalPath(rel)])
|
|
8437
|
+
);
|
|
7875
8438
|
stageStarted = Date.now();
|
|
7876
8439
|
const compileResult = await runCompileStage({
|
|
8440
|
+
urlByAbsentLocalPath,
|
|
7877
8441
|
projectDir,
|
|
7878
8442
|
workDir,
|
|
7879
8443
|
htmlPath,
|
|
@@ -7961,30 +8525,32 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7961
8525
|
abortSignal,
|
|
7962
8526
|
assertNotAborted,
|
|
7963
8527
|
materializeSymlinks: true,
|
|
7964
|
-
deferRangeExtraction: options.deferVideoExtraction
|
|
8528
|
+
deferRangeExtraction: options.deferVideoExtraction,
|
|
8529
|
+
urlByAbsentLocalPath
|
|
7965
8530
|
});
|
|
7966
8531
|
logPlanStage(log, "extract", stageStarted);
|
|
7967
8532
|
if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
|
|
7968
8533
|
if (extractResult.extractionResult) {
|
|
7969
8534
|
assertVideoExtractionSucceeded(extractResult.extractionResult);
|
|
7970
8535
|
}
|
|
7971
|
-
const stagedVideoFrames =
|
|
7972
|
-
const videoFramesDst =
|
|
7973
|
-
if (
|
|
7974
|
-
const videoSourcesDir =
|
|
7975
|
-
|
|
7976
|
-
if (
|
|
7977
|
-
|
|
8536
|
+
const stagedVideoFrames = join22(compiledDir, "__hyperframes_video_frames");
|
|
8537
|
+
const videoFramesDst = join22(planDir, "video-frames");
|
|
8538
|
+
if (existsSync15(videoFramesDst)) rmSync8(videoFramesDst, { recursive: true, force: true });
|
|
8539
|
+
const videoSourcesDir = join22(planDir, "video-sources");
|
|
8540
|
+
rmSync8(videoSourcesDir, { recursive: true, force: true });
|
|
8541
|
+
if (existsSync15(stagedVideoFrames)) {
|
|
8542
|
+
renameSync4(stagedVideoFrames, videoFramesDst);
|
|
7978
8543
|
} else {
|
|
7979
|
-
|
|
8544
|
+
mkdirSync10(videoFramesDst, { recursive: true });
|
|
7980
8545
|
}
|
|
7981
|
-
if (
|
|
7982
|
-
|
|
8546
|
+
if (existsSync15(finalCompiledDir)) rmSync8(finalCompiledDir, { recursive: true, force: true });
|
|
8547
|
+
renameSync4(compiledDir, finalCompiledDir);
|
|
7983
8548
|
const shipVideoSource = createVideoSourceShipper({
|
|
7984
8549
|
planDir,
|
|
7985
8550
|
projectDir,
|
|
7986
8551
|
workCompiledDir: compiledDir,
|
|
7987
|
-
workVideoFramesDir: stagedVideoFrames
|
|
8552
|
+
workVideoFramesDir: stagedVideoFrames,
|
|
8553
|
+
isRemote: (srcPath) => urlByAbsentLocalPath.has(srcPath)
|
|
7988
8554
|
});
|
|
7989
8555
|
const sourceRoots = [
|
|
7990
8556
|
[stagedVideoFrames, "video-frames"],
|
|
@@ -7994,19 +8560,30 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7994
8560
|
const planVideosJson = buildPlanVideosJson({
|
|
7995
8561
|
videos: composition.videos,
|
|
7996
8562
|
compositionEnd: job.duration ?? Number.NaN,
|
|
7997
|
-
extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) =>
|
|
7998
|
-
|
|
7999
|
-
|
|
8000
|
-
|
|
8001
|
-
|
|
8002
|
-
|
|
8003
|
-
|
|
8004
|
-
|
|
8005
|
-
|
|
8563
|
+
extracted: (extractResult.extractionResult?.extracted ?? []).map((ext) => {
|
|
8564
|
+
const sourcePath = localPathByUrl.get(ext.srcPath) ?? ext.srcPath;
|
|
8565
|
+
return {
|
|
8566
|
+
videoId: ext.videoId,
|
|
8567
|
+
srcPath: stableSourcePath(sourceRoots, sourcePath),
|
|
8568
|
+
framePattern: ext.framePattern,
|
|
8569
|
+
fps: ext.fps,
|
|
8570
|
+
totalFrames: ext.totalFrames,
|
|
8571
|
+
metadata: ext.metadata,
|
|
8572
|
+
...ext.deferredRange ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(sourcePath) } } : {}
|
|
8573
|
+
};
|
|
8574
|
+
})
|
|
8006
8575
|
});
|
|
8007
|
-
|
|
8008
|
-
|
|
8009
|
-
|
|
8576
|
+
mkdirSync10(join22(planDir, "meta"), { recursive: true });
|
|
8577
|
+
writePlanRemoteFiles(
|
|
8578
|
+
planDir,
|
|
8579
|
+
remoteFiles.map(([rel, { sha256, sizeBytes }]) => ({
|
|
8580
|
+
path: `compiled/${rel}`,
|
|
8581
|
+
sha256,
|
|
8582
|
+
sizeBytes
|
|
8583
|
+
}))
|
|
8584
|
+
);
|
|
8585
|
+
writeFileSync6(
|
|
8586
|
+
join22(planDir, PLAN_VIDEOS_META_RELATIVE_PATH),
|
|
8010
8587
|
JSON.stringify(planVideosJson, null, 2),
|
|
8011
8588
|
"utf-8"
|
|
8012
8589
|
);
|
|
@@ -8019,6 +8596,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
8019
8596
|
projectDir,
|
|
8020
8597
|
workDir,
|
|
8021
8598
|
compiledDir: finalCompiledDir,
|
|
8599
|
+
urlByAbsentLocalPath,
|
|
8022
8600
|
duration: audioSeconds,
|
|
8023
8601
|
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
|
|
8024
8602
|
audioGain: cfg.audioGain,
|
|
@@ -8079,11 +8657,11 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
8079
8657
|
if (audioResult.audioError) {
|
|
8080
8658
|
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
|
|
8081
8659
|
}
|
|
8082
|
-
const planAudioPath =
|
|
8083
|
-
if (audioResult.hasAudio &&
|
|
8084
|
-
|
|
8660
|
+
const planAudioPath = join22(planDir, PLAN_AUDIO_RELATIVE_PATH);
|
|
8661
|
+
if (audioResult.hasAudio && existsSync15(audioResult.audioOutputPath)) {
|
|
8662
|
+
renameSync4(audioResult.audioOutputPath, planAudioPath);
|
|
8085
8663
|
}
|
|
8086
|
-
if (chunksCarryAudio2 &&
|
|
8664
|
+
if (chunksCarryAudio2 && existsSync15(planAudioPath)) {
|
|
8087
8665
|
stageStarted = Date.now();
|
|
8088
8666
|
const peak = await constrainAacTruePeak({
|
|
8089
8667
|
audioPath: planAudioPath,
|
|
@@ -8093,6 +8671,16 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
8093
8671
|
});
|
|
8094
8672
|
if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
|
|
8095
8673
|
logPlanStage(log, "audio-peak", stageStarted);
|
|
8674
|
+
if (options.sliceChunkAudio) {
|
|
8675
|
+
stageStarted = Date.now();
|
|
8676
|
+
await writeChunkAudioSlices({
|
|
8677
|
+
audioPath: planAudioPath,
|
|
8678
|
+
planDir,
|
|
8679
|
+
chunks,
|
|
8680
|
+
fps: { num: config.fps, den: 1 }
|
|
8681
|
+
});
|
|
8682
|
+
logPlanStage(log, "audio-slices", stageStarted);
|
|
8683
|
+
}
|
|
8096
8684
|
}
|
|
8097
8685
|
const ffmpegVersion = await readFfmpegVersion();
|
|
8098
8686
|
const producerVersion = readProducerVersion();
|
|
@@ -8126,7 +8714,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
8126
8714
|
format: config.format
|
|
8127
8715
|
};
|
|
8128
8716
|
try {
|
|
8129
|
-
|
|
8717
|
+
rmSync8(workDir, { recursive: true, force: true });
|
|
8130
8718
|
} catch (err) {
|
|
8131
8719
|
log.warn("[plan] failed to remove temp work dir", {
|
|
8132
8720
|
workDir,
|
|
@@ -8197,35 +8785,35 @@ async function plan(projectDir, config, planDir, options = {}) {
|
|
|
8197
8785
|
// src/services/distributed/planV2.ts
|
|
8198
8786
|
import {
|
|
8199
8787
|
copyFileSync as copyFileSync7,
|
|
8200
|
-
existsSync as
|
|
8788
|
+
existsSync as existsSync17,
|
|
8201
8789
|
lstatSync as lstatSync2,
|
|
8202
|
-
mkdirSync as
|
|
8203
|
-
mkdtempSync as
|
|
8204
|
-
readFileSync as
|
|
8790
|
+
mkdirSync as mkdirSync12,
|
|
8791
|
+
mkdtempSync as mkdtempSync5,
|
|
8792
|
+
readFileSync as readFileSync10,
|
|
8205
8793
|
readdirSync as readdirSync7,
|
|
8206
|
-
renameSync as
|
|
8207
|
-
rmSync as
|
|
8794
|
+
renameSync as renameSync6,
|
|
8795
|
+
rmSync as rmSync10,
|
|
8208
8796
|
statSync as statSync6,
|
|
8209
|
-
writeFileSync as
|
|
8797
|
+
writeFileSync as writeFileSync8
|
|
8210
8798
|
} from "node:fs";
|
|
8211
8799
|
import { tmpdir } from "node:os";
|
|
8212
|
-
import { dirname as
|
|
8800
|
+
import { dirname as dirname11, isAbsolute as isAbsolute4, join as join25, relative as relative6, resolve as resolve8, sep as sep3 } from "node:path";
|
|
8213
8801
|
import { DISTRIBUTED_FPS, isDistributedFps } from "@hyperframes/core";
|
|
8214
8802
|
import { createFrameLookupTable as createFrameLookupTable4 } from "@hyperframes/engine";
|
|
8215
8803
|
|
|
8216
8804
|
// src/services/distributed/planV2Publisher.ts
|
|
8217
8805
|
import {
|
|
8218
8806
|
copyFileSync as copyFileSync6,
|
|
8219
|
-
existsSync as
|
|
8807
|
+
existsSync as existsSync16,
|
|
8220
8808
|
linkSync as linkSync2,
|
|
8221
|
-
mkdirSync as
|
|
8222
|
-
mkdtempSync as
|
|
8223
|
-
renameSync as
|
|
8224
|
-
rmSync as
|
|
8809
|
+
mkdirSync as mkdirSync11,
|
|
8810
|
+
mkdtempSync as mkdtempSync4,
|
|
8811
|
+
renameSync as renameSync5,
|
|
8812
|
+
rmSync as rmSync9,
|
|
8225
8813
|
statSync as statSync5,
|
|
8226
|
-
writeFileSync as
|
|
8814
|
+
writeFileSync as writeFileSync7
|
|
8227
8815
|
} from "node:fs";
|
|
8228
|
-
import { dirname as
|
|
8816
|
+
import { dirname as dirname10, join as join24 } from "node:path";
|
|
8229
8817
|
|
|
8230
8818
|
// src/services/distributed/planV2Errors.ts
|
|
8231
8819
|
var PLAN_V2_INTEGRITY_UNRECOVERABLE = "PLAN_V2_INTEGRITY_UNRECOVERABLE";
|
|
@@ -8240,7 +8828,7 @@ var PlanV2IntegrityError = class extends Error {
|
|
|
8240
8828
|
};
|
|
8241
8829
|
|
|
8242
8830
|
// src/services/distributed/planV2Layout.ts
|
|
8243
|
-
import { join as
|
|
8831
|
+
import { join as join23 } from "node:path";
|
|
8244
8832
|
var SHA256_PATTERN = /^[0-9a-f]{64}$/;
|
|
8245
8833
|
function assertPlanV2Sha256(value, field) {
|
|
8246
8834
|
if (typeof value !== "string" || !SHA256_PATTERN.test(value)) {
|
|
@@ -8250,7 +8838,7 @@ function assertPlanV2Sha256(value, field) {
|
|
|
8250
8838
|
}
|
|
8251
8839
|
function planV2BlobPath(root, sha256) {
|
|
8252
8840
|
const digest = assertPlanV2Sha256(sha256, "artifact sha256");
|
|
8253
|
-
return
|
|
8841
|
+
return join23(root, "artifacts", "sha256", digest.slice(0, 2), digest);
|
|
8254
8842
|
}
|
|
8255
8843
|
|
|
8256
8844
|
// src/services/distributed/planV2Publisher.ts
|
|
@@ -8285,13 +8873,13 @@ var LocalPlanV2ArtifactPublisher = class {
|
|
|
8285
8873
|
#linkFile;
|
|
8286
8874
|
#committed = false;
|
|
8287
8875
|
constructor(destinationDir, options = {}) {
|
|
8288
|
-
if (
|
|
8876
|
+
if (existsSync16(destinationDir)) {
|
|
8289
8877
|
throw new PlanV2IntegrityError(`output directory already exists: ${destinationDir}`);
|
|
8290
8878
|
}
|
|
8291
8879
|
this.destinationDir = destinationDir;
|
|
8292
8880
|
this.#linkFile = options.linkFile ?? linkSync2;
|
|
8293
|
-
|
|
8294
|
-
this.temporaryDir =
|
|
8881
|
+
mkdirSync11(dirname10(destinationDir), { recursive: true });
|
|
8882
|
+
this.temporaryDir = mkdtempSync4(join24(dirname10(destinationDir), ".plan-v2-publish-"));
|
|
8295
8883
|
}
|
|
8296
8884
|
async putBlob(blob) {
|
|
8297
8885
|
const digest = assertPlanV2Sha256(blob.sha256, "published blob sha256");
|
|
@@ -8302,10 +8890,10 @@ var LocalPlanV2ArtifactPublisher = class {
|
|
|
8302
8890
|
);
|
|
8303
8891
|
}
|
|
8304
8892
|
const destinationPath = planV2BlobPath(this.temporaryDir, digest);
|
|
8305
|
-
if (
|
|
8306
|
-
|
|
8307
|
-
const stagingDir =
|
|
8308
|
-
const temporaryPath =
|
|
8893
|
+
if (existsSync16(destinationPath)) return;
|
|
8894
|
+
mkdirSync11(dirname10(destinationPath), { recursive: true });
|
|
8895
|
+
const stagingDir = mkdtempSync4(join24(dirname10(destinationPath), ".plan-v2-blob-"));
|
|
8896
|
+
const temporaryPath = join24(stagingDir, "blob");
|
|
8309
8897
|
try {
|
|
8310
8898
|
try {
|
|
8311
8899
|
this.#linkFile(blob.sourcePath, temporaryPath);
|
|
@@ -8313,26 +8901,26 @@ var LocalPlanV2ArtifactPublisher = class {
|
|
|
8313
8901
|
if (!canFallbackToCopy(error)) throw error;
|
|
8314
8902
|
copyFileSync6(blob.sourcePath, temporaryPath);
|
|
8315
8903
|
}
|
|
8316
|
-
|
|
8904
|
+
renameSync5(temporaryPath, destinationPath);
|
|
8317
8905
|
} finally {
|
|
8318
|
-
|
|
8906
|
+
rmSync9(stagingDir, { recursive: true, force: true });
|
|
8319
8907
|
}
|
|
8320
8908
|
}
|
|
8321
8909
|
async commitManifest(manifestBytes) {
|
|
8322
8910
|
for (const digest of new Set(manifestDigests(manifestBytes))) {
|
|
8323
|
-
if (!
|
|
8911
|
+
if (!existsSync16(planV2BlobPath(this.temporaryDir, digest))) {
|
|
8324
8912
|
throw new PlanV2IntegrityError(
|
|
8325
8913
|
`cannot commit manifest before referenced blob is durable: ${digest}`
|
|
8326
8914
|
);
|
|
8327
8915
|
}
|
|
8328
8916
|
}
|
|
8329
|
-
|
|
8330
|
-
|
|
8917
|
+
writeFileSync7(join24(this.temporaryDir, "plan.json"), manifestBytes, "utf-8");
|
|
8918
|
+
renameSync5(this.temporaryDir, this.destinationDir);
|
|
8331
8919
|
this.#committed = true;
|
|
8332
8920
|
}
|
|
8333
8921
|
async abort() {
|
|
8334
8922
|
if (!this.#committed) {
|
|
8335
|
-
|
|
8923
|
+
rmSync9(this.temporaryDir, { recursive: true, force: true });
|
|
8336
8924
|
}
|
|
8337
8925
|
}
|
|
8338
8926
|
};
|
|
@@ -8351,7 +8939,7 @@ function isNonNegativeIntegerArray(value) {
|
|
|
8351
8939
|
return Array.isArray(value) && value.every((item) => typeof item === "number" && Number.isInteger(item) && item >= 0);
|
|
8352
8940
|
}
|
|
8353
8941
|
function readJsonFile(path, label) {
|
|
8354
|
-
const contents =
|
|
8942
|
+
const contents = readFileSync10(path, "utf-8");
|
|
8355
8943
|
try {
|
|
8356
8944
|
const parsed = JSON.parse(contents);
|
|
8357
8945
|
return parsed;
|
|
@@ -8371,7 +8959,7 @@ function listFiles(root) {
|
|
|
8371
8959
|
const rootResolved = resolve8(root);
|
|
8372
8960
|
function walk(dir) {
|
|
8373
8961
|
for (const entry of readdirSync7(dir, { withFileTypes: true })) {
|
|
8374
|
-
const absolutePath =
|
|
8962
|
+
const absolutePath = join25(dir, entry.name);
|
|
8375
8963
|
if (entry.isDirectory()) {
|
|
8376
8964
|
walk(absolutePath);
|
|
8377
8965
|
} else if (entry.isFile()) {
|
|
@@ -8398,13 +8986,13 @@ function assertValidExtractionCacheCompleteSentinel(path) {
|
|
|
8398
8986
|
}
|
|
8399
8987
|
}
|
|
8400
8988
|
function validateExtractionCacheCompleteSentinels(executionPlanDir) {
|
|
8401
|
-
const videoRoot =
|
|
8402
|
-
if (!
|
|
8989
|
+
const videoRoot = join25(executionPlanDir, "video-frames");
|
|
8990
|
+
if (!existsSync17(videoRoot)) return;
|
|
8403
8991
|
for (const videoEntry of readdirSync7(videoRoot, { withFileTypes: true })) {
|
|
8404
8992
|
if (!videoEntry.isDirectory()) continue;
|
|
8405
|
-
const videoDir =
|
|
8993
|
+
const videoDir = join25(videoRoot, videoEntry.name);
|
|
8406
8994
|
if (readdirSync7(videoDir).includes(EXTRACTION_CACHE_COMPLETE_SENTINEL)) {
|
|
8407
|
-
const sentinelPath =
|
|
8995
|
+
const sentinelPath = join25(videoDir, EXTRACTION_CACHE_COMPLETE_SENTINEL);
|
|
8408
8996
|
assertValidExtractionCacheCompleteSentinel(sentinelPath);
|
|
8409
8997
|
}
|
|
8410
8998
|
}
|
|
@@ -8421,31 +9009,37 @@ function resolveExtractedVideoOutputDir(planDir, videoId) {
|
|
|
8421
9009
|
}
|
|
8422
9010
|
return outputDir;
|
|
8423
9011
|
}
|
|
8424
|
-
function artifactTargets(path, videoDependencies,
|
|
9012
|
+
function artifactTargets(path, videoDependencies, chunkAudio) {
|
|
8425
9013
|
if (isPlanAudioArtifactPath(path)) {
|
|
8426
|
-
|
|
9014
|
+
if (chunkAudio === "none") return { chunks: [], assembler: true };
|
|
9015
|
+
return { chunks: chunkAudio === "whole" ? "all" : [], assembler: false };
|
|
8427
9016
|
}
|
|
9017
|
+
const slice = chunkAudioSliceTarget(path);
|
|
9018
|
+
if (slice !== null) return { chunks: slice === "all" ? "all" : [slice], assembler: false };
|
|
8428
9019
|
if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {
|
|
8429
9020
|
return { chunks: "all", assembler: true };
|
|
8430
9021
|
}
|
|
8431
|
-
if (path
|
|
8432
|
-
return { chunks: "all", assembler: false };
|
|
8433
|
-
}
|
|
8434
|
-
if (path.startsWith("video-frames/") || path.startsWith("video-sources/")) {
|
|
9022
|
+
if (path.startsWith("video-frames/") || path.startsWith("video-sources/") || path.startsWith("compiled/") && videoDependencies?.has(path)) {
|
|
8435
9023
|
return {
|
|
8436
9024
|
chunks: videoDependencies === null ? "all" : videoDependencies.get(path) ?? [],
|
|
8437
9025
|
assembler: false
|
|
8438
9026
|
};
|
|
8439
9027
|
}
|
|
9028
|
+
if (path === "meta/composition.json" || path === "meta/videos.json" || path.startsWith("compiled/")) {
|
|
9029
|
+
return { chunks: "all", assembler: false };
|
|
9030
|
+
}
|
|
8440
9031
|
return { chunks: "all", assembler: true };
|
|
8441
9032
|
}
|
|
9033
|
+
function isPlanV2DeferredSourceArtifact(artifact) {
|
|
9034
|
+
return artifact.path.startsWith("compiled/") && artifact.chunks !== "all";
|
|
9035
|
+
}
|
|
8442
9036
|
function listVideoFramePaths(executionPlanDir, videos) {
|
|
8443
9037
|
return videos.extracted.map((video) => {
|
|
8444
9038
|
const outputDir = resolveExtractedVideoOutputDir(executionPlanDir, video.videoId);
|
|
8445
9039
|
if (video.deferredRange) {
|
|
8446
9040
|
const framePaths2 = /* @__PURE__ */ new Map();
|
|
8447
9041
|
for (let index = 0; index < video.totalFrames; index++) {
|
|
8448
|
-
framePaths2.set(index,
|
|
9042
|
+
framePaths2.set(index, join25(outputDir, String(index)));
|
|
8449
9043
|
}
|
|
8450
9044
|
return { ...video, outputDir, framePaths: framePaths2, ownedByLookup: false };
|
|
8451
9045
|
}
|
|
@@ -8453,7 +9047,7 @@ function listVideoFramePaths(executionPlanDir, videos) {
|
|
|
8453
9047
|
const framePaths = /* @__PURE__ */ new Map();
|
|
8454
9048
|
for (const frameName of frameNames) {
|
|
8455
9049
|
if (frameName === EXTRACTION_CACHE_COMPLETE_SENTINEL) {
|
|
8456
|
-
assertValidExtractionCacheCompleteSentinel(
|
|
9050
|
+
assertValidExtractionCacheCompleteSentinel(join25(outputDir, frameName));
|
|
8457
9051
|
continue;
|
|
8458
9052
|
}
|
|
8459
9053
|
const match = /(\d+)(?=\.[^.]+$)/.exec(frameName);
|
|
@@ -8472,7 +9066,7 @@ function listVideoFramePaths(executionPlanDir, videos) {
|
|
|
8472
9066
|
`duplicate extracted frame index ${frameIndex} in ${outputDir}`
|
|
8473
9067
|
);
|
|
8474
9068
|
}
|
|
8475
|
-
framePaths.set(frameIndex,
|
|
9069
|
+
framePaths.set(frameIndex, join25(outputDir, frameName));
|
|
8476
9070
|
}
|
|
8477
9071
|
return {
|
|
8478
9072
|
...video,
|
|
@@ -8493,11 +9087,11 @@ function parsePlanVideosJson2(value) {
|
|
|
8493
9087
|
}
|
|
8494
9088
|
}
|
|
8495
9089
|
function materializeExtractedVideoDirectories(planDir) {
|
|
8496
|
-
const videosPath =
|
|
8497
|
-
if (!
|
|
9090
|
+
const videosPath = join25(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9091
|
+
if (!existsSync17(videosPath)) return;
|
|
8498
9092
|
const videos = parsePlanVideosJson2(readJsonFile(videosPath, PLAN_VIDEOS_META_RELATIVE_PATH));
|
|
8499
9093
|
for (const video of videos.extracted) {
|
|
8500
|
-
|
|
9094
|
+
mkdirSync12(resolveExtractedVideoOutputDir(planDir, video.videoId), { recursive: true });
|
|
8501
9095
|
}
|
|
8502
9096
|
}
|
|
8503
9097
|
function parseChunkSlices(value) {
|
|
@@ -8522,13 +9116,13 @@ function parseChunkSlices(value) {
|
|
|
8522
9116
|
});
|
|
8523
9117
|
}
|
|
8524
9118
|
function buildVideoChunkDependencies(executionPlanDir, dimensions) {
|
|
8525
|
-
const videoRoot =
|
|
8526
|
-
const hasExtractedFrames =
|
|
8527
|
-
const videosPath =
|
|
8528
|
-
if (!
|
|
9119
|
+
const videoRoot = join25(executionPlanDir, "video-frames");
|
|
9120
|
+
const hasExtractedFrames = existsSync17(videoRoot) && listFiles(videoRoot).length > 0;
|
|
9121
|
+
const videosPath = join25(executionPlanDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9122
|
+
if (!existsSync17(videosPath)) {
|
|
8529
9123
|
return hasExtractedFrames ? { mode: "full-source-pack", dependencies: null } : { mode: "exact-rendered-frames", dependencies: /* @__PURE__ */ new Map() };
|
|
8530
9124
|
}
|
|
8531
|
-
const chunksPath =
|
|
9125
|
+
const chunksPath = join25(executionPlanDir, "meta", "chunks.json");
|
|
8532
9126
|
const videos = readJsonFile(videosPath, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
8533
9127
|
const chunks = readJsonFile(chunksPath, "meta/chunks.json");
|
|
8534
9128
|
const parsedVideos = parsePlanVideosJson2(videos);
|
|
@@ -8552,850 +9146,552 @@ function buildVideoChunkDependencies(executionPlanDir, dimensions) {
|
|
|
8552
9146
|
chunks: parsedChunks,
|
|
8553
9147
|
fpsNum,
|
|
8554
9148
|
fpsDen,
|
|
8555
|
-
deferredSources
|
|
8556
|
-
});
|
|
8557
|
-
return {
|
|
8558
|
-
mode: deferredSources.size > 0 ? "source-extract" : "exact-rendered-frames",
|
|
8559
|
-
dependencies
|
|
8560
|
-
};
|
|
8561
|
-
}
|
|
8562
|
-
function collectChunkVideoDependencies(input) {
|
|
8563
|
-
const mutable = /* @__PURE__ */ new Map();
|
|
8564
|
-
for (const chunk of input.chunks) {
|
|
8565
|
-
for (let frame = chunk.startFrame; frame < chunk.endFrame; frame++) {
|
|
8566
|
-
const globalTime = frame * input.fpsDen / input.fpsNum;
|
|
8567
|
-
for (const [videoId, payload] of input.table.getActiveFramePayloads(globalTime)) {
|
|
8568
|
-
const path = input.deferredSources.get(videoId) ?? relative6(resolve8(input.executionPlanDir), payload.framePath).split(sep3).join("/");
|
|
8569
|
-
assertSafeRelativePath(path);
|
|
8570
|
-
const indexes = mutable.get(path) ?? /* @__PURE__ */ new Set();
|
|
8571
|
-
indexes.add(chunk.index);
|
|
8572
|
-
mutable.set(path, indexes);
|
|
8573
|
-
}
|
|
8574
|
-
}
|
|
8575
|
-
}
|
|
8576
|
-
return new Map([...mutable].map(([path, indexes]) => [path, [...indexes].sort((a, b) => a - b)]));
|
|
8577
|
-
}
|
|
8578
|
-
function manifestPayload(manifest) {
|
|
8579
|
-
return manifest;
|
|
8580
|
-
}
|
|
8581
|
-
function computeManifestHash(manifest) {
|
|
8582
|
-
return sha256Hex(`${PLAN_V2_HASH_PREFIX}${canonicalJsonStringify(manifestPayload(manifest))}`);
|
|
8583
|
-
}
|
|
8584
|
-
function writeBlob(sourcePath, destinationPath) {
|
|
8585
|
-
if (existsSync16(destinationPath)) return;
|
|
8586
|
-
mkdirSync10(dirname10(destinationPath), { recursive: true });
|
|
8587
|
-
const temporaryDir = mkdtempSync4(join23(dirname10(destinationPath), ".plan-v2-blob-"));
|
|
8588
|
-
const temporaryPath = join23(temporaryDir, "blob");
|
|
8589
|
-
try {
|
|
8590
|
-
copyFileSync7(sourcePath, temporaryPath);
|
|
8591
|
-
renameSync5(temporaryPath, destinationPath);
|
|
8592
|
-
} finally {
|
|
8593
|
-
rmSync9(temporaryDir, { recursive: true, force: true });
|
|
8594
|
-
}
|
|
8595
|
-
}
|
|
8596
|
-
function publishBlobFor(absolutePath, digests) {
|
|
8597
|
-
return {
|
|
8598
|
-
sourcePath: absolutePath,
|
|
8599
|
-
sha256: planFileSha256(absolutePath, digests),
|
|
8600
|
-
sizeBytes: statSync6(absolutePath).size
|
|
8601
|
-
};
|
|
8602
|
-
}
|
|
8603
|
-
async function putBlobs(publisher, blobs) {
|
|
8604
|
-
const uploadSlots = 32;
|
|
8605
|
-
let next = 0;
|
|
8606
|
-
let failure;
|
|
8607
|
-
const upload = async () => {
|
|
8608
|
-
while (!failure && next < blobs.length) {
|
|
8609
|
-
const blob = blobs[next++];
|
|
8610
|
-
try {
|
|
8611
|
-
await publisher.putBlob(blob);
|
|
8612
|
-
} catch (reason) {
|
|
8613
|
-
failure ??= { reason };
|
|
8614
|
-
}
|
|
8615
|
-
}
|
|
8616
|
-
};
|
|
8617
|
-
await Promise.all(Array.from({ length: uploadSlots }, upload));
|
|
8618
|
-
if (failure) throw failure.reason;
|
|
8619
|
-
}
|
|
8620
|
-
function buildPlanV2Publication(executionPlanDir, digests) {
|
|
8621
|
-
const executionPlanPath = join23(executionPlanDir, "plan.json");
|
|
8622
|
-
if (!existsSync16(executionPlanPath)) {
|
|
8623
|
-
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
|
|
8624
|
-
}
|
|
8625
|
-
const executionPlanValue = readJsonFile(executionPlanPath, "execution plan.json");
|
|
8626
|
-
if (!isRecord3(executionPlanValue)) {
|
|
8627
|
-
throw new PlanV2IntegrityError("execution plan.json must be an object");
|
|
8628
|
-
}
|
|
8629
|
-
const executionPlan = executionPlanValue;
|
|
8630
|
-
readPlanProtocolV1(executionPlan);
|
|
8631
|
-
const executionPlanHash = executionPlan.planHash;
|
|
8632
|
-
if (!isSha256(executionPlanHash)) {
|
|
8633
|
-
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
|
|
8634
|
-
}
|
|
8635
|
-
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);
|
|
8636
|
-
if (recomputedExecutionPlanHash !== executionPlanHash) {
|
|
8637
|
-
throw new PlanV2IntegrityError(
|
|
8638
|
-
`execution plan content fingerprint does not match plan.json.planHash: expected ${executionPlanHash}, recomputed ${recomputedExecutionPlanHash}`
|
|
8639
|
-
);
|
|
8640
|
-
}
|
|
8641
|
-
const artifacts = [];
|
|
8642
|
-
const blobs = /* @__PURE__ */ new Map();
|
|
8643
|
-
const dimensions = executionPlan.dimensions;
|
|
8644
|
-
if (!isRecord3(dimensions)) {
|
|
8645
|
-
throw new PlanV2IntegrityError("execution plan.json.dimensions must be an object");
|
|
8646
|
-
}
|
|
8647
|
-
validateExtractionCacheCompleteSentinels(executionPlanDir);
|
|
8648
|
-
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
|
|
8649
|
-
const encoder = readJsonFile(join23(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");
|
|
8650
|
-
const chunksCarryAudio2 = isRecord3(encoder) && encoder.chunksCarryAudio === true;
|
|
8651
|
-
for (const file of listFiles(executionPlanDir)) {
|
|
8652
|
-
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
|
|
8653
|
-
const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunksCarryAudio2);
|
|
8654
|
-
if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) && targets.chunks !== "all" && targets.chunks.length === 0 && !targets.assembler) {
|
|
8655
|
-
continue;
|
|
8656
|
-
}
|
|
8657
|
-
const blob = publishBlobFor(file.absolutePath, digests);
|
|
8658
|
-
if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
|
|
8659
|
-
artifacts.push({
|
|
8660
|
-
path: file.path,
|
|
8661
|
-
sha256: blob.sha256,
|
|
8662
|
-
sizeBytes: blob.sizeBytes,
|
|
8663
|
-
...targets
|
|
8664
|
-
});
|
|
8665
|
-
}
|
|
8666
|
-
const base = {
|
|
8667
|
-
protocol: PLAN_PROTOCOL_V2,
|
|
8668
|
-
// Retain the established manifest key byte-for-byte. It now acts as the
|
|
8669
|
-
// wire alias for the neutral local execution-plan hash.
|
|
8670
|
-
sourcePlanV1Hash: executionPlanHash,
|
|
8671
|
-
chunkCount: readPositiveInteger2(executionPlan.chunkCount, "chunkCount"),
|
|
8672
|
-
totalFrames: readPositiveInteger2(executionPlan.totalFrames, "totalFrames"),
|
|
8673
|
-
fps: readExecutionPlanFps(dimensions),
|
|
8674
|
-
width: readPositiveInteger2(dimensions.width, "dimensions.width"),
|
|
8675
|
-
height: readPositiveInteger2(dimensions.height, "dimensions.height"),
|
|
8676
|
-
format: readDistributedFormat(dimensions.format),
|
|
8677
|
-
ffmpegVersion: readString2(executionPlan.ffmpegVersion, "ffmpegVersion"),
|
|
8678
|
-
producerVersion: readString2(executionPlan.producerVersion, "producerVersion"),
|
|
8679
|
-
limitations: { videoDependencyMode: videoDependencyPlan.mode },
|
|
8680
|
-
artifacts
|
|
8681
|
-
};
|
|
8682
|
-
return {
|
|
8683
|
-
manifest: {
|
|
8684
|
-
...base,
|
|
8685
|
-
planHash: computeManifestHash(base)
|
|
8686
|
-
},
|
|
8687
|
-
blobs: [...blobs.values()]
|
|
8688
|
-
};
|
|
8689
|
-
}
|
|
8690
|
-
function createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir) {
|
|
8691
|
-
if (existsSync16(planV2Dir)) {
|
|
8692
|
-
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
8693
|
-
}
|
|
8694
|
-
const publication = buildPlanV2Publication(executionPlanDir);
|
|
8695
|
-
mkdirSync10(dirname10(planV2Dir), { recursive: true });
|
|
8696
|
-
const tempDir = mkdtempSync4(join23(dirname10(planV2Dir), ".plan-v2-build-"));
|
|
8697
|
-
try {
|
|
8698
|
-
for (const blob of publication.blobs) {
|
|
8699
|
-
writeBlob(blob.sourcePath, planV2BlobPath(tempDir, blob.sha256));
|
|
8700
|
-
}
|
|
8701
|
-
writeFileSync7(
|
|
8702
|
-
join23(tempDir, "plan.json"),
|
|
8703
|
-
canonicalJsonStringify(publication.manifest),
|
|
8704
|
-
"utf-8"
|
|
8705
|
-
);
|
|
8706
|
-
renameSync5(tempDir, planV2Dir);
|
|
8707
|
-
return resultFromManifest(planV2Dir, publication.manifest);
|
|
8708
|
-
} catch (error) {
|
|
8709
|
-
rmSync9(tempDir, { recursive: true, force: true });
|
|
8710
|
-
throw error;
|
|
8711
|
-
}
|
|
8712
|
-
}
|
|
8713
|
-
function createPlanV2FromV1(executionPlanDir, planV2Dir) {
|
|
8714
|
-
return createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir);
|
|
8715
|
-
}
|
|
8716
|
-
async function publishPlanV2FromExecutionPlan(executionPlanDir, publisher) {
|
|
8717
|
-
return publishExecutionPlan(executionPlanDir, publisher, {});
|
|
8718
|
-
}
|
|
8719
|
-
async function publishExecutionPlan(executionPlanDir, publisher, { digests, log = defaultLogger }) {
|
|
8720
|
-
try {
|
|
8721
|
-
let stageStarted = Date.now();
|
|
8722
|
-
const publication = buildPlanV2Publication(executionPlanDir, digests);
|
|
8723
|
-
logPlanStage(log, "publication", stageStarted);
|
|
8724
|
-
stageStarted = Date.now();
|
|
8725
|
-
await putBlobs(publisher, publication.blobs);
|
|
8726
|
-
logPlanStage(log, "upload", stageStarted);
|
|
8727
|
-
stageStarted = Date.now();
|
|
8728
|
-
await publisher.commitManifest(canonicalJsonStringify(publication.manifest));
|
|
8729
|
-
logPlanStage(log, "manifest", stageStarted);
|
|
8730
|
-
return publication.manifest;
|
|
8731
|
-
} catch (error) {
|
|
8732
|
-
try {
|
|
8733
|
-
await publisher.abort();
|
|
8734
|
-
} catch {
|
|
8735
|
-
}
|
|
8736
|
-
throw error;
|
|
8737
|
-
}
|
|
8738
|
-
}
|
|
8739
|
-
async function publishPlanV2FromV1(executionPlanDir, publisher) {
|
|
8740
|
-
return publishPlanV2FromExecutionPlan(executionPlanDir, publisher);
|
|
8741
|
-
}
|
|
8742
|
-
async function planV2WithPublisher(projectDir, config, publisher, options = {}) {
|
|
8743
|
-
const stagingRoot = mkdtempSync4(join23(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
|
|
8744
|
-
const digests = /* @__PURE__ */ new Map();
|
|
8745
|
-
try {
|
|
8746
|
-
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
|
|
8747
|
-
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
|
|
8748
|
-
deferVideoExtraction: true,
|
|
8749
|
-
digests,
|
|
8750
|
-
// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
|
|
8751
|
-
whileAudioMixes: () => putBlobs(
|
|
8752
|
-
publisher,
|
|
8753
|
-
listFiles(join23(stagingRoot, "compiled")).map(
|
|
8754
|
-
(file) => publishBlobFor(file.absolutePath, digests)
|
|
8755
|
-
)
|
|
8756
|
-
),
|
|
8757
|
-
chunksCarryAudio: options.chunksCarryAudio
|
|
8758
|
-
});
|
|
8759
|
-
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
|
|
8760
|
-
} catch (error) {
|
|
8761
|
-
try {
|
|
8762
|
-
await publisher.abort();
|
|
8763
|
-
} catch {
|
|
8764
|
-
}
|
|
8765
|
-
throw error;
|
|
8766
|
-
} finally {
|
|
8767
|
-
rmSync9(stagingRoot, { recursive: true, force: true });
|
|
8768
|
-
}
|
|
8769
|
-
}
|
|
8770
|
-
async function planV2(projectDir, config, planV2Dir) {
|
|
8771
|
-
if (existsSync16(planV2Dir)) {
|
|
8772
|
-
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
8773
|
-
}
|
|
8774
|
-
const publisher = new LocalPlanV2ArtifactPublisher(planV2Dir);
|
|
8775
|
-
const manifest = await planV2WithPublisher(projectDir, config, publisher, {
|
|
8776
|
-
stagingParentDir: dirname10(planV2Dir)
|
|
8777
|
-
});
|
|
8778
|
-
return resultFromManifest(planV2Dir, manifest);
|
|
8779
|
-
}
|
|
8780
|
-
function resultFromManifest(planV2Dir, manifest) {
|
|
8781
|
-
return {
|
|
8782
|
-
planDir: planV2Dir,
|
|
8783
|
-
manifestPath: join23(planV2Dir, "plan.json"),
|
|
8784
|
-
planProtocol: PLAN_PROTOCOL_V2,
|
|
8785
|
-
planHash: manifest.planHash,
|
|
8786
|
-
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
8787
|
-
chunkCount: manifest.chunkCount,
|
|
8788
|
-
totalFrames: manifest.totalFrames,
|
|
8789
|
-
fps: manifest.fps,
|
|
8790
|
-
width: manifest.width,
|
|
8791
|
-
height: manifest.height,
|
|
8792
|
-
format: manifest.format,
|
|
8793
|
-
ffmpegVersion: manifest.ffmpegVersion,
|
|
8794
|
-
producerVersion: manifest.producerVersion,
|
|
8795
|
-
limitations: manifest.limitations
|
|
9149
|
+
deferredSources
|
|
9150
|
+
});
|
|
9151
|
+
return {
|
|
9152
|
+
mode: deferredSources.size > 0 ? "source-extract" : "exact-rendered-frames",
|
|
9153
|
+
dependencies
|
|
8796
9154
|
};
|
|
8797
9155
|
}
|
|
8798
|
-
function
|
|
8799
|
-
|
|
8800
|
-
|
|
8801
|
-
|
|
8802
|
-
|
|
8803
|
-
|
|
8804
|
-
|
|
8805
|
-
|
|
8806
|
-
|
|
8807
|
-
|
|
8808
|
-
|
|
8809
|
-
|
|
8810
|
-
|
|
8811
|
-
return value;
|
|
8812
|
-
}
|
|
8813
|
-
function readNonNegativeInteger2(value, field) {
|
|
8814
|
-
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
8815
|
-
throw new PlanV2IntegrityError(`${field} must be a non-negative integer`);
|
|
9156
|
+
function collectChunkVideoDependencies(input) {
|
|
9157
|
+
const mutable = /* @__PURE__ */ new Map();
|
|
9158
|
+
for (const chunk of input.chunks) {
|
|
9159
|
+
for (let frame = chunk.startFrame; frame < chunk.endFrame; frame++) {
|
|
9160
|
+
const globalTime = frame * input.fpsDen / input.fpsNum;
|
|
9161
|
+
for (const [videoId, payload] of input.table.getActiveFramePayloads(globalTime)) {
|
|
9162
|
+
const path = input.deferredSources.get(videoId) ?? relative6(resolve8(input.executionPlanDir), payload.framePath).split(sep3).join("/");
|
|
9163
|
+
assertSafeRelativePath(path);
|
|
9164
|
+
const indexes = mutable.get(path) ?? /* @__PURE__ */ new Set();
|
|
9165
|
+
indexes.add(chunk.index);
|
|
9166
|
+
mutable.set(path, indexes);
|
|
9167
|
+
}
|
|
9168
|
+
}
|
|
8816
9169
|
}
|
|
8817
|
-
return
|
|
9170
|
+
return new Map([...mutable].map(([path, indexes]) => [path, [...indexes].sort((a, b) => a - b)]));
|
|
8818
9171
|
}
|
|
8819
|
-
function
|
|
8820
|
-
|
|
8821
|
-
throw new PlanV2IntegrityError(
|
|
8822
|
-
`dimensions.fpsNum must be one of ${DISTRIBUTED_FPS.join(", ")}`
|
|
8823
|
-
);
|
|
8824
|
-
}
|
|
8825
|
-
return value;
|
|
9172
|
+
function manifestPayload(manifest) {
|
|
9173
|
+
return manifest;
|
|
8826
9174
|
}
|
|
8827
|
-
function
|
|
8828
|
-
|
|
8829
|
-
if (fpsDen !== 1) {
|
|
8830
|
-
throw new PlanV2IntegrityError("dimensions.fpsDen must be 1 for plan v2");
|
|
8831
|
-
}
|
|
8832
|
-
return readSupportedFps(dimensions.fpsNum);
|
|
9175
|
+
function computeManifestHash(manifest) {
|
|
9176
|
+
return sha256Hex(`${PLAN_V2_HASH_PREFIX}${canonicalJsonStringify(manifestPayload(manifest))}`);
|
|
8833
9177
|
}
|
|
8834
|
-
function
|
|
8835
|
-
if (
|
|
8836
|
-
|
|
9178
|
+
function writeBlob(sourcePath, destinationPath) {
|
|
9179
|
+
if (existsSync17(destinationPath)) return;
|
|
9180
|
+
mkdirSync12(dirname11(destinationPath), { recursive: true });
|
|
9181
|
+
const temporaryDir = mkdtempSync5(join25(dirname11(destinationPath), ".plan-v2-blob-"));
|
|
9182
|
+
const temporaryPath = join25(temporaryDir, "blob");
|
|
9183
|
+
try {
|
|
9184
|
+
copyFileSync7(sourcePath, temporaryPath);
|
|
9185
|
+
renameSync6(temporaryPath, destinationPath);
|
|
9186
|
+
} finally {
|
|
9187
|
+
rmSync10(temporaryDir, { recursive: true, force: true });
|
|
8837
9188
|
}
|
|
8838
|
-
return value;
|
|
8839
9189
|
}
|
|
8840
|
-
function
|
|
8841
|
-
if (!isRecord3(value)) throw new PlanV2IntegrityError(`artifacts[${index}] must be an object`);
|
|
8842
|
-
const path = readString2(value.path, `artifacts[${index}].path`);
|
|
8843
|
-
assertSafeRelativePath(path);
|
|
8844
|
-
if (!isSha256(value.sha256)) {
|
|
8845
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].sha256 must be a lowercase sha256`);
|
|
8846
|
-
}
|
|
8847
|
-
const sizeBytes = value.sizeBytes;
|
|
8848
|
-
if (typeof sizeBytes !== "number" || !Number.isInteger(sizeBytes) || sizeBytes < 0) {
|
|
8849
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].sizeBytes must be a non-negative integer`);
|
|
8850
|
-
}
|
|
8851
|
-
let chunks;
|
|
8852
|
-
if (value.chunks === "all") {
|
|
8853
|
-
chunks = "all";
|
|
8854
|
-
} else if (isNonNegativeIntegerArray(value.chunks)) {
|
|
8855
|
-
chunks = value.chunks;
|
|
8856
|
-
} else {
|
|
8857
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].chunks is invalid`);
|
|
8858
|
-
}
|
|
8859
|
-
if (typeof value.assembler !== "boolean") {
|
|
8860
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].assembler must be boolean`);
|
|
8861
|
-
}
|
|
9190
|
+
function publishBlobFor(absolutePath, digests) {
|
|
8862
9191
|
return {
|
|
8863
|
-
|
|
8864
|
-
sha256:
|
|
8865
|
-
sizeBytes
|
|
8866
|
-
chunks,
|
|
8867
|
-
assembler: value.assembler
|
|
8868
|
-
};
|
|
8869
|
-
}
|
|
8870
|
-
function parsePlanV2Manifest(value) {
|
|
8871
|
-
if (!isRecord3(value)) throw new PlanV2IntegrityError("manifest must be an object");
|
|
8872
|
-
if (!isRecord3(value.protocol) || value.protocol.schemaVersion !== PLAN_PROTOCOL_V2.schemaVersion || value.protocol.artifactLayout !== PLAN_PROTOCOL_V2.artifactLayout || value.protocol.hashSchema !== PLAN_PROTOCOL_V2.hashSchema) {
|
|
8873
|
-
throw new PlanV2IntegrityError("manifest protocol is not the supported v2 descriptor");
|
|
8874
|
-
}
|
|
8875
|
-
if (!isSha256(value.planHash) || !isSha256(value.sourcePlanV1Hash)) {
|
|
8876
|
-
throw new PlanV2IntegrityError("manifest hashes must be lowercase sha256 digests");
|
|
8877
|
-
}
|
|
8878
|
-
if (!Array.isArray(value.artifacts)) throw new PlanV2IntegrityError("artifacts must be an array");
|
|
8879
|
-
const artifacts = value.artifacts.map(parseArtifact);
|
|
8880
|
-
const paths = /* @__PURE__ */ new Set();
|
|
8881
|
-
for (const artifact of artifacts) {
|
|
8882
|
-
if (paths.has(artifact.path))
|
|
8883
|
-
throw new PlanV2IntegrityError(`duplicate artifact path: ${artifact.path}`);
|
|
8884
|
-
paths.add(artifact.path);
|
|
8885
|
-
}
|
|
8886
|
-
if (!paths.has("plan.json"))
|
|
8887
|
-
throw new PlanV2IntegrityError("manifest must include the local execution plan.json artifact");
|
|
8888
|
-
if (!isRecord3(value.limitations) || value.limitations.videoDependencyMode !== "exact-rendered-frames" && value.limitations.videoDependencyMode !== "full-source-pack" && value.limitations.videoDependencyMode !== "source-extract") {
|
|
8889
|
-
throw new PlanV2IntegrityError("unsupported video dependency mode");
|
|
8890
|
-
}
|
|
8891
|
-
const parsed = {
|
|
8892
|
-
protocol: PLAN_PROTOCOL_V2,
|
|
8893
|
-
planHash: value.planHash,
|
|
8894
|
-
sourcePlanV1Hash: value.sourcePlanV1Hash,
|
|
8895
|
-
chunkCount: readPositiveInteger2(value.chunkCount, "chunkCount"),
|
|
8896
|
-
totalFrames: readPositiveInteger2(value.totalFrames, "totalFrames"),
|
|
8897
|
-
fps: readSupportedFps(value.fps),
|
|
8898
|
-
width: readPositiveInteger2(value.width, "width"),
|
|
8899
|
-
height: readPositiveInteger2(value.height, "height"),
|
|
8900
|
-
format: readDistributedFormat(value.format),
|
|
8901
|
-
ffmpegVersion: readString2(value.ffmpegVersion, "ffmpegVersion"),
|
|
8902
|
-
producerVersion: readString2(value.producerVersion, "producerVersion"),
|
|
8903
|
-
limitations: { videoDependencyMode: value.limitations.videoDependencyMode },
|
|
8904
|
-
artifacts
|
|
9192
|
+
sourcePath: absolutePath,
|
|
9193
|
+
sha256: planFileSha256(absolutePath, digests),
|
|
9194
|
+
sizeBytes: statSync6(absolutePath).size
|
|
8905
9195
|
};
|
|
8906
|
-
const { planHash: _planHash, ...payload } = parsed;
|
|
8907
|
-
const expectedHash = computeManifestHash(payload);
|
|
8908
|
-
if (expectedHash !== parsed.planHash) {
|
|
8909
|
-
throw new PlanV2IntegrityError("manifest integrity hash mismatch");
|
|
8910
|
-
}
|
|
8911
|
-
for (const [index, artifact] of parsed.artifacts.entries()) {
|
|
8912
|
-
if (artifact.chunks !== "all" && artifact.chunks.some((chunkIndex) => chunkIndex >= parsed.chunkCount)) {
|
|
8913
|
-
throw new PlanV2IntegrityError(`artifacts[${index}].chunks contains an out-of-range index`);
|
|
8914
|
-
}
|
|
8915
|
-
}
|
|
8916
|
-
return parsed;
|
|
8917
9196
|
}
|
|
8918
|
-
function
|
|
8919
|
-
|
|
8920
|
-
if (!existsSync16(manifestPath)) {
|
|
8921
|
-
throw new PlanV2IntegrityError(`missing v2 manifest: ${manifestPath}`);
|
|
8922
|
-
}
|
|
8923
|
-
const value = readJsonFile(manifestPath, "v2 manifest");
|
|
8924
|
-
return parsePlanV2Manifest(value);
|
|
9197
|
+
async function putBlobs(publisher, blobs) {
|
|
9198
|
+
await putTasks(blobs.map((blob) => () => publisher.putBlob(blob)));
|
|
8925
9199
|
}
|
|
8926
|
-
function
|
|
8927
|
-
|
|
8928
|
-
|
|
8929
|
-
|
|
9200
|
+
async function putTasks(tasks) {
|
|
9201
|
+
const uploadSlots = 32;
|
|
9202
|
+
let next = 0;
|
|
9203
|
+
let failure;
|
|
9204
|
+
const upload = async () => {
|
|
9205
|
+
while (!failure && next < tasks.length) {
|
|
9206
|
+
const task = tasks[next++];
|
|
9207
|
+
try {
|
|
9208
|
+
await task();
|
|
9209
|
+
} catch (reason) {
|
|
9210
|
+
failure ??= { reason };
|
|
9211
|
+
}
|
|
8930
9212
|
}
|
|
8931
|
-
|
|
8932
|
-
|
|
8933
|
-
|
|
8934
|
-
}
|
|
8935
|
-
return manifest.artifacts.filter((artifact) => artifact.assembler);
|
|
9213
|
+
};
|
|
9214
|
+
await Promise.all(Array.from({ length: uploadSlots }, upload));
|
|
9215
|
+
if (failure) throw failure.reason;
|
|
8936
9216
|
}
|
|
8937
|
-
function
|
|
8938
|
-
const
|
|
8939
|
-
if (!
|
|
8940
|
-
throw new PlanV2IntegrityError(`missing
|
|
9217
|
+
function buildPlanV2Publication(executionPlanDir, digests) {
|
|
9218
|
+
const executionPlanPath = join25(executionPlanDir, "plan.json");
|
|
9219
|
+
if (!existsSync17(executionPlanPath)) {
|
|
9220
|
+
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
|
|
8941
9221
|
}
|
|
8942
|
-
const
|
|
8943
|
-
if (!
|
|
8944
|
-
throw new PlanV2IntegrityError(
|
|
9222
|
+
const executionPlanValue = readJsonFile(executionPlanPath, "execution plan.json");
|
|
9223
|
+
if (!isRecord3(executionPlanValue)) {
|
|
9224
|
+
throw new PlanV2IntegrityError("execution plan.json must be an object");
|
|
8945
9225
|
}
|
|
8946
|
-
|
|
8947
|
-
|
|
9226
|
+
const executionPlan = executionPlanValue;
|
|
9227
|
+
readPlanProtocolV1(executionPlan);
|
|
9228
|
+
const executionPlanHash = executionPlan.planHash;
|
|
9229
|
+
if (!isSha256(executionPlanHash)) {
|
|
9230
|
+
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
|
|
8948
9231
|
}
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
8952
|
-
|
|
8953
|
-
|
|
9232
|
+
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);
|
|
9233
|
+
if (recomputedExecutionPlanHash !== executionPlanHash) {
|
|
9234
|
+
throw new PlanV2IntegrityError(
|
|
9235
|
+
`execution plan content fingerprint does not match plan.json.planHash: expected ${executionPlanHash}, recomputed ${recomputedExecutionPlanHash}`
|
|
9236
|
+
);
|
|
8954
9237
|
}
|
|
8955
|
-
const
|
|
8956
|
-
const
|
|
8957
|
-
const
|
|
8958
|
-
|
|
8959
|
-
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
const
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
|
|
9238
|
+
const artifacts = [];
|
|
9239
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
9240
|
+
const dimensions = executionPlan.dimensions;
|
|
9241
|
+
if (!isRecord3(dimensions)) {
|
|
9242
|
+
throw new PlanV2IntegrityError("execution plan.json.dimensions must be an object");
|
|
9243
|
+
}
|
|
9244
|
+
validateExtractionCacheCompleteSentinels(executionPlanDir);
|
|
9245
|
+
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
|
|
9246
|
+
const encoder = readJsonFile(join25(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");
|
|
9247
|
+
const files = listFiles(executionPlanDir);
|
|
9248
|
+
const chunkAudio = !isRecord3(encoder) || encoder.chunksCarryAudio !== true ? "none" : files.some((file) => chunkAudioSliceTarget(file.path) !== null) ? "sliced" : "whole";
|
|
9249
|
+
for (const file of files) {
|
|
9250
|
+
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
|
|
9251
|
+
const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunkAudio);
|
|
9252
|
+
if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) && targets.chunks !== "all" && targets.chunks.length === 0 && !targets.assembler) {
|
|
9253
|
+
continue;
|
|
8969
9254
|
}
|
|
8970
|
-
|
|
8971
|
-
|
|
8972
|
-
|
|
8973
|
-
|
|
8974
|
-
|
|
8975
|
-
|
|
8976
|
-
|
|
8977
|
-
|
|
8978
|
-
rmSync9(tempDir, { recursive: true, force: true });
|
|
8979
|
-
throw error;
|
|
9255
|
+
const blob = publishBlobFor(file.absolutePath, digests);
|
|
9256
|
+
if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
|
|
9257
|
+
artifacts.push({
|
|
9258
|
+
path: file.path,
|
|
9259
|
+
sha256: blob.sha256,
|
|
9260
|
+
sizeBytes: blob.sizeBytes,
|
|
9261
|
+
...targets
|
|
9262
|
+
});
|
|
8980
9263
|
}
|
|
8981
|
-
|
|
8982
|
-
|
|
8983
|
-
|
|
8984
|
-
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
|
|
8988
|
-
|
|
8989
|
-
|
|
8990
|
-
|
|
8991
|
-
|
|
9264
|
+
const storedBlobs = /* @__PURE__ */ new Map();
|
|
9265
|
+
for (const remote of readPlanRemoteFiles(executionPlanDir)) {
|
|
9266
|
+
const { sha256, sizeBytes } = remote;
|
|
9267
|
+
if (!blobs.has(sha256)) storedBlobs.set(sha256, { sha256, sizeBytes });
|
|
9268
|
+
artifacts.push({
|
|
9269
|
+
...remote,
|
|
9270
|
+
...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunkAudio)
|
|
9271
|
+
});
|
|
9272
|
+
}
|
|
9273
|
+
const base = {
|
|
9274
|
+
protocol: PLAN_PROTOCOL_V2,
|
|
9275
|
+
// Retain the established manifest key byte-for-byte. It now acts as the
|
|
9276
|
+
// wire alias for the neutral local execution-plan hash.
|
|
9277
|
+
sourcePlanV1Hash: executionPlanHash,
|
|
9278
|
+
chunkCount: readPositiveInteger2(executionPlan.chunkCount, "chunkCount"),
|
|
9279
|
+
totalFrames: readPositiveInteger2(executionPlan.totalFrames, "totalFrames"),
|
|
9280
|
+
fps: readExecutionPlanFps(dimensions),
|
|
9281
|
+
width: readPositiveInteger2(dimensions.width, "dimensions.width"),
|
|
9282
|
+
height: readPositiveInteger2(dimensions.height, "dimensions.height"),
|
|
9283
|
+
format: readDistributedFormat(dimensions.format),
|
|
9284
|
+
ffmpegVersion: readString2(executionPlan.ffmpegVersion, "ffmpegVersion"),
|
|
9285
|
+
producerVersion: readString2(executionPlan.producerVersion, "producerVersion"),
|
|
9286
|
+
limitations: { videoDependencyMode: videoDependencyPlan.mode },
|
|
9287
|
+
artifacts
|
|
9288
|
+
};
|
|
9289
|
+
return {
|
|
9290
|
+
manifest: {
|
|
9291
|
+
...base,
|
|
9292
|
+
planHash: computeManifestHash(base)
|
|
9293
|
+
},
|
|
9294
|
+
blobs: [...blobs.values()],
|
|
9295
|
+
storedBlobs: [...storedBlobs.values()]
|
|
8992
9296
|
};
|
|
8993
9297
|
}
|
|
8994
|
-
function
|
|
8995
|
-
|
|
8996
|
-
|
|
8997
|
-
const marker = readJsonFile(markerPath, "materialization marker");
|
|
8998
|
-
if (!isRecord3(marker) || !isRecord3(marker.manifest) || !isRecord3(marker.target)) {
|
|
8999
|
-
throw new PlanV2IntegrityError("malformed materialization marker");
|
|
9298
|
+
function createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir) {
|
|
9299
|
+
if (existsSync17(planV2Dir)) {
|
|
9300
|
+
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
9000
9301
|
}
|
|
9001
|
-
const
|
|
9002
|
-
if (
|
|
9003
|
-
throw new PlanV2IntegrityError(
|
|
9004
|
-
"materialization target does not match requested execution role"
|
|
9005
|
-
);
|
|
9302
|
+
const publication = buildPlanV2Publication(executionPlanDir);
|
|
9303
|
+
if (publication.storedBlobs.length > 0) {
|
|
9304
|
+
throw new PlanV2IntegrityError("a local plan v2 directory cannot reference remote files");
|
|
9006
9305
|
}
|
|
9007
|
-
|
|
9008
|
-
const
|
|
9009
|
-
|
|
9010
|
-
const
|
|
9011
|
-
|
|
9012
|
-
throw new PlanV2IntegrityError(
|
|
9013
|
-
`materialized artifact missing or truncated: ${artifact.path}`
|
|
9014
|
-
);
|
|
9015
|
-
}
|
|
9016
|
-
if (sha256File(path) !== artifact.sha256) {
|
|
9017
|
-
throw new PlanV2IntegrityError(`materialized artifact hash mismatch: ${artifact.path}`);
|
|
9306
|
+
mkdirSync12(dirname11(planV2Dir), { recursive: true });
|
|
9307
|
+
const tempDir = mkdtempSync5(join25(dirname11(planV2Dir), ".plan-v2-build-"));
|
|
9308
|
+
try {
|
|
9309
|
+
for (const blob of publication.blobs) {
|
|
9310
|
+
writeBlob(blob.sourcePath, planV2BlobPath(tempDir, blob.sha256));
|
|
9018
9311
|
}
|
|
9312
|
+
writeFileSync8(
|
|
9313
|
+
join25(tempDir, "plan.json"),
|
|
9314
|
+
canonicalJsonStringify(publication.manifest),
|
|
9315
|
+
"utf-8"
|
|
9316
|
+
);
|
|
9317
|
+
renameSync6(tempDir, planV2Dir);
|
|
9318
|
+
return resultFromManifest(planV2Dir, publication.manifest);
|
|
9319
|
+
} catch (error) {
|
|
9320
|
+
rmSync10(tempDir, { recursive: true, force: true });
|
|
9321
|
+
throw error;
|
|
9019
9322
|
}
|
|
9020
|
-
return manifest;
|
|
9021
|
-
}
|
|
9022
|
-
|
|
9023
|
-
// src/services/distributed/planV2Execution.ts
|
|
9024
|
-
import { mkdtempSync as mkdtempSync7, rmSync as rmSync13 } from "node:fs";
|
|
9025
|
-
import { tmpdir as tmpdir2 } from "node:os";
|
|
9026
|
-
import { join as join27 } from "node:path";
|
|
9027
|
-
|
|
9028
|
-
// src/services/distributed/assemble.ts
|
|
9029
|
-
import {
|
|
9030
|
-
cpSync as cpSync3,
|
|
9031
|
-
existsSync as existsSync17,
|
|
9032
|
-
mkdirSync as mkdirSync12,
|
|
9033
|
-
mkdtempSync as mkdtempSync6,
|
|
9034
|
-
readFileSync as readFileSync10,
|
|
9035
|
-
readdirSync as readdirSync8,
|
|
9036
|
-
renameSync as renameSync7,
|
|
9037
|
-
rmSync as rmSync11,
|
|
9038
|
-
statSync as statSync7,
|
|
9039
|
-
writeFileSync as writeFileSync8
|
|
9040
|
-
} from "node:fs";
|
|
9041
|
-
import { dirname as dirname12, join as join25 } from "node:path";
|
|
9042
|
-
import {
|
|
9043
|
-
appendRenderProvenanceArgs,
|
|
9044
|
-
describeFfmpegFailure as describeFfmpegFailure2,
|
|
9045
|
-
getEncoderPreset as getEncoderPreset5,
|
|
9046
|
-
MIXED_AUDIO_FILENAME as MIXED_AUDIO_FILENAME4,
|
|
9047
|
-
muxVideoWithAudio as muxVideoWithAudio2,
|
|
9048
|
-
resolveConfig as resolveConfig4,
|
|
9049
|
-
runFfmpeg as runFfmpeg5
|
|
9050
|
-
} from "@hyperframes/engine";
|
|
9051
|
-
import { fpsToFfmpegArg as fpsToFfmpegArg3 } from "@hyperframes/core";
|
|
9052
|
-
|
|
9053
|
-
// src/services/distributed/mp4Stitch.ts
|
|
9054
|
-
import {
|
|
9055
|
-
createReadStream as createReadStream4,
|
|
9056
|
-
createWriteStream,
|
|
9057
|
-
mkdirSync as mkdirSync11,
|
|
9058
|
-
mkdtempSync as mkdtempSync5,
|
|
9059
|
-
renameSync as renameSync6,
|
|
9060
|
-
rmSync as rmSync10
|
|
9061
|
-
} from "node:fs";
|
|
9062
|
-
import { open } from "node:fs/promises";
|
|
9063
|
-
import { dirname as dirname11, join as join24 } from "node:path";
|
|
9064
|
-
import { finished, pipeline } from "node:stream/promises";
|
|
9065
|
-
function boxAt(src, off, at, limit) {
|
|
9066
|
-
let size = src.readUInt32BE(off);
|
|
9067
|
-
let bodyStart = at + 8;
|
|
9068
|
-
if (size === 1) {
|
|
9069
|
-
size = Number(src.readBigUInt64BE(off + 8));
|
|
9070
|
-
bodyStart = at + 16;
|
|
9071
|
-
}
|
|
9072
|
-
if (size < 8 || at + size > limit) throw new Error(`[mp4Stitch] malformed box at ${at}`);
|
|
9073
|
-
return { type: src.toString("latin1", off + 4, off + 8), start: at, bodyStart, end: at + size };
|
|
9074
|
-
}
|
|
9075
|
-
function childBoxes(buf, parent) {
|
|
9076
|
-
const boxes = [];
|
|
9077
|
-
for (let at = parent.bodyStart; at < parent.end; at = boxes.at(-1).end) {
|
|
9078
|
-
boxes.push(boxAt(buf, at, at, parent.end));
|
|
9079
|
-
}
|
|
9080
|
-
return boxes;
|
|
9081
9323
|
}
|
|
9082
|
-
function
|
|
9083
|
-
return
|
|
9324
|
+
function createPlanV2FromV1(executionPlanDir, planV2Dir) {
|
|
9325
|
+
return createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir);
|
|
9084
9326
|
}
|
|
9085
|
-
function
|
|
9086
|
-
|
|
9087
|
-
if (!found) throw new Error(`[mp4Stitch] ${parent.type} has no ${type}`);
|
|
9088
|
-
return found;
|
|
9327
|
+
async function publishPlanV2FromExecutionPlan(executionPlanDir, publisher) {
|
|
9328
|
+
return publishExecutionPlan(executionPlanDir, publisher, {});
|
|
9089
9329
|
}
|
|
9090
|
-
|
|
9091
|
-
|
|
9092
|
-
|
|
9093
|
-
|
|
9094
|
-
|
|
9095
|
-
|
|
9096
|
-
|
|
9097
|
-
if (start >= bufAt && end <= bufAt + buf.length) return;
|
|
9098
|
-
buf = await read(start, Math.max(end, start + SCAN_BYTES));
|
|
9099
|
-
bufAt = start;
|
|
9100
|
-
};
|
|
9101
|
-
for (let at = 0; !(found.has("ftyp") && found.has("moov") && payload); ) {
|
|
9102
|
-
await load(at, at + 16);
|
|
9103
|
-
if (at + 8 > bufAt + buf.length) {
|
|
9104
|
-
throw new Error(`[mp4Stitch] chunk needs ftyp, moov and mdat; found ${[...found.keys()]}`);
|
|
9330
|
+
async function publishExecutionPlan(executionPlanDir, publisher, { digests, log = defaultLogger }) {
|
|
9331
|
+
try {
|
|
9332
|
+
let stageStarted = Date.now();
|
|
9333
|
+
const publication = buildPlanV2Publication(executionPlanDir, digests);
|
|
9334
|
+
logPlanStage(log, "publication", stageStarted);
|
|
9335
|
+
if (publication.storedBlobs.length > 0 && !publisher.referenceStoredBlob) {
|
|
9336
|
+
throw new PlanV2IntegrityError("this publisher cannot reference remote files");
|
|
9105
9337
|
}
|
|
9106
|
-
|
|
9107
|
-
|
|
9108
|
-
|
|
9109
|
-
|
|
9110
|
-
|
|
9338
|
+
stageStarted = Date.now();
|
|
9339
|
+
await putTasks([
|
|
9340
|
+
...publication.blobs.map((blob) => () => publisher.putBlob(blob)),
|
|
9341
|
+
...publication.storedBlobs.map((blob) => () => publisher.referenceStoredBlob(blob))
|
|
9342
|
+
]);
|
|
9343
|
+
logPlanStage(log, "upload", stageStarted);
|
|
9344
|
+
stageStarted = Date.now();
|
|
9345
|
+
await publisher.commitManifest(canonicalJsonStringify(publication.manifest));
|
|
9346
|
+
logPlanStage(log, "manifest", stageStarted);
|
|
9347
|
+
return publication.manifest;
|
|
9348
|
+
} catch (error) {
|
|
9349
|
+
try {
|
|
9350
|
+
await publisher.abort();
|
|
9351
|
+
} catch {
|
|
9111
9352
|
}
|
|
9112
|
-
|
|
9353
|
+
throw error;
|
|
9113
9354
|
}
|
|
9114
|
-
const [ftyp, moov] = [found.get("ftyp"), found.get("moov")];
|
|
9115
|
-
if (moov.length !== moovSize(moov)) throw new Error("[mp4Stitch] moov is cut off");
|
|
9116
|
-
return { ftyp, moov, payloadStart: payload.bodyStart, payloadEnd: payload.end };
|
|
9117
|
-
}
|
|
9118
|
-
var moovSize = (moov) => boxAt(moov, 0, 0, Number.MAX_SAFE_INTEGER).end;
|
|
9119
|
-
function readTracks(moov) {
|
|
9120
|
-
const root = boxAt(moov, 0, 0, moov.length);
|
|
9121
|
-
return childBoxes(moov, root).filter((box2) => box2.type === "trak").map((trak) => readTrack(moov, trak));
|
|
9122
9355
|
}
|
|
9123
|
-
function
|
|
9124
|
-
|
|
9125
|
-
const first = table.bodyStart + countAt + 4;
|
|
9126
|
-
return Array.from(
|
|
9127
|
-
{ length: count },
|
|
9128
|
-
(_, row) => Array.from(
|
|
9129
|
-
{ length: columns },
|
|
9130
|
-
(_2, col) => buf.readUInt32BE(first + (row * columns + col) * 4)
|
|
9131
|
-
)
|
|
9132
|
-
);
|
|
9356
|
+
async function publishPlanV2FromV1(executionPlanDir, publisher) {
|
|
9357
|
+
return publishPlanV2FromExecutionPlan(executionPlanDir, publisher);
|
|
9133
9358
|
}
|
|
9134
|
-
function
|
|
9135
|
-
const
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9359
|
+
async function planV2WithPublisher(projectDir, config, publisher, options = {}) {
|
|
9360
|
+
const stagingRoot = mkdtempSync5(join25(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
|
|
9361
|
+
const digests = /* @__PURE__ */ new Map();
|
|
9362
|
+
try {
|
|
9363
|
+
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
|
|
9364
|
+
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
|
|
9365
|
+
deferVideoExtraction: true,
|
|
9366
|
+
remoteFiles: options.remoteFiles,
|
|
9367
|
+
digests,
|
|
9368
|
+
// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
|
|
9369
|
+
whileAudioMixes: () => putBlobs(
|
|
9370
|
+
publisher,
|
|
9371
|
+
listFiles(join25(stagingRoot, "compiled")).map(
|
|
9372
|
+
(file) => publishBlobFor(file.absolutePath, digests)
|
|
9373
|
+
)
|
|
9374
|
+
),
|
|
9375
|
+
chunksCarryAudio: options.chunksCarryAudio,
|
|
9376
|
+
sliceChunkAudio: true
|
|
9377
|
+
});
|
|
9378
|
+
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
|
|
9379
|
+
} catch (error) {
|
|
9380
|
+
try {
|
|
9381
|
+
await publisher.abort();
|
|
9382
|
+
} catch {
|
|
9383
|
+
}
|
|
9384
|
+
throw error;
|
|
9385
|
+
} finally {
|
|
9386
|
+
rmSync10(stagingRoot, { recursive: true, force: true });
|
|
9144
9387
|
}
|
|
9145
|
-
return 0;
|
|
9146
|
-
}
|
|
9147
|
-
function readSizes(moov, stsz) {
|
|
9148
|
-
const fixedSize = moov.readUInt32BE(stsz.bodyStart + 4);
|
|
9149
|
-
if (!fixedSize) return rows(moov, stsz, 1, 8).map(([size]) => size);
|
|
9150
|
-
return Array(moov.readUInt32BE(stsz.bodyStart + 8)).fill(fixedSize);
|
|
9151
|
-
}
|
|
9152
|
-
function readSync2(moov, stbl, sampleCount) {
|
|
9153
|
-
const stss = findChild(moov, stbl, "stss");
|
|
9154
|
-
if (!stss) return null;
|
|
9155
|
-
const sync = Array(sampleCount).fill(false);
|
|
9156
|
-
for (const [n] of rows(moov, stss, 1)) sync[n - 1] = true;
|
|
9157
|
-
return sync;
|
|
9158
|
-
}
|
|
9159
|
-
function readChunkOffsets(moov, stbl) {
|
|
9160
|
-
const stco = findChild(moov, stbl, "stco");
|
|
9161
|
-
if (stco) return rows(moov, stco, 1).map(([offset]) => offset);
|
|
9162
|
-
return rows(moov, child(moov, stbl, "co64"), 2).map(([hi, lo]) => hi * 2 ** 32 + lo);
|
|
9163
9388
|
}
|
|
9164
|
-
function
|
|
9165
|
-
|
|
9166
|
-
|
|
9167
|
-
if (description !== 1) throw new Error("[mp4Stitch] chunk uses more than one sample entry");
|
|
9168
|
-
const nextFirst = stsc[i + 1]?.[0] ?? chunkCount + 1;
|
|
9169
|
-
return Array(nextFirst - firstChunk).fill(perChunk);
|
|
9170
|
-
});
|
|
9171
|
-
}
|
|
9172
|
-
function readTrack(moov, trak) {
|
|
9173
|
-
const mdia = child(moov, trak, "mdia");
|
|
9174
|
-
const hdlr = child(moov, mdia, "hdlr");
|
|
9175
|
-
const stbl = child(moov, child(moov, mdia, "minf"), "stbl");
|
|
9176
|
-
if (findChild(moov, stbl, "ctts")) throw new Error("[mp4Stitch] chunk has ctts (B-frames)");
|
|
9177
|
-
const stts = rows(moov, child(moov, stbl, "stts"), 2);
|
|
9178
|
-
const durations = stts.flatMap(([n, delta]) => Array(n).fill(delta));
|
|
9179
|
-
const sizes = readSizes(moov, child(moov, stbl, "stsz"));
|
|
9180
|
-
const chunkOffsets = readChunkOffsets(moov, stbl);
|
|
9181
|
-
const samplesPerChunk = readSamplesPerChunk(moov, stbl, chunkOffsets.length);
|
|
9182
|
-
if (durations.length !== sizes.length || sum(samplesPerChunk) !== sizes.length) {
|
|
9183
|
-
throw new Error("[mp4Stitch] sample tables disagree on the sample count");
|
|
9389
|
+
async function planV2(projectDir, config, planV2Dir) {
|
|
9390
|
+
if (existsSync17(planV2Dir)) {
|
|
9391
|
+
throw new PlanV2IntegrityError(`output directory already exists: ${planV2Dir}`);
|
|
9184
9392
|
}
|
|
9185
|
-
const
|
|
9186
|
-
|
|
9187
|
-
|
|
9188
|
-
|
|
9189
|
-
|
|
9190
|
-
mediaTime: readMediaTime(moov, trak),
|
|
9191
|
-
stsd: moov.subarray(stsd.start, stsd.end),
|
|
9192
|
-
durations,
|
|
9193
|
-
sizes,
|
|
9194
|
-
sync: readSync2(moov, stbl, sizes.length),
|
|
9195
|
-
samplesPerChunk,
|
|
9196
|
-
chunkOffsets
|
|
9197
|
-
};
|
|
9198
|
-
}
|
|
9199
|
-
function sum(values) {
|
|
9200
|
-
return values.reduce((total, value) => total + value, 0);
|
|
9201
|
-
}
|
|
9202
|
-
function box(type, ...body) {
|
|
9203
|
-
const header = Buffer.alloc(8);
|
|
9204
|
-
header.writeUInt32BE(8 + sum(body.map((part) => part.length)));
|
|
9205
|
-
header.write(type, 4, "latin1");
|
|
9206
|
-
return Buffer.concat([header, ...body]);
|
|
9393
|
+
const publisher = new LocalPlanV2ArtifactPublisher(planV2Dir);
|
|
9394
|
+
const manifest = await planV2WithPublisher(projectDir, config, publisher, {
|
|
9395
|
+
stagingParentDir: dirname11(planV2Dir)
|
|
9396
|
+
});
|
|
9397
|
+
return resultFromManifest(planV2Dir, manifest);
|
|
9207
9398
|
}
|
|
9208
|
-
function
|
|
9209
|
-
return
|
|
9399
|
+
function resultFromManifest(planV2Dir, manifest) {
|
|
9400
|
+
return {
|
|
9401
|
+
planDir: planV2Dir,
|
|
9402
|
+
manifestPath: join25(planV2Dir, "plan.json"),
|
|
9403
|
+
planProtocol: PLAN_PROTOCOL_V2,
|
|
9404
|
+
planHash: manifest.planHash,
|
|
9405
|
+
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
9406
|
+
chunkCount: manifest.chunkCount,
|
|
9407
|
+
totalFrames: manifest.totalFrames,
|
|
9408
|
+
fps: manifest.fps,
|
|
9409
|
+
width: manifest.width,
|
|
9410
|
+
height: manifest.height,
|
|
9411
|
+
format: manifest.format,
|
|
9412
|
+
ffmpegVersion: manifest.ffmpegVersion,
|
|
9413
|
+
producerVersion: manifest.producerVersion,
|
|
9414
|
+
limitations: manifest.limitations
|
|
9415
|
+
};
|
|
9210
9416
|
}
|
|
9211
|
-
function
|
|
9212
|
-
|
|
9213
|
-
values.forEach((value, i) => buf.writeUInt32BE(value, i * 4));
|
|
9214
|
-
return buf;
|
|
9417
|
+
function getPlanV2ExecutionPlanHash(plan2) {
|
|
9418
|
+
return plan2.sourcePlanV1Hash;
|
|
9215
9419
|
}
|
|
9216
|
-
function
|
|
9217
|
-
|
|
9218
|
-
|
|
9219
|
-
const last = out.at(-1);
|
|
9220
|
-
if (last && last[1] === value) last[0]++;
|
|
9221
|
-
else out.push([1, value]);
|
|
9420
|
+
function readString2(value, field) {
|
|
9421
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
9422
|
+
throw new PlanV2IntegrityError(`${field} must be a non-empty string`);
|
|
9222
9423
|
}
|
|
9223
|
-
return
|
|
9424
|
+
return value;
|
|
9224
9425
|
}
|
|
9225
|
-
function
|
|
9226
|
-
|
|
9227
|
-
|
|
9228
|
-
|
|
9229
|
-
return
|
|
9426
|
+
function readPositiveInteger2(value, field) {
|
|
9427
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
9428
|
+
throw new PlanV2IntegrityError(`${field} must be a positive integer`);
|
|
9429
|
+
}
|
|
9430
|
+
return value;
|
|
9230
9431
|
}
|
|
9231
|
-
|
|
9232
|
-
|
|
9233
|
-
|
|
9234
|
-
|
|
9235
|
-
|
|
9236
|
-
function withDuration(buf, target, duration) {
|
|
9237
|
-
const copy = Buffer.from(buf.subarray(target.start, target.end));
|
|
9238
|
-
const body = target.bodyStart - target.start;
|
|
9239
|
-
const wide = copy[body] === 1;
|
|
9240
|
-
const at = body + DURATION_AT[target.type][wide ? 1 : 0];
|
|
9241
|
-
if (wide) copy.writeBigUInt64BE(BigInt(duration), at);
|
|
9242
|
-
else if (duration <= 4294967295) copy.writeUInt32BE(duration, at);
|
|
9243
|
-
else throw new Error(`[mp4Stitch] ${target.type} duration ${duration} needs a version 1 box`);
|
|
9244
|
-
return copy;
|
|
9432
|
+
function readNonNegativeInteger2(value, field) {
|
|
9433
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
9434
|
+
throw new PlanV2IntegrityError(`${field} must be a non-negative integer`);
|
|
9435
|
+
}
|
|
9436
|
+
return value;
|
|
9245
9437
|
}
|
|
9246
|
-
function
|
|
9247
|
-
|
|
9438
|
+
function readSupportedFps(value) {
|
|
9439
|
+
if (!isDistributedFps(value)) {
|
|
9440
|
+
throw new PlanV2IntegrityError(
|
|
9441
|
+
`dimensions.fpsNum must be one of ${DISTRIBUTED_FPS.join(", ")}`
|
|
9442
|
+
);
|
|
9443
|
+
}
|
|
9444
|
+
return value;
|
|
9248
9445
|
}
|
|
9249
|
-
function
|
|
9250
|
-
const
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
for (const [count, perChunk] of runs(track.samplesPerChunk)) {
|
|
9254
|
-
stsc.push([firstChunk, perChunk, 1]);
|
|
9255
|
-
firstChunk += count;
|
|
9446
|
+
function readExecutionPlanFps(dimensions) {
|
|
9447
|
+
const fpsDen = readPositiveInteger2(dimensions.fpsDen, "dimensions.fpsDen");
|
|
9448
|
+
if (fpsDen !== 1) {
|
|
9449
|
+
throw new PlanV2IntegrityError("dimensions.fpsDen must be 1 for plan v2");
|
|
9256
9450
|
}
|
|
9257
|
-
|
|
9258
|
-
co64.writeUInt32BE(track.chunkOffsets.length);
|
|
9259
|
-
track.chunkOffsets.forEach((offset, i) => co64.writeBigUInt64BE(BigInt(offset), 4 + i * 8));
|
|
9260
|
-
const syncSamples = track.sync?.flatMap((isSync, i) => isSync ? [i + 1] : []) ?? null;
|
|
9261
|
-
return box(
|
|
9262
|
-
"stbl",
|
|
9263
|
-
stsd,
|
|
9264
|
-
fullBox("stts", 0, u32s([stts.length, ...stts.flat()])),
|
|
9265
|
-
...syncSamples ? [fullBox("stss", 0, u32s([syncSamples.length, ...syncSamples]))] : [],
|
|
9266
|
-
fullBox("stsc", 0, u32s([stsc.length, ...stsc.flat()])),
|
|
9267
|
-
fullBox("stsz", 0, u32s([0, track.sizes.length, ...track.sizes])),
|
|
9268
|
-
fullBox("co64", 0, co64)
|
|
9269
|
-
);
|
|
9451
|
+
return readSupportedFps(dimensions.fpsNum);
|
|
9270
9452
|
}
|
|
9271
|
-
|
|
9272
|
-
|
|
9273
|
-
|
|
9274
|
-
let next = 0;
|
|
9275
|
-
const matched = plan2.map((planTrack) => {
|
|
9276
|
-
const track = tracks[next];
|
|
9277
|
-
if (track && sameTrack(track, planTrack)) return tracks[next++];
|
|
9278
|
-
if (planTrack.handler === "soun") return null;
|
|
9279
|
-
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
9280
|
-
});
|
|
9281
|
-
if (next !== tracks.length)
|
|
9282
|
-
throw new Error(`[mp4Stitch] chunk ${index} tracks differ from chunk 0`);
|
|
9283
|
-
for (const track of tracks) {
|
|
9284
|
-
if (track.handler === "vide" && track.mediaTime !== 0) {
|
|
9285
|
-
throw new Error(`[mp4Stitch] chunk ${index} video starts at media time ${track.mediaTime}`);
|
|
9286
|
-
}
|
|
9453
|
+
function readDistributedFormat(value) {
|
|
9454
|
+
if (value !== "mp4" && value !== "mov" && value !== "webm" && value !== "png-sequence") {
|
|
9455
|
+
throw new PlanV2IntegrityError("dimensions.format is unsupported");
|
|
9287
9456
|
}
|
|
9288
|
-
return
|
|
9457
|
+
return value;
|
|
9289
9458
|
}
|
|
9290
|
-
function
|
|
9291
|
-
|
|
9292
|
-
const
|
|
9293
|
-
|
|
9294
|
-
|
|
9295
|
-
|
|
9296
|
-
payloadBases.push(payloadBytes);
|
|
9297
|
-
payloadBytes += layout.payloadEnd - layout.payloadStart;
|
|
9459
|
+
function parseArtifact(value, index) {
|
|
9460
|
+
if (!isRecord3(value)) throw new PlanV2IntegrityError(`artifacts[${index}] must be an object`);
|
|
9461
|
+
const path = readString2(value.path, `artifacts[${index}].path`);
|
|
9462
|
+
assertSafeRelativePath(path);
|
|
9463
|
+
if (!isSha256(value.sha256)) {
|
|
9464
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].sha256 must be a lowercase sha256`);
|
|
9298
9465
|
}
|
|
9299
|
-
const
|
|
9300
|
-
|
|
9301
|
-
|
|
9302
|
-
|
|
9303
|
-
|
|
9304
|
-
|
|
9305
|
-
|
|
9306
|
-
|
|
9307
|
-
|
|
9308
|
-
|
|
9309
|
-
|
|
9310
|
-
|
|
9311
|
-
|
|
9312
|
-
|
|
9313
|
-
|
|
9314
|
-
|
|
9315
|
-
|
|
9316
|
-
|
|
9317
|
-
|
|
9318
|
-
|
|
9319
|
-
|
|
9320
|
-
return offset - payloadStart + payloadBases[i] + dataStart2;
|
|
9321
|
-
})
|
|
9322
|
-
)
|
|
9323
|
-
};
|
|
9324
|
-
const mediaDuration = sum(merged.durations);
|
|
9325
|
-
const shown = Math.round(
|
|
9326
|
-
(mediaDuration - track.mediaTime) * movieTimescale / track.timescale
|
|
9327
|
-
);
|
|
9328
|
-
const elst = Buffer.alloc(24);
|
|
9329
|
-
elst.writeUInt32BE(1);
|
|
9330
|
-
elst.writeBigUInt64BE(BigInt(shown), 4);
|
|
9331
|
-
elst.writeBigInt64BE(BigInt(track.mediaTime), 12);
|
|
9332
|
-
elst.writeUInt32BE(65536, 20);
|
|
9333
|
-
const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);
|
|
9334
|
-
const trak = rebuild(template, track.trak, {
|
|
9335
|
-
tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),
|
|
9336
|
-
edts: () => Buffer.alloc(0),
|
|
9337
|
-
mdia: (mdia) => rebuild(template, mdia, {
|
|
9338
|
-
mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),
|
|
9339
|
-
minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) })
|
|
9340
|
-
})
|
|
9341
|
-
});
|
|
9342
|
-
return { trak, shown };
|
|
9343
|
-
});
|
|
9344
|
-
let next = 0;
|
|
9345
|
-
return rebuild(template, root, {
|
|
9346
|
-
mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),
|
|
9347
|
-
trak: () => traks[next++].trak
|
|
9348
|
-
});
|
|
9466
|
+
const sizeBytes = value.sizeBytes;
|
|
9467
|
+
if (typeof sizeBytes !== "number" || !Number.isInteger(sizeBytes) || sizeBytes < 0) {
|
|
9468
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].sizeBytes must be a non-negative integer`);
|
|
9469
|
+
}
|
|
9470
|
+
let chunks;
|
|
9471
|
+
if (value.chunks === "all") {
|
|
9472
|
+
chunks = "all";
|
|
9473
|
+
} else if (isNonNegativeIntegerArray(value.chunks)) {
|
|
9474
|
+
chunks = value.chunks;
|
|
9475
|
+
} else {
|
|
9476
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].chunks is invalid`);
|
|
9477
|
+
}
|
|
9478
|
+
if (typeof value.assembler !== "boolean") {
|
|
9479
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].assembler must be boolean`);
|
|
9480
|
+
}
|
|
9481
|
+
return {
|
|
9482
|
+
path,
|
|
9483
|
+
sha256: value.sha256,
|
|
9484
|
+
sizeBytes,
|
|
9485
|
+
chunks,
|
|
9486
|
+
assembler: value.assembler
|
|
9349
9487
|
};
|
|
9350
|
-
const ftyp = layouts[0].ftyp;
|
|
9351
|
-
const dataStart = ftyp.length + buildMoov(0).length + 16;
|
|
9352
|
-
const mdat = Buffer.alloc(16);
|
|
9353
|
-
mdat.writeUInt32BE(1);
|
|
9354
|
-
mdat.write("mdat", 4, "latin1");
|
|
9355
|
-
mdat.writeBigUInt64BE(BigInt(16 + payloadBytes), 8);
|
|
9356
|
-
return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);
|
|
9357
9488
|
}
|
|
9358
|
-
function
|
|
9359
|
-
|
|
9360
|
-
|
|
9361
|
-
|
|
9362
|
-
|
|
9363
|
-
|
|
9364
|
-
|
|
9365
|
-
|
|
9366
|
-
|
|
9367
|
-
|
|
9489
|
+
function parsePlanV2Manifest(value) {
|
|
9490
|
+
if (!isRecord3(value)) throw new PlanV2IntegrityError("manifest must be an object");
|
|
9491
|
+
if (!isRecord3(value.protocol) || value.protocol.schemaVersion !== PLAN_PROTOCOL_V2.schemaVersion || value.protocol.artifactLayout !== PLAN_PROTOCOL_V2.artifactLayout || value.protocol.hashSchema !== PLAN_PROTOCOL_V2.hashSchema) {
|
|
9492
|
+
throw new PlanV2IntegrityError("manifest protocol is not the supported v2 descriptor");
|
|
9493
|
+
}
|
|
9494
|
+
if (!isSha256(value.planHash) || !isSha256(value.sourcePlanV1Hash)) {
|
|
9495
|
+
throw new PlanV2IntegrityError("manifest hashes must be lowercase sha256 digests");
|
|
9496
|
+
}
|
|
9497
|
+
if (!Array.isArray(value.artifacts)) throw new PlanV2IntegrityError("artifacts must be an array");
|
|
9498
|
+
const artifacts = value.artifacts.map(parseArtifact);
|
|
9499
|
+
const paths = /* @__PURE__ */ new Set();
|
|
9500
|
+
for (const artifact of artifacts) {
|
|
9501
|
+
if (paths.has(artifact.path))
|
|
9502
|
+
throw new PlanV2IntegrityError(`duplicate artifact path: ${artifact.path}`);
|
|
9503
|
+
paths.add(artifact.path);
|
|
9504
|
+
}
|
|
9505
|
+
if (!paths.has("plan.json"))
|
|
9506
|
+
throw new PlanV2IntegrityError("manifest must include the local execution plan.json artifact");
|
|
9507
|
+
if (!isRecord3(value.limitations) || value.limitations.videoDependencyMode !== "exact-rendered-frames" && value.limitations.videoDependencyMode !== "full-source-pack" && value.limitations.videoDependencyMode !== "source-extract") {
|
|
9508
|
+
throw new PlanV2IntegrityError("unsupported video dependency mode");
|
|
9509
|
+
}
|
|
9510
|
+
const parsed = {
|
|
9511
|
+
protocol: PLAN_PROTOCOL_V2,
|
|
9512
|
+
planHash: value.planHash,
|
|
9513
|
+
sourcePlanV1Hash: value.sourcePlanV1Hash,
|
|
9514
|
+
chunkCount: readPositiveInteger2(value.chunkCount, "chunkCount"),
|
|
9515
|
+
totalFrames: readPositiveInteger2(value.totalFrames, "totalFrames"),
|
|
9516
|
+
fps: readSupportedFps(value.fps),
|
|
9517
|
+
width: readPositiveInteger2(value.width, "width"),
|
|
9518
|
+
height: readPositiveInteger2(value.height, "height"),
|
|
9519
|
+
format: readDistributedFormat(value.format),
|
|
9520
|
+
ffmpegVersion: readString2(value.ffmpegVersion, "ffmpegVersion"),
|
|
9521
|
+
producerVersion: readString2(value.producerVersion, "producerVersion"),
|
|
9522
|
+
limitations: { videoDependencyMode: value.limitations.videoDependencyMode },
|
|
9523
|
+
artifacts
|
|
9368
9524
|
};
|
|
9525
|
+
const { planHash: _planHash, ...payload } = parsed;
|
|
9526
|
+
const expectedHash = computeManifestHash(payload);
|
|
9527
|
+
if (expectedHash !== parsed.planHash) {
|
|
9528
|
+
throw new PlanV2IntegrityError("manifest integrity hash mismatch");
|
|
9529
|
+
}
|
|
9530
|
+
for (const [index, artifact] of parsed.artifacts.entries()) {
|
|
9531
|
+
if (artifact.chunks !== "all" && artifact.chunks.some((chunkIndex) => chunkIndex >= parsed.chunkCount)) {
|
|
9532
|
+
throw new PlanV2IntegrityError(`artifacts[${index}].chunks contains an out-of-range index`);
|
|
9533
|
+
}
|
|
9534
|
+
}
|
|
9535
|
+
return parsed;
|
|
9369
9536
|
}
|
|
9370
|
-
|
|
9371
|
-
const
|
|
9372
|
-
|
|
9373
|
-
|
|
9374
|
-
|
|
9375
|
-
|
|
9376
|
-
|
|
9537
|
+
function readPlanV2Manifest(planV2Dir) {
|
|
9538
|
+
const manifestPath = join25(planV2Dir, "plan.json");
|
|
9539
|
+
if (!existsSync17(manifestPath)) {
|
|
9540
|
+
throw new PlanV2IntegrityError(`missing v2 manifest: ${manifestPath}`);
|
|
9541
|
+
}
|
|
9542
|
+
const value = readJsonFile(manifestPath, "v2 manifest");
|
|
9543
|
+
return parsePlanV2Manifest(value);
|
|
9544
|
+
}
|
|
9545
|
+
function listPlanV2ArtifactsForTarget(manifest, target) {
|
|
9546
|
+
if (target.role === "chunk") {
|
|
9547
|
+
if (!Number.isInteger(target.chunkIndex) || target.chunkIndex < 0 || target.chunkIndex >= manifest.chunkCount) {
|
|
9548
|
+
throw new PlanV2IntegrityError(`chunkIndex ${String(target.chunkIndex)} is out of range`);
|
|
9549
|
+
}
|
|
9550
|
+
return manifest.artifacts.filter(
|
|
9551
|
+
(artifact) => artifact.chunks === "all" || artifact.chunks.some((chunkIndex) => chunkIndex === target.chunkIndex)
|
|
9377
9552
|
);
|
|
9378
9553
|
}
|
|
9554
|
+
return manifest.artifacts.filter((artifact) => artifact.assembler);
|
|
9555
|
+
}
|
|
9556
|
+
function verifyBlob(planV2Dir, artifact) {
|
|
9557
|
+
const sourcePath = planV2BlobPath(planV2Dir, artifact.sha256);
|
|
9558
|
+
if (!existsSync17(sourcePath)) {
|
|
9559
|
+
throw new PlanV2IntegrityError(`missing content-addressed artifact ${artifact.sha256}`);
|
|
9560
|
+
}
|
|
9561
|
+
const stats = statSync6(sourcePath);
|
|
9562
|
+
if (!stats.isFile() || stats.size !== artifact.sizeBytes) {
|
|
9563
|
+
throw new PlanV2IntegrityError(`artifact size mismatch for ${artifact.path}`);
|
|
9564
|
+
}
|
|
9565
|
+
if (sha256File(sourcePath) !== artifact.sha256) {
|
|
9566
|
+
throw new PlanV2IntegrityError(`artifact hash mismatch for ${artifact.path}`);
|
|
9567
|
+
}
|
|
9568
|
+
return sourcePath;
|
|
9379
9569
|
}
|
|
9380
|
-
|
|
9381
|
-
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
const
|
|
9385
|
-
const
|
|
9570
|
+
function materializePlanV2Target(planV2Dir, target, destinationDir, options = {}) {
|
|
9571
|
+
if (existsSync17(destinationDir)) {
|
|
9572
|
+
throw new PlanV2IntegrityError(`materialization destination already exists: ${destinationDir}`);
|
|
9573
|
+
}
|
|
9574
|
+
const manifest = readPlanV2Manifest(planV2Dir);
|
|
9575
|
+
const artifacts = listPlanV2ArtifactsForTarget(manifest, target);
|
|
9576
|
+
const remoteSources = options.remoteSources ?? {};
|
|
9577
|
+
assertRemoteSources(remoteSources, artifacts);
|
|
9578
|
+
const audioArtifact = artifacts.find((artifact) => isPlanAudioArtifactPath(artifact.path));
|
|
9579
|
+
const verified = artifacts.filter((artifact) => !Object.hasOwn(remoteSources, artifact.path)).map((artifact) => ({
|
|
9580
|
+
artifact,
|
|
9581
|
+
sourcePath: verifyBlob(planV2Dir, artifact)
|
|
9582
|
+
}));
|
|
9583
|
+
mkdirSync12(dirname11(destinationDir), { recursive: true });
|
|
9584
|
+
const tempDir = mkdtempSync5(join25(dirname11(destinationDir), ".plan-v2-materialize-"));
|
|
9386
9585
|
try {
|
|
9387
|
-
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9392
|
-
|
|
9393
|
-
|
|
9394
|
-
|
|
9586
|
+
for (const { artifact, sourcePath } of verified) {
|
|
9587
|
+
const destinationPath = join25(tempDir, ...artifact.path.split("/"));
|
|
9588
|
+
mkdirSync12(dirname11(destinationPath), { recursive: true });
|
|
9589
|
+
copyFileSync7(sourcePath, destinationPath);
|
|
9590
|
+
}
|
|
9591
|
+
if (target.role === "chunk") materializeExtractedVideoDirectories(tempDir);
|
|
9592
|
+
writeFileSync8(
|
|
9593
|
+
join25(tempDir, PLAN_V2_MATERIALIZATION_MARKER),
|
|
9594
|
+
canonicalJsonStringify({
|
|
9595
|
+
manifest,
|
|
9596
|
+
target,
|
|
9597
|
+
...Object.keys(remoteSources).length > 0 ? { remoteSources } : {}
|
|
9598
|
+
}),
|
|
9599
|
+
"utf-8"
|
|
9600
|
+
);
|
|
9601
|
+
renameSync6(tempDir, destinationDir);
|
|
9602
|
+
} catch (error) {
|
|
9603
|
+
rmSync10(tempDir, { recursive: true, force: true });
|
|
9604
|
+
throw error;
|
|
9605
|
+
}
|
|
9606
|
+
return {
|
|
9607
|
+
planDir: destinationDir,
|
|
9608
|
+
target,
|
|
9609
|
+
planHash: manifest.planHash,
|
|
9610
|
+
sourcePlanV1Hash: getPlanV2ExecutionPlanHash(manifest),
|
|
9611
|
+
artifactCount: artifacts.length,
|
|
9612
|
+
sizeBytes: artifacts.reduce((sum2, artifact) => sum2 + artifact.sizeBytes, 0),
|
|
9613
|
+
// Join the artifact's OWN name, not the current constant: a plan written
|
|
9614
|
+
// before the container change carries the legacy name, and materializing it
|
|
9615
|
+
// under the new one would point at a file that was never written.
|
|
9616
|
+
audioPath: target.role === "assembler" ? audioArtifact ? join25(destinationDir, audioArtifact.path) : null : null
|
|
9617
|
+
};
|
|
9618
|
+
}
|
|
9619
|
+
function assertRemoteSources(remoteSources, artifacts) {
|
|
9620
|
+
if (!isRecord3(remoteSources)) throw new PlanV2IntegrityError("remoteSources must be an object");
|
|
9621
|
+
for (const [path, url] of Object.entries(remoteSources)) {
|
|
9622
|
+
const artifact = artifacts.find((candidate) => candidate.path === path);
|
|
9623
|
+
if (!artifact || !isPlanV2DeferredSourceArtifact(artifact)) {
|
|
9624
|
+
throw new PlanV2IntegrityError(`remote source is not a selected deferred video: ${path}`);
|
|
9625
|
+
}
|
|
9626
|
+
if (typeof url !== "string" || !/^https?:\/\//.test(url)) {
|
|
9627
|
+
throw new PlanV2IntegrityError(`remote source for ${path} must be an http(s) URL`);
|
|
9628
|
+
}
|
|
9629
|
+
}
|
|
9630
|
+
}
|
|
9631
|
+
function validatePlanV2MaterializedTarget(planDir, expectedTarget) {
|
|
9632
|
+
return readValidatedPlanV2Materialization(planDir, expectedTarget)?.manifest ?? null;
|
|
9633
|
+
}
|
|
9634
|
+
function readValidatedPlanV2Materialization(planDir, expectedTarget) {
|
|
9635
|
+
const markerPath = join25(planDir, PLAN_V2_MATERIALIZATION_MARKER);
|
|
9636
|
+
if (!existsSync17(markerPath)) return null;
|
|
9637
|
+
const marker = readJsonFile(markerPath, "materialization marker");
|
|
9638
|
+
if (!isRecord3(marker) || !isRecord3(marker.manifest) || !isRecord3(marker.target)) {
|
|
9639
|
+
throw new PlanV2IntegrityError("malformed materialization marker");
|
|
9640
|
+
}
|
|
9641
|
+
const target = marker.target;
|
|
9642
|
+
if (target.role !== expectedTarget.role || expectedTarget.role === "chunk" && target.chunkIndex !== expectedTarget.chunkIndex) {
|
|
9643
|
+
throw new PlanV2IntegrityError(
|
|
9644
|
+
"materialization target does not match requested execution role"
|
|
9645
|
+
);
|
|
9646
|
+
}
|
|
9647
|
+
const manifest = parsePlanV2Manifest(marker.manifest);
|
|
9648
|
+
const required = listPlanV2ArtifactsForTarget(manifest, expectedTarget);
|
|
9649
|
+
const remoteSources = marker.remoteSources ?? {};
|
|
9650
|
+
assertRemoteSources(remoteSources, required);
|
|
9651
|
+
for (const artifact of required) {
|
|
9652
|
+
if (Object.hasOwn(remoteSources, artifact.path)) continue;
|
|
9653
|
+
const path = join25(planDir, ...artifact.path.split("/"));
|
|
9654
|
+
if (!existsSync17(path) || statSync6(path).size !== artifact.sizeBytes) {
|
|
9655
|
+
throw new PlanV2IntegrityError(
|
|
9656
|
+
`materialized artifact missing or truncated: ${artifact.path}`
|
|
9657
|
+
);
|
|
9658
|
+
}
|
|
9659
|
+
if (sha256File(path) !== artifact.sha256) {
|
|
9660
|
+
throw new PlanV2IntegrityError(`materialized artifact hash mismatch: ${artifact.path}`);
|
|
9661
|
+
}
|
|
9395
9662
|
}
|
|
9663
|
+
return { manifest, remoteSources };
|
|
9396
9664
|
}
|
|
9397
9665
|
|
|
9666
|
+
// src/services/distributed/planV2Execution.ts
|
|
9667
|
+
import { mkdtempSync as mkdtempSync7, rmSync as rmSync13 } from "node:fs";
|
|
9668
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
9669
|
+
import { join as join28 } from "node:path";
|
|
9670
|
+
|
|
9398
9671
|
// src/services/distributed/assemble.ts
|
|
9672
|
+
import {
|
|
9673
|
+
cpSync as cpSync3,
|
|
9674
|
+
existsSync as existsSync18,
|
|
9675
|
+
mkdirSync as mkdirSync13,
|
|
9676
|
+
mkdtempSync as mkdtempSync6,
|
|
9677
|
+
readFileSync as readFileSync11,
|
|
9678
|
+
readdirSync as readdirSync8,
|
|
9679
|
+
renameSync as renameSync7,
|
|
9680
|
+
rmSync as rmSync11,
|
|
9681
|
+
statSync as statSync7,
|
|
9682
|
+
writeFileSync as writeFileSync9
|
|
9683
|
+
} from "node:fs";
|
|
9684
|
+
import { dirname as dirname12, join as join26 } from "node:path";
|
|
9685
|
+
import {
|
|
9686
|
+
appendRenderProvenanceArgs as appendRenderProvenanceArgs2,
|
|
9687
|
+
describeFfmpegFailure as describeFfmpegFailure3,
|
|
9688
|
+
getEncoderPreset as getEncoderPreset5,
|
|
9689
|
+
MIXED_AUDIO_FILENAME as MIXED_AUDIO_FILENAME4,
|
|
9690
|
+
muxVideoWithAudio as muxVideoWithAudio2,
|
|
9691
|
+
resolveConfig as resolveConfig4,
|
|
9692
|
+
runFfmpeg as runFfmpeg6
|
|
9693
|
+
} from "@hyperframes/engine";
|
|
9694
|
+
import { fpsToFfmpegArg as fpsToFfmpegArg3 } from "@hyperframes/core";
|
|
9399
9695
|
function cfrRateControlArgs(locked) {
|
|
9400
9696
|
const preset = getEncoderPreset5(locked.quality ?? "standard", "mp4");
|
|
9401
9697
|
if (locked.crf == null && locked.bitrate) {
|
|
@@ -9422,8 +9718,8 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9422
9718
|
}
|
|
9423
9719
|
const format = plan2.dimensions.format;
|
|
9424
9720
|
const cfrEncoder = cfr ? readCfrEncoder(planDir, format) : null;
|
|
9425
|
-
if (!
|
|
9426
|
-
|
|
9721
|
+
if (!existsSync18(dirname12(outputPath))) {
|
|
9722
|
+
mkdirSync13(dirname12(outputPath), { recursive: true });
|
|
9427
9723
|
}
|
|
9428
9724
|
const workDir = mkdtempSync6(`${outputPath}.assemble-work-`);
|
|
9429
9725
|
const fps = { num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen };
|
|
@@ -9445,8 +9741,8 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9445
9741
|
};
|
|
9446
9742
|
try {
|
|
9447
9743
|
const videoSources = cfrEncoder ? [await reencodeCfr(ctx, chunkPaths, cfrEncoder)] : chunkPaths;
|
|
9448
|
-
const finalPath =
|
|
9449
|
-
const audio = audioPath !== null &&
|
|
9744
|
+
const finalPath = join26(workDir, `output.${format}`);
|
|
9745
|
+
const audio = audioPath !== null && existsSync18(audioPath) ? audioPath : null;
|
|
9450
9746
|
await writeJoined(ctx, videoSources, audio, finalPath);
|
|
9451
9747
|
renameSync7(finalPath, outputPath);
|
|
9452
9748
|
log.info("[assemble] stage timings", {
|
|
@@ -9464,7 +9760,7 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9464
9760
|
});
|
|
9465
9761
|
}
|
|
9466
9762
|
}
|
|
9467
|
-
const fileSize =
|
|
9763
|
+
const fileSize = existsSync18(outputPath) ? statSync7(outputPath).size : 0;
|
|
9468
9764
|
return {
|
|
9469
9765
|
outputPath,
|
|
9470
9766
|
durationMs: Date.now() - start,
|
|
@@ -9473,25 +9769,25 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
9473
9769
|
};
|
|
9474
9770
|
}
|
|
9475
9771
|
function readAssemblePlan(planDir, chunkPaths) {
|
|
9476
|
-
const planJsonPath =
|
|
9477
|
-
const chunksJsonPath =
|
|
9478
|
-
if (!
|
|
9772
|
+
const planJsonPath = join26(planDir, "plan.json");
|
|
9773
|
+
const chunksJsonPath = join26(planDir, "meta", "chunks.json");
|
|
9774
|
+
if (!existsSync18(planJsonPath)) {
|
|
9479
9775
|
throw new Error(`[assemble] planDir missing plan.json: ${planJsonPath}`);
|
|
9480
9776
|
}
|
|
9481
|
-
const plan2 = JSON.parse(
|
|
9777
|
+
const plan2 = JSON.parse(readFileSync11(planJsonPath, "utf-8"));
|
|
9482
9778
|
readPlanProtocolV1(plan2, DISTRIBUTED_RENDER_CAPABILITIES.roles.assembler);
|
|
9483
9779
|
validatePlanV2MaterializedTarget(planDir, { role: "assembler" });
|
|
9484
|
-
if (!
|
|
9780
|
+
if (!existsSync18(chunksJsonPath)) {
|
|
9485
9781
|
throw new Error(`[assemble] planDir missing meta/chunks.json: ${chunksJsonPath}`);
|
|
9486
9782
|
}
|
|
9487
|
-
const chunks = JSON.parse(
|
|
9783
|
+
const chunks = JSON.parse(readFileSync11(chunksJsonPath, "utf-8"));
|
|
9488
9784
|
if (chunkPaths.length !== chunks.length) {
|
|
9489
9785
|
throw new Error(
|
|
9490
9786
|
`[assemble] chunkPaths length (${chunkPaths.length}) does not match chunks.json length (${chunks.length}). Adapters must pass one path per chunk, ordered by index.`
|
|
9491
9787
|
);
|
|
9492
9788
|
}
|
|
9493
9789
|
for (const path of chunkPaths) {
|
|
9494
|
-
if (!
|
|
9790
|
+
if (!existsSync18(path)) {
|
|
9495
9791
|
throw new Error(`[assemble] chunk path does not exist: ${path}`);
|
|
9496
9792
|
}
|
|
9497
9793
|
}
|
|
@@ -9501,7 +9797,7 @@ function videoInputArgs(ctx, videoSources) {
|
|
|
9501
9797
|
return videoSources.length === 1 ? ["-i", videoSources[0]] : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);
|
|
9502
9798
|
}
|
|
9503
9799
|
async function reencodeCfr(ctx, chunkPaths, encoder) {
|
|
9504
|
-
const cfrOutputPath =
|
|
9800
|
+
const cfrOutputPath = join26(ctx.workDir, `cfr.${ctx.format}`);
|
|
9505
9801
|
const cfrArgs = [
|
|
9506
9802
|
...videoInputArgs(ctx, chunkPaths),
|
|
9507
9803
|
"-c:v",
|
|
@@ -9514,7 +9810,7 @@ async function reencodeCfr(ctx, chunkPaths, encoder) {
|
|
|
9514
9810
|
"-r",
|
|
9515
9811
|
ctx.fpsArg
|
|
9516
9812
|
];
|
|
9517
|
-
|
|
9813
|
+
appendRenderProvenanceArgs2(cfrArgs, cfrOutputPath);
|
|
9518
9814
|
cfrArgs.push("-y", cfrOutputPath);
|
|
9519
9815
|
await ctx.timed(
|
|
9520
9816
|
"cfrMs",
|
|
@@ -9531,7 +9827,7 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9531
9827
|
if (audioPath === null) {
|
|
9532
9828
|
const joinArgs = [...videoInputArgs(ctx, videoSources), "-c", "copy"];
|
|
9533
9829
|
if (ctx.format === "mp4") joinArgs.push("-movflags", "+faststart");
|
|
9534
|
-
|
|
9830
|
+
appendRenderProvenanceArgs2(joinArgs, finalPath);
|
|
9535
9831
|
joinArgs.push("-r", ctx.fpsArg, "-y", finalPath);
|
|
9536
9832
|
await ctx.timed(
|
|
9537
9833
|
"joinMs",
|
|
@@ -9539,7 +9835,7 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9539
9835
|
);
|
|
9540
9836
|
return;
|
|
9541
9837
|
}
|
|
9542
|
-
const paddedAudioPath =
|
|
9838
|
+
const paddedAudioPath = join26(ctx.workDir, "audio-padded.m4a");
|
|
9543
9839
|
const padTrimResult = await ctx.timed(
|
|
9544
9840
|
"audioMs",
|
|
9545
9841
|
() => padOrTrimAudioToVideoFrameCount({
|
|
@@ -9579,9 +9875,9 @@ async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
|
9579
9875
|
}
|
|
9580
9876
|
}
|
|
9581
9877
|
function concatDemuxerInput(chunkPaths, workDir, fpsArg) {
|
|
9582
|
-
const listPath =
|
|
9878
|
+
const listPath = join26(workDir, "concat-list.txt");
|
|
9583
9879
|
const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");
|
|
9584
|
-
|
|
9880
|
+
writeFileSync9(listPath, `${body}
|
|
9585
9881
|
`, "utf-8");
|
|
9586
9882
|
return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath];
|
|
9587
9883
|
}
|
|
@@ -9596,10 +9892,10 @@ async function probeJoinedFrameInfo(videoPaths, fps, signal) {
|
|
|
9596
9892
|
};
|
|
9597
9893
|
}
|
|
9598
9894
|
async function runAssembleFfmpeg(args, step, signal, timeout) {
|
|
9599
|
-
const result = await
|
|
9895
|
+
const result = await runFfmpeg6(args, { signal, timeout });
|
|
9600
9896
|
if (!result.success) {
|
|
9601
9897
|
throw encoderFailureError(`[assemble] ffmpeg ${step} failed`, {
|
|
9602
|
-
error:
|
|
9898
|
+
error: describeFfmpegFailure3(result, timeout),
|
|
9603
9899
|
failureReason: result.failureReason
|
|
9604
9900
|
});
|
|
9605
9901
|
}
|
|
@@ -9612,9 +9908,9 @@ function chunksAreStitchedMp4(planDir, cfr) {
|
|
|
9612
9908
|
return true;
|
|
9613
9909
|
}
|
|
9614
9910
|
function chunksCarryAudio(planDir) {
|
|
9615
|
-
const encoderJsonPath =
|
|
9616
|
-
if (!
|
|
9617
|
-
return JSON.parse(
|
|
9911
|
+
const encoderJsonPath = join26(planDir, "meta", "encoder.json");
|
|
9912
|
+
if (!existsSync18(encoderJsonPath)) return false;
|
|
9913
|
+
return JSON.parse(readFileSync11(encoderJsonPath, "utf-8")).chunksCarryAudio === true;
|
|
9618
9914
|
}
|
|
9619
9915
|
function readCfrEncoder(planDir, format) {
|
|
9620
9916
|
if (format !== "mp4") {
|
|
@@ -9622,11 +9918,11 @@ function readCfrEncoder(planDir, format) {
|
|
|
9622
9918
|
`[assemble] cfr=true is only supported for format="mp4" (got "${format}"). Stream-copy paths for webm and mov already produce exact avg_frame_rate; cfr re-encode is not needed.`
|
|
9623
9919
|
);
|
|
9624
9920
|
}
|
|
9625
|
-
const encoderJsonPath =
|
|
9626
|
-
if (!
|
|
9921
|
+
const encoderJsonPath = join26(planDir, "meta", "encoder.json");
|
|
9922
|
+
if (!existsSync18(encoderJsonPath)) {
|
|
9627
9923
|
throw new Error(`[assemble] planDir missing meta/encoder.json: ${encoderJsonPath}`);
|
|
9628
9924
|
}
|
|
9629
|
-
const encoderJson = JSON.parse(
|
|
9925
|
+
const encoderJson = JSON.parse(readFileSync11(encoderJsonPath, "utf-8"));
|
|
9630
9926
|
if (encoderJson.encoder === "libx265-software") {
|
|
9631
9927
|
throw new Error(
|
|
9632
9928
|
`[assemble] cfr=true is not yet supported with codec: "h265". The cfr re-encode pass uses libx264 and would silently transcode the h265 chunks. Either disable cfr or render with codec: "h264".`
|
|
@@ -9635,8 +9931,8 @@ function readCfrEncoder(planDir, format) {
|
|
|
9635
9931
|
return encoderJson;
|
|
9636
9932
|
}
|
|
9637
9933
|
function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, startTimeMs) {
|
|
9638
|
-
if (
|
|
9639
|
-
|
|
9934
|
+
if (existsSync18(outputPath)) rmSync11(outputPath, { recursive: true, force: true });
|
|
9935
|
+
mkdirSync13(outputPath, { recursive: true });
|
|
9640
9936
|
let globalIdx = 0;
|
|
9641
9937
|
for (const chunkDir of chunkPaths) {
|
|
9642
9938
|
if (!statSync7(chunkDir).isDirectory()) {
|
|
@@ -9649,8 +9945,8 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9649
9945
|
throw new Error(`[assemble] png-sequence chunk has no frames: ${chunkDir}`);
|
|
9650
9946
|
}
|
|
9651
9947
|
for (const frame of frames) {
|
|
9652
|
-
const dst =
|
|
9653
|
-
cpSync3(
|
|
9948
|
+
const dst = join26(outputPath, formatExportFrameName(globalIdx, "png"));
|
|
9949
|
+
cpSync3(join26(chunkDir, frame), dst);
|
|
9654
9950
|
globalIdx += 1;
|
|
9655
9951
|
}
|
|
9656
9952
|
}
|
|
@@ -9659,14 +9955,14 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9659
9955
|
`[assemble] png-sequence frame count mismatch: merged ${globalIdx} frames vs plan.totalFrames=${totalFrames}. Using on-disk count.`
|
|
9660
9956
|
);
|
|
9661
9957
|
}
|
|
9662
|
-
if (audioPath !== null &&
|
|
9663
|
-
const sidecar =
|
|
9958
|
+
if (audioPath !== null && existsSync18(audioPath)) {
|
|
9959
|
+
const sidecar = join26(outputPath, MIXED_AUDIO_FILENAME4);
|
|
9664
9960
|
cpSync3(audioPath, sidecar);
|
|
9665
9961
|
}
|
|
9666
9962
|
let fileSize = 0;
|
|
9667
9963
|
for (const name of readdirSync8(outputPath)) {
|
|
9668
9964
|
try {
|
|
9669
|
-
fileSize += statSync7(
|
|
9965
|
+
fileSize += statSync7(join26(outputPath, name)).size;
|
|
9670
9966
|
} catch {
|
|
9671
9967
|
}
|
|
9672
9968
|
}
|
|
@@ -9680,8 +9976,8 @@ function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, start
|
|
|
9680
9976
|
|
|
9681
9977
|
// src/services/distributed/renderChunk.ts
|
|
9682
9978
|
import { randomBytes } from "node:crypto";
|
|
9683
|
-
import { existsSync as
|
|
9684
|
-
import { join as
|
|
9979
|
+
import { existsSync as existsSync19, mkdirSync as mkdirSync14, readFileSync as readFileSync12, readdirSync as readdirSync9, rmSync as rmSync12, writeFileSync as writeFileSync10 } from "node:fs";
|
|
9980
|
+
import { join as join27 } from "node:path";
|
|
9685
9981
|
import {
|
|
9686
9982
|
assertSwiftShader,
|
|
9687
9983
|
BROWSER_GPU_NOT_SOFTWARE as BROWSER_GPU_NOT_SOFTWARE4,
|
|
@@ -9697,104 +9993,6 @@ import {
|
|
|
9697
9993
|
readWebGlVendorInfoFromCanvas,
|
|
9698
9994
|
resolveConfig as resolveConfig5
|
|
9699
9995
|
} from "@hyperframes/engine";
|
|
9700
|
-
|
|
9701
|
-
// src/services/distributed/chunkAudio.ts
|
|
9702
|
-
import { appendRenderProvenanceArgs as appendRenderProvenanceArgs2, describeFfmpegFailure as describeFfmpegFailure3, runFfmpeg as runFfmpeg6 } from "@hyperframes/engine";
|
|
9703
|
-
function audioCutForChunk(grid, chunks, index, fps) {
|
|
9704
|
-
const firstPacketAt = (frame) => Math.min(
|
|
9705
|
-
grid.packetCount,
|
|
9706
|
-
Math.max(
|
|
9707
|
-
0,
|
|
9708
|
-
Math.ceil(
|
|
9709
|
-
(frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) / (grid.packetSamples * fps.num)
|
|
9710
|
-
)
|
|
9711
|
-
)
|
|
9712
|
-
);
|
|
9713
|
-
const first = index === 0 ? 0 : firstPacketAt(chunks[index].startFrame);
|
|
9714
|
-
const end = index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1].startFrame);
|
|
9715
|
-
if (end <= first) return null;
|
|
9716
|
-
const cutSeconds = (packet) => (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;
|
|
9717
|
-
const seekSeconds = index === 0 ? null : cutSeconds(first);
|
|
9718
|
-
const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);
|
|
9719
|
-
return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
|
|
9720
|
-
}
|
|
9721
|
-
async function probeAacPacketGrid(audioPath) {
|
|
9722
|
-
const probe = await runFfprobeJson([
|
|
9723
|
-
"-v",
|
|
9724
|
-
"error",
|
|
9725
|
-
"-select_streams",
|
|
9726
|
-
"a:0",
|
|
9727
|
-
"-show_entries",
|
|
9728
|
-
"stream=sample_rate,time_base,nb_frames:packet=pts,duration",
|
|
9729
|
-
"-read_intervals",
|
|
9730
|
-
"%+#1",
|
|
9731
|
-
"-of",
|
|
9732
|
-
"json",
|
|
9733
|
-
"--",
|
|
9734
|
-
audioPath
|
|
9735
|
-
]);
|
|
9736
|
-
const stream = probe.streams?.[0] ?? {};
|
|
9737
|
-
const packet = probe.packets?.[0] ?? {};
|
|
9738
|
-
const sampleRate = Number(stream.sample_rate);
|
|
9739
|
-
const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);
|
|
9740
|
-
const toSamples = (ticks) => Number(ticks) * sampleRate * tbNum / tbDen;
|
|
9741
|
-
const grid = {
|
|
9742
|
-
sampleRate,
|
|
9743
|
-
firstStart: toSamples(packet.pts),
|
|
9744
|
-
packetSamples: toSamples(packet.duration),
|
|
9745
|
-
packetCount: Number(stream.nb_frames)
|
|
9746
|
-
};
|
|
9747
|
-
const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];
|
|
9748
|
-
if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {
|
|
9749
|
-
throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);
|
|
9750
|
-
}
|
|
9751
|
-
return grid;
|
|
9752
|
-
}
|
|
9753
|
-
function chunkMuxArgs(input, cut) {
|
|
9754
|
-
if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];
|
|
9755
|
-
const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];
|
|
9756
|
-
const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];
|
|
9757
|
-
const exact = seek.length ? ["-copypriorss", "0"] : [];
|
|
9758
|
-
return [
|
|
9759
|
-
...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath],
|
|
9760
|
-
...["-map", "0:v:0", "-map", "1:a:0", ...exact]
|
|
9761
|
-
];
|
|
9762
|
-
}
|
|
9763
|
-
async function muxChunkWithAudio(input) {
|
|
9764
|
-
const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;
|
|
9765
|
-
const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;
|
|
9766
|
-
const args = [...chunkMuxArgs(input, cut), "-c", "copy"];
|
|
9767
|
-
appendRenderProvenanceArgs2(args, input.outputPath);
|
|
9768
|
-
args.push("-y", input.outputPath);
|
|
9769
|
-
const result = await runFfmpeg6(args, { signal: input.signal, timeout: input.timeoutMs });
|
|
9770
|
-
if (!result.success) {
|
|
9771
|
-
throw new Error(
|
|
9772
|
-
`[chunkAudio] chunk mux failed: ${describeFfmpegFailure3(result, input.timeoutMs)}`
|
|
9773
|
-
);
|
|
9774
|
-
}
|
|
9775
|
-
const slice = input.chunks[input.chunkIndex];
|
|
9776
|
-
const expected = { video: slice.endFrame - slice.startFrame };
|
|
9777
|
-
if (cut) expected.audio = cut.packetCount;
|
|
9778
|
-
const probe = await runFfprobeJson([
|
|
9779
|
-
"-v",
|
|
9780
|
-
"error",
|
|
9781
|
-
"-count_packets",
|
|
9782
|
-
"-show_entries",
|
|
9783
|
-
"stream=codec_type,nb_read_packets",
|
|
9784
|
-
"-of",
|
|
9785
|
-
"json",
|
|
9786
|
-
"--",
|
|
9787
|
-
input.outputPath
|
|
9788
|
-
]);
|
|
9789
|
-
for (const [type, count] of Object.entries(expected)) {
|
|
9790
|
-
const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);
|
|
9791
|
-
if (packets !== count) {
|
|
9792
|
-
throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);
|
|
9793
|
-
}
|
|
9794
|
-
}
|
|
9795
|
-
}
|
|
9796
|
-
|
|
9797
|
-
// src/services/distributed/renderChunk.ts
|
|
9798
9996
|
import { readWebGlVendorInfoFromCanvas as readWebGlVendorInfoFromCanvas2 } from "@hyperframes/engine";
|
|
9799
9997
|
var LEGACY_DISTRIBUTED_VP9_CPU_USED = 2;
|
|
9800
9998
|
function validatePlanVideosForChunk(value) {
|
|
@@ -9860,10 +10058,10 @@ async function beginFrameSessionNeedsScreenshotFallback(session, probe = probeBe
|
|
|
9860
10058
|
return !await probe(session.page, timeoutMs, probeTick, session.beginFrameIntervalMs);
|
|
9861
10059
|
}
|
|
9862
10060
|
function hashChunkOutput(outputPath, kind) {
|
|
9863
|
-
if (kind === "file") return sha256Hex(
|
|
10061
|
+
if (kind === "file") return sha256Hex(readFileSync12(outputPath));
|
|
9864
10062
|
const entries = readdirSync9(outputPath).filter((name) => /\.(png|jpg|jpeg)$/i.test(name)).sort();
|
|
9865
10063
|
const lines = entries.map(
|
|
9866
|
-
(name) => `${name}\0${sha256Hex(
|
|
10064
|
+
(name) => `${name}\0${sha256Hex(readFileSync12(join27(outputPath, name)))}`
|
|
9867
10065
|
);
|
|
9868
10066
|
return sha256Hex(lines.join("\0"));
|
|
9869
10067
|
}
|
|
@@ -9880,37 +10078,39 @@ function resolveLockedVp9CpuUsed(lockedEncoder) {
|
|
|
9880
10078
|
async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
9881
10079
|
const start = Date.now();
|
|
9882
10080
|
const log = defaultLogger;
|
|
9883
|
-
const planJsonPath =
|
|
9884
|
-
const encoderJsonPath =
|
|
9885
|
-
const chunksJsonPath =
|
|
9886
|
-
if (!
|
|
10081
|
+
const planJsonPath = join27(planDir, "plan.json");
|
|
10082
|
+
const encoderJsonPath = join27(planDir, "meta", "encoder.json");
|
|
10083
|
+
const chunksJsonPath = join27(planDir, "meta", "chunks.json");
|
|
10084
|
+
if (!existsSync19(planJsonPath)) {
|
|
9887
10085
|
throw new RenderChunkValidationError(
|
|
9888
10086
|
MISSING_PLAN_ARTIFACT,
|
|
9889
10087
|
`[renderChunk] planDir is missing required artifact: ${planJsonPath}`
|
|
9890
10088
|
);
|
|
9891
10089
|
}
|
|
9892
|
-
const plan2 = JSON.parse(
|
|
10090
|
+
const plan2 = JSON.parse(readFileSync12(planJsonPath, "utf-8"));
|
|
9893
10091
|
readPlanProtocolV1(plan2, DISTRIBUTED_RENDER_CAPABILITIES.roles.chunk);
|
|
9894
10092
|
const planHashStarted = Date.now();
|
|
9895
|
-
const
|
|
10093
|
+
const v2Materialization = readValidatedPlanV2Materialization(planDir, {
|
|
9896
10094
|
role: "chunk",
|
|
9897
10095
|
chunkIndex
|
|
9898
10096
|
});
|
|
10097
|
+
const v2Manifest = v2Materialization?.manifest ?? null;
|
|
10098
|
+
const remoteSources = v2Materialization?.remoteSources ?? {};
|
|
9899
10099
|
for (const required of [encoderJsonPath, chunksJsonPath]) {
|
|
9900
|
-
if (!
|
|
10100
|
+
if (!existsSync19(required)) {
|
|
9901
10101
|
throw new RenderChunkValidationError(
|
|
9902
10102
|
MISSING_PLAN_ARTIFACT,
|
|
9903
10103
|
`[renderChunk] planDir is missing required artifact: ${required}`
|
|
9904
10104
|
);
|
|
9905
10105
|
}
|
|
9906
10106
|
}
|
|
9907
|
-
const encoder = JSON.parse(
|
|
9908
|
-
const chunks = JSON.parse(
|
|
9909
|
-
const videosJsonPath =
|
|
10107
|
+
const encoder = JSON.parse(readFileSync12(encoderJsonPath, "utf-8"));
|
|
10108
|
+
const chunks = JSON.parse(readFileSync12(chunksJsonPath, "utf-8"));
|
|
10109
|
+
const videosJsonPath = join27(planDir, PLAN_VIDEOS_META_RELATIVE_PATH);
|
|
9910
10110
|
let planVideos = null;
|
|
9911
|
-
if (
|
|
10111
|
+
if (existsSync19(videosJsonPath)) {
|
|
9912
10112
|
try {
|
|
9913
|
-
planVideos = validatePlanVideosForChunk(JSON.parse(
|
|
10113
|
+
planVideos = validatePlanVideosForChunk(JSON.parse(readFileSync12(videosJsonPath, "utf-8")));
|
|
9914
10114
|
} catch (err) {
|
|
9915
10115
|
if (err instanceof RenderChunkValidationError) throw err;
|
|
9916
10116
|
throw new RenderChunkValidationError(
|
|
@@ -9936,8 +10136,8 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
9936
10136
|
`[renderChunk] chunk ${chunkIndex} has non-positive frame count: ${framesInChunk}`
|
|
9937
10137
|
);
|
|
9938
10138
|
}
|
|
9939
|
-
const compiledDir =
|
|
9940
|
-
if (!
|
|
10139
|
+
const compiledDir = join27(planDir, "compiled");
|
|
10140
|
+
if (!existsSync19(compiledDir)) {
|
|
9941
10141
|
throw new RenderChunkValidationError(
|
|
9942
10142
|
MISSING_PLAN_ARTIFACT,
|
|
9943
10143
|
`[renderChunk] planDir missing compiled/ directory: ${compiledDir}`
|
|
@@ -9993,10 +10193,28 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
9993
10193
|
forceScreenshotExplicitlyOptedOut: !encoder.forceScreenshot
|
|
9994
10194
|
};
|
|
9995
10195
|
const workDir = `${outputChunkPath}.work.${process.pid}.${randomBytes(4).toString("hex")}`;
|
|
9996
|
-
|
|
9997
|
-
const framesDir =
|
|
9998
|
-
|
|
10196
|
+
mkdirSync14(workDir, { recursive: true });
|
|
10197
|
+
const framesDir = join27(workDir, "captured-frames");
|
|
10198
|
+
mkdirSync14(framesDir, { recursive: true });
|
|
9999
10199
|
const videoFramesDir = `${workDir}.video-frames`;
|
|
10200
|
+
const deferredVideoFrames = planVideos ? await extractDeferredVideoFramesForChunk({
|
|
10201
|
+
planDir,
|
|
10202
|
+
planVideos,
|
|
10203
|
+
frames: Array.from(
|
|
10204
|
+
{ length: slice.endFrame - slice.startFrame },
|
|
10205
|
+
(_, i) => slice.startFrame + i
|
|
10206
|
+
),
|
|
10207
|
+
fps: { num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen },
|
|
10208
|
+
outputRoot: videoFramesDir,
|
|
10209
|
+
cfg,
|
|
10210
|
+
remoteSources
|
|
10211
|
+
}) : [];
|
|
10212
|
+
const remoteVideoIds = new Set(
|
|
10213
|
+
planVideos?.extracted.filter(
|
|
10214
|
+
(video) => video.deferredRange && Object.hasOwn(remoteSources, video.deferredRange.sourcePath)
|
|
10215
|
+
).map((video) => video.videoId)
|
|
10216
|
+
);
|
|
10217
|
+
const remoteVideos = deferredVideoFrames.filter((video) => remoteVideoIds.has(video.videoId));
|
|
10000
10218
|
const videoFrameLookup = planVideos && planVideos.extracted.length > 0 ? createFrameLookupTable5(
|
|
10001
10219
|
planVideos.videos,
|
|
10002
10220
|
[
|
|
@@ -10005,17 +10223,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10005
10223
|
planVideos.extracted.filter((video) => !video.deferredRange),
|
|
10006
10224
|
v2Manifest === null ? "dense-v1" : "sparse-v2"
|
|
10007
10225
|
),
|
|
10008
|
-
...
|
|
10009
|
-
planDir,
|
|
10010
|
-
planVideos,
|
|
10011
|
-
frames: Array.from(
|
|
10012
|
-
{ length: slice.endFrame - slice.startFrame },
|
|
10013
|
-
(_, i) => slice.startFrame + i
|
|
10014
|
-
),
|
|
10015
|
-
fps: { num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen },
|
|
10016
|
-
outputRoot: videoFramesDir,
|
|
10017
|
-
cfg
|
|
10018
|
-
})
|
|
10226
|
+
...deferredVideoFrames
|
|
10019
10227
|
],
|
|
10020
10228
|
resolveRenderFpsConfig(job.config.fps).value
|
|
10021
10229
|
) : null;
|
|
@@ -10033,7 +10241,10 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10033
10241
|
dimensions: plan2.dimensions,
|
|
10034
10242
|
encoder,
|
|
10035
10243
|
videoCount: planVideos?.videos.length ?? 0,
|
|
10036
|
-
html:
|
|
10244
|
+
html: readFileSync12(join27(compiledDir, "index.html"), "utf-8"),
|
|
10245
|
+
// The plan ships a deferred source only to chunks that show it; don't wait on the rest.
|
|
10246
|
+
skipReadinessVideoIds: deferredVideoFrames.filter((video) => video.framePaths.size === 0 || remoteVideoIds.has(video.videoId)).map((video) => video.videoId),
|
|
10247
|
+
videoMetadataHints: collectVideoMetadataHints(remoteVideos)
|
|
10037
10248
|
});
|
|
10038
10249
|
const chunkWorkerCount = calculateOptimalWorkers2(framesInChunk, void 0, cfg);
|
|
10039
10250
|
let session = null;
|
|
@@ -10164,7 +10375,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10164
10375
|
},
|
|
10165
10376
|
resetForScreenshotRetry: () => {
|
|
10166
10377
|
rmSync12(workDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
10167
|
-
|
|
10378
|
+
mkdirSync14(framesDir, { recursive: true });
|
|
10168
10379
|
capturePerfs.length = 0;
|
|
10169
10380
|
captureAttempts.length = 0;
|
|
10170
10381
|
job.framesRendered = 0;
|
|
@@ -10195,10 +10406,10 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10195
10406
|
const effectiveBitrate = encoder.crf != null ? void 0 : encoder.bitrate;
|
|
10196
10407
|
const videoOnlyPath = encoder.chunksCarryAudio ? `${outputChunkPath}.video.mp4` : outputChunkPath;
|
|
10197
10408
|
if (isPngSequence) {
|
|
10198
|
-
if (!
|
|
10409
|
+
if (!existsSync19(outputChunkPath)) mkdirSync14(outputChunkPath, { recursive: true });
|
|
10199
10410
|
} else {
|
|
10200
|
-
const outDir =
|
|
10201
|
-
if (!
|
|
10411
|
+
const outDir = join27(outputChunkPath, "..");
|
|
10412
|
+
if (!existsSync19(outDir)) mkdirSync14(outDir, { recursive: true });
|
|
10202
10413
|
}
|
|
10203
10414
|
const encodeStarted = Date.now();
|
|
10204
10415
|
await runEncodeStage({
|
|
@@ -10241,9 +10452,10 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10241
10452
|
gopSize: framesInChunk
|
|
10242
10453
|
});
|
|
10243
10454
|
if (encoder.chunksCarryAudio) {
|
|
10455
|
+
const audioScratchPath = `${outputChunkPath}.audio.m4a`;
|
|
10244
10456
|
await muxChunkWithAudio({
|
|
10245
10457
|
videoPath: videoOnlyPath,
|
|
10246
|
-
audioPath:
|
|
10458
|
+
audioPath: chunkAudioSource(planDir, chunkIndex, audioScratchPath),
|
|
10247
10459
|
outputPath: outputChunkPath,
|
|
10248
10460
|
chunks,
|
|
10249
10461
|
chunkIndex,
|
|
@@ -10251,6 +10463,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10251
10463
|
timeoutMs: cfg.ffmpegProcessTimeout
|
|
10252
10464
|
});
|
|
10253
10465
|
rmSync12(videoOnlyPath, { force: true });
|
|
10466
|
+
rmSync12(audioScratchPath, { force: true });
|
|
10254
10467
|
}
|
|
10255
10468
|
encodeStageMs = Date.now() - encodeStarted;
|
|
10256
10469
|
} finally {
|
|
@@ -10289,7 +10502,7 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10289
10502
|
producerVersion: plan2.producerVersion,
|
|
10290
10503
|
ffmpegVersion
|
|
10291
10504
|
};
|
|
10292
|
-
|
|
10505
|
+
writeFileSync10(perfPath, `${JSON.stringify(perfPayload, null, 2)}
|
|
10293
10506
|
`, "utf-8");
|
|
10294
10507
|
try {
|
|
10295
10508
|
rmSync12(workDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
@@ -10321,8 +10534,8 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
10321
10534
|
|
|
10322
10535
|
// src/services/distributed/planV2Execution.ts
|
|
10323
10536
|
async function renderChunkV2(planV2Dir, chunkIndex, outputChunkPath) {
|
|
10324
|
-
const workRoot = mkdtempSync7(
|
|
10325
|
-
const materializedPlanDir =
|
|
10537
|
+
const workRoot = mkdtempSync7(join28(tmpdir2(), "hf-plan-v2-chunk-"));
|
|
10538
|
+
const materializedPlanDir = join28(workRoot, "plan");
|
|
10326
10539
|
try {
|
|
10327
10540
|
materializePlanV2Target(planV2Dir, { role: "chunk", chunkIndex }, materializedPlanDir);
|
|
10328
10541
|
return await renderChunk(materializedPlanDir, chunkIndex, outputChunkPath);
|
|
@@ -10331,8 +10544,8 @@ async function renderChunkV2(planV2Dir, chunkIndex, outputChunkPath) {
|
|
|
10331
10544
|
}
|
|
10332
10545
|
}
|
|
10333
10546
|
async function assembleV2(planV2Dir, chunkPaths, outputPath, options) {
|
|
10334
|
-
const workRoot = mkdtempSync7(
|
|
10335
|
-
const materializedPlanDir =
|
|
10547
|
+
const workRoot = mkdtempSync7(join28(tmpdir2(), "hf-plan-v2-assembler-"));
|
|
10548
|
+
const materializedPlanDir = join28(workRoot, "plan");
|
|
10336
10549
|
try {
|
|
10337
10550
|
const materialized = materializePlanV2Target(
|
|
10338
10551
|
planV2Dir,
|
|
@@ -10594,28 +10807,46 @@ function validateIntDimension(field, value) {
|
|
|
10594
10807
|
}
|
|
10595
10808
|
|
|
10596
10809
|
// src/services/distributed/projectHash.ts
|
|
10597
|
-
import { readdirSync as readdirSync10, readFileSync as
|
|
10810
|
+
import { readdirSync as readdirSync10, readFileSync as readFileSync13, realpathSync as realpathSync2, statSync as statSync8 } from "node:fs";
|
|
10598
10811
|
import { createHash as createHash6 } from "node:crypto";
|
|
10599
|
-
import { join as
|
|
10600
|
-
|
|
10601
|
-
|
|
10812
|
+
import { join as join29 } from "node:path";
|
|
10813
|
+
var SITE_ID_SCHEME = "site-v2\0";
|
|
10814
|
+
function listSiteFiles(projectDir) {
|
|
10602
10815
|
const files = [];
|
|
10603
|
-
|
|
10604
|
-
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
const
|
|
10609
|
-
|
|
10610
|
-
|
|
10816
|
+
const ancestors = /* @__PURE__ */ new Set([realpathSync2(projectDir)]);
|
|
10817
|
+
function walk(dir, rel) {
|
|
10818
|
+
for (const name of readdirSync10(dir).sort()) {
|
|
10819
|
+
if (rel === "" && PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(name)) continue;
|
|
10820
|
+
const full = join29(dir, name);
|
|
10821
|
+
const relPath = rel === "" ? name : `${rel}/${name}`;
|
|
10822
|
+
let stat;
|
|
10823
|
+
try {
|
|
10824
|
+
stat = statSync8(full);
|
|
10825
|
+
} catch (err) {
|
|
10826
|
+
throw new Error(`[listSiteFiles] cannot read ${relPath} (dangling symlink?)`, {
|
|
10827
|
+
cause: err
|
|
10828
|
+
});
|
|
10829
|
+
}
|
|
10830
|
+
if (stat.isFile()) {
|
|
10831
|
+
files.push(relPath);
|
|
10832
|
+
} else if (stat.isDirectory()) {
|
|
10833
|
+
const real = realpathSync2(full);
|
|
10834
|
+
if (ancestors.has(real)) continue;
|
|
10835
|
+
ancestors.add(real);
|
|
10836
|
+
walk(full, relPath);
|
|
10837
|
+
ancestors.delete(real);
|
|
10838
|
+
}
|
|
10611
10839
|
}
|
|
10612
10840
|
}
|
|
10613
|
-
walk(projectDir,
|
|
10614
|
-
|
|
10615
|
-
|
|
10841
|
+
walk(projectDir, "");
|
|
10842
|
+
return files;
|
|
10843
|
+
}
|
|
10844
|
+
function hashProjectDir(projectDir, files = listSiteFiles(projectDir)) {
|
|
10845
|
+
const hash = createHash6("sha256").update(SITE_ID_SCHEME);
|
|
10846
|
+
for (const rel of files) {
|
|
10616
10847
|
hash.update(rel);
|
|
10617
10848
|
hash.update("\0");
|
|
10618
|
-
hash.update(
|
|
10849
|
+
hash.update(readFileSync13(join29(projectDir, rel)));
|
|
10619
10850
|
}
|
|
10620
10851
|
return hash.digest("hex").slice(0, 16);
|
|
10621
10852
|
}
|
|
@@ -10670,7 +10901,9 @@ export {
|
|
|
10670
10901
|
getPlanV2ExecutionPlanHash,
|
|
10671
10902
|
hashProjectDir,
|
|
10672
10903
|
isPlanAudioArtifactPath,
|
|
10904
|
+
isPlanV2DeferredSourceArtifact,
|
|
10673
10905
|
listPlanV2ArtifactsForTarget,
|
|
10906
|
+
listSiteFiles,
|
|
10674
10907
|
materializePlanV2Target,
|
|
10675
10908
|
measurePlanDirBytes,
|
|
10676
10909
|
muxChunkWithAudio,
|