@hyperframes/producer 0.8.167 → 0.8.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/distributed.d.ts +1 -1
- package/dist/distributed.js +417 -263
- package/dist/hyperframe.manifest.json +1 -1
- package/dist/hyperframe.runtime.iife.js +73 -71
- package/dist/index.js +2881 -2707
- package/dist/plan-parity-contract.d.ts +2 -1
- package/dist/public-server.js +114 -18
- package/dist/regression-harness-distributed.d.ts +5 -5
- package/dist/regression-harness-lambda-local-types.d.ts +2 -1
- package/dist/services/distributed/assemble.d.ts +4 -8
- package/dist/services/distributed/plan.d.ts +37 -9
- package/dist/services/distributed/planV2.d.ts +3 -2
- package/dist/services/render/audioPadTrim.d.ts +3 -2
- package/dist/services/render/outputScaleRoute.d.ts +8 -0
- package/dist/services/render/shared.d.ts +4 -6
- package/dist/services/render/stages/freezePlan.d.ts +7 -1
- package/dist/services/render/stages/planHash.d.ts +3 -0
- package/dist/services/renderOrchestrator.d.ts +1 -1
- package/package.json +7 -7
package/dist/distributed.js
CHANGED
|
@@ -876,7 +876,7 @@ function resolveDeviceScaleFactor(input) {
|
|
|
876
876
|
});
|
|
877
877
|
if (!compat.ok) throw new Error(compat.message);
|
|
878
878
|
const target = CANVAS_DIMENSIONS[input.outputResolution];
|
|
879
|
-
return target.width / input.compositionWidth;
|
|
879
|
+
return Math.max(target.width / input.compositionWidth, target.height / input.compositionHeight);
|
|
880
880
|
}
|
|
881
881
|
function writeCompiledArtifacts(compiled, workDir, includeSummary) {
|
|
882
882
|
const compileDir = join3(workDir, "compiled");
|
|
@@ -3766,10 +3766,10 @@ async function runCompileStage(input) {
|
|
|
3766
3766
|
hdrRequested: job.config.hdrMode === "force-hdr",
|
|
3767
3767
|
alphaRequested: needsAlpha
|
|
3768
3768
|
});
|
|
3769
|
-
const outputWidth = width * deviceScaleFactor;
|
|
3770
|
-
const outputHeight = height * deviceScaleFactor;
|
|
3771
|
-
if (deviceScaleFactor
|
|
3772
|
-
log.info("
|
|
3769
|
+
const outputWidth = Math.round(width * deviceScaleFactor);
|
|
3770
|
+
const outputHeight = Math.round(height * deviceScaleFactor);
|
|
3771
|
+
if (deviceScaleFactor !== 1) {
|
|
3772
|
+
log.info("Scaling composition via deviceScaleFactor", {
|
|
3773
3773
|
compositionWidth: width,
|
|
3774
3774
|
compositionHeight: height,
|
|
3775
3775
|
outputResolution: effectiveResolution,
|
|
@@ -6450,6 +6450,7 @@ function readPlanProtocolV1(planJson, capabilities = DISTRIBUTED_RENDER_CAPABILI
|
|
|
6450
6450
|
|
|
6451
6451
|
// src/services/render/stages/planHash.ts
|
|
6452
6452
|
import { createHash as createHash4 } from "node:crypto";
|
|
6453
|
+
import { closeSync as closeSync3, createReadStream as createReadStream3, openSync as openSync3, readSync } from "node:fs";
|
|
6453
6454
|
var PLAN_HASH_SCHEMA_PREFIX = "hyperframes-plan-hash-v1\0";
|
|
6454
6455
|
var FIELD_DELIMITER = Buffer.from([0]);
|
|
6455
6456
|
function computePlanHash(input) {
|
|
@@ -6504,6 +6505,28 @@ function sha256Hex(bytes) {
|
|
|
6504
6505
|
h.update(bytes);
|
|
6505
6506
|
return h.digest("hex");
|
|
6506
6507
|
}
|
|
6508
|
+
function sha256File(path) {
|
|
6509
|
+
const hash = createHash4("sha256");
|
|
6510
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
6511
|
+
const fd = openSync3(path, "r");
|
|
6512
|
+
try {
|
|
6513
|
+
let bytesRead = 0;
|
|
6514
|
+
do {
|
|
6515
|
+
bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);
|
|
6516
|
+
if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));
|
|
6517
|
+
} while (bytesRead > 0);
|
|
6518
|
+
} finally {
|
|
6519
|
+
closeSync3(fd);
|
|
6520
|
+
}
|
|
6521
|
+
return hash.digest("hex");
|
|
6522
|
+
}
|
|
6523
|
+
async function sha256FileAsync(path) {
|
|
6524
|
+
const hash = createHash4("sha256");
|
|
6525
|
+
for await (const chunk of createReadStream3(path, { highWaterMark: 1024 * 1024 })) {
|
|
6526
|
+
hash.update(chunk);
|
|
6527
|
+
}
|
|
6528
|
+
return hash.digest("hex");
|
|
6529
|
+
}
|
|
6507
6530
|
|
|
6508
6531
|
// src/services/render/stages/freezePlan.ts
|
|
6509
6532
|
var COMPILED_INDEX_RELATIVE_PATH = "compiled/index.html";
|
|
@@ -6512,6 +6535,20 @@ var HASH_EXCLUDED_PLAN_FILES = /* @__PURE__ */ new Set([
|
|
|
6512
6535
|
"meta/encoder.json",
|
|
6513
6536
|
COMPILED_INDEX_RELATIVE_PATH
|
|
6514
6537
|
]);
|
|
6538
|
+
function planFileSha256(absolutePath, digests) {
|
|
6539
|
+
let sha256 = digests?.get(absolutePath);
|
|
6540
|
+
if (sha256 === void 0) {
|
|
6541
|
+
sha256 = sha256File(absolutePath);
|
|
6542
|
+
digests?.set(absolutePath, sha256);
|
|
6543
|
+
}
|
|
6544
|
+
return sha256;
|
|
6545
|
+
}
|
|
6546
|
+
async function prehashPlanFiles(dir, digests) {
|
|
6547
|
+
if (!existsSync11(dir)) return;
|
|
6548
|
+
for (const { absolutePath } of listPlanFiles(dir)) {
|
|
6549
|
+
if (!digests.has(absolutePath)) digests.set(absolutePath, await sha256FileAsync(absolutePath));
|
|
6550
|
+
}
|
|
6551
|
+
}
|
|
6515
6552
|
function stripUndefined(value) {
|
|
6516
6553
|
if (Array.isArray(value)) return value.map(stripUndefined);
|
|
6517
6554
|
if (value !== null && typeof value === "object") {
|
|
@@ -6547,7 +6584,7 @@ function listPlanFiles(planDir) {
|
|
|
6547
6584
|
results.sort((a, b) => a.planRelativePath < b.planRelativePath ? -1 : 1);
|
|
6548
6585
|
return results;
|
|
6549
6586
|
}
|
|
6550
|
-
function collectPlanAssetShas(planDir) {
|
|
6587
|
+
function collectPlanAssetShas(planDir, digests) {
|
|
6551
6588
|
const files = listPlanFiles(planDir);
|
|
6552
6589
|
let compositionHtml = null;
|
|
6553
6590
|
const assets = [];
|
|
@@ -6557,8 +6594,10 @@ function collectPlanAssetShas(planDir) {
|
|
|
6557
6594
|
continue;
|
|
6558
6595
|
}
|
|
6559
6596
|
if (HASH_EXCLUDED_PLAN_FILES.has(file.planRelativePath)) continue;
|
|
6560
|
-
|
|
6561
|
-
|
|
6597
|
+
assets.push({
|
|
6598
|
+
path: file.planRelativePath,
|
|
6599
|
+
sha256: planFileSha256(file.absolutePath, digests)
|
|
6600
|
+
});
|
|
6562
6601
|
}
|
|
6563
6602
|
if (compositionHtml === null) {
|
|
6564
6603
|
throw new Error(
|
|
@@ -6567,7 +6606,7 @@ function collectPlanAssetShas(planDir) {
|
|
|
6567
6606
|
}
|
|
6568
6607
|
return { compositionHtml, assets };
|
|
6569
6608
|
}
|
|
6570
|
-
function recomputePlanHashFromPlanDir(planDir) {
|
|
6609
|
+
function recomputePlanHashFromPlanDir(planDir, digests) {
|
|
6571
6610
|
const planJsonPath = join15(planDir, "plan.json");
|
|
6572
6611
|
const encoderJsonPath = join15(planDir, "meta", "encoder.json");
|
|
6573
6612
|
if (!existsSync11(planJsonPath)) {
|
|
@@ -6578,7 +6617,7 @@ function recomputePlanHashFromPlanDir(planDir) {
|
|
|
6578
6617
|
}
|
|
6579
6618
|
const planJson = JSON.parse(readFileSync7(planJsonPath, "utf-8"));
|
|
6580
6619
|
const encoderConfigCanonicalJson = readFileSync7(encoderJsonPath, "utf-8");
|
|
6581
|
-
const { compositionHtml, assets } = collectPlanAssetShas(planDir);
|
|
6620
|
+
const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);
|
|
6582
6621
|
return computePlanHash({
|
|
6583
6622
|
compositionHtml,
|
|
6584
6623
|
assets,
|
|
@@ -6600,7 +6639,8 @@ async function freezePlan(input) {
|
|
|
6600
6639
|
fontSnapshotSha,
|
|
6601
6640
|
durationSeconds,
|
|
6602
6641
|
totalFrames,
|
|
6603
|
-
hasAudio
|
|
6642
|
+
hasAudio,
|
|
6643
|
+
digests
|
|
6604
6644
|
} = input;
|
|
6605
6645
|
if (!existsSync11(planDir)) {
|
|
6606
6646
|
throw new Error(`[freezePlan] planDir does not exist: ${planDir}`);
|
|
@@ -6618,7 +6658,7 @@ async function freezePlan(input) {
|
|
|
6618
6658
|
writeFileSync4(join15(metaDir, "encoder.json"), encoderConfigCanonicalJson, "utf-8");
|
|
6619
6659
|
writeFileSync4(join15(metaDir, "chunks.json"), `${JSON.stringify(chunks, null, 2)}
|
|
6620
6660
|
`, "utf-8");
|
|
6621
|
-
const { compositionHtml, assets } = collectPlanAssetShas(planDir);
|
|
6661
|
+
const { compositionHtml, assets } = collectPlanAssetShas(planDir, digests);
|
|
6622
6662
|
const planHash = computePlanHash({
|
|
6623
6663
|
compositionHtml,
|
|
6624
6664
|
assets,
|
|
@@ -7108,6 +7148,9 @@ function measurePlanSizeBreakdown(rootDir, rootKind = "plan") {
|
|
|
7108
7148
|
}
|
|
7109
7149
|
|
|
7110
7150
|
// src/services/distributed/plan.ts
|
|
7151
|
+
function logPlanStage(log, stage, startedAt) {
|
|
7152
|
+
log.info("[plan] stage", { stage, ms: Date.now() - startedAt });
|
|
7153
|
+
}
|
|
7111
7154
|
function applyDistributedAudioWarningPolicy(job, audioError, audioFailures = [], log = defaultLogger) {
|
|
7112
7155
|
const failureOwner = audioFailures.length === 0 ? void 0 : audioFailures.some((failure) => failure.owner === "system") ? "system" : "user";
|
|
7113
7156
|
const retryable = audioFailures.length === 0 ? void 0 : audioFailures.every((failure) => failure.retryable);
|
|
@@ -7143,6 +7186,58 @@ var PLAN_PROJECT_DIR_SKIP_SEGMENTS = /* @__PURE__ */ new Set([
|
|
|
7143
7186
|
var DEFAULT_CHUNK_SIZE = 240;
|
|
7144
7187
|
var DEFAULT_MAX_PARALLEL_CHUNKS = 16;
|
|
7145
7188
|
var MIN_CHUNK_SIZE = 10;
|
|
7189
|
+
var MAX_PARALLEL_CHUNKS_CEILING = 256;
|
|
7190
|
+
var ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME = 0.11;
|
|
7191
|
+
var ESTIMATED_CHUNK_OVERHEAD_SECONDS = 60;
|
|
7192
|
+
var WORKER_TIMEOUT_MARGIN = 0.75;
|
|
7193
|
+
function estimateChunkSeconds(frames, megapixels) {
|
|
7194
|
+
return ESTIMATED_CHUNK_OVERHEAD_SECONDS + frames * ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels;
|
|
7195
|
+
}
|
|
7196
|
+
var CHUNK_EXCEEDS_WORKER_TIMEOUT = "CHUNK_EXCEEDS_WORKER_TIMEOUT";
|
|
7197
|
+
var ChunkExceedsWorkerTimeoutError = class extends Error {
|
|
7198
|
+
// fallow-ignore-next-line unused-class-member
|
|
7199
|
+
code = CHUNK_EXCEEDS_WORKER_TIMEOUT;
|
|
7200
|
+
constructor(predictedChunkSeconds, workerTimeoutSeconds) {
|
|
7201
|
+
super(
|
|
7202
|
+
`[plan] each chunk is predicted to take about ${predictedChunkSeconds} s even split ${MAX_PARALLEL_CHUNKS_CEILING} ways, over the ${workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN} s budget (${WORKER_TIMEOUT_MARGIN * 100}% of the ${workerTimeoutSeconds} s worker time limit) (${CHUNK_EXCEEDS_WORKER_TIMEOUT}). Shorten the video or lower its resolution, raise the worker time limit, or pass an explicit chunkSize or maxParallelChunks to override the estimate.`
|
|
7203
|
+
);
|
|
7204
|
+
this.name = "ChunkExceedsWorkerTimeoutError";
|
|
7205
|
+
}
|
|
7206
|
+
};
|
|
7207
|
+
function resolveMaxParallelChunks(config, totalFrames, megapixels, log = defaultLogger) {
|
|
7208
|
+
if (config.maxParallelChunks !== void 0) return config.maxParallelChunks;
|
|
7209
|
+
const { workerTimeoutSeconds, workerConcurrency } = config;
|
|
7210
|
+
const pinsChunking = config.chunkSize !== void 0 || config.targetChunkFrames !== void 0;
|
|
7211
|
+
if (pinsChunking || workerTimeoutSeconds === void 0 && workerConcurrency === void 0) {
|
|
7212
|
+
return DEFAULT_MAX_PARALLEL_CHUNKS;
|
|
7213
|
+
}
|
|
7214
|
+
const secondsPerFrame = ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels;
|
|
7215
|
+
const wave = Math.min(
|
|
7216
|
+
workerConcurrency ?? DEFAULT_MAX_PARALLEL_CHUNKS,
|
|
7217
|
+
MAX_PARALLEL_CHUNKS_CEILING
|
|
7218
|
+
);
|
|
7219
|
+
const minFramesPerChunk = Math.ceil(ESTIMATED_CHUNK_OVERHEAD_SECONDS / secondsPerFrame);
|
|
7220
|
+
const wide = Math.max(1, Math.min(wave, Math.floor(totalFrames / minFramesPerChunk)));
|
|
7221
|
+
if (workerTimeoutSeconds === void 0) return wide;
|
|
7222
|
+
const maxFramesPerChunk = Math.floor(
|
|
7223
|
+
(workerTimeoutSeconds * WORKER_TIMEOUT_MARGIN - ESTIMATED_CHUNK_OVERHEAD_SECONDS) / secondsPerFrame
|
|
7224
|
+
);
|
|
7225
|
+
const needed = maxFramesPerChunk < 1 ? Infinity : Math.ceil(totalFrames / maxFramesPerChunk);
|
|
7226
|
+
if (needed > MAX_PARALLEL_CHUNKS_CEILING) {
|
|
7227
|
+
const framesAtCeiling = Math.ceil(totalFrames / MAX_PARALLEL_CHUNKS_CEILING);
|
|
7228
|
+
throw new ChunkExceedsWorkerTimeoutError(
|
|
7229
|
+
Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels)),
|
|
7230
|
+
workerTimeoutSeconds
|
|
7231
|
+
);
|
|
7232
|
+
}
|
|
7233
|
+
if (needed > wave) {
|
|
7234
|
+
log.warn(
|
|
7235
|
+
`[plan] ${needed} chunks are needed to keep each under the ${workerTimeoutSeconds} s worker time limit, but the plan assumes ${wave} workers at once, so the render runs in ${Math.ceil(needed / wave)} waves. Raise --concurrency to ${needed} to render in one wave (Lambda: hyperframes lambda deploy --concurrency ${needed}).`,
|
|
7236
|
+
{ needed, workerConcurrency: wave, workerTimeoutSeconds }
|
|
7237
|
+
);
|
|
7238
|
+
}
|
|
7239
|
+
return Math.max(wide, needed);
|
|
7240
|
+
}
|
|
7146
7241
|
var PLAN_DIR_SIZE_LIMIT_BYTES = 2 * 1024 * 1024 * 1024;
|
|
7147
7242
|
var PLAN_TOO_LARGE = "PLAN_TOO_LARGE";
|
|
7148
7243
|
var PlanTooLargeError = class extends Error {
|
|
@@ -7435,6 +7530,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7435
7530
|
mkdirSync8(workDir, { recursive: true });
|
|
7436
7531
|
const compiledDir = join18(workDir, "compiled");
|
|
7437
7532
|
mkdirSync8(compiledDir, { recursive: true });
|
|
7533
|
+
let stageStarted = Date.now();
|
|
7438
7534
|
cpSync2(projectDir, compiledDir, {
|
|
7439
7535
|
recursive: true,
|
|
7440
7536
|
dereference: true,
|
|
@@ -7445,8 +7541,10 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7445
7541
|
return firstSegment === void 0 || !PLAN_PROJECT_DIR_SKIP_SEGMENTS.has(firstSegment);
|
|
7446
7542
|
}
|
|
7447
7543
|
});
|
|
7544
|
+
logPlanStage(log, "copy", stageStarted);
|
|
7448
7545
|
const finalCompiledDir = join18(planDir, "compiled");
|
|
7449
7546
|
const needsAlpha = outputNeedsAlpha(config.format);
|
|
7547
|
+
stageStarted = Date.now();
|
|
7450
7548
|
const compileResult = await runCompileStage({
|
|
7451
7549
|
projectDir,
|
|
7452
7550
|
workDir,
|
|
@@ -7466,6 +7564,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7466
7564
|
allowSystemFontCapture: false,
|
|
7467
7565
|
variables: config.variables
|
|
7468
7566
|
});
|
|
7567
|
+
logPlanStage(log, "compile", stageStarted);
|
|
7469
7568
|
let compiled = compileResult.compiled;
|
|
7470
7569
|
const composition = compileResult.composition;
|
|
7471
7570
|
const { deviceScaleFactor } = compileResult;
|
|
@@ -7475,6 +7574,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7475
7574
|
if (config.rejectOnSystemFonts !== false) {
|
|
7476
7575
|
validateNoSystemFonts(compiled.html);
|
|
7477
7576
|
}
|
|
7577
|
+
stageStarted = Date.now();
|
|
7478
7578
|
const probeResult = await runProbeStage({
|
|
7479
7579
|
projectDir,
|
|
7480
7580
|
workDir,
|
|
@@ -7492,6 +7592,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7492
7592
|
deviceScaleFactor,
|
|
7493
7593
|
subTimelineWaitMemo: {}
|
|
7494
7594
|
});
|
|
7595
|
+
logPlanStage(log, "probe", stageStarted);
|
|
7495
7596
|
compiled = probeResult.compiled;
|
|
7496
7597
|
job.duration = probeResult.duration;
|
|
7497
7598
|
job.totalFrames = probeResult.totalFrames;
|
|
@@ -7512,6 +7613,19 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7512
7613
|
});
|
|
7513
7614
|
}
|
|
7514
7615
|
}
|
|
7616
|
+
const maxParallel = resolveMaxParallelChunks(
|
|
7617
|
+
config,
|
|
7618
|
+
totalFrames,
|
|
7619
|
+
compileResult.outputWidth * compileResult.outputHeight / 1e6,
|
|
7620
|
+
log
|
|
7621
|
+
);
|
|
7622
|
+
const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
|
|
7623
|
+
totalFrames,
|
|
7624
|
+
config.chunkSize,
|
|
7625
|
+
maxParallel,
|
|
7626
|
+
config.targetChunkFrames
|
|
7627
|
+
);
|
|
7628
|
+
const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
|
|
7515
7629
|
assertPlanSizeWithinLimit({
|
|
7516
7630
|
rootDir: compiledDir,
|
|
7517
7631
|
rootKind: "compiled",
|
|
@@ -7519,6 +7633,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7519
7633
|
observedAt: "pre-extract",
|
|
7520
7634
|
log
|
|
7521
7635
|
});
|
|
7636
|
+
stageStarted = Date.now();
|
|
7522
7637
|
const extractResult = await runExtractVideosStage({
|
|
7523
7638
|
projectDir,
|
|
7524
7639
|
compiledDir,
|
|
@@ -7530,29 +7645,16 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7530
7645
|
materializeSymlinks: true,
|
|
7531
7646
|
deferRangeExtraction: options.deferVideoExtraction
|
|
7532
7647
|
});
|
|
7648
|
+
logPlanStage(log, "extract", stageStarted);
|
|
7533
7649
|
if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
|
|
7534
7650
|
if (extractResult.extractionResult) {
|
|
7535
7651
|
assertVideoExtractionSucceeded(extractResult.extractionResult);
|
|
7536
7652
|
}
|
|
7537
|
-
const audioResult = await runAudioStage({
|
|
7538
|
-
projectDir,
|
|
7539
|
-
workDir,
|
|
7540
|
-
compiledDir,
|
|
7541
|
-
duration: job.duration,
|
|
7542
|
-
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
|
|
7543
|
-
audioGain: cfg.audioGain,
|
|
7544
|
-
audios: composition.audios,
|
|
7545
|
-
masterFxChain: composition.masterFxChain,
|
|
7546
|
-
abortSignal,
|
|
7547
|
-
assertNotAborted
|
|
7548
|
-
});
|
|
7549
|
-
if (audioResult.audioError) {
|
|
7550
|
-
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
|
|
7551
|
-
}
|
|
7552
7653
|
const stagedVideoFrames = join18(compiledDir, "__hyperframes_video_frames");
|
|
7553
7654
|
const videoFramesDst = join18(planDir, "video-frames");
|
|
7554
7655
|
if (existsSync13(videoFramesDst)) rmSync7(videoFramesDst, { recursive: true, force: true });
|
|
7555
|
-
|
|
7656
|
+
const videoSourcesDir = join18(planDir, "video-sources");
|
|
7657
|
+
rmSync7(videoSourcesDir, { recursive: true, force: true });
|
|
7556
7658
|
if (existsSync13(stagedVideoFrames)) {
|
|
7557
7659
|
renameSync3(stagedVideoFrames, videoFramesDst);
|
|
7558
7660
|
} else {
|
|
@@ -7585,18 +7687,43 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7585
7687
|
JSON.stringify(planVideosJson, null, 2),
|
|
7586
7688
|
"utf-8"
|
|
7587
7689
|
);
|
|
7690
|
+
const digests = options.digests ?? /* @__PURE__ */ new Map();
|
|
7691
|
+
stageStarted = Date.now();
|
|
7692
|
+
const [audioOutcome, overlapOutcome] = await Promise.allSettled([
|
|
7693
|
+
runAudioStage({
|
|
7694
|
+
projectDir,
|
|
7695
|
+
workDir,
|
|
7696
|
+
compiledDir: finalCompiledDir,
|
|
7697
|
+
duration: probeResult.duration,
|
|
7698
|
+
ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
|
|
7699
|
+
audioGain: cfg.audioGain,
|
|
7700
|
+
audios: composition.audios,
|
|
7701
|
+
masterFxChain: composition.masterFxChain,
|
|
7702
|
+
abortSignal,
|
|
7703
|
+
assertNotAborted
|
|
7704
|
+
}).finally(() => logPlanStage(log, "audio", stageStarted)),
|
|
7705
|
+
(async () => {
|
|
7706
|
+
for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {
|
|
7707
|
+
await prehashPlanFiles(dir, digests);
|
|
7708
|
+
}
|
|
7709
|
+
logPlanStage(log, "prehash", stageStarted);
|
|
7710
|
+
if (options.whileAudioMixes) {
|
|
7711
|
+
const prepublishStarted = Date.now();
|
|
7712
|
+
await options.whileAudioMixes();
|
|
7713
|
+
logPlanStage(log, "prepublish", prepublishStarted);
|
|
7714
|
+
}
|
|
7715
|
+
})()
|
|
7716
|
+
]);
|
|
7717
|
+
if (audioOutcome.status === "rejected") throw audioOutcome.reason;
|
|
7718
|
+
if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;
|
|
7719
|
+
const audioResult = audioOutcome.value;
|
|
7720
|
+
if (audioResult.audioError) {
|
|
7721
|
+
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
|
|
7722
|
+
}
|
|
7588
7723
|
const planAudioPath = join18(planDir, PLAN_AUDIO_RELATIVE_PATH);
|
|
7589
7724
|
if (audioResult.hasAudio && existsSync13(audioResult.audioOutputPath)) {
|
|
7590
7725
|
renameSync3(audioResult.audioOutputPath, planAudioPath);
|
|
7591
7726
|
}
|
|
7592
|
-
const maxParallel = config.maxParallelChunks ?? DEFAULT_MAX_PARALLEL_CHUNKS;
|
|
7593
|
-
const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
|
|
7594
|
-
totalFrames,
|
|
7595
|
-
config.chunkSize,
|
|
7596
|
-
maxParallel,
|
|
7597
|
-
config.targetChunkFrames
|
|
7598
|
-
);
|
|
7599
|
-
const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
|
|
7600
7727
|
const ffmpegVersion = await readFfmpegVersion();
|
|
7601
7728
|
const producerVersion = readProducerVersion();
|
|
7602
7729
|
const fontSnapshotSha = await readFontSnapshotSha();
|
|
@@ -7643,6 +7770,7 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7643
7770
|
observedAt: "pre-freeze",
|
|
7644
7771
|
log
|
|
7645
7772
|
});
|
|
7773
|
+
stageStarted = Date.now();
|
|
7646
7774
|
const freezeResult = await freezePlan({
|
|
7647
7775
|
planDir,
|
|
7648
7776
|
composition: compositionJson,
|
|
@@ -7653,8 +7781,10 @@ async function buildLocalExecutionPlan(projectDir, config, executionPlanDir, opt
|
|
|
7653
7781
|
fontSnapshotSha,
|
|
7654
7782
|
durationSeconds: job.duration ?? 0,
|
|
7655
7783
|
totalFrames,
|
|
7656
|
-
hasAudio: audioResult.hasAudio
|
|
7784
|
+
hasAudio: audioResult.hasAudio,
|
|
7785
|
+
digests
|
|
7657
7786
|
});
|
|
7787
|
+
logPlanStage(log, "freeze", stageStarted);
|
|
7658
7788
|
const planHash = freezeResult.planHash;
|
|
7659
7789
|
assertPlanSizeWithinLimit({
|
|
7660
7790
|
rootDir: planDir,
|
|
@@ -7695,24 +7825,21 @@ async function plan(projectDir, config, planDir) {
|
|
|
7695
7825
|
|
|
7696
7826
|
// src/services/distributed/planV2.ts
|
|
7697
7827
|
import {
|
|
7698
|
-
closeSync as closeSync3,
|
|
7699
7828
|
copyFileSync as copyFileSync7,
|
|
7700
7829
|
existsSync as existsSync15,
|
|
7701
7830
|
lstatSync as lstatSync2,
|
|
7702
7831
|
mkdirSync as mkdirSync10,
|
|
7703
7832
|
mkdtempSync as mkdtempSync4,
|
|
7704
|
-
openSync as openSync3,
|
|
7705
7833
|
readFileSync as readFileSync9,
|
|
7706
|
-
readSync,
|
|
7707
7834
|
readdirSync as readdirSync6,
|
|
7708
7835
|
renameSync as renameSync5,
|
|
7709
7836
|
rmSync as rmSync9,
|
|
7710
7837
|
statSync as statSync6,
|
|
7711
7838
|
writeFileSync as writeFileSync7
|
|
7712
7839
|
} from "node:fs";
|
|
7713
|
-
import { createHash as createHash6 } from "node:crypto";
|
|
7714
7840
|
import { tmpdir } from "node:os";
|
|
7715
7841
|
import { dirname as dirname10, isAbsolute as isAbsolute4, join as join21, relative as relative6, resolve as resolve8, sep as sep3 } from "node:path";
|
|
7842
|
+
import { DISTRIBUTED_FPS, isDistributedFps } from "@hyperframes/core";
|
|
7716
7843
|
import { createFrameLookupTable as createFrameLookupTable2 } from "@hyperframes/engine";
|
|
7717
7844
|
|
|
7718
7845
|
// src/services/distributed/planV2Publisher.ts
|
|
@@ -7891,21 +8018,6 @@ function listFiles(root) {
|
|
|
7891
8018
|
walk(rootResolved);
|
|
7892
8019
|
return files.sort((a, b) => a.path.localeCompare(b.path));
|
|
7893
8020
|
}
|
|
7894
|
-
function sha256File(path) {
|
|
7895
|
-
const hash = createHash6("sha256");
|
|
7896
|
-
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
7897
|
-
const fd = openSync3(path, "r");
|
|
7898
|
-
try {
|
|
7899
|
-
let bytesRead = 0;
|
|
7900
|
-
do {
|
|
7901
|
-
bytesRead = readSync(fd, buffer, 0, buffer.byteLength, null);
|
|
7902
|
-
if (bytesRead > 0) hash.update(buffer.subarray(0, bytesRead));
|
|
7903
|
-
} while (bytesRead > 0);
|
|
7904
|
-
} finally {
|
|
7905
|
-
closeSync3(fd);
|
|
7906
|
-
}
|
|
7907
|
-
return hash.digest("hex");
|
|
7908
|
-
}
|
|
7909
8021
|
function assertValidExtractionCacheCompleteSentinel(path) {
|
|
7910
8022
|
const sentinel = lstatSync2(path);
|
|
7911
8023
|
if (!sentinel.isFile() || sentinel.size !== 0) {
|
|
@@ -8108,7 +8220,31 @@ function writeBlob(sourcePath, destinationPath) {
|
|
|
8108
8220
|
rmSync9(temporaryDir, { recursive: true, force: true });
|
|
8109
8221
|
}
|
|
8110
8222
|
}
|
|
8111
|
-
function
|
|
8223
|
+
function publishBlobFor(absolutePath, digests) {
|
|
8224
|
+
return {
|
|
8225
|
+
sourcePath: absolutePath,
|
|
8226
|
+
sha256: planFileSha256(absolutePath, digests),
|
|
8227
|
+
sizeBytes: statSync6(absolutePath).size
|
|
8228
|
+
};
|
|
8229
|
+
}
|
|
8230
|
+
async function putBlobs(publisher, blobs) {
|
|
8231
|
+
const uploadSlots = 32;
|
|
8232
|
+
let next = 0;
|
|
8233
|
+
let failure;
|
|
8234
|
+
const upload = async () => {
|
|
8235
|
+
while (!failure && next < blobs.length) {
|
|
8236
|
+
const blob = blobs[next++];
|
|
8237
|
+
try {
|
|
8238
|
+
await publisher.putBlob(blob);
|
|
8239
|
+
} catch (reason) {
|
|
8240
|
+
failure ??= { reason };
|
|
8241
|
+
}
|
|
8242
|
+
}
|
|
8243
|
+
};
|
|
8244
|
+
await Promise.all(Array.from({ length: uploadSlots }, upload));
|
|
8245
|
+
if (failure) throw failure.reason;
|
|
8246
|
+
}
|
|
8247
|
+
function buildPlanV2Publication(executionPlanDir, digests) {
|
|
8112
8248
|
const executionPlanPath = join21(executionPlanDir, "plan.json");
|
|
8113
8249
|
if (!existsSync15(executionPlanPath)) {
|
|
8114
8250
|
throw new PlanV2IntegrityError(`execution plan is missing plan.json: ${executionPlanPath}`);
|
|
@@ -8123,7 +8259,7 @@ function buildPlanV2Publication(executionPlanDir) {
|
|
|
8123
8259
|
if (!isSha256(executionPlanHash)) {
|
|
8124
8260
|
throw new PlanV2IntegrityError("execution plan.json.planHash must be a sha256 digest");
|
|
8125
8261
|
}
|
|
8126
|
-
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir);
|
|
8262
|
+
const recomputedExecutionPlanHash = recomputePlanHashFromPlanDir(executionPlanDir, digests);
|
|
8127
8263
|
if (recomputedExecutionPlanHash !== executionPlanHash) {
|
|
8128
8264
|
throw new PlanV2IntegrityError(
|
|
8129
8265
|
`execution plan content fingerprint does not match plan.json.planHash: expected ${executionPlanHash}, recomputed ${recomputedExecutionPlanHash}`
|
|
@@ -8143,15 +8279,12 @@ function buildPlanV2Publication(executionPlanDir) {
|
|
|
8143
8279
|
if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) && targets.chunks !== "all" && targets.chunks.length === 0 && !targets.assembler) {
|
|
8144
8280
|
continue;
|
|
8145
8281
|
}
|
|
8146
|
-
const
|
|
8147
|
-
|
|
8148
|
-
if (!blobs.has(sha256)) {
|
|
8149
|
-
blobs.set(sha256, { sourcePath: file.absolutePath, sha256, sizeBytes });
|
|
8150
|
-
}
|
|
8282
|
+
const blob = publishBlobFor(file.absolutePath, digests);
|
|
8283
|
+
if (!blobs.has(blob.sha256)) blobs.set(blob.sha256, blob);
|
|
8151
8284
|
artifacts.push({
|
|
8152
8285
|
path: file.path,
|
|
8153
|
-
sha256,
|
|
8154
|
-
sizeBytes,
|
|
8286
|
+
sha256: blob.sha256,
|
|
8287
|
+
sizeBytes: blob.sizeBytes,
|
|
8155
8288
|
...targets
|
|
8156
8289
|
});
|
|
8157
8290
|
}
|
|
@@ -8206,24 +8339,19 @@ function createPlanV2FromV1(executionPlanDir, planV2Dir) {
|
|
|
8206
8339
|
return createPlanV2FromExecutionPlan(executionPlanDir, planV2Dir);
|
|
8207
8340
|
}
|
|
8208
8341
|
async function publishPlanV2FromExecutionPlan(executionPlanDir, publisher) {
|
|
8342
|
+
return publishExecutionPlan(executionPlanDir, publisher, {});
|
|
8343
|
+
}
|
|
8344
|
+
async function publishExecutionPlan(executionPlanDir, publisher, { digests, log = defaultLogger }) {
|
|
8209
8345
|
try {
|
|
8210
|
-
|
|
8211
|
-
const
|
|
8212
|
-
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8216
|
-
|
|
8217
|
-
try {
|
|
8218
|
-
await publisher.putBlob(blob);
|
|
8219
|
-
} catch (reason) {
|
|
8220
|
-
failure ??= { reason };
|
|
8221
|
-
}
|
|
8222
|
-
}
|
|
8223
|
-
};
|
|
8224
|
-
await Promise.all(Array.from({ length: uploadSlots }, upload));
|
|
8225
|
-
if (failure) throw failure.reason;
|
|
8346
|
+
let stageStarted = Date.now();
|
|
8347
|
+
const publication = buildPlanV2Publication(executionPlanDir, digests);
|
|
8348
|
+
logPlanStage(log, "publication", stageStarted);
|
|
8349
|
+
stageStarted = Date.now();
|
|
8350
|
+
await putBlobs(publisher, publication.blobs);
|
|
8351
|
+
logPlanStage(log, "upload", stageStarted);
|
|
8352
|
+
stageStarted = Date.now();
|
|
8226
8353
|
await publisher.commitManifest(canonicalJsonStringify(publication.manifest));
|
|
8354
|
+
logPlanStage(log, "manifest", stageStarted);
|
|
8227
8355
|
return publication.manifest;
|
|
8228
8356
|
} catch (error) {
|
|
8229
8357
|
try {
|
|
@@ -8238,12 +8366,21 @@ async function publishPlanV2FromV1(executionPlanDir, publisher) {
|
|
|
8238
8366
|
}
|
|
8239
8367
|
async function planV2WithPublisher(projectDir, config, publisher, options = {}) {
|
|
8240
8368
|
const stagingRoot = mkdtempSync4(join21(options.stagingParentDir ?? tmpdir(), ".execution-plan-"));
|
|
8369
|
+
const digests = /* @__PURE__ */ new Map();
|
|
8241
8370
|
try {
|
|
8242
8371
|
await buildLocalExecutionPlan(projectDir, config, stagingRoot, {
|
|
8243
8372
|
executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
|
|
8244
|
-
deferVideoExtraction: true
|
|
8373
|
+
deferVideoExtraction: true,
|
|
8374
|
+
digests,
|
|
8375
|
+
// artifactTargets ships every compiled/ file to all chunks, so each is in the manifest.
|
|
8376
|
+
whileAudioMixes: () => putBlobs(
|
|
8377
|
+
publisher,
|
|
8378
|
+
listFiles(join21(stagingRoot, "compiled")).map(
|
|
8379
|
+
(file) => publishBlobFor(file.absolutePath, digests)
|
|
8380
|
+
)
|
|
8381
|
+
)
|
|
8245
8382
|
});
|
|
8246
|
-
return await
|
|
8383
|
+
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
|
|
8247
8384
|
} catch (error) {
|
|
8248
8385
|
try {
|
|
8249
8386
|
await publisher.abort();
|
|
@@ -8304,8 +8441,10 @@ function readNonNegativeInteger2(value, field) {
|
|
|
8304
8441
|
return value;
|
|
8305
8442
|
}
|
|
8306
8443
|
function readSupportedFps(value) {
|
|
8307
|
-
if (value
|
|
8308
|
-
throw new PlanV2IntegrityError(
|
|
8444
|
+
if (!isDistributedFps(value)) {
|
|
8445
|
+
throw new PlanV2IntegrityError(
|
|
8446
|
+
`dimensions.fpsNum must be one of ${DISTRIBUTED_FPS.join(", ")}`
|
|
8447
|
+
);
|
|
8309
8448
|
}
|
|
8310
8449
|
return value;
|
|
8311
8450
|
}
|
|
@@ -8518,6 +8657,7 @@ import {
|
|
|
8518
8657
|
mkdtempSync as mkdtempSync5,
|
|
8519
8658
|
readFileSync as readFileSync10,
|
|
8520
8659
|
readdirSync as readdirSync7,
|
|
8660
|
+
renameSync as renameSync6,
|
|
8521
8661
|
rmSync as rmSync10,
|
|
8522
8662
|
statSync as statSync7,
|
|
8523
8663
|
writeFileSync as writeFileSync8
|
|
@@ -8525,7 +8665,6 @@ import {
|
|
|
8525
8665
|
import { dirname as dirname11, join as join22 } from "node:path";
|
|
8526
8666
|
import {
|
|
8527
8667
|
appendRenderProvenanceArgs,
|
|
8528
|
-
applyFaststart as applyFaststart2,
|
|
8529
8668
|
describeFfmpegFailure as describeFfmpegFailure2,
|
|
8530
8669
|
getEncoderPreset as getEncoderPreset5,
|
|
8531
8670
|
MIXED_AUDIO_FILENAME as MIXED_AUDIO_FILENAME4,
|
|
@@ -8544,9 +8683,64 @@ function cfrRateControlArgs(locked) {
|
|
|
8544
8683
|
async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
8545
8684
|
const start = Date.now();
|
|
8546
8685
|
const log = options?.logger ?? defaultLogger;
|
|
8547
|
-
const abortSignal = options?.abortSignal;
|
|
8548
8686
|
const cfr = options?.cfr === true;
|
|
8549
|
-
const
|
|
8687
|
+
const plan2 = readAssemblePlan(planDir, chunkPaths);
|
|
8688
|
+
if (plan2.dimensions.format === "png-sequence") {
|
|
8689
|
+
return mergePngFrameDirs(chunkPaths, outputPath, plan2.totalFrames, audioPath, start);
|
|
8690
|
+
}
|
|
8691
|
+
const format = plan2.dimensions.format;
|
|
8692
|
+
const cfrEncoder = cfr ? readCfrEncoder(planDir, format) : null;
|
|
8693
|
+
if (!existsSync16(dirname11(outputPath))) {
|
|
8694
|
+
mkdirSync11(dirname11(outputPath), { recursive: true });
|
|
8695
|
+
}
|
|
8696
|
+
const workDir = mkdtempSync5(`${outputPath}.assemble-work-`);
|
|
8697
|
+
const fps = { num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen };
|
|
8698
|
+
const stageMs = {};
|
|
8699
|
+
const ctx = {
|
|
8700
|
+
workDir,
|
|
8701
|
+
format,
|
|
8702
|
+
fps,
|
|
8703
|
+
fpsArg: fpsToFfmpegArg3(fps),
|
|
8704
|
+
signal: options?.abortSignal,
|
|
8705
|
+
timeout: resolveConfig4().ffmpegProcessTimeout,
|
|
8706
|
+
log,
|
|
8707
|
+
timed: async (stage, run) => {
|
|
8708
|
+
const stageStart = Date.now();
|
|
8709
|
+
const result = await run();
|
|
8710
|
+
stageMs[stage] = Date.now() - stageStart;
|
|
8711
|
+
return result;
|
|
8712
|
+
}
|
|
8713
|
+
};
|
|
8714
|
+
try {
|
|
8715
|
+
const videoSources = cfrEncoder ? [await reencodeCfr(ctx, chunkPaths, cfrEncoder)] : chunkPaths;
|
|
8716
|
+
const finalPath = join22(workDir, `output.${format}`);
|
|
8717
|
+
const audio = audioPath !== null && existsSync16(audioPath) ? audioPath : null;
|
|
8718
|
+
await writeJoined(ctx, videoSources, audio, finalPath);
|
|
8719
|
+
renameSync6(finalPath, outputPath);
|
|
8720
|
+
log.info("[assemble] stage timings", {
|
|
8721
|
+
...stageMs,
|
|
8722
|
+
chunks: chunkPaths.length,
|
|
8723
|
+
totalMs: Date.now() - start
|
|
8724
|
+
});
|
|
8725
|
+
} finally {
|
|
8726
|
+
try {
|
|
8727
|
+
rmSync10(workDir, { recursive: true, force: true });
|
|
8728
|
+
} catch (err) {
|
|
8729
|
+
log.warn("[assemble] failed to remove work dir", {
|
|
8730
|
+
workDir,
|
|
8731
|
+
error: err instanceof Error ? err.message : String(err)
|
|
8732
|
+
});
|
|
8733
|
+
}
|
|
8734
|
+
}
|
|
8735
|
+
const fileSize = existsSync16(outputPath) ? statSync7(outputPath).size : 0;
|
|
8736
|
+
return {
|
|
8737
|
+
outputPath,
|
|
8738
|
+
durationMs: Date.now() - start,
|
|
8739
|
+
framesEncoded: plan2.totalFrames,
|
|
8740
|
+
fileSize
|
|
8741
|
+
};
|
|
8742
|
+
}
|
|
8743
|
+
function readAssemblePlan(planDir, chunkPaths) {
|
|
8550
8744
|
const planJsonPath = join22(planDir, "plan.json");
|
|
8551
8745
|
const chunksJsonPath = join22(planDir, "meta", "chunks.json");
|
|
8552
8746
|
if (!existsSync16(planJsonPath)) {
|
|
@@ -8569,180 +8763,133 @@ async function assemble(planDir, chunkPaths, audioPath, outputPath, options) {
|
|
|
8569
8763
|
throw new Error(`[assemble] chunk path does not exist: ${path}`);
|
|
8570
8764
|
}
|
|
8571
8765
|
}
|
|
8572
|
-
|
|
8573
|
-
|
|
8766
|
+
return plan2;
|
|
8767
|
+
}
|
|
8768
|
+
function videoInputArgs(ctx, videoSources) {
|
|
8769
|
+
return videoSources.length === 1 ? ["-i", videoSources[0]] : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);
|
|
8770
|
+
}
|
|
8771
|
+
async function reencodeCfr(ctx, chunkPaths, encoder) {
|
|
8772
|
+
const cfrOutputPath = join22(ctx.workDir, `cfr.${ctx.format}`);
|
|
8773
|
+
const cfrArgs = [
|
|
8774
|
+
...videoInputArgs(ctx, chunkPaths),
|
|
8775
|
+
"-c:v",
|
|
8776
|
+
"libx264",
|
|
8777
|
+
...cfrRateControlArgs(encoder),
|
|
8778
|
+
"-pix_fmt",
|
|
8779
|
+
"yuv420p",
|
|
8780
|
+
"-fps_mode",
|
|
8781
|
+
"cfr",
|
|
8782
|
+
"-r",
|
|
8783
|
+
ctx.fpsArg
|
|
8784
|
+
];
|
|
8785
|
+
appendRenderProvenanceArgs(cfrArgs, cfrOutputPath);
|
|
8786
|
+
cfrArgs.push("-y", cfrOutputPath);
|
|
8787
|
+
await ctx.timed(
|
|
8788
|
+
"cfrMs",
|
|
8789
|
+
() => runAssembleFfmpeg(cfrArgs, "cfr re-encode", ctx.signal, ctx.timeout)
|
|
8790
|
+
);
|
|
8791
|
+
ctx.log.info("[assemble] cfr re-encode applied", {
|
|
8792
|
+
format: ctx.format,
|
|
8793
|
+
fpsNum: ctx.fps.num,
|
|
8794
|
+
fpsDen: ctx.fps.den
|
|
8795
|
+
});
|
|
8796
|
+
return cfrOutputPath;
|
|
8797
|
+
}
|
|
8798
|
+
async function writeJoined(ctx, videoSources, audioPath, finalPath) {
|
|
8799
|
+
if (audioPath === null) {
|
|
8800
|
+
const joinArgs = [...videoInputArgs(ctx, videoSources), "-c", "copy"];
|
|
8801
|
+
if (ctx.format === "mp4") joinArgs.push("-movflags", "+faststart");
|
|
8802
|
+
appendRenderProvenanceArgs(joinArgs, finalPath);
|
|
8803
|
+
joinArgs.push("-r", ctx.fpsArg, "-y", finalPath);
|
|
8804
|
+
await ctx.timed(
|
|
8805
|
+
"joinMs",
|
|
8806
|
+
() => runAssembleFfmpeg(joinArgs, "concat-copy", ctx.signal, ctx.timeout)
|
|
8807
|
+
);
|
|
8808
|
+
return;
|
|
8574
8809
|
}
|
|
8575
|
-
|
|
8576
|
-
|
|
8810
|
+
const paddedAudioPath = join22(ctx.workDir, "audio-padded.m4a");
|
|
8811
|
+
const padTrimResult = await ctx.timed(
|
|
8812
|
+
"audioMs",
|
|
8813
|
+
() => padOrTrimAudioToVideoFrameCount({
|
|
8814
|
+
videoPath: videoSources[0],
|
|
8815
|
+
probeVideoFrameInfo: () => probeJoinedFrameInfo(videoSources, ctx.fps, ctx.signal),
|
|
8816
|
+
audioPath,
|
|
8817
|
+
outputPath: paddedAudioPath,
|
|
8818
|
+
signal: ctx.signal,
|
|
8819
|
+
timeoutMs: ctx.timeout
|
|
8820
|
+
})
|
|
8821
|
+
);
|
|
8822
|
+
if (!padTrimResult.success) {
|
|
8823
|
+
throw encoderFailureError("[assemble] audio pad/trim failed", padTrimResult);
|
|
8577
8824
|
}
|
|
8578
|
-
|
|
8579
|
-
|
|
8580
|
-
|
|
8581
|
-
|
|
8582
|
-
|
|
8583
|
-
|
|
8584
|
-
|
|
8585
|
-
|
|
8586
|
-
|
|
8587
|
-
|
|
8588
|
-
|
|
8589
|
-
|
|
8590
|
-
signal: abortSignal,
|
|
8591
|
-
timeout: ffmpegProcessTimeout
|
|
8592
|
-
});
|
|
8593
|
-
if (!remuxResult.success) {
|
|
8594
|
-
throw encoderFailureError("[assemble] ffmpeg single-chunk remux failed", {
|
|
8595
|
-
error: describeFfmpegFailure2(remuxResult, ffmpegProcessTimeout),
|
|
8596
|
-
failureReason: remuxResult.failureReason
|
|
8597
|
-
});
|
|
8598
|
-
}
|
|
8599
|
-
} else {
|
|
8600
|
-
const concatListPath = join22(workDir, "concat-list.txt");
|
|
8601
|
-
const concatBody = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");
|
|
8602
|
-
writeFileSync8(concatListPath, `${concatBody}
|
|
8603
|
-
`, "utf-8");
|
|
8604
|
-
const concatArgs = [
|
|
8605
|
-
"-r",
|
|
8606
|
-
fpsArg,
|
|
8607
|
-
"-f",
|
|
8608
|
-
"concat",
|
|
8609
|
-
"-safe",
|
|
8610
|
-
"0",
|
|
8611
|
-
"-i",
|
|
8612
|
-
concatListPath,
|
|
8613
|
-
"-c",
|
|
8614
|
-
"copy"
|
|
8615
|
-
];
|
|
8616
|
-
appendRenderProvenanceArgs(concatArgs, concatOutputPath);
|
|
8617
|
-
concatArgs.push("-y", concatOutputPath);
|
|
8618
|
-
const concatResult = await runFfmpeg5(concatArgs, {
|
|
8619
|
-
signal: abortSignal,
|
|
8620
|
-
timeout: ffmpegProcessTimeout
|
|
8621
|
-
});
|
|
8622
|
-
if (!concatResult.success) {
|
|
8623
|
-
throw encoderFailureError("[assemble] ffmpeg concat-copy failed", {
|
|
8624
|
-
error: describeFfmpegFailure2(concatResult, ffmpegProcessTimeout),
|
|
8625
|
-
failureReason: concatResult.failureReason
|
|
8626
|
-
});
|
|
8627
|
-
}
|
|
8628
|
-
}
|
|
8629
|
-
let postConcatPath = concatOutputPath;
|
|
8630
|
-
if (cfr) {
|
|
8631
|
-
if (plan2.dimensions.format !== "mp4") {
|
|
8632
|
-
throw new Error(
|
|
8633
|
-
`[assemble] cfr=true is only supported for format="mp4" (got "${plan2.dimensions.format}"). Stream-copy paths for webm and mov already produce exact avg_frame_rate; cfr re-encode is not needed.`
|
|
8634
|
-
);
|
|
8635
|
-
}
|
|
8636
|
-
const encoderJsonPath = join22(planDir, "meta", "encoder.json");
|
|
8637
|
-
if (!existsSync16(encoderJsonPath)) {
|
|
8638
|
-
throw new Error(`[assemble] planDir missing meta/encoder.json: ${encoderJsonPath}`);
|
|
8639
|
-
}
|
|
8640
|
-
const encoderJson = JSON.parse(
|
|
8641
|
-
readFileSync10(encoderJsonPath, "utf-8")
|
|
8642
|
-
);
|
|
8643
|
-
if (encoderJson.encoder === "libx265-software") {
|
|
8644
|
-
throw new Error(
|
|
8645
|
-
`[assemble] cfr=true is not yet supported with codec: "h265". The cfr re-encode pass uses libx264 and would silently transcode the h265 chunks. Either disable cfr or render with codec: "h264".`
|
|
8646
|
-
);
|
|
8647
|
-
}
|
|
8648
|
-
const cfrOutputPath = join22(workDir, `cfr.${plan2.dimensions.format}`);
|
|
8649
|
-
const cfrArgs = [
|
|
8650
|
-
"-i",
|
|
8651
|
-
concatOutputPath,
|
|
8652
|
-
"-c:v",
|
|
8653
|
-
"libx264",
|
|
8654
|
-
...cfrRateControlArgs(encoderJson),
|
|
8655
|
-
"-pix_fmt",
|
|
8656
|
-
"yuv420p",
|
|
8657
|
-
"-fps_mode",
|
|
8658
|
-
"cfr",
|
|
8659
|
-
"-r",
|
|
8660
|
-
fpsArg
|
|
8661
|
-
];
|
|
8662
|
-
appendRenderProvenanceArgs(cfrArgs, cfrOutputPath);
|
|
8663
|
-
cfrArgs.push("-y", cfrOutputPath);
|
|
8664
|
-
const cfrResult = await runFfmpeg5(cfrArgs, {
|
|
8665
|
-
signal: abortSignal,
|
|
8666
|
-
timeout: ffmpegProcessTimeout
|
|
8667
|
-
});
|
|
8668
|
-
if (!cfrResult.success) {
|
|
8669
|
-
throw encoderFailureError("[assemble] ffmpeg cfr re-encode failed", {
|
|
8670
|
-
error: describeFfmpegFailure2(cfrResult, ffmpegProcessTimeout),
|
|
8671
|
-
failureReason: cfrResult.failureReason
|
|
8672
|
-
});
|
|
8673
|
-
}
|
|
8674
|
-
postConcatPath = cfrOutputPath;
|
|
8675
|
-
log.info("[assemble] cfr re-encode applied", {
|
|
8676
|
-
format: plan2.dimensions.format,
|
|
8677
|
-
fpsNum: plan2.dimensions.fpsNum,
|
|
8678
|
-
fpsDen: plan2.dimensions.fpsDen
|
|
8679
|
-
});
|
|
8680
|
-
}
|
|
8681
|
-
let normalizedAudioPath = null;
|
|
8682
|
-
if (audioPath !== null && existsSync16(audioPath)) {
|
|
8683
|
-
const paddedAudioPath = join22(workDir, "audio-padded.m4a");
|
|
8684
|
-
const padTrimResult = await padOrTrimAudioToVideoFrameCount({
|
|
8685
|
-
videoPath: postConcatPath,
|
|
8686
|
-
audioPath,
|
|
8687
|
-
outputPath: paddedAudioPath,
|
|
8688
|
-
signal: abortSignal,
|
|
8689
|
-
timeoutMs: ffmpegProcessTimeout
|
|
8690
|
-
});
|
|
8691
|
-
if (!padTrimResult.success) {
|
|
8692
|
-
throw encoderFailureError("[assemble] audio pad/trim failed", padTrimResult);
|
|
8693
|
-
}
|
|
8694
|
-
normalizedAudioPath = paddedAudioPath;
|
|
8695
|
-
log.info("[assemble] audio normalized for mux", {
|
|
8696
|
-
operation: padTrimResult.operation,
|
|
8697
|
-
targetDurationSeconds: padTrimResult.targetDurationSeconds,
|
|
8698
|
-
sourceDurationSeconds: padTrimResult.sourceDurationSeconds
|
|
8699
|
-
});
|
|
8700
|
-
}
|
|
8701
|
-
const muxOutputPath = normalizedAudioPath !== null ? join22(workDir, `mux.${plan2.dimensions.format}`) : postConcatPath;
|
|
8702
|
-
if (normalizedAudioPath !== null) {
|
|
8703
|
-
const muxResult = await muxVideoWithAudio2(
|
|
8704
|
-
postConcatPath,
|
|
8705
|
-
normalizedAudioPath,
|
|
8706
|
-
muxOutputPath,
|
|
8707
|
-
abortSignal,
|
|
8708
|
-
{ audioCodec: "aac", ffmpegProcessTimeout },
|
|
8709
|
-
{ num: plan2.dimensions.fpsNum, den: plan2.dimensions.fpsDen }
|
|
8710
|
-
);
|
|
8711
|
-
if (!muxResult.success) {
|
|
8712
|
-
throw encoderFailureError("[assemble] audio mux failed", muxResult);
|
|
8713
|
-
}
|
|
8714
|
-
}
|
|
8715
|
-
const faststartResult = await applyFaststart2(
|
|
8716
|
-
muxOutputPath,
|
|
8717
|
-
outputPath,
|
|
8718
|
-
abortSignal,
|
|
8719
|
-
{ ffmpegProcessTimeout },
|
|
8825
|
+
ctx.log.info("[assemble] audio normalized for mux", {
|
|
8826
|
+
operation: padTrimResult.operation,
|
|
8827
|
+
targetDurationSeconds: padTrimResult.targetDurationSeconds,
|
|
8828
|
+
sourceDurationSeconds: padTrimResult.sourceDurationSeconds
|
|
8829
|
+
});
|
|
8830
|
+
const muxResult = await ctx.timed(
|
|
8831
|
+
"joinMs",
|
|
8832
|
+
() => muxVideoWithAudio2(
|
|
8833
|
+
videoSources[0],
|
|
8834
|
+
paddedAudioPath,
|
|
8835
|
+
finalPath,
|
|
8836
|
+
ctx.signal,
|
|
8720
8837
|
{
|
|
8721
|
-
|
|
8722
|
-
|
|
8723
|
-
|
|
8724
|
-
|
|
8725
|
-
|
|
8726
|
-
|
|
8727
|
-
|
|
8728
|
-
|
|
8729
|
-
|
|
8730
|
-
rmSync10(workDir, { recursive: true, force: true });
|
|
8731
|
-
} catch (err) {
|
|
8732
|
-
log.warn("[assemble] failed to remove work dir", {
|
|
8733
|
-
workDir,
|
|
8734
|
-
error: err instanceof Error ? err.message : String(err)
|
|
8735
|
-
});
|
|
8736
|
-
}
|
|
8838
|
+
audioCodec: "aac",
|
|
8839
|
+
ffmpegProcessTimeout: ctx.timeout,
|
|
8840
|
+
videoInputArgs: videoInputArgs(ctx, videoSources)
|
|
8841
|
+
},
|
|
8842
|
+
ctx.fps
|
|
8843
|
+
)
|
|
8844
|
+
);
|
|
8845
|
+
if (!muxResult.success) {
|
|
8846
|
+
throw encoderFailureError("[assemble] audio mux failed", muxResult);
|
|
8737
8847
|
}
|
|
8738
|
-
|
|
8848
|
+
}
|
|
8849
|
+
function concatDemuxerInput(chunkPaths, workDir, fpsArg) {
|
|
8850
|
+
const listPath = join22(workDir, "concat-list.txt");
|
|
8851
|
+
const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");
|
|
8852
|
+
writeFileSync8(listPath, `${body}
|
|
8853
|
+
`, "utf-8");
|
|
8854
|
+
return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath];
|
|
8855
|
+
}
|
|
8856
|
+
async function probeJoinedFrameInfo(videoPaths, fps, signal) {
|
|
8857
|
+
const infos = await Promise.all(
|
|
8858
|
+
videoPaths.map((path) => defaultProbeVideoFrameInfo(path, signal))
|
|
8859
|
+
);
|
|
8739
8860
|
return {
|
|
8740
|
-
|
|
8741
|
-
|
|
8742
|
-
|
|
8743
|
-
fileSize
|
|
8861
|
+
frameCount: infos.reduce((sum, info) => sum + info.frameCount, 0),
|
|
8862
|
+
fpsNum: fps.num,
|
|
8863
|
+
fpsDen: fps.den
|
|
8744
8864
|
};
|
|
8745
8865
|
}
|
|
8866
|
+
async function runAssembleFfmpeg(args, step, signal, timeout) {
|
|
8867
|
+
const result = await runFfmpeg5(args, { signal, timeout });
|
|
8868
|
+
if (!result.success) {
|
|
8869
|
+
throw encoderFailureError(`[assemble] ffmpeg ${step} failed`, {
|
|
8870
|
+
error: describeFfmpegFailure2(result, timeout),
|
|
8871
|
+
failureReason: result.failureReason
|
|
8872
|
+
});
|
|
8873
|
+
}
|
|
8874
|
+
}
|
|
8875
|
+
function readCfrEncoder(planDir, format) {
|
|
8876
|
+
if (format !== "mp4") {
|
|
8877
|
+
throw new Error(
|
|
8878
|
+
`[assemble] cfr=true is only supported for format="mp4" (got "${format}"). Stream-copy paths for webm and mov already produce exact avg_frame_rate; cfr re-encode is not needed.`
|
|
8879
|
+
);
|
|
8880
|
+
}
|
|
8881
|
+
const encoderJsonPath = join22(planDir, "meta", "encoder.json");
|
|
8882
|
+
if (!existsSync16(encoderJsonPath)) {
|
|
8883
|
+
throw new Error(`[assemble] planDir missing meta/encoder.json: ${encoderJsonPath}`);
|
|
8884
|
+
}
|
|
8885
|
+
const encoderJson = JSON.parse(readFileSync10(encoderJsonPath, "utf-8"));
|
|
8886
|
+
if (encoderJson.encoder === "libx265-software") {
|
|
8887
|
+
throw new Error(
|
|
8888
|
+
`[assemble] cfr=true is not yet supported with codec: "h265". The cfr re-encode pass uses libx264 and would silently transcode the h265 chunks. Either disable cfr or render with codec: "h264".`
|
|
8889
|
+
);
|
|
8890
|
+
}
|
|
8891
|
+
return encoderJson;
|
|
8892
|
+
}
|
|
8746
8893
|
function mergePngFrameDirs(chunkPaths, outputPath, totalFrames, audioPath, startTimeMs) {
|
|
8747
8894
|
if (existsSync16(outputPath)) rmSync10(outputPath, { recursive: true, force: true });
|
|
8748
8895
|
mkdirSync11(outputPath, { recursive: true });
|
|
@@ -9372,8 +9519,8 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath) {
|
|
|
9372
9519
|
outputPath: outputChunkPath,
|
|
9373
9520
|
framesDir,
|
|
9374
9521
|
videoOnlyPath,
|
|
9375
|
-
width: plan2.dimensions.width * encoder.deviceScaleFactor,
|
|
9376
|
-
height: plan2.dimensions.height * encoder.deviceScaleFactor,
|
|
9522
|
+
width: Math.round(plan2.dimensions.width * encoder.deviceScaleFactor),
|
|
9523
|
+
height: Math.round(plan2.dimensions.height * encoder.deviceScaleFactor),
|
|
9377
9524
|
needsAlpha: plan2.dimensions.format !== "mp4",
|
|
9378
9525
|
captureImageFormat: captureOptions.format ?? "jpeg",
|
|
9379
9526
|
// Each chunk produces video only — audio is muxed once at assemble
|
|
@@ -9510,6 +9657,7 @@ import {
|
|
|
9510
9657
|
isVideoFrameFormat,
|
|
9511
9658
|
validateEngineConfigSnapshot
|
|
9512
9659
|
} from "@hyperframes/engine";
|
|
9660
|
+
import { DISTRIBUTED_FPS as DISTRIBUTED_FPS2, isDistributedFps as isDistributedFps2 } from "@hyperframes/core";
|
|
9513
9661
|
var InvalidConfigError = class extends Error {
|
|
9514
9662
|
// Read via Error.prototype.toString; fallow can't see it.
|
|
9515
9663
|
// fallow-ignore-next-line unused-class-member
|
|
@@ -9521,7 +9669,6 @@ var InvalidConfigError = class extends Error {
|
|
|
9521
9669
|
this.field = field;
|
|
9522
9670
|
}
|
|
9523
9671
|
};
|
|
9524
|
-
var ALLOWED_FPS = [24, 30, 60];
|
|
9525
9672
|
var ALLOWED_FORMATS = [
|
|
9526
9673
|
"mp4",
|
|
9527
9674
|
"mov",
|
|
@@ -9535,15 +9682,14 @@ var ALLOWED_HDR_MODES = ["auto", "force-sdr"];
|
|
|
9535
9682
|
var MAX_DIMENSION = 7680;
|
|
9536
9683
|
var MIN_DIMENSION = 16;
|
|
9537
9684
|
var MAX_CHUNK_SIZE = 3600;
|
|
9538
|
-
var MAX_PARALLEL_CHUNKS_CEILING = 256;
|
|
9539
9685
|
function validateDistributedRenderConfig(config) {
|
|
9540
9686
|
if (config === null || typeof config !== "object") {
|
|
9541
9687
|
throw new InvalidConfigError("config", "must be an object");
|
|
9542
9688
|
}
|
|
9543
|
-
if (!
|
|
9689
|
+
if (!isDistributedFps2(config.fps)) {
|
|
9544
9690
|
throw new InvalidConfigError(
|
|
9545
9691
|
"config.fps",
|
|
9546
|
-
`must be one of ${
|
|
9692
|
+
`must be one of ${DISTRIBUTED_FPS2.join(", ")}; got ${String(config.fps)}`
|
|
9547
9693
|
);
|
|
9548
9694
|
}
|
|
9549
9695
|
validateIntDimension("config.width", config.width);
|
|
@@ -9634,6 +9780,12 @@ function validateDistributedRenderConfig(config) {
|
|
|
9634
9780
|
);
|
|
9635
9781
|
}
|
|
9636
9782
|
}
|
|
9783
|
+
for (const field of ["workerTimeoutSeconds", "workerConcurrency"]) {
|
|
9784
|
+
const value = config[field];
|
|
9785
|
+
if (value !== void 0 && (!Number.isInteger(value) || value < 1)) {
|
|
9786
|
+
throw new InvalidConfigError(`config.${field}`, `must be a positive integer; got ${value}`);
|
|
9787
|
+
}
|
|
9788
|
+
}
|
|
9637
9789
|
if (config.runtimeCap !== void 0 && !ALLOWED_RUNTIME_CAPS.includes(config.runtimeCap)) {
|
|
9638
9790
|
throw new InvalidConfigError(
|
|
9639
9791
|
"config.runtimeCap",
|
|
@@ -9743,10 +9895,10 @@ function validateIntDimension(field, value) {
|
|
|
9743
9895
|
|
|
9744
9896
|
// src/services/distributed/projectHash.ts
|
|
9745
9897
|
import { readdirSync as readdirSync9, readFileSync as readFileSync12 } from "node:fs";
|
|
9746
|
-
import { createHash as
|
|
9898
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
9747
9899
|
import { join as join25, relative as relative7 } from "node:path";
|
|
9748
9900
|
function hashProjectDir(projectDir) {
|
|
9749
|
-
const hash =
|
|
9901
|
+
const hash = createHash6("sha256");
|
|
9750
9902
|
const files = [];
|
|
9751
9903
|
function walk(dir, isRoot) {
|
|
9752
9904
|
for (const entry of readdirSync9(dir, { withFileTypes: true }).sort(
|
|
@@ -9768,7 +9920,9 @@ function hashProjectDir(projectDir) {
|
|
|
9768
9920
|
return hash.digest("hex").slice(0, 16);
|
|
9769
9921
|
}
|
|
9770
9922
|
export {
|
|
9923
|
+
CHUNK_EXCEEDS_WORKER_TIMEOUT,
|
|
9771
9924
|
CURRENT_PLAN_PROTOCOL,
|
|
9925
|
+
ChunkExceedsWorkerTimeoutError,
|
|
9772
9926
|
DEFAULT_CHUNK_SIZE,
|
|
9773
9927
|
DEFAULT_MAX_PARALLEL_CHUNKS,
|
|
9774
9928
|
DISTRIBUTED_DURATION_OUT_OF_RANGE,
|