multiplane 0.3.7 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +17 -6
- package/README.md +17 -6
- package/bin/multiplane +24 -14
- package/package.json +3 -2
- package/skills/multiplane/SKILL.md +4 -1
- package/src/artifacts.mjs +14 -7
- package/src/capture-watchdog.mjs +29 -0
- package/src/capture.mjs +22 -6
- package/src/errors.mjs +1 -0
- package/src/events.mjs +7 -5
- package/src/hosted.mjs +2 -2
- package/src/index.mjs +1 -0
- package/src/input-records.mjs +33 -0
- package/src/jobs.mjs +3 -1
- package/src/post.mjs +18 -2
- package/src/render-pool.mjs +39 -12
- package/src/render-worker.mjs +13 -2
- package/src/render.mjs +148 -52
- package/src/resume.mjs +23 -0
- package/src/segment-encoder.mjs +91 -0
- package/src/segments.mjs +123 -0
- package/src/server.mjs +4 -0
package/src/render-pool.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import { MultiplaneError } from './errors.mjs';
|
|
|
4
4
|
import { availableParallelism, freemem, totalmem } from 'node:os';
|
|
5
5
|
import { Worker } from 'node:worker_threads';
|
|
6
6
|
import { blurSampleReport } from './post.mjs';
|
|
7
|
+
import { createCaptureWatchdog } from './capture-watchdog.mjs';
|
|
7
8
|
|
|
8
9
|
export function defaultWorkers(scale = 1, hardware = {}) {
|
|
9
10
|
const cores = hardware.cores ?? availableParallelism();
|
|
@@ -12,24 +13,50 @@ export function defaultWorkers(scale = 1, hardware = {}) {
|
|
|
12
13
|
return Math.max(1, Math.min(12, Math.floor(cores / 2), Math.floor((free - 2 * 1024 ** 3) / workerBytes)));
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
export function createRenderPool(shot, options, fps, workers, size, onWarning, onCaptured) {
|
|
16
|
+
export function createRenderPool(shot, options, fps, workers, size, onWarning, onCaptured, onInput) {
|
|
16
17
|
options = { ...options, _browserRecovery: options._browserRecovery ?? recoveryBudget() };
|
|
17
18
|
reuseState(options);
|
|
18
19
|
const slots = [], stats = { frames: 0, samples: 0, staticFrames: 0 };
|
|
19
20
|
const histogram = {}, dofHistogram = {};
|
|
20
21
|
const captured = new Set();
|
|
21
|
-
let failure;
|
|
22
|
+
let failure, stopping = false, closing;
|
|
23
|
+
function close() {
|
|
24
|
+
if (closing) return closing;
|
|
25
|
+
stopping = true; watchdog.stop();
|
|
26
|
+
for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'close' });
|
|
27
|
+
// A blocked page cannot finish its frame. Close only browsers owned by
|
|
28
|
+
// these workers after the grace period, letting their normal cleanup run.
|
|
29
|
+
const timer = setTimeout(() => {
|
|
30
|
+
for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'cancel' });
|
|
31
|
+
}, options._stopGraceMs ?? 5000); // Internal override for short fault tests.
|
|
32
|
+
closing = Promise.all(slots.map(s => s.exited)).finally(() => clearTimeout(timer));
|
|
33
|
+
return closing;
|
|
34
|
+
}
|
|
35
|
+
// Internal deadline override for short fault-injection tests; no CLI option.
|
|
36
|
+
const watchdog = createCaptureWatchdog(slots, (slot, details) => {
|
|
37
|
+
if (failure || stopping) return;
|
|
38
|
+
try {
|
|
39
|
+
onWarning?.({ code: 'worker_stalled', severity: 'warning', message: 'Rendering worker stalled.', details });
|
|
40
|
+
slot.worker.postMessage({ type: 'stalled', details });
|
|
41
|
+
} catch (error) { fail(error); }
|
|
42
|
+
}, { minimumMs: options._stallTimeoutMs });
|
|
43
|
+
const fail = error => {
|
|
44
|
+
failure ??= error;
|
|
45
|
+
watchdog.stop();
|
|
46
|
+
for (const s of slots) { s.pending?.reject(failure); s.pending = null; }
|
|
47
|
+
void close();
|
|
48
|
+
};
|
|
22
49
|
for (let i = 0; i < workers; i++) {
|
|
23
50
|
const worker = new Worker(new URL('./render-worker.mjs', import.meta.url), { workerData: { shot, options, fps, size } });
|
|
24
51
|
const slot = { worker, pending: null, closed: false };
|
|
25
|
-
const fail = error => {
|
|
26
|
-
failure ??= error;
|
|
27
|
-
for (const s of slots) { s.pending?.reject(error); s.pending = null; }
|
|
28
|
-
};
|
|
29
52
|
slot.ready = new Promise((resolve, reject) => { slot.pending = { resolve, reject }; });
|
|
30
53
|
slot.ready.catch(() => {});
|
|
31
54
|
worker.on('message', message => {
|
|
55
|
+
if (message.type === 'frame-start') { slot.frame = message.frame; slot.frameStarted = performance.now(); return; }
|
|
56
|
+
if (message.type === 'recovered') { watchdog.recovered(slot); return; }
|
|
57
|
+
if (message.type === 'input') { try { onInput?.(message.file, message.record); } catch (error) { fail(error); } return; }
|
|
32
58
|
if (message.type === 'captured') {
|
|
59
|
+
watchdog.completed(slot);
|
|
33
60
|
if (!captured.has(message.frame)) {
|
|
34
61
|
captured.add(message.frame);
|
|
35
62
|
try { onCaptured?.(captured.size); } catch (error) { fail(error); }
|
|
@@ -39,6 +66,7 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
|
|
|
39
66
|
if (message.type === 'warning') { onWarning?.(message.warning); return; }
|
|
40
67
|
if (message.type === 'failure') { fail(new MultiplaneError(message.code ?? 'worker_crashed', message.error, { details: message.details, hint: message.hint })); return; }
|
|
41
68
|
const pending = slot.pending; slot.pending = null;
|
|
69
|
+
slot.job = null;
|
|
42
70
|
pending?.resolve(message);
|
|
43
71
|
});
|
|
44
72
|
worker.on('error', error => fail(new MultiplaneError('worker_crashed', undefined, { cause: error })));
|
|
@@ -66,6 +94,8 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
|
|
|
66
94
|
next = Math.min(end, next + chunk, reuseEnabled(options) ? (Math.floor(begin / REUSE_INTERVAL) + 1) * REUSE_INTERVAL + 1 : end);
|
|
67
95
|
const promise = new Promise((resolve, reject) => { slot.pending = { resolve, reject }; });
|
|
68
96
|
promise.catch(() => {});
|
|
97
|
+
slot.job = { first: begin, end: next }; slot.frame = begin;
|
|
98
|
+
slot.lastDelivery = performance.now(); slot.frameStarted = undefined;
|
|
69
99
|
slot.worker.postMessage({ type: 'render', first: begin, end: next });
|
|
70
100
|
queue.push({ slot, promise });
|
|
71
101
|
};
|
|
@@ -86,13 +116,10 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
|
|
|
86
116
|
if (options.transport !== 'raw') schedule(slot);
|
|
87
117
|
}
|
|
88
118
|
},
|
|
89
|
-
|
|
90
|
-
for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'close' });
|
|
91
|
-
await Promise.all(slots.map(s => s.exited));
|
|
92
|
-
},
|
|
119
|
+
close,
|
|
93
120
|
async cancel() {
|
|
94
|
-
|
|
95
|
-
await
|
|
121
|
+
fail(new MultiplaneError('cancelled'));
|
|
122
|
+
await close();
|
|
96
123
|
},
|
|
97
124
|
};
|
|
98
125
|
const frames = api.frames.bind(api);
|
package/src/render-worker.mjs
CHANGED
|
@@ -3,8 +3,11 @@ import { MultiplaneError, asError } from './errors.mjs';
|
|
|
3
3
|
import { parentPort, workerData, isMarkedAsUntransferable } from 'node:worker_threads';
|
|
4
4
|
import { openShot } from './capture.mjs';
|
|
5
5
|
import { createPostCapture, resolveScale } from './post.mjs';
|
|
6
|
+
import { inputRecords } from './input-records.mjs';
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
if (workerData.options._recordInputs) inputRecords.enterWith((file, record) => parentPort.postMessage({ type: 'input', file, record }));
|
|
9
|
+
|
|
10
|
+
let capture, closing = false, running = false, activeFrame;
|
|
8
11
|
const sendFailure = error => {
|
|
9
12
|
const failure = asError(error);
|
|
10
13
|
parentPort.postMessage({ type: 'failure', error: failure.message, code: failure.code, details: failure.details, hint: failure.hint });
|
|
@@ -16,12 +19,17 @@ async function close() {
|
|
|
16
19
|
parentPort.close();
|
|
17
20
|
}
|
|
18
21
|
try {
|
|
19
|
-
capture = await openShot(workerData.shot, resolveScale(workerData.options).renderScale, { ...workerData.options, onWarning: warning => parentPort.postMessage({ type: 'warning', warning }), check: () => { if (closing) throw new MultiplaneError('cancelled'); } });
|
|
22
|
+
capture = await openShot(workerData.shot, resolveScale(workerData.options).renderScale, { ...workerData.options, onRecovered: () => parentPort.postMessage({ type: 'recovered' }), onWarning: warning => parentPort.postMessage({ type: 'warning', warning }), check: () => { if (closing) throw new MultiplaneError('cancelled'); } });
|
|
20
23
|
if (workerData.size && (capture.width !== workerData.size.width || capture.height !== workerData.size.height)) throw new MultiplaneError('internal_error', 'Worker stage dimensions differ from the main process.');
|
|
21
24
|
const processed = createPostCapture(capture, { ...workerData.options, onWarning: warning => parentPort.postMessage({ type: 'warning', warning }) }, workerData.fps);
|
|
22
25
|
parentPort.on('message', async message => {
|
|
23
26
|
if (message.type === 'cancel') { closing = true; await capture?.close(); if (!running) parentPort.close(); return; }
|
|
24
27
|
if (message.type === 'close') { await close(); return; }
|
|
28
|
+
if (message.type === 'stalled') {
|
|
29
|
+
if (!closing && activeFrame === message.details.frame) await capture.stall(message.details).catch(sendFailure);
|
|
30
|
+
else parentPort.postMessage({ type: 'recovered' });
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
25
33
|
if (closing) return;
|
|
26
34
|
running = true;
|
|
27
35
|
try {
|
|
@@ -29,7 +37,10 @@ try {
|
|
|
29
37
|
const source = async function* (first, end) {
|
|
30
38
|
for (let index = first; index < end && !closing; index++) {
|
|
31
39
|
const frame = message.first + index;
|
|
40
|
+
activeFrame = frame;
|
|
41
|
+
parentPort.postMessage({ type: 'frame-start', frame });
|
|
32
42
|
const png = await processed.frame(frame / workerData.fps, frame, index === end - 1);
|
|
43
|
+
activeFrame = undefined;
|
|
33
44
|
parentPort.postMessage({ type: 'captured', frame });
|
|
34
45
|
yield { png, hash: processed.lastHash, blurSamples: processed.lastBlurSamples, dofSamples: processed.lastDofSamples };
|
|
35
46
|
}
|
package/src/render.mjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { readBenchmark, benchmarkMachine, selectWorkers, classifyShot } from './workers.mjs';
|
|
2
2
|
import { recoveryBudget } from './browser-recovery.mjs';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { mkdir, rm, writeFile, rename } from 'node:fs/promises';
|
|
5
5
|
import { createHash } from 'node:crypto';
|
|
6
|
-
import { spawn } from 'node:child_process';
|
|
7
6
|
import { stopChild } from './process.mjs';
|
|
8
7
|
import { inspectFFmpeg } from './ffmpeg.mjs';
|
|
9
8
|
import { openShot, number } from './capture.mjs';
|
|
@@ -11,19 +10,28 @@ import { createPostCapture, resolveScale } from './post.mjs';
|
|
|
11
10
|
import { createRenderPool } from './render-pool.mjs';
|
|
12
11
|
import { resolveFormat, videoEncoding } from './format.mjs';
|
|
13
12
|
import { checkAlphaVideo } from './output-alpha.mjs';
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
13
|
+
import { inputRecords, recordInput } from './input-records.mjs';
|
|
14
|
+
import { atomicJSON, lockSegments, manifestStore, checkInputs, checkEnvironment, reusableSegments, readManifest, unavailable } from './segments.mjs';
|
|
15
|
+
import { segmentEncoder, assembleSegments } from './segment-encoder.mjs';
|
|
16
|
+
import { version, runCommand } from './events.mjs';
|
|
16
17
|
import { loaded, observeBrowser, environment } from './environment.mjs';
|
|
17
|
-
import {
|
|
18
|
+
import { invalidOption } from './errors.mjs';
|
|
18
19
|
import { requireNewOutput, publishVideo } from './artifacts.mjs';
|
|
19
20
|
import { createPreview } from './preview.mjs';
|
|
20
21
|
import { checkVideo } from './output-check.mjs';
|
|
21
22
|
import { frameRate } from './frame-rate.mjs';
|
|
22
23
|
|
|
24
|
+
const persistedOptions = options => Object.fromEntries(Object.entries(options).filter(([key, value]) =>
|
|
25
|
+
!key.startsWith('_') && typeof value !== 'function' && !['job-dir-exact', 'hosted-executor'].includes(key)));
|
|
26
|
+
|
|
23
27
|
export async function renderShot(shot, options = {}) {
|
|
24
|
-
return runCommand('render', shot, options, async events => {
|
|
28
|
+
return runCommand('render', shot, options, async events => inputRecords.run(null, async () => {
|
|
25
29
|
options = { ...events.options };
|
|
26
30
|
delete options.onEvent;
|
|
31
|
+
const resume = options._resume; delete options._resume;
|
|
32
|
+
const hook = options._segmentHook; delete options._segmentHook;
|
|
33
|
+
const segmentFrames = options._segmentFrames; delete options._segmentFrames;
|
|
34
|
+
delete options._effectiveOptions;
|
|
27
35
|
const probe = options._probeVideo;
|
|
28
36
|
delete options._probeVideo;
|
|
29
37
|
const inspect = options._inspectFFmpeg ?? inspectFFmpeg;
|
|
@@ -32,25 +40,56 @@ export async function renderShot(shot, options = {}) {
|
|
|
32
40
|
const name = path.basename(shot, path.extname(shot));
|
|
33
41
|
let result;
|
|
34
42
|
|
|
35
|
-
let capture, pool,
|
|
43
|
+
let capture, pool, temporary, cleaning, store, directory, release, movedSource;
|
|
44
|
+
const encoders = new Set(), pending = [];
|
|
45
|
+
const inputs = {};
|
|
46
|
+
const onInput = (file, record) => { if (store) store.input(file, record); else inputs[file] = record; };
|
|
47
|
+
inputRecords.enterWith(onInput);
|
|
36
48
|
function cleanup() { return cleaning ??= cleanResources(); }
|
|
37
49
|
async function cleanResources() {
|
|
38
50
|
try {
|
|
39
|
-
|
|
51
|
+
await Promise.all([...encoders].map(encoder => stopChild(encoder)));
|
|
52
|
+
await Promise.allSettled(pending);
|
|
40
53
|
} finally {
|
|
41
54
|
try { await capture?.close(); }
|
|
42
|
-
finally { try { await pool?.close(); } finally {
|
|
55
|
+
finally { try { await pool?.close(); } finally {
|
|
56
|
+
try {
|
|
57
|
+
if (temporary && movedSource) await rename(temporary, movedSource);
|
|
58
|
+
else if (temporary) await rm(temporary, { force: true });
|
|
59
|
+
} finally { release?.(); }
|
|
60
|
+
} }
|
|
43
61
|
}
|
|
44
62
|
}
|
|
45
63
|
|
|
46
64
|
try {
|
|
65
|
+
if (resume) {
|
|
66
|
+
directory = resume.directory;
|
|
67
|
+
release ??= lockSegments(directory);
|
|
68
|
+
resume.manifest = readManifest(directory);
|
|
69
|
+
if (!resume.manifest) throw unavailable('missing_manifest');
|
|
70
|
+
atomicJSON(path.join(events.directory, 'resume.json'), { directory });
|
|
71
|
+
await checkInputs(resume.manifest);
|
|
72
|
+
}
|
|
47
73
|
const out = path.resolve(options.out ?? path.join(events.directory, `${name}.${options.format}`));
|
|
74
|
+
if (!resume) {
|
|
75
|
+
directory = path.join(events.directory, 'segments');
|
|
76
|
+
release = lockSegments(directory);
|
|
77
|
+
// A launch failure can be retried even before metadata or a first frame
|
|
78
|
+
// exists. Replace this provisional record with effective values below.
|
|
79
|
+
store = manifestStore(directory, { schema: 'multiplane-segments/1', jobId: events.jobId,
|
|
80
|
+
shot: path.resolve(options.root, shot), options: persistedOptions({ ...options, out,
|
|
81
|
+
audio: options.audio ? path.resolve(options.audio) : undefined }),
|
|
82
|
+
environment: null, first: 0, end: 0, total: 0, segmentFrames: segmentFrames ?? null, inputs: {}, segments: [] });
|
|
83
|
+
store.save();
|
|
84
|
+
await recordInput(path.resolve(options.root, shot));
|
|
85
|
+
if (options.audio) await recordInput(path.resolve(options.audio));
|
|
86
|
+
}
|
|
48
87
|
if (!options.overwrite) await requireNewOutput(out);
|
|
49
88
|
events.check();
|
|
50
89
|
events.progress('launch');
|
|
51
90
|
const { scale, renderScale } = resolveScale(options);
|
|
52
91
|
const encoding = videoEncoding(options);
|
|
53
|
-
|
|
92
|
+
|
|
54
93
|
const ffmpegInfo = await inspect(options.ffmpeg, process.env, { x264: 'libx264', 'nvenc-h264': 'h264_nvenc', 'nvenc-hevc': 'hevc_nvenc' }[encoding.encoder] ?? encoding.encoder);
|
|
55
94
|
const ffmpeg = ffmpegInfo.path;
|
|
56
95
|
options.transport = 'raw';
|
|
@@ -74,8 +113,7 @@ export async function renderShot(shot, options = {}) {
|
|
|
74
113
|
options.alpha = events.options.alpha = capture.alpha;
|
|
75
114
|
resolveFormat(options);
|
|
76
115
|
Object.assign(encoding, videoEncoding(options));
|
|
77
|
-
|
|
78
|
-
await loaded(events, capture, ffmpegInfo);
|
|
116
|
+
|
|
79
117
|
const fps = number(options.fps ?? capture.fps, '--fps', { positive: true });
|
|
80
118
|
const rate = frameRate(fps);
|
|
81
119
|
const processed = createPostCapture(capture, options, fps);
|
|
@@ -87,57 +125,110 @@ export async function renderShot(shot, options = {}) {
|
|
|
87
125
|
const total = end - first;
|
|
88
126
|
if (!total) throw invalidOption('--from', 'Range contains no frames.');
|
|
89
127
|
const { width, height } = capture;
|
|
90
|
-
events.
|
|
128
|
+
directory = resume?.directory ?? path.join(events.directory, 'segments');
|
|
129
|
+
release ??= lockSegments(directory);
|
|
130
|
+
const info = await environment(capture.browser);
|
|
131
|
+
const currentEnvironment = { multiplane: version, chromium: info.chromium, raster: options.raster,
|
|
132
|
+
renderer: info.gpu.renderer, ffmpeg: ffmpegInfo.version, encoder: encoding.encoder };
|
|
133
|
+
const effective = { ...options, fps, from, to, out, ffmpeg, audio: options.audio ? path.resolve(options.audio) : undefined,
|
|
134
|
+
'motion-blur': processed.post.motionBlur, 'max-motion-blur': processed.post.maxMotionBlur,
|
|
135
|
+
shutter: processed.post.shutter, grain: processed.post.grain };
|
|
136
|
+
const savedOptions = persistedOptions(effective);
|
|
137
|
+
let manifest;
|
|
138
|
+
if (resume) {
|
|
139
|
+
manifest = resume.manifest;
|
|
140
|
+
checkEnvironment(manifest.environment, currentEnvironment);
|
|
141
|
+
if (manifest.environment && manifest.environment.renderer !== currentEnvironment.renderer) {
|
|
142
|
+
events.warning({ code: 'resume_renderer_changed', message: 'GPU renderer changed since the original render.',
|
|
143
|
+
details: { before: manifest.environment.renderer, after: currentEnvironment.renderer } });
|
|
144
|
+
}
|
|
145
|
+
await checkInputs(manifest);
|
|
146
|
+
manifest.segments = await reusableSegments(directory, manifest);
|
|
147
|
+
if (!manifest.environment) Object.assign(manifest, { environment: currentEnvironment, options: savedOptions,
|
|
148
|
+
first, end, total, segmentFrames: manifest.segmentFrames ?? Math.max(1, Math.round(fps * 10)) });
|
|
149
|
+
} else {
|
|
150
|
+
manifest = { schema: 'multiplane-segments/1', jobId: events.jobId, shot: path.resolve(options.root, shot),
|
|
151
|
+
options: savedOptions, environment: currentEnvironment, first, end, total,
|
|
152
|
+
segmentFrames: segmentFrames ?? Math.max(1, Math.round(fps * 10)), inputs: store.manifest.inputs, segments: [] };
|
|
153
|
+
}
|
|
154
|
+
if (!Number.isSafeInteger(manifest.segmentFrames) || manifest.segmentFrames < 1) throw invalidOption('_segmentFrames', 'Segment size must be a positive integer.');
|
|
155
|
+
store = manifestStore(directory, manifest);
|
|
156
|
+
for (const [file, record] of Object.entries(inputs)) store.input(file, record);
|
|
157
|
+
if (options.audio) await recordInput(path.resolve(options.audio));
|
|
158
|
+
store.save();
|
|
159
|
+
const framesReused = manifest.segments.reduce((sum, segment) => sum + segment.end - segment.first, 0);
|
|
160
|
+
const resumeData = resume ? { resumedFrom: resume.jobId, resume: { segmentsReused: manifest.segments.length, framesReused } } : {};
|
|
161
|
+
events.resumeData = resumeData;
|
|
162
|
+
if (resume) atomicJSON(path.join(events.directory, 'resume.json'), { directory });
|
|
163
|
+
await loaded(events, capture, ffmpegInfo);
|
|
164
|
+
events.progress('capture', framesReused, total);
|
|
91
165
|
await capture.close(); capture = undefined;
|
|
92
166
|
events.check();
|
|
93
|
-
const actualWorkers = Math.min(workers, Math.ceil(total / 4));
|
|
167
|
+
const actualWorkers = Math.min(workers, Math.max(1, Math.ceil((total - framesReused) / 4)));
|
|
94
168
|
events.resources.setWorkers(actualWorkers);
|
|
95
169
|
const started = performance.now();
|
|
96
170
|
const frameHashes = process.env.RENDER_FRAME_HASHES ? [] : null;
|
|
171
|
+
options._recordInputs = true;
|
|
97
172
|
pool = createRenderPool(shot, options, fps, actualWorkers, { width, height }, events.warning,
|
|
98
|
-
done => events.progress('capture', done, total));
|
|
173
|
+
done => events.progress('capture', done + framesReused, total), onInput);
|
|
99
174
|
events.onCancel(() => pool?.cancel());
|
|
100
175
|
await mkdir(path.dirname(out), { recursive: true });
|
|
101
|
-
events.
|
|
102
|
-
temporary = `${out}.${events.jobId}.partial.${options.format}`;
|
|
103
|
-
const audioArgs = options.audio
|
|
104
|
-
? ['-i', path.resolve(options.audio), '-map', '0:v:0', '-map', '1:a:0', '-af', `atrim=start=${audioOffset + rate.seconds(first)},asetpts=PTS-STARTPTS,apad,atrim=duration=${rate.seconds(total)}`, ...encoding.audio]
|
|
105
|
-
: ['-an'];
|
|
106
|
-
// Supported since FFmpeg 5.1; older FFmpeg uses the global -vsync 0.
|
|
107
|
-
// Do not add output -r: passthrough must neither drop nor duplicate frames.
|
|
108
|
-
encoder = spawn(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', '-f', 'rawvideo', '-pixel_format', 'rgba', '-video_size', `${width * scale}x${height * scale}`, '-framerate', rate.argument, '-i', 'pipe:0', ...audioArgs, ...videoArgs, '-fps_mode', 'passthrough', '-pix_fmt', encoding.pixelFormat, ...encoding.mux, temporary], { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });
|
|
109
|
-
encoder.once('close', () => { encoderClosed = true; });
|
|
110
|
-
events.onCancel(() => stopChild(encoder));
|
|
111
|
-
let stderr = '', failure;
|
|
112
|
-
encoder.stderr.on('data', data => { stderr = (stderr + data).slice(-8000); });
|
|
113
|
-
encoder.stdin.on('error', error => { failure = error; });
|
|
114
|
-
const finished = new Promise((resolve, reject) => {
|
|
115
|
-
encoder.once('error', cause => reject(new MultiplaneError('encode_failed', undefined, { cause, details: { reason: cause.message } })));
|
|
116
|
-
encoder.once('close', code => code === 0 ? resolve() : reject(new MultiplaneError('encode_failed', undefined, { details: { exitCode: code, stderr } })));
|
|
117
|
-
});
|
|
118
|
-
finished.catch(error => { failure = error; });
|
|
176
|
+
temporary = out + '.' + events.jobId + '.partial.' + options.format;
|
|
119
177
|
const alphaReferences = new Map();
|
|
120
|
-
let encoderWriteMs = 0, framesSent =
|
|
121
|
-
// Internal switch for the pixel-equivalence and overhead tests, not a CLI option.
|
|
178
|
+
let encoderWriteMs = 0, framesSent = total, sealingFailure;
|
|
122
179
|
const preview = createPreview(events, width * scale, height * scale, { enabled: options._preview !== false });
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
180
|
+
const remember = segment => {
|
|
181
|
+
frameHashes?.push(...(segment.frameHashes ?? []));
|
|
182
|
+
for (const [frame, bytes] of Object.entries(segment.alphaReferences ?? {})) alphaReferences.set(Number(frame), Buffer.from(bytes, 'base64'));
|
|
183
|
+
};
|
|
184
|
+
// Keep workers busy while the preceding encoder drains and seals.
|
|
185
|
+
const ranges = [];
|
|
186
|
+
for (let start = first, index = 0; start < end; start += manifest.segmentFrames, index++) {
|
|
187
|
+
const stop = Math.min(end, start + manifest.segmentFrames);
|
|
188
|
+
const reused = manifest.segments.find(item => item.first === start && item.end === stop);
|
|
189
|
+
if (reused) { remember(reused); continue; }
|
|
190
|
+
ranges.push({ first: start, end: stop, index });
|
|
191
|
+
}
|
|
192
|
+
for (let cursor = 0; cursor < ranges.length;) {
|
|
193
|
+
let last = cursor;
|
|
194
|
+
while (last + 1 < ranges.length && ranges[last].end === ranges[last + 1].first) last++;
|
|
195
|
+
let active, hashes, references, rangeIndex = cursor;
|
|
196
|
+
for await (const { frame, png } of pool.frames(ranges[cursor].first, ranges[last].end)) {
|
|
197
|
+
events.check();
|
|
198
|
+
if (sealingFailure) throw sealingFailure;
|
|
199
|
+
if (!active) {
|
|
200
|
+
active = segmentEncoder({ ffmpeg, directory, ...ranges[rangeIndex], width: width * scale, height: height * scale,
|
|
201
|
+
rate, encoding, format: options.format, events, store, hook });
|
|
202
|
+
encoders.add(active.child);
|
|
203
|
+
active.finished.finally(() => encoders.delete(active.child)).catch(() => {});
|
|
204
|
+
hashes = []; references = {};
|
|
205
|
+
}
|
|
206
|
+
await preview(png, frame);
|
|
207
|
+
const relative = frame - first;
|
|
208
|
+
if (options.alpha && (relative === 0 || relative === Math.floor(total / 2))) references[relative] = Buffer.from(png.filter((_, i) => i % 4 === 3)).toString('base64');
|
|
209
|
+
if (frameHashes) hashes.push({ frame, sha256: createHash('sha256').update(png).digest('hex') });
|
|
210
|
+
const writeStarted = performance.now();
|
|
211
|
+
await active.write(png);
|
|
212
|
+
encoderWriteMs += performance.now() - writeStarted;
|
|
213
|
+
if (frame + 1 === ranges[rangeIndex].end) {
|
|
214
|
+
const sealed = active.seal({ ...(frameHashes ? { frameHashes: hashes } : {}), ...(options.alpha ? { alphaReferences: references } : {}) }).then(remember);
|
|
215
|
+
sealed.catch(error => { sealingFailure ??= error; });
|
|
216
|
+
pending.push(sealed);
|
|
217
|
+
// Hooks model interruption at an exact durable boundary in tests.
|
|
218
|
+
if (hook) await sealed;
|
|
219
|
+
active = undefined; rangeIndex++;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
cursor = last + 1;
|
|
135
223
|
}
|
|
136
224
|
const drainStarted = performance.now();
|
|
137
225
|
events.progress('encode', total, total);
|
|
138
|
-
|
|
139
|
-
|
|
226
|
+
await Promise.all(pending);
|
|
227
|
+
// Finish all resource requests before the final input audit/publication.
|
|
228
|
+
await pool.close();
|
|
140
229
|
events.check();
|
|
230
|
+
await checkInputs(manifest);
|
|
231
|
+
movedSource = await assembleSegments({ ffmpeg, directory, manifest, temporary, encoding, rate, audioOffset, events });
|
|
141
232
|
const encoderDrainMs = performance.now() - drainStarted;
|
|
142
233
|
const probeStarted = performance.now();
|
|
143
234
|
const outputCheck = await checkVideo(ffmpeg, temporary, { frames: framesSent, durationSeconds: rate.seconds(framesSent), width: width * scale, height: height * scale, fps: rate.fps, ...(options.alpha ? { alpha: true, format: options.format } : {}) }, events, probe);
|
|
@@ -145,13 +236,18 @@ export async function renderShot(shot, options = {}) {
|
|
|
145
236
|
const outputCheckMs = performance.now() - probeStarted;
|
|
146
237
|
await publishVideo(temporary, out, options.overwrite);
|
|
147
238
|
temporary = undefined;
|
|
239
|
+
// Invalidate the checkpoint while still holding the lock, so another
|
|
240
|
+
// resume cannot reuse a manifest whose output has already been published.
|
|
241
|
+
await rm(path.join(directory, 'manifest.json'), { force: true });
|
|
242
|
+
await rm(directory, { recursive: true, force: true });
|
|
243
|
+
release(); release = undefined;
|
|
148
244
|
const elapsed = (performance.now() - started) / 1000;
|
|
149
|
-
if (frameHashes) await writeFile(process.env.RENDER_FRAME_HASHES, JSON.stringify(frameHashes, null, 2));
|
|
245
|
+
if (frameHashes) await writeFile(process.env.RENDER_FRAME_HASHES, JSON.stringify(frameHashes.sort((a, b) => a.frame - b.frame), null, 2));
|
|
150
246
|
if (process.env.RENDER_PROFILE) await writeFile(process.env.RENDER_PROFILE, JSON.stringify({elapsed,frames:total,workers:actualWorkers,options,stats:pool.stats,encoderWriteMs,encoderDrainMs,outputCheckMs},null,2));
|
|
151
247
|
events.artifact(out, 'video');
|
|
152
|
-
result = { staticReuse: pool.staticReuse, blurSamples: pool.blurSamples, dofSamples: pool.dofSamples, output: out, frames: framesSent, fps, from, to, width: width * scale, height: height * scale, encoder: encoding.encoder, format: options.format, alpha: options.alpha, post: { ...processed.post, adaptiveBlur: Number(options['adaptive-blur'] ?? 0.5) }, renderScale, scale, outputCheck };
|
|
248
|
+
result = { ...resumeData, staticReuse: pool.staticReuse, blurSamples: pool.blurSamples, dofSamples: pool.dofSamples, output: out, frames: framesSent, fps, from, to, width: width * scale, height: height * scale, encoder: encoding.encoder, format: options.format, alpha: options.alpha, post: { ...processed.post, adaptiveBlur: Number(options['adaptive-blur'] ?? 0.5) }, renderScale, scale, outputCheck };
|
|
153
249
|
} finally { await cleanup(); }
|
|
154
250
|
|
|
155
251
|
return options.draft ? { ...result, draft: true } : result;
|
|
156
|
-
});
|
|
252
|
+
}));
|
|
157
253
|
}
|
package/src/resume.mjs
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { renderShot } from './render.mjs';
|
|
2
|
+
import { readStatus, resolveJob } from './jobs.mjs';
|
|
3
|
+
import { readManifest, segmentDirectory, unavailable, checkInputs } from './segments.mjs';
|
|
4
|
+
import { invalidOption } from './errors.mjs';
|
|
5
|
+
import { loadUserConfig } from './user-home.mjs';
|
|
6
|
+
|
|
7
|
+
export async function resumeRender(target, options = {}) {
|
|
8
|
+
const allowed = ['workers', 'json', 'job-dir', 'no-hosted', 'onEvent', '_hostedCLI', '_segmentHook', '_inspectFFmpeg', '_probeVideo', 'hosted-executor', 'job-dir-exact'];
|
|
9
|
+
for (const key of Object.keys(options)) if (!allowed.includes(key)) throw invalidOption('--' + key, 'This option cannot be changed when resuming.');
|
|
10
|
+
const configured = loadUserConfig('status', options);
|
|
11
|
+
const jobDir = resolveJob(target, configured.options);
|
|
12
|
+
const status = await readStatus(jobDir);
|
|
13
|
+
if (status.command !== 'render') throw unavailable('not_render');
|
|
14
|
+
if (status.state === 'succeeded') throw unavailable('succeeded');
|
|
15
|
+
if (!['failed', 'cancelled'].includes(status.state)) throw unavailable('running');
|
|
16
|
+
const directory = segmentDirectory(jobDir), manifest = readManifest(directory);
|
|
17
|
+
if (!manifest) throw unavailable('missing_manifest');
|
|
18
|
+
await checkInputs(manifest);
|
|
19
|
+
return renderShot(manifest.shot, { ...manifest.options, ...options,
|
|
20
|
+
_resume: { directory, jobId: status.jobId }, _effectiveOptions: true,
|
|
21
|
+
// Only hosting policy and job lookup use current configuration.
|
|
22
|
+
hosted: loadUserConfig('render', {}).options.hosted, 'no-hosted': options['no-hosted'] ?? false });
|
|
23
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { createInterface } from 'node:readline';
|
|
3
|
+
import { rename, stat, writeFile, open } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { stopChild } from './process.mjs';
|
|
6
|
+
import { writeFrame } from './video-pipe.mjs';
|
|
7
|
+
import { MultiplaneError } from './errors.mjs';
|
|
8
|
+
import { fileDigest } from './input-records.mjs';
|
|
9
|
+
|
|
10
|
+
function childProcess(ffmpeg, args, events, input = false, onLine) {
|
|
11
|
+
events.check();
|
|
12
|
+
const child = spawn(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', ...args],
|
|
13
|
+
{ windowsHide: true, stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
|
|
14
|
+
let stderr = '', stdout = '', failure;
|
|
15
|
+
const unsubscribe = events.onCancel(() => stopChild(child));
|
|
16
|
+
child.stderr.on('data', bytes => { stderr = (stderr + bytes).slice(-8000); });
|
|
17
|
+
if (onLine) createInterface({ input: child.stdout, crlfDelay: Infinity }).on('line', onLine);
|
|
18
|
+
else child.stdout.on('data', bytes => { stdout = (stdout + bytes).slice(-8000); });
|
|
19
|
+
child.stdin?.on('error', error => { failure = error; });
|
|
20
|
+
const finished = new Promise((resolve, reject) => {
|
|
21
|
+
child.once('error', cause => reject(new MultiplaneError('encode_failed', undefined, { cause })));
|
|
22
|
+
child.once('close', code => {
|
|
23
|
+
unsubscribe();
|
|
24
|
+
if (code === 0) resolve(stdout);
|
|
25
|
+
else reject(new MultiplaneError('encode_failed', undefined, { details: { exitCode: code, stderr } }));
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
finished.catch(error => { failure = error; });
|
|
29
|
+
return { child, finished, async write(bytes) {
|
|
30
|
+
try {
|
|
31
|
+
if (failure) throw failure;
|
|
32
|
+
await writeFrame(child.stdin, bytes);
|
|
33
|
+
} catch (cause) {
|
|
34
|
+
throw cause instanceof MultiplaneError ? cause : new MultiplaneError('encode_failed', undefined,
|
|
35
|
+
{ cause, details: { reason: cause.message } });
|
|
36
|
+
}
|
|
37
|
+
} };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function segmentEncoder({ ffmpeg, directory, index, first, end, width, height, rate, encoding, format, events, store, hook }) {
|
|
41
|
+
const file = `segment-${String(index).padStart(6, '0')}.${format}`;
|
|
42
|
+
const temporary = path.join(directory, file + '.partial.' + format);
|
|
43
|
+
const encoder = childProcess(ffmpeg, ['-f', 'rawvideo', '-pixel_format', 'rgba', '-video_size', `${width}x${height}`,
|
|
44
|
+
'-framerate', rate.argument, '-i', 'pipe:0', '-an', ...encoding.args, '-fps_mode', 'passthrough',
|
|
45
|
+
'-pix_fmt', encoding.pixelFormat, ...encoding.mux, temporary], events, true);
|
|
46
|
+
return { ...encoder, async seal(extra = {}) {
|
|
47
|
+
encoder.child.stdin.end();
|
|
48
|
+
await encoder.finished;
|
|
49
|
+
events.check();
|
|
50
|
+
// Count copied video packets without decoding or relying on version-specific progress fields.
|
|
51
|
+
let frames = 0;
|
|
52
|
+
const probe = childProcess(ffmpeg, ['-i', temporary, '-map', '0:v:0', '-an', '-c', 'copy', '-f', 'framecrc', '-'], events, false, line => {
|
|
53
|
+
if (line.trim() && !line.trimStart().startsWith('#')) frames++;
|
|
54
|
+
});
|
|
55
|
+
await probe.finished;
|
|
56
|
+
if (!frames) throw new MultiplaneError('output_invalid', 'Segment frame count could not be read.', { details: { expected: end - first, frames: null } });
|
|
57
|
+
if (frames !== end - first) throw new MultiplaneError('output_invalid', 'Segment frame count differs.', { details: { expected: end - first, frames } });
|
|
58
|
+
const handle = await open(temporary, 'r+');
|
|
59
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
60
|
+
await hook?.('beforeSeal', index);
|
|
61
|
+
await rename(temporary, path.join(directory, file));
|
|
62
|
+
const segment = { file, first, end, bytes: (await stat(path.join(directory, file))).size,
|
|
63
|
+
sha256: await fileDigest(path.join(directory, file)), ...extra };
|
|
64
|
+
store.manifest.segments = store.manifest.segments.filter(item => item.first !== first).concat(segment).sort((a, b) => a.first - b.first);
|
|
65
|
+
store.save();
|
|
66
|
+
await hook?.('sealed', index);
|
|
67
|
+
return segment;
|
|
68
|
+
} };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export async function assembleSegments({ ffmpeg, directory, manifest, temporary, encoding, rate, audioOffset, events }) {
|
|
72
|
+
const segments = [...manifest.segments].sort((a, b) => a.first - b.first);
|
|
73
|
+
const { options, first, total } = manifest;
|
|
74
|
+
if (segments.length === 1 && !options.audio) {
|
|
75
|
+
const source = path.join(directory, segments[0].file);
|
|
76
|
+
try { await rename(source, temporary); return source; }
|
|
77
|
+
catch (error) {
|
|
78
|
+
if (error.code !== 'EXDEV') throw error;
|
|
79
|
+
const { copyFile } = await import('node:fs/promises');
|
|
80
|
+
await copyFile(source, temporary);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const list = path.join(directory, 'concat.txt');
|
|
85
|
+
// Relative generated names avoid quoting user paths in the concat language.
|
|
86
|
+
await writeFile(list, segments.map(segment => `file '${segment.file}'\nduration ${rate.seconds(segment.end - segment.first)}\n`).join(''));
|
|
87
|
+
const input = segments.length === 1 ? ['-i', path.join(directory, segments[0].file)] : ['-f', 'concat', '-safe', '1', '-i', list];
|
|
88
|
+
const audio = options.audio ? ['-i', options.audio, '-map', '0:v:0', '-map', '1:a:0', '-af',
|
|
89
|
+
`atrim=start=${audioOffset + rate.seconds(first)},asetpts=PTS-STARTPTS,apad,atrim=duration=${rate.seconds(total)}`, ...encoding.audio] : ['-map', '0:v:0', '-an'];
|
|
90
|
+
await childProcess(ffmpeg, [...input, ...audio, '-c:v', 'copy', ...encoding.mux, temporary], events).finished;
|
|
91
|
+
}
|