multiplane 0.3.7 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,7 @@ import { MultiplaneError } from './errors.mjs';
4
4
  import { availableParallelism, freemem, totalmem } from 'node:os';
5
5
  import { Worker } from 'node:worker_threads';
6
6
  import { blurSampleReport } from './post.mjs';
7
+ import { createCaptureWatchdog } from './capture-watchdog.mjs';
7
8
 
8
9
  export function defaultWorkers(scale = 1, hardware = {}) {
9
10
  const cores = hardware.cores ?? availableParallelism();
@@ -12,24 +13,50 @@ export function defaultWorkers(scale = 1, hardware = {}) {
12
13
  return Math.max(1, Math.min(12, Math.floor(cores / 2), Math.floor((free - 2 * 1024 ** 3) / workerBytes)));
13
14
  }
14
15
 
15
- export function createRenderPool(shot, options, fps, workers, size, onWarning, onCaptured) {
16
+ export function createRenderPool(shot, options, fps, workers, size, onWarning, onCaptured, onInput) {
16
17
  options = { ...options, _browserRecovery: options._browserRecovery ?? recoveryBudget() };
17
18
  reuseState(options);
18
19
  const slots = [], stats = { frames: 0, samples: 0, staticFrames: 0 };
19
20
  const histogram = {}, dofHistogram = {};
20
21
  const captured = new Set();
21
- let failure;
22
+ let failure, stopping = false, closing;
23
+ function close() {
24
+ if (closing) return closing;
25
+ stopping = true; watchdog.stop();
26
+ for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'close' });
27
+ // A blocked page cannot finish its frame. Close only browsers owned by
28
+ // these workers after the grace period, letting their normal cleanup run.
29
+ const timer = setTimeout(() => {
30
+ for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'cancel' });
31
+ }, options._stopGraceMs ?? 5000); // Internal override for short fault tests.
32
+ closing = Promise.all(slots.map(s => s.exited)).finally(() => clearTimeout(timer));
33
+ return closing;
34
+ }
35
+ // Internal deadline override for short fault-injection tests; no CLI option.
36
+ const watchdog = createCaptureWatchdog(slots, (slot, details) => {
37
+ if (failure || stopping) return;
38
+ try {
39
+ onWarning?.({ code: 'worker_stalled', severity: 'warning', message: 'Rendering worker stalled.', details });
40
+ slot.worker.postMessage({ type: 'stalled', details });
41
+ } catch (error) { fail(error); }
42
+ }, { minimumMs: options._stallTimeoutMs });
43
+ const fail = error => {
44
+ failure ??= error;
45
+ watchdog.stop();
46
+ for (const s of slots) { s.pending?.reject(failure); s.pending = null; }
47
+ void close();
48
+ };
22
49
  for (let i = 0; i < workers; i++) {
23
50
  const worker = new Worker(new URL('./render-worker.mjs', import.meta.url), { workerData: { shot, options, fps, size } });
24
51
  const slot = { worker, pending: null, closed: false };
25
- const fail = error => {
26
- failure ??= error;
27
- for (const s of slots) { s.pending?.reject(error); s.pending = null; }
28
- };
29
52
  slot.ready = new Promise((resolve, reject) => { slot.pending = { resolve, reject }; });
30
53
  slot.ready.catch(() => {});
31
54
  worker.on('message', message => {
55
+ if (message.type === 'frame-start') { slot.frame = message.frame; slot.frameStarted = performance.now(); return; }
56
+ if (message.type === 'recovered') { watchdog.recovered(slot); return; }
57
+ if (message.type === 'input') { try { onInput?.(message.file, message.record); } catch (error) { fail(error); } return; }
32
58
  if (message.type === 'captured') {
59
+ watchdog.completed(slot);
33
60
  if (!captured.has(message.frame)) {
34
61
  captured.add(message.frame);
35
62
  try { onCaptured?.(captured.size); } catch (error) { fail(error); }
@@ -39,6 +66,7 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
39
66
  if (message.type === 'warning') { onWarning?.(message.warning); return; }
40
67
  if (message.type === 'failure') { fail(new MultiplaneError(message.code ?? 'worker_crashed', message.error, { details: message.details, hint: message.hint })); return; }
41
68
  const pending = slot.pending; slot.pending = null;
69
+ slot.job = null;
42
70
  pending?.resolve(message);
43
71
  });
44
72
  worker.on('error', error => fail(new MultiplaneError('worker_crashed', undefined, { cause: error })));
@@ -66,6 +94,8 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
66
94
  next = Math.min(end, next + chunk, reuseEnabled(options) ? (Math.floor(begin / REUSE_INTERVAL) + 1) * REUSE_INTERVAL + 1 : end);
67
95
  const promise = new Promise((resolve, reject) => { slot.pending = { resolve, reject }; });
68
96
  promise.catch(() => {});
97
+ slot.job = { first: begin, end: next }; slot.frame = begin;
98
+ slot.lastDelivery = performance.now(); slot.frameStarted = undefined;
69
99
  slot.worker.postMessage({ type: 'render', first: begin, end: next });
70
100
  queue.push({ slot, promise });
71
101
  };
@@ -86,13 +116,10 @@ export function createRenderPool(shot, options, fps, workers, size, onWarning, o
86
116
  if (options.transport !== 'raw') schedule(slot);
87
117
  }
88
118
  },
89
- async close() {
90
- for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'close' });
91
- await Promise.all(slots.map(s => s.exited));
92
- },
119
+ close,
93
120
  async cancel() {
94
- for (const slot of slots) if (!slot.closed) slot.worker.postMessage({ type: 'cancel' });
95
- await Promise.all(slots.map(s => s.exited));
121
+ fail(new MultiplaneError('cancelled'));
122
+ await close();
96
123
  },
97
124
  };
98
125
  const frames = api.frames.bind(api);
@@ -3,8 +3,11 @@ import { MultiplaneError, asError } from './errors.mjs';
3
3
  import { parentPort, workerData, isMarkedAsUntransferable } from 'node:worker_threads';
4
4
  import { openShot } from './capture.mjs';
5
5
  import { createPostCapture, resolveScale } from './post.mjs';
6
+ import { inputRecords } from './input-records.mjs';
6
7
 
7
- let capture, closing = false, running = false;
8
+ if (workerData.options._recordInputs) inputRecords.enterWith((file, record) => parentPort.postMessage({ type: 'input', file, record }));
9
+
10
+ let capture, closing = false, running = false, activeFrame;
8
11
  const sendFailure = error => {
9
12
  const failure = asError(error);
10
13
  parentPort.postMessage({ type: 'failure', error: failure.message, code: failure.code, details: failure.details, hint: failure.hint });
@@ -16,12 +19,17 @@ async function close() {
16
19
  parentPort.close();
17
20
  }
18
21
  try {
19
- capture = await openShot(workerData.shot, resolveScale(workerData.options).renderScale, { ...workerData.options, onWarning: warning => parentPort.postMessage({ type: 'warning', warning }), check: () => { if (closing) throw new MultiplaneError('cancelled'); } });
22
+ capture = await openShot(workerData.shot, resolveScale(workerData.options).renderScale, { ...workerData.options, onRecovered: () => parentPort.postMessage({ type: 'recovered' }), onWarning: warning => parentPort.postMessage({ type: 'warning', warning }), check: () => { if (closing) throw new MultiplaneError('cancelled'); } });
20
23
  if (workerData.size && (capture.width !== workerData.size.width || capture.height !== workerData.size.height)) throw new MultiplaneError('internal_error', 'Worker stage dimensions differ from the main process.');
21
24
  const processed = createPostCapture(capture, { ...workerData.options, onWarning: warning => parentPort.postMessage({ type: 'warning', warning }) }, workerData.fps);
22
25
  parentPort.on('message', async message => {
23
26
  if (message.type === 'cancel') { closing = true; await capture?.close(); if (!running) parentPort.close(); return; }
24
27
  if (message.type === 'close') { await close(); return; }
28
+ if (message.type === 'stalled') {
29
+ if (!closing && activeFrame === message.details.frame) await capture.stall(message.details).catch(sendFailure);
30
+ else parentPort.postMessage({ type: 'recovered' });
31
+ return;
32
+ }
25
33
  if (closing) return;
26
34
  running = true;
27
35
  try {
@@ -29,7 +37,10 @@ try {
29
37
  const source = async function* (first, end) {
30
38
  for (let index = first; index < end && !closing; index++) {
31
39
  const frame = message.first + index;
40
+ activeFrame = frame;
41
+ parentPort.postMessage({ type: 'frame-start', frame });
32
42
  const png = await processed.frame(frame / workerData.fps, frame, index === end - 1);
43
+ activeFrame = undefined;
33
44
  parentPort.postMessage({ type: 'captured', frame });
34
45
  yield { png, hash: processed.lastHash, blurSamples: processed.lastBlurSamples, dofSamples: processed.lastDofSamples };
35
46
  }
package/src/render.mjs CHANGED
@@ -1,9 +1,8 @@
1
1
  import { readBenchmark, benchmarkMachine, selectWorkers, classifyShot } from './workers.mjs';
2
2
  import { recoveryBudget } from './browser-recovery.mjs';
3
3
  import path from 'node:path';
4
- import { mkdir, rm, writeFile } from 'node:fs/promises';
4
+ import { mkdir, rm, writeFile, rename } from 'node:fs/promises';
5
5
  import { createHash } from 'node:crypto';
6
- import { spawn } from 'node:child_process';
7
6
  import { stopChild } from './process.mjs';
8
7
  import { inspectFFmpeg } from './ffmpeg.mjs';
9
8
  import { openShot, number } from './capture.mjs';
@@ -11,19 +10,28 @@ import { createPostCapture, resolveScale } from './post.mjs';
11
10
  import { createRenderPool } from './render-pool.mjs';
12
11
  import { resolveFormat, videoEncoding } from './format.mjs';
13
12
  import { checkAlphaVideo } from './output-alpha.mjs';
14
- import { writeFrame } from './video-pipe.mjs';
15
- import { runCommand } from './events.mjs';
13
+ import { inputRecords, recordInput } from './input-records.mjs';
14
+ import { atomicJSON, lockSegments, manifestStore, checkInputs, checkEnvironment, reusableSegments, readManifest, unavailable } from './segments.mjs';
15
+ import { segmentEncoder, assembleSegments } from './segment-encoder.mjs';
16
+ import { version, runCommand } from './events.mjs';
16
17
  import { loaded, observeBrowser, environment } from './environment.mjs';
17
- import { MultiplaneError, invalidOption } from './errors.mjs';
18
+ import { invalidOption } from './errors.mjs';
18
19
  import { requireNewOutput, publishVideo } from './artifacts.mjs';
19
20
  import { createPreview } from './preview.mjs';
20
21
  import { checkVideo } from './output-check.mjs';
21
22
  import { frameRate } from './frame-rate.mjs';
22
23
 
24
+ const persistedOptions = options => Object.fromEntries(Object.entries(options).filter(([key, value]) =>
25
+ !key.startsWith('_') && typeof value !== 'function' && !['job-dir-exact', 'hosted-executor'].includes(key)));
26
+
23
27
  export async function renderShot(shot, options = {}) {
24
- return runCommand('render', shot, options, async events => {
28
+ return runCommand('render', shot, options, async events => inputRecords.run(null, async () => {
25
29
  options = { ...events.options };
26
30
  delete options.onEvent;
31
+ const resume = options._resume; delete options._resume;
32
+ const hook = options._segmentHook; delete options._segmentHook;
33
+ const segmentFrames = options._segmentFrames; delete options._segmentFrames;
34
+ delete options._effectiveOptions;
27
35
  const probe = options._probeVideo;
28
36
  delete options._probeVideo;
29
37
  const inspect = options._inspectFFmpeg ?? inspectFFmpeg;
@@ -32,25 +40,56 @@ export async function renderShot(shot, options = {}) {
32
40
  const name = path.basename(shot, path.extname(shot));
33
41
  let result;
34
42
 
35
- let capture, pool, encoder, temporary, cleaning, encoderClosed = false;
43
+ let capture, pool, temporary, cleaning, store, directory, release, movedSource;
44
+ const encoders = new Set(), pending = [];
45
+ const inputs = {};
46
+ const onInput = (file, record) => { if (store) store.input(file, record); else inputs[file] = record; };
47
+ inputRecords.enterWith(onInput);
36
48
  function cleanup() { return cleaning ??= cleanResources(); }
37
49
  async function cleanResources() {
38
50
  try {
39
- if (encoder && !encoderClosed) await stopChild(encoder);
51
+ await Promise.all([...encoders].map(encoder => stopChild(encoder)));
52
+ await Promise.allSettled(pending);
40
53
  } finally {
41
54
  try { await capture?.close(); }
42
- finally { try { await pool?.close(); } finally { if (temporary) await rm(temporary, { force: true }); } }
55
+ finally { try { await pool?.close(); } finally {
56
+ try {
57
+ if (temporary && movedSource) await rename(temporary, movedSource);
58
+ else if (temporary) await rm(temporary, { force: true });
59
+ } finally { release?.(); }
60
+ } }
43
61
  }
44
62
  }
45
63
 
46
64
  try {
65
+ if (resume) {
66
+ directory = resume.directory;
67
+ release ??= lockSegments(directory);
68
+ resume.manifest = readManifest(directory);
69
+ if (!resume.manifest) throw unavailable('missing_manifest');
70
+ atomicJSON(path.join(events.directory, 'resume.json'), { directory });
71
+ await checkInputs(resume.manifest);
72
+ }
47
73
  const out = path.resolve(options.out ?? path.join(events.directory, `${name}.${options.format}`));
74
+ if (!resume) {
75
+ directory = path.join(events.directory, 'segments');
76
+ release = lockSegments(directory);
77
+ // A launch failure can be retried even before metadata or a first frame
78
+ // exists. Replace this provisional record with effective values below.
79
+ store = manifestStore(directory, { schema: 'multiplane-segments/1', jobId: events.jobId,
80
+ shot: path.resolve(options.root, shot), options: persistedOptions({ ...options, out,
81
+ audio: options.audio ? path.resolve(options.audio) : undefined }),
82
+ environment: null, first: 0, end: 0, total: 0, segmentFrames: segmentFrames ?? null, inputs: {}, segments: [] });
83
+ store.save();
84
+ await recordInput(path.resolve(options.root, shot));
85
+ if (options.audio) await recordInput(path.resolve(options.audio));
86
+ }
48
87
  if (!options.overwrite) await requireNewOutput(out);
49
88
  events.check();
50
89
  events.progress('launch');
51
90
  const { scale, renderScale } = resolveScale(options);
52
91
  const encoding = videoEncoding(options);
53
- let videoArgs = encoding.args;
92
+
54
93
  const ffmpegInfo = await inspect(options.ffmpeg, process.env, { x264: 'libx264', 'nvenc-h264': 'h264_nvenc', 'nvenc-hevc': 'hevc_nvenc' }[encoding.encoder] ?? encoding.encoder);
55
94
  const ffmpeg = ffmpegInfo.path;
56
95
  options.transport = 'raw';
@@ -74,8 +113,7 @@ export async function renderShot(shot, options = {}) {
74
113
  options.alpha = events.options.alpha = capture.alpha;
75
114
  resolveFormat(options);
76
115
  Object.assign(encoding, videoEncoding(options));
77
- videoArgs = encoding.args;
78
- await loaded(events, capture, ffmpegInfo);
116
+
79
117
  const fps = number(options.fps ?? capture.fps, '--fps', { positive: true });
80
118
  const rate = frameRate(fps);
81
119
  const processed = createPostCapture(capture, options, fps);
@@ -87,57 +125,110 @@ export async function renderShot(shot, options = {}) {
87
125
  const total = end - first;
88
126
  if (!total) throw invalidOption('--from', 'Range contains no frames.');
89
127
  const { width, height } = capture;
90
- events.progress('capture', 0, total);
128
+ directory = resume?.directory ?? path.join(events.directory, 'segments');
129
+ release ??= lockSegments(directory);
130
+ const info = await environment(capture.browser);
131
+ const currentEnvironment = { multiplane: version, chromium: info.chromium, raster: options.raster,
132
+ renderer: info.gpu.renderer, ffmpeg: ffmpegInfo.version, encoder: encoding.encoder };
133
+ const effective = { ...options, fps, from, to, out, ffmpeg, audio: options.audio ? path.resolve(options.audio) : undefined,
134
+ 'motion-blur': processed.post.motionBlur, 'max-motion-blur': processed.post.maxMotionBlur,
135
+ shutter: processed.post.shutter, grain: processed.post.grain };
136
+ const savedOptions = persistedOptions(effective);
137
+ let manifest;
138
+ if (resume) {
139
+ manifest = resume.manifest;
140
+ checkEnvironment(manifest.environment, currentEnvironment);
141
+ if (manifest.environment && manifest.environment.renderer !== currentEnvironment.renderer) {
142
+ events.warning({ code: 'resume_renderer_changed', message: 'GPU renderer changed since the original render.',
143
+ details: { before: manifest.environment.renderer, after: currentEnvironment.renderer } });
144
+ }
145
+ await checkInputs(manifest);
146
+ manifest.segments = await reusableSegments(directory, manifest);
147
+ if (!manifest.environment) Object.assign(manifest, { environment: currentEnvironment, options: savedOptions,
148
+ first, end, total, segmentFrames: manifest.segmentFrames ?? Math.max(1, Math.round(fps * 10)) });
149
+ } else {
150
+ manifest = { schema: 'multiplane-segments/1', jobId: events.jobId, shot: path.resolve(options.root, shot),
151
+ options: savedOptions, environment: currentEnvironment, first, end, total,
152
+ segmentFrames: segmentFrames ?? Math.max(1, Math.round(fps * 10)), inputs: store.manifest.inputs, segments: [] };
153
+ }
154
+ if (!Number.isSafeInteger(manifest.segmentFrames) || manifest.segmentFrames < 1) throw invalidOption('_segmentFrames', 'Segment size must be a positive integer.');
155
+ store = manifestStore(directory, manifest);
156
+ for (const [file, record] of Object.entries(inputs)) store.input(file, record);
157
+ if (options.audio) await recordInput(path.resolve(options.audio));
158
+ store.save();
159
+ const framesReused = manifest.segments.reduce((sum, segment) => sum + segment.end - segment.first, 0);
160
+ const resumeData = resume ? { resumedFrom: resume.jobId, resume: { segmentsReused: manifest.segments.length, framesReused } } : {};
161
+ events.resumeData = resumeData;
162
+ if (resume) atomicJSON(path.join(events.directory, 'resume.json'), { directory });
163
+ await loaded(events, capture, ffmpegInfo);
164
+ events.progress('capture', framesReused, total);
91
165
  await capture.close(); capture = undefined;
92
166
  events.check();
93
- const actualWorkers = Math.min(workers, Math.ceil(total / 4));
167
+ const actualWorkers = Math.min(workers, Math.max(1, Math.ceil((total - framesReused) / 4)));
94
168
  events.resources.setWorkers(actualWorkers);
95
169
  const started = performance.now();
96
170
  const frameHashes = process.env.RENDER_FRAME_HASHES ? [] : null;
171
+ options._recordInputs = true;
97
172
  pool = createRenderPool(shot, options, fps, actualWorkers, { width, height }, events.warning,
98
- done => events.progress('capture', done, total));
173
+ done => events.progress('capture', done + framesReused, total), onInput);
99
174
  events.onCancel(() => pool?.cancel());
100
175
  await mkdir(path.dirname(out), { recursive: true });
101
- events.check();
102
- temporary = `${out}.${events.jobId}.partial.${options.format}`;
103
- const audioArgs = options.audio
104
- ? ['-i', path.resolve(options.audio), '-map', '0:v:0', '-map', '1:a:0', '-af', `atrim=start=${audioOffset + rate.seconds(first)},asetpts=PTS-STARTPTS,apad,atrim=duration=${rate.seconds(total)}`, ...encoding.audio]
105
- : ['-an'];
106
- // Supported since FFmpeg 5.1; older FFmpeg uses the global -vsync 0.
107
- // Do not add output -r: passthrough must neither drop nor duplicate frames.
108
- encoder = spawn(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', '-f', 'rawvideo', '-pixel_format', 'rgba', '-video_size', `${width * scale}x${height * scale}`, '-framerate', rate.argument, '-i', 'pipe:0', ...audioArgs, ...videoArgs, '-fps_mode', 'passthrough', '-pix_fmt', encoding.pixelFormat, ...encoding.mux, temporary], { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });
109
- encoder.once('close', () => { encoderClosed = true; });
110
- events.onCancel(() => stopChild(encoder));
111
- let stderr = '', failure;
112
- encoder.stderr.on('data', data => { stderr = (stderr + data).slice(-8000); });
113
- encoder.stdin.on('error', error => { failure = error; });
114
- const finished = new Promise((resolve, reject) => {
115
- encoder.once('error', cause => reject(new MultiplaneError('encode_failed', undefined, { cause, details: { reason: cause.message } })));
116
- encoder.once('close', code => code === 0 ? resolve() : reject(new MultiplaneError('encode_failed', undefined, { details: { exitCode: code, stderr } })));
117
- });
118
- finished.catch(error => { failure = error; });
176
+ temporary = out + '.' + events.jobId + '.partial.' + options.format;
119
177
  const alphaReferences = new Map();
120
- let encoderWriteMs = 0, framesSent = 0;
121
- // Internal switch for the pixel-equivalence and overhead tests, not a CLI option.
178
+ let encoderWriteMs = 0, framesSent = total, sealingFailure;
122
179
  const preview = createPreview(events, width * scale, height * scale, { enabled: options._preview !== false });
123
- for await (const { frame, png } of pool.frames(first, end, async ({ frame, png }) => {
124
- events.check();
125
- await preview(png, frame);
126
- })) {
127
- events.check();
128
- if (failure) throw new MultiplaneError('encode_failed', undefined, { cause: failure });
129
- if (options.alpha && (framesSent === 0 || framesSent === Math.floor(total / 2))) alphaReferences.set(framesSent, Buffer.from(png.filter((_, i) => i % 4 === 3)));
130
- frameHashes?.push({ frame, sha256: createHash('sha256').update(png).digest('hex') });
131
- const writeStarted = performance.now();
132
- await writeFrame(encoder.stdin, png).catch(cause => { throw new MultiplaneError('encode_failed', undefined, { cause, details: { reason: cause.message } }); });
133
- encoderWriteMs += performance.now() - writeStarted;
134
- framesSent++;
180
+ const remember = segment => {
181
+ frameHashes?.push(...(segment.frameHashes ?? []));
182
+ for (const [frame, bytes] of Object.entries(segment.alphaReferences ?? {})) alphaReferences.set(Number(frame), Buffer.from(bytes, 'base64'));
183
+ };
184
+ // Keep workers busy while the preceding encoder drains and seals.
185
+ const ranges = [];
186
+ for (let start = first, index = 0; start < end; start += manifest.segmentFrames, index++) {
187
+ const stop = Math.min(end, start + manifest.segmentFrames);
188
+ const reused = manifest.segments.find(item => item.first === start && item.end === stop);
189
+ if (reused) { remember(reused); continue; }
190
+ ranges.push({ first: start, end: stop, index });
191
+ }
192
+ for (let cursor = 0; cursor < ranges.length;) {
193
+ let last = cursor;
194
+ while (last + 1 < ranges.length && ranges[last].end === ranges[last + 1].first) last++;
195
+ let active, hashes, references, rangeIndex = cursor;
196
+ for await (const { frame, png } of pool.frames(ranges[cursor].first, ranges[last].end)) {
197
+ events.check();
198
+ if (sealingFailure) throw sealingFailure;
199
+ if (!active) {
200
+ active = segmentEncoder({ ffmpeg, directory, ...ranges[rangeIndex], width: width * scale, height: height * scale,
201
+ rate, encoding, format: options.format, events, store, hook });
202
+ encoders.add(active.child);
203
+ active.finished.finally(() => encoders.delete(active.child)).catch(() => {});
204
+ hashes = []; references = {};
205
+ }
206
+ await preview(png, frame);
207
+ const relative = frame - first;
208
+ if (options.alpha && (relative === 0 || relative === Math.floor(total / 2))) references[relative] = Buffer.from(png.filter((_, i) => i % 4 === 3)).toString('base64');
209
+ if (frameHashes) hashes.push({ frame, sha256: createHash('sha256').update(png).digest('hex') });
210
+ const writeStarted = performance.now();
211
+ await active.write(png);
212
+ encoderWriteMs += performance.now() - writeStarted;
213
+ if (frame + 1 === ranges[rangeIndex].end) {
214
+ const sealed = active.seal({ ...(frameHashes ? { frameHashes: hashes } : {}), ...(options.alpha ? { alphaReferences: references } : {}) }).then(remember);
215
+ sealed.catch(error => { sealingFailure ??= error; });
216
+ pending.push(sealed);
217
+ // Hooks model interruption at an exact durable boundary in tests.
218
+ if (hook) await sealed;
219
+ active = undefined; rangeIndex++;
220
+ }
221
+ }
222
+ cursor = last + 1;
135
223
  }
136
224
  const drainStarted = performance.now();
137
225
  events.progress('encode', total, total);
138
- encoder.stdin.end();
139
- await finished;
226
+ await Promise.all(pending);
227
+ // Finish all resource requests before the final input audit/publication.
228
+ await pool.close();
140
229
  events.check();
230
+ await checkInputs(manifest);
231
+ movedSource = await assembleSegments({ ffmpeg, directory, manifest, temporary, encoding, rate, audioOffset, events });
141
232
  const encoderDrainMs = performance.now() - drainStarted;
142
233
  const probeStarted = performance.now();
143
234
  const outputCheck = await checkVideo(ffmpeg, temporary, { frames: framesSent, durationSeconds: rate.seconds(framesSent), width: width * scale, height: height * scale, fps: rate.fps, ...(options.alpha ? { alpha: true, format: options.format } : {}) }, events, probe);
@@ -145,13 +236,18 @@ export async function renderShot(shot, options = {}) {
145
236
  const outputCheckMs = performance.now() - probeStarted;
146
237
  await publishVideo(temporary, out, options.overwrite);
147
238
  temporary = undefined;
239
+ // Invalidate the checkpoint while still holding the lock, so another
240
+ // resume cannot reuse a manifest whose output has already been published.
241
+ await rm(path.join(directory, 'manifest.json'), { force: true });
242
+ await rm(directory, { recursive: true, force: true });
243
+ release(); release = undefined;
148
244
  const elapsed = (performance.now() - started) / 1000;
149
- if (frameHashes) await writeFile(process.env.RENDER_FRAME_HASHES, JSON.stringify(frameHashes, null, 2));
245
+ if (frameHashes) await writeFile(process.env.RENDER_FRAME_HASHES, JSON.stringify(frameHashes.sort((a, b) => a.frame - b.frame), null, 2));
150
246
  if (process.env.RENDER_PROFILE) await writeFile(process.env.RENDER_PROFILE, JSON.stringify({elapsed,frames:total,workers:actualWorkers,options,stats:pool.stats,encoderWriteMs,encoderDrainMs,outputCheckMs},null,2));
151
247
  events.artifact(out, 'video');
152
- result = { staticReuse: pool.staticReuse, blurSamples: pool.blurSamples, dofSamples: pool.dofSamples, output: out, frames: framesSent, fps, from, to, width: width * scale, height: height * scale, encoder: encoding.encoder, format: options.format, alpha: options.alpha, post: { ...processed.post, adaptiveBlur: Number(options['adaptive-blur'] ?? 0.5) }, renderScale, scale, outputCheck };
248
+ result = { ...resumeData, staticReuse: pool.staticReuse, blurSamples: pool.blurSamples, dofSamples: pool.dofSamples, output: out, frames: framesSent, fps, from, to, width: width * scale, height: height * scale, encoder: encoding.encoder, format: options.format, alpha: options.alpha, post: { ...processed.post, adaptiveBlur: Number(options['adaptive-blur'] ?? 0.5) }, renderScale, scale, outputCheck };
153
249
  } finally { await cleanup(); }
154
250
 
155
251
  return options.draft ? { ...result, draft: true } : result;
156
- });
252
+ }));
157
253
  }
package/src/resume.mjs ADDED
@@ -0,0 +1,23 @@
1
+ import { renderShot } from './render.mjs';
2
+ import { readStatus, resolveJob } from './jobs.mjs';
3
+ import { readManifest, segmentDirectory, unavailable, checkInputs } from './segments.mjs';
4
+ import { invalidOption } from './errors.mjs';
5
+ import { loadUserConfig } from './user-home.mjs';
6
+
7
+ export async function resumeRender(target, options = {}) {
8
+ const allowed = ['workers', 'json', 'job-dir', 'no-hosted', 'onEvent', '_hostedCLI', '_segmentHook', '_inspectFFmpeg', '_probeVideo', 'hosted-executor', 'job-dir-exact'];
9
+ for (const key of Object.keys(options)) if (!allowed.includes(key)) throw invalidOption('--' + key, 'This option cannot be changed when resuming.');
10
+ const configured = loadUserConfig('status', options);
11
+ const jobDir = resolveJob(target, configured.options);
12
+ const status = await readStatus(jobDir);
13
+ if (status.command !== 'render') throw unavailable('not_render');
14
+ if (status.state === 'succeeded') throw unavailable('succeeded');
15
+ if (!['failed', 'cancelled'].includes(status.state)) throw unavailable('running');
16
+ const directory = segmentDirectory(jobDir), manifest = readManifest(directory);
17
+ if (!manifest) throw unavailable('missing_manifest');
18
+ await checkInputs(manifest);
19
+ return renderShot(manifest.shot, { ...manifest.options, ...options,
20
+ _resume: { directory, jobId: status.jobId }, _effectiveOptions: true,
21
+ // Only hosting policy and job lookup use current configuration.
22
+ hosted: loadUserConfig('render', {}).options.hosted, 'no-hosted': options['no-hosted'] ?? false });
23
+ }
@@ -0,0 +1,91 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { createInterface } from 'node:readline';
3
+ import { rename, stat, writeFile, open } from 'node:fs/promises';
4
+ import path from 'node:path';
5
+ import { stopChild } from './process.mjs';
6
+ import { writeFrame } from './video-pipe.mjs';
7
+ import { MultiplaneError } from './errors.mjs';
8
+ import { fileDigest } from './input-records.mjs';
9
+
10
+ function childProcess(ffmpeg, args, events, input = false, onLine) {
11
+ events.check();
12
+ const child = spawn(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', ...args],
13
+ { windowsHide: true, stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
14
+ let stderr = '', stdout = '', failure;
15
+ const unsubscribe = events.onCancel(() => stopChild(child));
16
+ child.stderr.on('data', bytes => { stderr = (stderr + bytes).slice(-8000); });
17
+ if (onLine) createInterface({ input: child.stdout, crlfDelay: Infinity }).on('line', onLine);
18
+ else child.stdout.on('data', bytes => { stdout = (stdout + bytes).slice(-8000); });
19
+ child.stdin?.on('error', error => { failure = error; });
20
+ const finished = new Promise((resolve, reject) => {
21
+ child.once('error', cause => reject(new MultiplaneError('encode_failed', undefined, { cause })));
22
+ child.once('close', code => {
23
+ unsubscribe();
24
+ if (code === 0) resolve(stdout);
25
+ else reject(new MultiplaneError('encode_failed', undefined, { details: { exitCode: code, stderr } }));
26
+ });
27
+ });
28
+ finished.catch(error => { failure = error; });
29
+ return { child, finished, async write(bytes) {
30
+ try {
31
+ if (failure) throw failure;
32
+ await writeFrame(child.stdin, bytes);
33
+ } catch (cause) {
34
+ throw cause instanceof MultiplaneError ? cause : new MultiplaneError('encode_failed', undefined,
35
+ { cause, details: { reason: cause.message } });
36
+ }
37
+ } };
38
+ }
39
+
40
+ export function segmentEncoder({ ffmpeg, directory, index, first, end, width, height, rate, encoding, format, events, store, hook }) {
41
+ const file = `segment-${String(index).padStart(6, '0')}.${format}`;
42
+ const temporary = path.join(directory, file + '.partial.' + format);
43
+ const encoder = childProcess(ffmpeg, ['-f', 'rawvideo', '-pixel_format', 'rgba', '-video_size', `${width}x${height}`,
44
+ '-framerate', rate.argument, '-i', 'pipe:0', '-an', ...encoding.args, '-fps_mode', 'passthrough',
45
+ '-pix_fmt', encoding.pixelFormat, ...encoding.mux, temporary], events, true);
46
+ return { ...encoder, async seal(extra = {}) {
47
+ encoder.child.stdin.end();
48
+ await encoder.finished;
49
+ events.check();
50
+ // Count copied video packets without decoding or relying on version-specific progress fields.
51
+ let frames = 0;
52
+ const probe = childProcess(ffmpeg, ['-i', temporary, '-map', '0:v:0', '-an', '-c', 'copy', '-f', 'framecrc', '-'], events, false, line => {
53
+ if (line.trim() && !line.trimStart().startsWith('#')) frames++;
54
+ });
55
+ await probe.finished;
56
+ if (!frames) throw new MultiplaneError('output_invalid', 'Segment frame count could not be read.', { details: { expected: end - first, frames: null } });
57
+ if (frames !== end - first) throw new MultiplaneError('output_invalid', 'Segment frame count differs.', { details: { expected: end - first, frames } });
58
+ const handle = await open(temporary, 'r+');
59
+ try { await handle.sync(); } finally { await handle.close(); }
60
+ await hook?.('beforeSeal', index);
61
+ await rename(temporary, path.join(directory, file));
62
+ const segment = { file, first, end, bytes: (await stat(path.join(directory, file))).size,
63
+ sha256: await fileDigest(path.join(directory, file)), ...extra };
64
+ store.manifest.segments = store.manifest.segments.filter(item => item.first !== first).concat(segment).sort((a, b) => a.first - b.first);
65
+ store.save();
66
+ await hook?.('sealed', index);
67
+ return segment;
68
+ } };
69
+ }
70
+
71
+ export async function assembleSegments({ ffmpeg, directory, manifest, temporary, encoding, rate, audioOffset, events }) {
72
+ const segments = [...manifest.segments].sort((a, b) => a.first - b.first);
73
+ const { options, first, total } = manifest;
74
+ if (segments.length === 1 && !options.audio) {
75
+ const source = path.join(directory, segments[0].file);
76
+ try { await rename(source, temporary); return source; }
77
+ catch (error) {
78
+ if (error.code !== 'EXDEV') throw error;
79
+ const { copyFile } = await import('node:fs/promises');
80
+ await copyFile(source, temporary);
81
+ return;
82
+ }
83
+ }
84
+ const list = path.join(directory, 'concat.txt');
85
+ // Relative generated names avoid quoting user paths in the concat language.
86
+ await writeFile(list, segments.map(segment => `file '${segment.file}'\nduration ${rate.seconds(segment.end - segment.first)}\n`).join(''));
87
+ const input = segments.length === 1 ? ['-i', path.join(directory, segments[0].file)] : ['-f', 'concat', '-safe', '1', '-i', list];
88
+ const audio = options.audio ? ['-i', options.audio, '-map', '0:v:0', '-map', '1:a:0', '-af',
89
+ `atrim=start=${audioOffset + rate.seconds(first)},asetpts=PTS-STARTPTS,apad,atrim=duration=${rate.seconds(total)}`, ...encoding.audio] : ['-map', '0:v:0', '-an'];
90
+ await childProcess(ffmpeg, [...input, ...audio, '-c:v', 'copy', ...encoding.mux, temporary], events).finished;
91
+ }