@torrent-tv/proxy 2.97.0 → 2.97.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/services/encode/EncodeInputs.js +60 -47
- package/services/encode/EncodeOrchestrator.js +29 -38
- package/services/encode/EncodeRun.js +26 -5
- package/services/encode/EncodeRuns.js +15 -48
- package/services/encode/RunClock.js +27 -0
- package/services/encode/run-costs.js +47 -2
- package/services/server/wire-outputs.js +3 -0
- package/services/storage/processor-report.js +48 -35
package/package.json
CHANGED
|
@@ -33,8 +33,13 @@ export class EncodeInputs {
|
|
|
33
33
|
#wanted = new Map();
|
|
34
34
|
/** Bytes per millisecond of recent original-input copies, read where the copy is made. */
|
|
35
35
|
#copyRates = [];
|
|
36
|
-
/**
|
|
37
|
-
|
|
36
|
+
/**
|
|
37
|
+
* How long an encoder takes to open an output's original input: asked of
|
|
38
|
+
* the one owner of start readings (`RunCosts`, torrent-tv/meta#184).
|
|
39
|
+
*
|
|
40
|
+
* @type {(output: object) => { seconds: number, measured: boolean } | null}
|
|
41
|
+
*/
|
|
42
|
+
#openOf;
|
|
38
43
|
#resolve;
|
|
39
44
|
#read;
|
|
40
45
|
#heldRanges;
|
|
@@ -46,7 +51,9 @@ export class EncodeInputs {
|
|
|
46
51
|
#log;
|
|
47
52
|
#now;
|
|
48
53
|
|
|
49
|
-
constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
54
|
+
constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
55
|
+
openOf = () => null, log = () => {}, now = Date.now }) {
|
|
56
|
+
this.#openOf = openOf;
|
|
50
57
|
this.#resolve = resolve;
|
|
51
58
|
this.#read = readRanges;
|
|
52
59
|
this.#heldRanges = heldRanges;
|
|
@@ -130,10 +137,26 @@ export class EncodeInputs {
|
|
|
130
137
|
return null;
|
|
131
138
|
}
|
|
132
139
|
|
|
133
|
-
|
|
140
|
+
/**
|
|
141
|
+
* Withdraw what this output no longer needs: a request no wanted window
|
|
142
|
+
* covers, and — when `starts` is given — a copy already held whose start is
|
|
143
|
+
* neither where a live run ends nor where one is being started. A next
|
|
144
|
+
* stretch is copied for the run working before it (torrent-tv/meta#158);
|
|
145
|
+
* once that run has gone and the plan starts elsewhere, after a seek, the
|
|
146
|
+
* copy held memory nobody would use (torrent-tv/meta#184).
|
|
147
|
+
*
|
|
148
|
+
* @param {object} output
|
|
149
|
+
* @param {{ from: number, to: number }[]} windows
|
|
150
|
+
* @param {Set<number> | null} [starts]
|
|
151
|
+
*/
|
|
152
|
+
retain(output, windows, starts = null) {
|
|
134
153
|
for (const [key, request] of this.#requests) {
|
|
135
|
-
if (request.output !== output
|
|
136
|
-
|
|
154
|
+
if (request.output !== output) continue;
|
|
155
|
+
const covered = windows.some(window => request.from <= window.to && request.to >= window.from);
|
|
156
|
+
const unused = starts !== null && request.result?.kind === "result" && !starts.has(request.from);
|
|
157
|
+
if (covered && !unused) continue;
|
|
158
|
+
this.#note(request, covered ? "withdrawn: no live run ends before it and no start is placed there"
|
|
159
|
+
: "withdrawn: no wanted window covers it");
|
|
137
160
|
this.#requests.delete(key);
|
|
138
161
|
this.#wanted.delete(key);
|
|
139
162
|
request.result?.release?.();
|
|
@@ -154,16 +177,13 @@ export class EncodeInputs {
|
|
|
154
177
|
let existing = this.#requests.get(key);
|
|
155
178
|
if (existing && existing.to !== to) {
|
|
156
179
|
// The plan's bound for this start moved. A copy already made beyond it
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
180
|
+
// holds the shorter stretch whole, so the run is commanded over its
|
|
181
|
+
// prefix and the copy is kept until the run ends: copying it again cost
|
|
182
|
+
// as long as the first copy (torrent-tv/meta#184). One being made is
|
|
183
|
+
// narrowed when it is done, and one waiting for bytes or memory takes the
|
|
184
|
+
// new bound when its own event prepares it again.
|
|
160
185
|
existing.to = to;
|
|
161
|
-
if (existing.result?.kind === "result" && existing.result.to > to)
|
|
162
|
-
this.#requests.delete(key);
|
|
163
|
-
this.#wanted.delete(key);
|
|
164
|
-
existing.result.release();
|
|
165
|
-
existing = undefined;
|
|
166
|
-
}
|
|
186
|
+
if (existing.result?.kind === "result" && existing.result.to > to) existing.result.to = Math.max(existing.from, to);
|
|
167
187
|
}
|
|
168
188
|
if (existing?.result?.kind === "result") {
|
|
169
189
|
this.#requests.delete(key);
|
|
@@ -200,44 +220,38 @@ export class EncodeInputs {
|
|
|
200
220
|
this.#wanted.delete(key);
|
|
201
221
|
request.result?.release?.();
|
|
202
222
|
}
|
|
203
|
-
this.#opens.delete(addressOf(output));
|
|
204
223
|
}
|
|
205
224
|
|
|
206
225
|
/**
|
|
207
|
-
* How
|
|
208
|
-
*
|
|
209
|
-
*
|
|
226
|
+
* How many bytes a stretch may hold so that copying it takes no longer than
|
|
227
|
+
* opening the input it replaces: the recent copy rate times this output's
|
|
228
|
+
* recent open. Beyond it the open's share of each piece falls below the
|
|
229
|
+
* copy's, which is the same for every piece; a longer stretch buys less and
|
|
230
|
+
* delays the run's start more. Null until a copy rate is measured and an
|
|
231
|
+
* open is known, and the run then gets one piece. An output that has not
|
|
232
|
+
* opened yet is sized by the host's startup reading, which is said: at a
|
|
233
|
+
* cold start the second run was one piece too for want of it, and cost
|
|
234
|
+
* another open (6.34 s, torrent-tv/meta#166 run 12).
|
|
210
235
|
*
|
|
211
236
|
* @param {object} output
|
|
212
|
-
* @
|
|
237
|
+
* @returns {{ bytes: number, openMeasured: boolean } | null}
|
|
213
238
|
*/
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const
|
|
217
|
-
|
|
218
|
-
this.#opens.set(addressOf(output), readings);
|
|
239
|
+
stretchFor(output) {
|
|
240
|
+
const rate = middleOf(this.#copyRates);
|
|
241
|
+
const open = this.#openOf(output);
|
|
242
|
+
return rate === null || !open ? null : { bytes: Math.floor(rate * open.seconds * 1000), openMeasured: open.measured };
|
|
219
243
|
}
|
|
220
244
|
|
|
221
245
|
/**
|
|
222
|
-
* How many bytes a stretch may hold so that copying it takes no longer than
|
|
223
|
-
* opening the input it replaces: the recent copy rate times this output's
|
|
224
|
-
* recent open. Beyond it the open's share of each piece falls below the
|
|
225
|
-
* copy's, which is the same for every piece; a longer stretch buys less and
|
|
226
|
-
* delays the run's start more. Null until both have been measured, and the
|
|
227
|
-
* run then gets one piece, as it did before either reading existed.
|
|
228
|
-
*
|
|
229
246
|
* @param {object} output
|
|
230
247
|
* @returns {number | null}
|
|
231
248
|
*/
|
|
232
249
|
stretchBytesFor(output) {
|
|
233
|
-
|
|
234
|
-
const open = middleOf(this.#opens.get(addressOf(output)) ?? []);
|
|
235
|
-
return rate === null || open === null ? null : Math.floor(rate * open * 1000);
|
|
250
|
+
return this.stretchFor(output)?.bytes ?? null;
|
|
236
251
|
}
|
|
237
252
|
|
|
238
253
|
#prepare(request) {
|
|
239
254
|
request.pending = true;
|
|
240
|
-
request.stale = false;
|
|
241
255
|
request.revision = this.#revision;
|
|
242
256
|
request.memoryRevision = this.#memoryRevision;
|
|
243
257
|
// Where the time from the request to its input goes, across every attempt:
|
|
@@ -260,13 +274,9 @@ export class EncodeInputs {
|
|
|
260
274
|
if (result.kind === "result") Object.assign(result, { from: request.from, to: request.from });
|
|
261
275
|
}
|
|
262
276
|
if (this.#requests.get(request.key) !== request) { result.release?.(); return; }
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
request.result = null;
|
|
267
|
-
request.stale = true;
|
|
268
|
-
return;
|
|
269
|
-
}
|
|
277
|
+
// The plan narrowed this start while the copy was being made: the run
|
|
278
|
+
// is commanded over its prefix.
|
|
279
|
+
if (result.kind === "result" && result.to > request.to) result.to = Math.max(request.from, request.to);
|
|
270
280
|
request.result = result;
|
|
271
281
|
this.#note(request, result.kind === "result" ? `ready #${result.from}..#${result.to}, ${result.bytes ?? "?"} bytes` +
|
|
272
282
|
(Number.isFinite(result.copyMs) ? `, copied in ${result.copyMs}ms` : "")
|
|
@@ -281,8 +291,8 @@ export class EncodeInputs {
|
|
|
281
291
|
}).finally(() => {
|
|
282
292
|
request.pending = false;
|
|
283
293
|
if (request.result?.kind !== "needs-memory") this.#wanted.delete(request.key);
|
|
284
|
-
const changed = request.
|
|
285
|
-
? request.memoryRevision !== this.#memoryRevision : request.revision !== this.#revision
|
|
294
|
+
const changed = request.result?.kind === "needs-memory"
|
|
295
|
+
? request.memoryRevision !== this.#memoryRevision : request.revision !== this.#revision;
|
|
286
296
|
if (this.#requests.get(request.key) === request && changed &&
|
|
287
297
|
request.result?.kind !== "result" && request.result?.kind !== "terminal") this.#prepare(request);
|
|
288
298
|
});
|
|
@@ -360,7 +370,9 @@ export class EncodeInputs {
|
|
|
360
370
|
}
|
|
361
371
|
return good;
|
|
362
372
|
};
|
|
363
|
-
const
|
|
373
|
+
const sizing = this.stretchFor(output);
|
|
374
|
+
const target = sizing?.bytes ?? null;
|
|
375
|
+
if (sizing && !sizing.openMeasured) request.timing.openApproximate = true;
|
|
364
376
|
let end = target === null || this.#heldRanges === null ? from : await longest(target);
|
|
365
377
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
366
378
|
this.#wanted.set(request.key, { wanted: stretches.get(end).bytes, required: minimum });
|
|
@@ -429,7 +441,8 @@ function describeTiming(timing, now) {
|
|
|
429
441
|
return `; asked ${Math.max(0, now - timing.askedAt)}ms ago over ${timing.attempts} attempt(s): ` +
|
|
430
442
|
`${timing.resolutions} resolution(s) ${timing.resolveMs}ms ` +
|
|
431
443
|
`(${timing.reads} file read(s): queued ${timing.queuedMs}ms, reading ${timing.readMs}ms), ` +
|
|
432
|
-
`held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms
|
|
444
|
+
`held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms` +
|
|
445
|
+
(timing.openApproximate ? "; sized by the host's startup open, this output has not opened yet" : "");
|
|
433
446
|
}
|
|
434
447
|
|
|
435
448
|
function addressOf(output) { return output.outputKey ?? output.id; }
|
|
@@ -145,6 +145,30 @@ export class EncodeOrchestrator {
|
|
|
145
145
|
this.#costs.noteStartup(measured);
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
+
/**
|
|
149
|
+
* One output's original-input open, and what sizes its stretch: the one
|
|
150
|
+
* owner of start readings is this class's `RunCosts` (torrent-tv/meta#184).
|
|
151
|
+
*
|
|
152
|
+
* @param {string} address
|
|
153
|
+
* @param {number} seconds
|
|
154
|
+
*/
|
|
155
|
+
noteOpen(address, seconds) {
|
|
156
|
+
this.#costs.noteOpen(address, seconds);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* @param {string} address
|
|
161
|
+
* @returns {{ seconds: number, measured: boolean } | null}
|
|
162
|
+
*/
|
|
163
|
+
openOf(address) {
|
|
164
|
+
return this.#costs.openOf(address);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** @param {string} address */
|
|
168
|
+
forgetOpens(address) {
|
|
169
|
+
this.#costs.forgetOpens(address);
|
|
170
|
+
}
|
|
171
|
+
|
|
148
172
|
/**
|
|
149
173
|
* What a stop and a start cost on this host, in seconds, as the plan reads
|
|
150
174
|
* them — for anyone else pricing a fresh encoder by the same figures.
|
|
@@ -289,41 +313,6 @@ export class EncodeOrchestrator {
|
|
|
289
313
|
return coverage;
|
|
290
314
|
}
|
|
291
315
|
|
|
292
|
-
/**
|
|
293
|
-
* Where a run started here must stop: the free stretch in front of it.
|
|
294
|
-
*
|
|
295
|
-
* Asked of the one map, brought up to date first. It used to be worked out by
|
|
296
|
-
* the session manager, which reached into this layer for the map and into the
|
|
297
|
-
* store for what was on the disk and put the two together itself — one fact
|
|
298
|
-
* with two owners and a third party carrying it between them, which is how the
|
|
299
|
-
* two came to disagree.
|
|
300
|
-
*
|
|
301
|
-
* @param {object} params
|
|
302
|
-
* @param {string} params.address
|
|
303
|
-
* @param {number} params.from - Where the run will start.
|
|
304
|
-
* @param {object | null} [params.exceptRun] - The run being replaced, whose
|
|
305
|
-
* own claim is not somebody else's material.
|
|
306
|
-
* @param {number} [params.segmentCount] - The output's length, when known.
|
|
307
|
-
* @returns {number} The last number to work through, or `-1` for the end of
|
|
308
|
-
* the film.
|
|
309
|
-
*/
|
|
310
|
-
freeStretchEnd({ address, from, exceptRun = null, segmentCount = 0 }) {
|
|
311
|
-
if (!address) {
|
|
312
|
-
return -1;
|
|
313
|
-
}
|
|
314
|
-
if (segmentCount > 0) {
|
|
315
|
-
this.coverageOf(address).setSegmentCount(segmentCount);
|
|
316
|
-
}
|
|
317
|
-
const coverage = this.#upToDateCoverage(address);
|
|
318
|
-
const start = Math.max(0, from);
|
|
319
|
-
const free = coverage.freeRunFrom(start, exceptRun);
|
|
320
|
-
if (!Number.isFinite(free)) {
|
|
321
|
-
return -1;
|
|
322
|
-
}
|
|
323
|
-
const end = start + Math.max(1, free) - 1;
|
|
324
|
-
return segmentCount > 0 && end >= segmentCount - 1 ? -1 : end;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
316
|
/**
|
|
328
317
|
* @param {string} address
|
|
329
318
|
* @returns {import("../encode/EncodeRun.js").EncodeRun[]}
|
|
@@ -974,9 +963,11 @@ export class EncodeOrchestrator {
|
|
|
974
963
|
// were two steps, and two owners each performed the second one.
|
|
975
964
|
//
|
|
976
965
|
// A run whose input is one copied stretch can make nothing past it, so it
|
|
977
|
-
// claims that stretch and not the plan's longer bound:
|
|
978
|
-
// for the next run (torrent-tv/meta#158).
|
|
979
|
-
|
|
966
|
+
// claims that stretch — its own `to` — and not the plan's longer bound:
|
|
967
|
+
// the rest stays free for the next run (torrent-tv/meta#158). A run whose
|
|
968
|
+
// input grows a segment at a time claims the plan's bound, as it may reach
|
|
969
|
+
// it (torrent-tv/meta#184).
|
|
970
|
+
const through = run.originalInput ? run.to : to;
|
|
980
971
|
this.coverageOf(address).claim(run, from, endOfRun({ from, to: through }));
|
|
981
972
|
}
|
|
982
973
|
|
|
@@ -64,8 +64,14 @@ const MICROSECONDS_PER_SECOND = 1_000_000;
|
|
|
64
64
|
* it finished none.
|
|
65
65
|
* @property {number} livedMs
|
|
66
66
|
* @property {number | null} [firstNamedMs] - Spawn to the moment the encoder
|
|
67
|
-
* named its first closed piece
|
|
68
|
-
* encoding, without the wait for the
|
|
67
|
+
* named its first closed piece, less the time it was paused before then: the
|
|
68
|
+
* open of its input plus that one piece's encoding, without the wait for the
|
|
69
|
+
* following piece that publication adds.
|
|
70
|
+
* @property {number | null} [firstNamedWallMs] - The same on the wall clock,
|
|
71
|
+
* pauses included: what the viewer waited.
|
|
72
|
+
* @property {number | null} [firstOutputWallMs] - Spawn to the first output on
|
|
73
|
+
* the wall clock; `firstOutputMs` leaves the pauses out.
|
|
74
|
+
* @property {number} [stoppedMs] - How long the run was paused in all.
|
|
69
75
|
* @property {boolean} normal - Whether this ending is the expected one.
|
|
70
76
|
* @property {string} lastError - The last thing ffmpeg said on stderr.
|
|
71
77
|
* @property {string | null} provenName - The last piece this run named while it
|
|
@@ -166,6 +172,10 @@ export class EncodeRun {
|
|
|
166
172
|
*/
|
|
167
173
|
#firstNamedAt = 0;
|
|
168
174
|
|
|
175
|
+
/** How long the run had been paused when each of the two above was taken. */
|
|
176
|
+
#stoppedBeforeFirstOutput = 0;
|
|
177
|
+
#stoppedBeforeFirstNamed = 0;
|
|
178
|
+
|
|
169
179
|
/** The film made and the run's own working time, as of the last progress report. */
|
|
170
180
|
#workSample = null;
|
|
171
181
|
|
|
@@ -569,6 +579,7 @@ export class EncodeRun {
|
|
|
569
579
|
this.#produced.add(index);
|
|
570
580
|
if (this.#firstOutputAt === 0) {
|
|
571
581
|
this.#firstOutputAt = this.now();
|
|
582
|
+
this.#stoppedBeforeFirstOutput = this.clock.stoppedMs(this.#firstOutputAt);
|
|
572
583
|
}
|
|
573
584
|
if (this.#state === ENCODE_RUN_STATE.STARTING) {
|
|
574
585
|
this.#transition(ENCODE_RUN_EVENT.FIRST_SEGMENT);
|
|
@@ -607,7 +618,10 @@ export class EncodeRun {
|
|
|
607
618
|
}
|
|
608
619
|
if (this.#publicationError) break;
|
|
609
620
|
const index = this.indexOfName(name);
|
|
610
|
-
if (this.#firstNamedAt === 0 && Number.isInteger(index) && index >= this.from)
|
|
621
|
+
if (this.#firstNamedAt === 0 && Number.isInteger(index) && index >= this.from) {
|
|
622
|
+
this.#firstNamedAt = this.now();
|
|
623
|
+
this.#stoppedBeforeFirstNamed = this.clock.stoppedMs(this.#firstNamedAt);
|
|
624
|
+
}
|
|
611
625
|
const endOfWork = Number.isFinite(endOfRun(this)) ? this.to : this.lastSegmentIndex();
|
|
612
626
|
// A packet crossing the final cut can flush a following file. It may
|
|
613
627
|
// complete its predecessor, but no unadmitted interval can be published.
|
|
@@ -920,11 +934,18 @@ export class EncodeRun {
|
|
|
920
934
|
// Null where the run was never told to stop, or never produced anything:
|
|
921
935
|
// an absent measurement says so rather than reading as zero.
|
|
922
936
|
dyingMs: this.#stopOrderedAt > 0 ? this.now() - this.#stopOrderedAt : null,
|
|
937
|
+
// Without the time this run was paused before then: a pause is our own
|
|
938
|
+
// decision, not what starting cost (torrent-tv/meta#184). The wall-clock
|
|
939
|
+
// figures beside them are what the viewer waited.
|
|
923
940
|
firstOutputMs:
|
|
924
941
|
this.#firstOutputAt > 0 && this.#startedAt > 0
|
|
925
|
-
? this.#firstOutputAt - this.#startedAt
|
|
942
|
+
? this.#firstOutputAt - this.#startedAt - this.#stoppedBeforeFirstOutput
|
|
926
943
|
: null,
|
|
927
|
-
|
|
944
|
+
firstOutputWallMs: this.#firstOutputAt > 0 && this.#startedAt > 0 ? this.#firstOutputAt - this.#startedAt : null,
|
|
945
|
+
firstNamedMs: this.#firstNamedAt > 0 && this.#startedAt > 0
|
|
946
|
+
? this.#firstNamedAt - this.#startedAt - this.#stoppedBeforeFirstNamed : null,
|
|
947
|
+
firstNamedWallMs: this.#firstNamedAt > 0 && this.#startedAt > 0 ? this.#firstNamedAt - this.#startedAt : null,
|
|
948
|
+
stoppedMs: this.clock.stoppedMs(),
|
|
928
949
|
normal: ending === ENCODE_EXIT.COMPLETE,
|
|
929
950
|
lastError: this.lastError
|
|
930
951
|
};
|
|
@@ -582,6 +582,7 @@ export class EncodeRuns {
|
|
|
582
582
|
this.#host.encodeInputs?.forget(output);
|
|
583
583
|
if (this.#host.outputs.outputsOn(output.outputKey).length === 0) {
|
|
584
584
|
this.#inputFailures.forget(output.outputKey);
|
|
585
|
+
this.#host.encodeOrchestrator.forgetOpens?.(output.outputKey);
|
|
585
586
|
this.#host.encodeOrchestrator.forgetOutput(output.outputKey);
|
|
586
587
|
}
|
|
587
588
|
}
|
|
@@ -793,40 +794,6 @@ export class EncodeRuns {
|
|
|
793
794
|
);
|
|
794
795
|
}
|
|
795
796
|
|
|
796
|
-
/**
|
|
797
|
-
* How far a run starting here may work before it meets somebody else's
|
|
798
|
-
* material — read off the ONE coverage map, never worked out again here.
|
|
799
|
-
*
|
|
800
|
-
* This replaced `planRunInterval`, which was a second authority over the same
|
|
801
|
-
* question and answered it by different rules: it walked the whole track for
|
|
802
|
-
* the first free number, MOVED the start there itself, and counted every live
|
|
803
|
-
* run as claiming up to `head + look-ahead`. Measured in the field
|
|
804
|
-
* 2026-09-05: the plan commanded a start at #46, this moved it to #78 — the
|
|
805
|
-
* "run moved forward from #46 to #78" line — and the plan, seeing a run
|
|
806
|
-
* outside the window it had asked for, killed it as unwanted and commanded
|
|
807
|
-
* the same start again, 350-700ms per cycle, dozens of times, the viewer's
|
|
808
|
-
* picture stopped for 125 seconds.
|
|
809
|
-
*
|
|
810
|
-
* Where a run STARTS is the plan's decision and arrives here as an argument.
|
|
811
|
-
* All this answers is where it must stop, and the answer is a fact of the
|
|
812
|
-
* map: the free stretch from that number on. `-1` means the end of the track,
|
|
813
|
-
* which is what "no end" is written as everywhere here.
|
|
814
|
-
*
|
|
815
|
-
* @param {object} session
|
|
816
|
-
* @param {number} startIndex
|
|
817
|
-
* @param {object | null} exceptRun - The run being replaced, whose own claim
|
|
818
|
-
* is not somebody else's material.
|
|
819
|
-
* @returns {number} The last number to work through, or `-1` for the end.
|
|
820
|
-
*/
|
|
821
|
-
#runEndFrom(session, startIndex, exceptRun = null) {
|
|
822
|
-
return this.#host.encodeOrchestrator.freeStretchEnd({
|
|
823
|
-
address: session.outputKey ?? "",
|
|
824
|
-
from: startIndex,
|
|
825
|
-
exceptRun,
|
|
826
|
-
segmentCount: Number(session.timeline?.segmentCount) || 0
|
|
827
|
-
});
|
|
828
|
-
}
|
|
829
|
-
|
|
830
797
|
// Returns the encoder it built, or nothing when there was nothing to build.
|
|
831
798
|
// It waits for nothing: the one thing it used to wait for was the death of
|
|
832
799
|
// the run it was replacing, and that killing is gone.
|
|
@@ -834,7 +801,10 @@ export class EncodeRuns {
|
|
|
834
801
|
if (!this.isLive(session)) {
|
|
835
802
|
return null;
|
|
836
803
|
}
|
|
837
|
-
|
|
804
|
+
// A copy held for a start that no live run ends before and that is not
|
|
805
|
+
// being started now goes back (torrent-tv/meta#184).
|
|
806
|
+
this.#host.encodeInputs?.retain(session, this.#host.encodeOrchestrator.wantedSegmentsOn(session.outputKey),
|
|
807
|
+
new Set([startIndex, ...this.liveRunsOf(session).map((run) => run.to + 1)]));
|
|
838
808
|
// THE LAST NUMBER THIS RUN MAY REACH is the plan's; how far its input
|
|
839
809
|
// really reaches is the input's answer. An original input is one copied
|
|
840
810
|
// stretch, and the run is given exactly that stretch (torrent-tv/meta#158).
|
|
@@ -847,13 +817,13 @@ export class EncodeRuns {
|
|
|
847
817
|
// The stretch is part of an original input's identity: two stretches from
|
|
848
818
|
// one start can hold the same padded bytes, and a stretch that failed is
|
|
849
819
|
// refused only while it is the same stretch.
|
|
850
|
-
const inputParameters =
|
|
820
|
+
const inputParameters = { outputKey: session.outputKey, encoder: this.#host.videoEncoder.name,
|
|
851
821
|
width: session.output.encodeWidth, height: session.output.encodeHeight, fps: session.output.outputFps,
|
|
852
822
|
preset: session.output.softwarePreset, tonemap: session.output.applyTonemap,
|
|
853
823
|
rateControl: session.spec.video?.encode?.rateControl ?? null,
|
|
854
|
-
...(admittedInput.original ? { interval: [startIndex, inputEnd] } : {}) }
|
|
855
|
-
const inputKey =
|
|
856
|
-
if (
|
|
824
|
+
...(admittedInput.original ? { interval: [startIndex, inputEnd] } : {}) };
|
|
825
|
+
const inputKey = this.#inputFailures.key(admittedInput, inputParameters);
|
|
826
|
+
if (this.#inputFailures.failure(session.outputKey, startIndex, inputKey)) {
|
|
857
827
|
admittedInput.release();
|
|
858
828
|
return null;
|
|
859
829
|
}
|
|
@@ -879,9 +849,10 @@ export class EncodeRuns {
|
|
|
879
849
|
// an argument. Reading it off the coverage map a second time was the last
|
|
880
850
|
// remaining second answer to that question: the plan computed `to`, passed
|
|
881
851
|
// it, and the parameter list did not name it.
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
852
|
+
// A run ends where its input does: an original input is one copied
|
|
853
|
+
// stretch, and a packet input grows one segment at a time from its first
|
|
854
|
+
// (`run.to` follows it below).
|
|
855
|
+
const runEnd = inputEnd;
|
|
885
856
|
// The restart backs off a segment or two from what was asked for, so the
|
|
886
857
|
// request that prompted it is recorded under a HIGHER index than the run
|
|
887
858
|
// starts at. Looking it up by the start index alone found nothing and the
|
|
@@ -1056,7 +1027,7 @@ export class EncodeRuns {
|
|
|
1056
1027
|
const since = (at) => at ? `${at - run.startedAt}ms` : "never";
|
|
1057
1028
|
this.#host.logger.info(`encode input of run #${run.from}..#${run.to} on ${session.outputKey}: ` +
|
|
1058
1029
|
`${reads.count} read(s), ${reads.bytes} bytes, first at ${since(reads.firstAt)}, last at ${since(reads.lastAt)}, ` +
|
|
1059
|
-
`ended at ${Date.now() - run.startedAt}ms` + describeTaken(reads.answers, run.startedAt, ended.
|
|
1030
|
+
`ended at ${Date.now() - run.startedAt}ms` + describeTaken(reads.answers, run.startedAt, ended.firstNamedWallMs));
|
|
1060
1031
|
}
|
|
1061
1032
|
this.#noteOpen(session, run, ended);
|
|
1062
1033
|
admittedInput.release();
|
|
@@ -1072,10 +1043,6 @@ export class EncodeRuns {
|
|
|
1072
1043
|
// Every number of the stretch answers to its one key, so a failure
|
|
1073
1044
|
// anywhere inside it is recorded against the stretch that was commanded.
|
|
1074
1045
|
run.admittedInputKeys = new Map(Array.from({ length: inputEnd - safeIndex + 1 }, (_, offset) => [safeIndex + offset, inputKey]));
|
|
1075
|
-
// What this run can make is what its input holds, and that is what it
|
|
1076
|
-
// claims; the plan's longer bound would read as made by a run that
|
|
1077
|
-
// cannot reach it.
|
|
1078
|
-
run.inputThrough = inputEnd;
|
|
1079
1046
|
run.process.stdin.end();
|
|
1080
1047
|
// THE NEXT STRETCH IS PREPARED WHILE THIS ONE IS BEING MADE. Asked for
|
|
1081
1048
|
// only when this run had ended, its copy fell between the two runs and
|
|
@@ -1159,7 +1126,7 @@ export class EncodeRuns {
|
|
|
1159
1126
|
const speed = run.speedX > 0 ? run.speedX : this.#host.encodeOrchestrator.speedOn?.(session.outputKey) ?? 0;
|
|
1160
1127
|
if (!Number.isFinite(ended.firstNamedMs) || !(pieceSeconds > 0) || !(speed > 0)) return;
|
|
1161
1128
|
const openSeconds = Math.max(0, ended.firstNamedMs / 1000 - pieceSeconds / speed);
|
|
1162
|
-
this.#host.
|
|
1129
|
+
this.#host.encodeOrchestrator.noteOpen?.(session.outputKey ?? "", openSeconds);
|
|
1163
1130
|
this.#host.logger.info(`encode input of run #${run.from}..#${run.to} on ${session.outputKey}: opened in ` +
|
|
1164
1131
|
`${openSeconds.toFixed(2)}s (first piece named at ${ended.firstNamedMs}ms, ${pieceSeconds.toFixed(3)}s of film at ${speed.toFixed(2)}x)`);
|
|
1165
1132
|
}
|
|
@@ -28,6 +28,15 @@ export class RunClock {
|
|
|
28
28
|
#idleCauses = 0;
|
|
29
29
|
#idleSince = 0;
|
|
30
30
|
#idleMs = 0;
|
|
31
|
+
/**
|
|
32
|
+
* Stopped time alone, apart from input waits: a pause is a decision of
|
|
33
|
+
* ours, while a wait for input is part of what starting a run costs, so the
|
|
34
|
+
* readings of a start leave out the first and keep the second
|
|
35
|
+
* (torrent-tv/meta#184).
|
|
36
|
+
*/
|
|
37
|
+
#stoppedSince = 0;
|
|
38
|
+
#stoppedMs = 0;
|
|
39
|
+
#isStopped = false;
|
|
31
40
|
|
|
32
41
|
/**
|
|
33
42
|
* @param {{ now?: () => number }} [options]
|
|
@@ -67,11 +76,29 @@ export class RunClock {
|
|
|
67
76
|
/** The process is stopped. */
|
|
68
77
|
stopped() {
|
|
69
78
|
this.#idleBegins();
|
|
79
|
+
if (!this.#isStopped) {
|
|
80
|
+
this.#isStopped = true;
|
|
81
|
+
this.#stoppedSince = this.#now();
|
|
82
|
+
}
|
|
70
83
|
}
|
|
71
84
|
|
|
72
85
|
/** The process runs again. */
|
|
73
86
|
continued() {
|
|
74
87
|
this.#idleEnds();
|
|
88
|
+
if (this.#isStopped) {
|
|
89
|
+
this.#isStopped = false;
|
|
90
|
+
this.#stoppedMs += this.#now() - this.#stoppedSince;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Milliseconds the process has been stopped so far.
|
|
96
|
+
*
|
|
97
|
+
* @param {number} [at]
|
|
98
|
+
* @returns {number}
|
|
99
|
+
*/
|
|
100
|
+
stoppedMs(at = this.#now()) {
|
|
101
|
+
return this.#stoppedMs + (this.#isStopped ? Math.max(0, at - this.#stoppedSince) : 0);
|
|
75
102
|
}
|
|
76
103
|
|
|
77
104
|
/**
|
|
@@ -100,8 +100,10 @@ export class RunCosts {
|
|
|
100
100
|
// Without it the figure that prices a move could only ever be learned from
|
|
101
101
|
// runs that survived, so the state in which moves are ruinous was exactly
|
|
102
102
|
// the state in which their cost stayed unknown.
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
// Its pauses are left out: a pause is our decision, not what a start costs.
|
|
104
|
+
const livedWorking = Number(ended?.livedMs) - Math.max(0, Number(ended?.stoppedMs) || 0);
|
|
105
|
+
if (!Number.isFinite(ended?.firstOutputMs) && Number.isFinite(livedWorking) && livedWorking > 0) {
|
|
106
|
+
RunCosts.#keep(this.#firstOutput, livedWorking);
|
|
105
107
|
}
|
|
106
108
|
if (Number.isFinite(ended?.dyingMs)) {
|
|
107
109
|
RunCosts.#keep(this.#dying, /** @type {number} */ (ended.dyingMs));
|
|
@@ -111,6 +113,49 @@ export class RunCosts {
|
|
|
111
113
|
}
|
|
112
114
|
}
|
|
113
115
|
|
|
116
|
+
/**
|
|
117
|
+
* How long an encoder took to open one output's original input, in seconds:
|
|
118
|
+
* from its spawn to the name of its first closed piece, less that piece's
|
|
119
|
+
* encoding and any pause (torrent-tv/meta#184). Kept per output, beside the
|
|
120
|
+
* host's time to first output above and apart from it: the two are related
|
|
121
|
+
* and not the same quantity, and the stretch of an original input is sized
|
|
122
|
+
* from this one.
|
|
123
|
+
*
|
|
124
|
+
* @type {Map<string, number[]>}
|
|
125
|
+
*/
|
|
126
|
+
#opens = new Map();
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* @param {string} address
|
|
130
|
+
* @param {number} seconds
|
|
131
|
+
*/
|
|
132
|
+
noteOpen(address, seconds) {
|
|
133
|
+
if (!address || !Number.isFinite(seconds) || seconds < 0) return;
|
|
134
|
+
const readings = this.#opens.get(address) ?? [];
|
|
135
|
+
RunCosts.#keep(readings, seconds);
|
|
136
|
+
this.#opens.set(address, readings);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The open of this output: its own readings' median where it has any; else
|
|
141
|
+
* the host's startup reading of spawn to first piece, said to be an
|
|
142
|
+
* approximation — it opened a generated picture, not this file; else null.
|
|
143
|
+
*
|
|
144
|
+
* @param {string} address
|
|
145
|
+
* @returns {{ seconds: number, measured: boolean } | null}
|
|
146
|
+
*/
|
|
147
|
+
openOf(address) {
|
|
148
|
+
const own = middleOf(this.#opens.get(address) ?? []);
|
|
149
|
+
if (own !== null) return { seconds: own, measured: true };
|
|
150
|
+
const startup = this.#atStartup?.firstByteWaitSec;
|
|
151
|
+
return Number.isFinite(startup) && startup > 0 ? { seconds: startup, measured: false } : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** @param {string} address */
|
|
155
|
+
forgetOpens(address) {
|
|
156
|
+
this.#opens.delete(address);
|
|
157
|
+
}
|
|
158
|
+
|
|
114
159
|
/**
|
|
115
160
|
* @param {number[]} readings
|
|
116
161
|
* @param {number} value
|
|
@@ -975,6 +975,9 @@ export function wireOutputs({
|
|
|
975
975
|
heldRanges: readEncodeHeldRanges,
|
|
976
976
|
reviseBudget: () => parts.machineBudget.revise(),
|
|
977
977
|
capacity: () => parts.machineBudget.capacityOf("memory"),
|
|
978
|
+
// How long this output's input takes to open, from the one owner of start
|
|
979
|
+
// readings (torrent-tv/meta#184).
|
|
980
|
+
openOf: (output) => parts.encodeOrchestrator?.openOf?.(output.outputKey ?? "") ?? null,
|
|
978
981
|
urgent: (output, index) => parts.encodeOrchestrator.wantedSegmentsOn(output.outputKey).some(window =>
|
|
979
982
|
index >= window.from && index <= window.to && window.urgent === true),
|
|
980
983
|
log: line => logger.info(line),
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* process figure alone is said.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import {
|
|
19
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
20
20
|
|
|
21
21
|
/** Clock ticks per second of `/proc/<pid>/stat`, as in `encode/host-load.js`. */
|
|
22
22
|
const CLOCK_TICKS_PER_SECOND = 100;
|
|
@@ -58,26 +58,46 @@ export function partOfThread(tid, name, pid) {
|
|
|
58
58
|
return "other";
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
/**
|
|
62
|
+
* One `/proc` stat file, or null when it cannot be read.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} file
|
|
65
|
+
* @returns {{ name: string, seconds: number } | null}
|
|
66
|
+
*/
|
|
67
|
+
function statOf(file) {
|
|
68
|
+
try {
|
|
69
|
+
return parseStat(readFileSync(file, "utf8"));
|
|
70
|
+
} catch {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
61
75
|
/**
|
|
62
76
|
* CPU seconds of this process's threads, by part, or null where `/proc` cannot
|
|
63
77
|
* be read.
|
|
64
78
|
*
|
|
79
|
+
* READ SYNCHRONOUSLY, with the clock read beside it: a reading is only a share
|
|
80
|
+
* when every counter in it and its time belong to one instant. Read as
|
|
81
|
+
* promises on the field host (2.97.0), the counters were taken while the main
|
|
82
|
+
* thread was busy and the time after, and a second read "process 1238 %" of a
|
|
83
|
+
* core (torrent-tv/meta#177). About a hundred small files a second, kernel
|
|
84
|
+
* memory, no disk.
|
|
85
|
+
*
|
|
65
86
|
* @param {number} [pid]
|
|
66
|
-
* @returns {
|
|
87
|
+
* @returns {{ main: number, worker: number, other: number } | null}
|
|
67
88
|
*/
|
|
68
|
-
export
|
|
89
|
+
export function readThreadSeconds(pid = process.pid) {
|
|
69
90
|
let tids;
|
|
70
91
|
try {
|
|
71
|
-
tids =
|
|
92
|
+
tids = readdirSync(`/proc/${pid}/task`);
|
|
72
93
|
} catch {
|
|
73
94
|
return null;
|
|
74
95
|
}
|
|
75
96
|
const parts = { main: 0, worker: 0, other: 0 };
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}));
|
|
97
|
+
for (const entry of tids) {
|
|
98
|
+
const stat = statOf(`/proc/${pid}/task/${entry}/stat`);
|
|
99
|
+
if (stat) parts[partOfThread(Number(entry), stat.name, pid)] += stat.seconds;
|
|
100
|
+
}
|
|
81
101
|
return parts;
|
|
82
102
|
}
|
|
83
103
|
|
|
@@ -85,14 +105,14 @@ export async function readThreadSeconds(pid = process.pid) {
|
|
|
85
105
|
* CPU seconds of each of these processes that can be read now.
|
|
86
106
|
*
|
|
87
107
|
* @param {number[]} pids
|
|
88
|
-
* @returns {
|
|
108
|
+
* @returns {Map<number, number>}
|
|
89
109
|
*/
|
|
90
|
-
export
|
|
110
|
+
export function readProcessesSeconds(pids) {
|
|
91
111
|
const seconds = new Map();
|
|
92
|
-
|
|
93
|
-
const stat =
|
|
112
|
+
for (const pid of pids) {
|
|
113
|
+
const stat = statOf(`/proc/${pid}/stat`);
|
|
94
114
|
if (stat) seconds.set(pid, stat.seconds);
|
|
95
|
-
}
|
|
115
|
+
}
|
|
96
116
|
return seconds;
|
|
97
117
|
}
|
|
98
118
|
|
|
@@ -158,29 +178,22 @@ export function startProcessorReport({ log, ffmpegPids, detailed = () => false,
|
|
|
158
178
|
let previous = null;
|
|
159
179
|
let series = [];
|
|
160
180
|
let since = Date.now();
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
series
|
|
173
|
-
|
|
174
|
-
if (now.at - since >= sayEveryMs) {
|
|
175
|
-
log(`processor: ${describeShares(series)}`);
|
|
176
|
-
series = [];
|
|
177
|
-
since = now.at;
|
|
178
|
-
}
|
|
179
|
-
} finally {
|
|
180
|
-
reading = false;
|
|
181
|
+
const tick = () => {
|
|
182
|
+
const at = Date.now();
|
|
183
|
+
const usage = process.cpuUsage();
|
|
184
|
+
const now = { at, process: (usage.user + usage.system) / 1e6, threads: readThreadSeconds(), ffmpeg: readProcessesSeconds(ffmpegPids()) };
|
|
185
|
+
const shares = previous ? sharesBetween(previous, now) : null;
|
|
186
|
+
previous = now;
|
|
187
|
+
if (!shares) return;
|
|
188
|
+
series.push(shares);
|
|
189
|
+
if (detailed()) log(`processor: ${describeShares([shares])}`);
|
|
190
|
+
if (now.at - since >= sayEveryMs) {
|
|
191
|
+
log(`processor: ${describeShares(series)}`);
|
|
192
|
+
series = [];
|
|
193
|
+
since = now.at;
|
|
181
194
|
}
|
|
182
195
|
};
|
|
183
|
-
const timer = setInterval(
|
|
196
|
+
const timer = setInterval(tick, everyMs);
|
|
184
197
|
if (typeof timer.unref === "function") timer.unref();
|
|
185
198
|
return { stop: () => clearInterval(timer) };
|
|
186
199
|
}
|