@torrent-tv/proxy 2.97.8 → 2.97.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/services/encode/EncodeInputs.js +36 -5
- package/services/encode/EncodeRun.js +23 -4
- package/services/encode/EncodeRuns.js +26 -3
- package/services/encode/input-deadline.js +52 -0
- package/services/encode/quality/HostLoad.js +40 -22
- package/services/server/wire-outputs.js +14 -0
- package/services/torrent/download/throttle.js +31 -0
- package/services/torrent/torrent-pool.js +3 -1
- package/services/torrent/worker/worker.js +12 -1
package/package.json
CHANGED
|
@@ -51,11 +51,13 @@ export class EncodeInputs {
|
|
|
51
51
|
#capacity;
|
|
52
52
|
#urgent;
|
|
53
53
|
#rankAt;
|
|
54
|
+
#deadlineOf;
|
|
55
|
+
#firstPieceIn;
|
|
54
56
|
#log;
|
|
55
57
|
#now;
|
|
56
58
|
|
|
57
59
|
constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
58
|
-
rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now }) {
|
|
60
|
+
rankAt = () => null, deadlineOf = () => null, firstPieceIn = () => null, openOf = () => null, log = () => {}, now = Date.now }) {
|
|
59
61
|
this.#openOf = openOf;
|
|
60
62
|
this.#resolve = resolve;
|
|
61
63
|
this.#read = readRanges;
|
|
@@ -66,6 +68,8 @@ export class EncodeInputs {
|
|
|
66
68
|
this.#capacity = capacity;
|
|
67
69
|
this.#urgent = urgent;
|
|
68
70
|
this.#rankAt = rankAt;
|
|
71
|
+
this.#deadlineOf = deadlineOf;
|
|
72
|
+
this.#firstPieceIn = firstPieceIn;
|
|
69
73
|
this.#log = log;
|
|
70
74
|
this.#now = now;
|
|
71
75
|
}
|
|
@@ -364,7 +368,7 @@ export class EncodeInputs {
|
|
|
364
368
|
*/
|
|
365
369
|
async #admitStretch(request, firstSources) {
|
|
366
370
|
const { output, from } = request;
|
|
367
|
-
|
|
371
|
+
let minimum = originalInputBytes(firstSources);
|
|
368
372
|
const stretches = new Map([[from, { sources: firstSources, bytes: minimum }]]);
|
|
369
373
|
const held = new Map();
|
|
370
374
|
const at = async end => {
|
|
@@ -383,8 +387,17 @@ export class EncodeInputs {
|
|
|
383
387
|
};
|
|
384
388
|
// Grows by doubling and then halves back: a handful of resolutions find
|
|
385
389
|
// the end whether it is a piece away or the rest of the film.
|
|
390
|
+
// THE LEAST THAT MAKES THE PIECE IT IS FOR. An input read from the keyframe
|
|
391
|
+
// before starts one piece early; holding that piece alone would not make
|
|
392
|
+
// the one the viewer wants (torrent-tv/meta#178).
|
|
393
|
+
const serves = Math.max(from, Math.min(request.serves ?? from, request.to));
|
|
394
|
+
let floor = from;
|
|
395
|
+
if (serves > from) {
|
|
396
|
+
const covering = await at(serves);
|
|
397
|
+
if (covering) { floor = serves; minimum = covering.bytes; }
|
|
398
|
+
}
|
|
386
399
|
const longest = async maxBytes => {
|
|
387
|
-
let good =
|
|
400
|
+
let good = floor, bad = null, step = 1;
|
|
388
401
|
while (bad === null) {
|
|
389
402
|
const end = good + step;
|
|
390
403
|
if (end > request.to) bad = request.to + 1;
|
|
@@ -398,9 +411,27 @@ export class EncodeInputs {
|
|
|
398
411
|
return good;
|
|
399
412
|
};
|
|
400
413
|
const sizing = this.stretchFor(output);
|
|
401
|
-
|
|
414
|
+
let target = sizing?.bytes ?? null;
|
|
402
415
|
if (sizing && !sizing.openMeasured) request.timing.openApproximate = true;
|
|
403
|
-
|
|
416
|
+
// NO LONGER THAN THE VIEWER CAN WAIT. A stretch is sized so its copy takes
|
|
417
|
+
// no longer than an open; for a piece the viewer reaches soon, the copy and
|
|
418
|
+
// the time to that piece's being served must both fit before they reach it.
|
|
419
|
+
// Field 2026-10-12: after a seek back, 222.4 MB were copied for 7.7 s ahead
|
|
420
|
+
// of a piece needed in about 8 s, and the picture stood 13.2 s. Unknown
|
|
421
|
+
// terms leave the stretch as it was.
|
|
422
|
+
const deadline = this.#deadlineOf(output, serves);
|
|
423
|
+
const firstPiece = this.#firstPieceIn(output, from, serves);
|
|
424
|
+
const copyRate = middleOf(this.#copyRates);
|
|
425
|
+
if (target !== null && Number.isFinite(deadline) && Number.isFinite(firstPiece) && copyRate !== null) {
|
|
426
|
+
const fitting = Math.floor(copyRate * Math.max(0, deadline - firstPiece) * 1000);
|
|
427
|
+
if (fitting < target) {
|
|
428
|
+
const capped = Math.max(minimum, fitting);
|
|
429
|
+
this.#log(`encode input #${from}..#${request.to} of ${output.outputKey ?? output.id}: copy held to ${capped} of ${target} bytes ` +
|
|
430
|
+
`— #${serves} is needed in ${deadline.toFixed(1)}s and its first piece takes ${firstPiece.toFixed(1)}s to be served`);
|
|
431
|
+
target = capped;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
let end = target === null || this.#heldRanges === null ? floor : await longest(target);
|
|
404
435
|
if (this.#requests.get(request.key) !== request) return undefined;
|
|
405
436
|
this.#wanted.set(request.key, { wanted: stretches.get(end).bytes, required: minimum });
|
|
406
437
|
await this.#spent(request.timing, "budgetMs", () => this.#revise());
|
|
@@ -232,6 +232,10 @@ export class EncodeRun {
|
|
|
232
232
|
* encoder itself names it on its own channel, and answers with the name that
|
|
233
233
|
* piece is served under — because making it servable is a rename, and only
|
|
234
234
|
* whoever owns the disk can perform one.
|
|
235
|
+
* @param {boolean} [params.publishesAlone] - Its pieces take nothing from the
|
|
236
|
+
* file closed after them (a re-encoded picture, or sound served on its own),
|
|
237
|
+
* so each is published on its own line once the completeness check proves it
|
|
238
|
+
* whole, instead of when the next file closes (torrent-tv/meta#178).
|
|
235
239
|
* @param {(name: string) => void} [params.onTail] - Called with the working
|
|
236
240
|
* name of a file this run closed but will not publish: past its last cut, or
|
|
237
241
|
* flushed when it was stopped. It begins with the next keyframe.
|
|
@@ -271,6 +275,7 @@ export class EncodeRun {
|
|
|
271
275
|
onEnded,
|
|
272
276
|
onSpeedMeasured,
|
|
273
277
|
onClosed,
|
|
278
|
+
publishesAlone = false,
|
|
274
279
|
onTail,
|
|
275
280
|
onProgress,
|
|
276
281
|
onProduced,
|
|
@@ -303,6 +308,7 @@ export class EncodeRun {
|
|
|
303
308
|
// proof it is whole; nothing else here can prove that.
|
|
304
309
|
this.onClosed = typeof onClosed === "function" ? onClosed : (name) => name;
|
|
305
310
|
this.onTail = typeof onTail === "function" ? onTail : () => {};
|
|
311
|
+
this.publishesAlone = publishesAlone === true;
|
|
306
312
|
this.lastSegmentIndex = typeof lastSegmentIndex === "function" ? lastSegmentIndex : () => null;
|
|
307
313
|
this.inputUnavailable = typeof inputUnavailable === "function" ? inputUnavailable : () => false;
|
|
308
314
|
this.argsDescribed = argsDescribed;
|
|
@@ -653,19 +659,29 @@ export class EncodeRun {
|
|
|
653
659
|
this.#pendingClosed = null;
|
|
654
660
|
continue;
|
|
655
661
|
}
|
|
662
|
+
// A PIECE THAT NEEDS NOTHING AFTER IT does not wait for the next file:
|
|
663
|
+
// the completeness check reads its own bytes and decides whether it
|
|
664
|
+
// reached its cut. Waiting for the next file cost the viewer that file's
|
|
665
|
+
// whole encode — 5.6 s after a seek on the field's AVI, where the picture
|
|
666
|
+
// stood 13.2 s (torrent-tv/meta#178). One it cannot prove yet waits as
|
|
667
|
+
// before; the file after it publishes it or the stop judges it.
|
|
668
|
+
if (this.publishesAlone && this.#publishClosed(name, null, { tentative: true })) {
|
|
669
|
+
this.#pendingClosed = null;
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
656
672
|
this.#pendingClosed = name;
|
|
657
673
|
}
|
|
658
674
|
}
|
|
659
675
|
|
|
660
|
-
#publishClosed(name, following = null, { quiet = false } = {}) {
|
|
661
|
-
this.#measureSpeed();
|
|
676
|
+
#publishClosed(name, following = null, { quiet = false, tentative = false } = {}) {
|
|
677
|
+
if (!tentative) this.#measureSpeed();
|
|
662
678
|
// ITS SERVED NAME, which is what whoever owns the disk gives it in answer.
|
|
663
679
|
// ffmpeg writes a piece under a working name and reports that; the piece
|
|
664
680
|
// becomes servable by being renamed, and everything below works in the
|
|
665
681
|
// name a request can actually ask for.
|
|
666
682
|
let served;
|
|
667
683
|
try {
|
|
668
|
-
served = this.onClosed(name, following, this.#closedTimings.get(name) ?? null);
|
|
684
|
+
served = this.onClosed(name, following, this.#closedTimings.get(name) ?? null, { tentative });
|
|
669
685
|
} catch (error) {
|
|
670
686
|
if (quiet) {
|
|
671
687
|
// A piece closed before our stop that cannot be proved whole is not
|
|
@@ -684,10 +700,13 @@ export class EncodeRun {
|
|
|
684
700
|
this.#signal("SIGTERM");
|
|
685
701
|
return false;
|
|
686
702
|
} finally {
|
|
687
|
-
|
|
703
|
+
// A piece not proved whole on its own keeps its timing for the file
|
|
704
|
+
// after it.
|
|
705
|
+
if (!tentative || served) this.#closedTimings.delete(name);
|
|
688
706
|
}
|
|
689
707
|
|
|
690
708
|
if (!served) return false;
|
|
709
|
+
if (tentative) this.#measureSpeed();
|
|
691
710
|
if (!this.#stopping) {
|
|
692
711
|
this.#provenName = served;
|
|
693
712
|
}
|
|
@@ -386,7 +386,7 @@ export class EncodeRuns {
|
|
|
386
386
|
* @param {string | null} [following] - The working name of the file the muxer closed after it.
|
|
387
387
|
* @returns {object | null | false}
|
|
388
388
|
*/
|
|
389
|
-
#wholeClosedPiece(session, name, index, admitted = false, bytes = null, run = null, timing = null, following = null) {
|
|
389
|
+
#wholeClosedPiece(session, name, index, admitted = false, bytes = null, run = null, timing = null, following = null, { tentative = false } = {}) {
|
|
390
390
|
|
|
391
391
|
const format = session.segmentFormat;
|
|
392
392
|
if (!format?.readMediaRanges || !Number.isInteger(index) || index < 0) {
|
|
@@ -427,6 +427,9 @@ export class EncodeRuns {
|
|
|
427
427
|
if (mediaRanges && run.cutShiftSeconds !== null) mediaRanges.cutShiftSeconds = run.cutShiftSeconds;
|
|
428
428
|
}
|
|
429
429
|
const { whole, throughSeconds, reason } = judgePiece(format, mediaRanges, cut, interval);
|
|
430
|
+
// A piece tried before the next file closed is judged again then; its
|
|
431
|
+
// refusal now is not a fault and is not said.
|
|
432
|
+
if (!whole && tentative) return false;
|
|
430
433
|
if (!whole) {
|
|
431
434
|
this.#host.logger.warn(
|
|
432
435
|
`encode: not publishing piece ${index} of ${address.slice(0, 60)}: ` +
|
|
@@ -460,6 +463,7 @@ export class EncodeRuns {
|
|
|
460
463
|
continue;
|
|
461
464
|
}
|
|
462
465
|
if (!judged.whole) {
|
|
466
|
+
if (tentative) return false;
|
|
463
467
|
this.#host.logger.warn(`encode: not publishing piece ${index} of ${address}: ${judged.reason}`);
|
|
464
468
|
return false;
|
|
465
469
|
}
|
|
@@ -617,6 +621,18 @@ export class EncodeRuns {
|
|
|
617
621
|
return byOutput;
|
|
618
622
|
}
|
|
619
623
|
|
|
624
|
+
/**
|
|
625
|
+
* Whether a piece of this output takes nothing from the file closed after it:
|
|
626
|
+
* no sound moved into a piece of the picture, and no open-GOP leading
|
|
627
|
+
* pictures to read from it — a re-encoded picture, or sound served as its own
|
|
628
|
+
* output. Such a piece is published on its own line (torrent-tv/meta#178).
|
|
629
|
+
*/
|
|
630
|
+
publishesAlone(session) {
|
|
631
|
+
return (session.spec.transcodesVideo || session.spec.carries === "audio-only") &&
|
|
632
|
+
!(session.segmentFormat?.id === "fmp4" && session.spec.audio && session.spec.carries !== "audio-only" &&
|
|
633
|
+
!this.#host.servesAudioSeparately(session));
|
|
634
|
+
}
|
|
635
|
+
|
|
620
636
|
/** Whether a run copies the picture from the original file it reads. */
|
|
621
637
|
#copiesOriginalPicture(session, admittedInput) {
|
|
622
638
|
return Boolean(admittedInput?.original) && !session.spec.transcodesVideo && session.spec.carries !== "audio-only";
|
|
@@ -1082,7 +1098,12 @@ export class EncodeRuns {
|
|
|
1082
1098
|
// Why this encoder exists, recorded with its argument list. It used to be
|
|
1083
1099
|
// handed to a separate `start` call; there is no separate call now.
|
|
1084
1100
|
because,
|
|
1085
|
-
|
|
1101
|
+
// ITS PIECES TAKE NOTHING FROM THE FILE CLOSED AFTER THEM: no sound moved
|
|
1102
|
+
// into a piece of the picture, and no open-GOP leading pictures to read
|
|
1103
|
+
// from it — a re-encoded picture, or sound served as its own output. Such
|
|
1104
|
+
// a piece is published on its own line (torrent-tv/meta#178).
|
|
1105
|
+
publishesAlone: this.publishesAlone(session),
|
|
1106
|
+
onClosed: (name, following, timing, { tentative = false } = {}) => {
|
|
1086
1107
|
|
|
1087
1108
|
const index = session.segmentFormat.segmentIndexFromName(
|
|
1088
1109
|
session.segmentFormat.servedNameOf?.(name) ?? name);
|
|
@@ -1110,7 +1131,9 @@ export class EncodeRuns {
|
|
|
1110
1131
|
throw error;
|
|
1111
1132
|
}
|
|
1112
1133
|
}
|
|
1113
|
-
const mediaRanges = this.#wholeClosedPiece(session, name, index, admittedInput, bytes, run, timing, following);
|
|
1134
|
+
const mediaRanges = this.#wholeClosedPiece(session, name, index, admittedInput, bytes, run, timing, following, { tentative });
|
|
1135
|
+
// Tried on its own line and not proved whole: it waits for the next file.
|
|
1136
|
+
if (mediaRanges === false && tentative) return null;
|
|
1114
1137
|
if (mediaRanges === false) throw Object.assign(new Error(`Closed piece #${index} has incomplete media.`),
|
|
1115
1138
|
{ code: "ERR_INCOMPLETE_MEDIA" });
|
|
1116
1139
|
const published = this.#host.segmentFiles.publish(address, name, session.segmentFormat, { mediaRanges, bytes });
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file When the viewer reaches a piece, and how long a run takes to serve it:
|
|
3
|
+
* the two terms that keep an encoder input's copy from making the viewer wait
|
|
4
|
+
* (torrent-tv/meta#178).
|
|
5
|
+
*
|
|
6
|
+
* Field 2026-10-12, an AVI on Home Assistant: after a seek back to 100 s the
|
|
7
|
+
* input of run `#27..#224` — 222.4 MB, copied in 7.7 s — was prepared ahead of
|
|
8
|
+
* a piece the viewer reached about 8 s later, and the picture stood 13.2 s. A
|
|
9
|
+
* stretch is sized so that its copy takes no longer than an open; for a piece
|
|
10
|
+
* the viewer reaches soon the copy has to leave room for the run to serve it.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Seconds until the viewer reaches piece `index`, from the output's map.
|
|
15
|
+
*
|
|
16
|
+
* A zone states `withinSeconds` for its near edge (`output/Timeline.js`); a
|
|
17
|
+
* piece inside it is that much film further on. The nearest zone holding the
|
|
18
|
+
* piece decides. Null where no zone holds it or the grid is unknown.
|
|
19
|
+
*
|
|
20
|
+
* @param {{ from: number, to: number, withinSeconds?: number }[]} zones
|
|
21
|
+
* @param {number[] | null | undefined} grid - Where each piece begins, in seconds of film.
|
|
22
|
+
* @param {number} index
|
|
23
|
+
* @returns {number | null}
|
|
24
|
+
*/
|
|
25
|
+
export function secondsUntilPiece(zones, grid, index) {
|
|
26
|
+
if (!Array.isArray(grid) || !Number.isFinite(grid[index])) return null;
|
|
27
|
+
let nearest = null;
|
|
28
|
+
for (const zone of zones) {
|
|
29
|
+
if (!(index >= zone.from && index <= zone.to) || !Number.isFinite(zone.withinSeconds) || !Number.isFinite(grid[zone.from])) continue;
|
|
30
|
+
const seconds = Math.max(0, zone.withinSeconds) + (grid[index] - grid[zone.from]);
|
|
31
|
+
if (nearest === null || seconds < nearest) nearest = seconds;
|
|
32
|
+
}
|
|
33
|
+
return nearest;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Seconds from a run's start at `from` to piece `serves` being served.
|
|
38
|
+
*
|
|
39
|
+
* The open (spawn and first input, without encoding its first piece, as
|
|
40
|
+
* `EncodeRuns` measures it), the film from `from` through `serves` at the
|
|
41
|
+
* output's speed, and — where a piece is published only when the file after it
|
|
42
|
+
* closes — that next piece too. Null when a term is unknown.
|
|
43
|
+
*
|
|
44
|
+
* @param {{ open?: number | null, speed?: number | null, grid?: number[] | null, from: number, serves: number, waitsForNext?: boolean }} params
|
|
45
|
+
* @returns {number | null}
|
|
46
|
+
*/
|
|
47
|
+
export function secondsToServe({ open, speed, grid, from, serves, waitsForNext = false }) {
|
|
48
|
+
if (!Number.isFinite(open) || open < 0 || !Number.isFinite(speed) || !(speed > 0) || !Array.isArray(grid)) return null;
|
|
49
|
+
const last = waitsForNext && Number.isFinite(grid[serves + 2]) ? serves + 2 : serves + 1;
|
|
50
|
+
const film = grid[last] - grid[from];
|
|
51
|
+
return Number.isFinite(film) && film > 0 ? open + film / speed : null;
|
|
52
|
+
}
|
|
@@ -71,13 +71,24 @@ export class HostLoad {
|
|
|
71
71
|
fileLengthByKey = new Map();
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
* The machine's seconds per megabyte downloaded,
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
74
|
+
* The machine's seconds per megabyte downloaded, as the processor share and
|
|
75
|
+
* the quality offer read it. Null: no established price, so the download is
|
|
76
|
+
* not limited and the offer counts no torrent cost.
|
|
77
|
+
*
|
|
78
|
+
* Not set from the torrent thread's reading below. On 2.97.8 that reading
|
|
79
|
+
* carried the thread's other work — 1287, 217 and 146 ms per MB at the AVI's
|
|
80
|
+
* start, median 263 against 12.5 on the MKV — and held the AVI download at
|
|
81
|
+
* its own film rate; until the price is established it is kept for the log
|
|
82
|
+
* only (torrent-tv/meta#178, control run).
|
|
78
83
|
*/
|
|
79
84
|
observedTorrentCostPerMegabyte = null;
|
|
80
85
|
|
|
86
|
+
/**
|
|
87
|
+
* The torrent thread's processor per downloaded megabyte, its base draw taken
|
|
88
|
+
* off: a reading for the log, used by nothing.
|
|
89
|
+
*/
|
|
90
|
+
torrentThreadCostPerMegabyte = null;
|
|
91
|
+
|
|
81
92
|
/** @type {number[]} Recent readings behind that median. */
|
|
82
93
|
#torrentCostReadings = [];
|
|
83
94
|
|
|
@@ -172,6 +183,8 @@ export class HostLoad {
|
|
|
172
183
|
takenAt: Date.now(),
|
|
173
184
|
cpuSeconds: this.#host.readProxyCpuSeconds(),
|
|
174
185
|
bytes: totals?.downloaded ?? null,
|
|
186
|
+
uploaded: totals?.uploaded ?? null,
|
|
187
|
+
served: totals?.served ?? null,
|
|
175
188
|
threadCpuSeconds: totals?.threadCpuSeconds ?? null,
|
|
176
189
|
metadata: this.#host.readMetadataActivity?.()
|
|
177
190
|
};
|
|
@@ -202,7 +215,7 @@ export class HostLoad {
|
|
|
202
215
|
* nothing, a price where it downloaded something, and what it moved against
|
|
203
216
|
* the limit in force, said while anything moves or is limited.
|
|
204
217
|
*
|
|
205
|
-
* @param {{ takenAt: number, bytes: number | null, threadCpuSeconds: number | null }} now
|
|
218
|
+
* @param {{ takenAt: number, bytes: number | null, uploaded: number | null, served: number | null, threadCpuSeconds: number | null }} now
|
|
206
219
|
*/
|
|
207
220
|
#learnThreadCost(now) {
|
|
208
221
|
const previous = this.#torrentSample;
|
|
@@ -216,9 +229,17 @@ export class HostLoad {
|
|
|
216
229
|
if (!(elapsedSec > 0) || megabytes < 0 || now.threadCpuSeconds < previous.threadCpuSeconds) return;
|
|
217
230
|
const cpuSeconds = (now.threadCpuSeconds - previous.threadCpuSeconds) / Math.max(1, os.cpus().length);
|
|
218
231
|
const limit = this.downloadLimit;
|
|
219
|
-
|
|
232
|
+
// What else the thread moved over the same interval, across every torrent:
|
|
233
|
+
// sent back to the swarm, and handed to readers (encoder inputs, container
|
|
234
|
+
// reads, `/stream`). Said beside the download so that the thread's
|
|
235
|
+
// processor can be set against each kind of work (torrent-tv/meta#178).
|
|
236
|
+
const movedOf = (key) => Number.isFinite(now[key]) && Number.isFinite(previous[key]) && now[key] >= previous[key]
|
|
237
|
+
? (now[key] - previous[key]) / 1e6 : null;
|
|
238
|
+
const rateOf = (key) => movedOf(key) === null ? "n/a" : `${(movedOf(key) / elapsedSec).toFixed(2)} MB/s`;
|
|
239
|
+
if (megabytes > 0 || Number.isFinite(limit) || movedOf("served") > 0 || movedOf("uploaded") > 0) {
|
|
220
240
|
logger.info(
|
|
221
|
-
`torrent: downloaded ${(megabytes / elapsedSec).toFixed(2)} MB/s
|
|
241
|
+
`torrent: downloaded ${(megabytes / elapsedSec).toFixed(2)} MB/s, sent to peers ${rateOf("uploaded")}, ` +
|
|
242
|
+
`handed to readers ${rateOf("served")} over ${elapsedSec.toFixed(1)}s, ` +
|
|
222
243
|
`limit ${Number.isFinite(limit) ? `${(limit / 1e6).toFixed(2)} MB/s` : "none"}; ` +
|
|
223
244
|
`its thread drew ${Math.round(cpuSeconds / elapsedSec * 100)}% of the machine`
|
|
224
245
|
);
|
|
@@ -242,14 +263,15 @@ export class HostLoad {
|
|
|
242
263
|
const readings = [...this.#torrentCostReadings, costPerMegabyte].slice(-READINGS_KEPT);
|
|
243
264
|
this.#torrentCostReadings = readings;
|
|
244
265
|
const median = medianOf(readings);
|
|
245
|
-
if (!movedBeyondScatter(this.
|
|
266
|
+
if (!movedBeyondScatter(this.torrentThreadCostPerMegabyte, median, readings)) {
|
|
246
267
|
return;
|
|
247
268
|
}
|
|
248
|
-
this.
|
|
269
|
+
this.torrentThreadCostPerMegabyte = median;
|
|
249
270
|
logger.info(
|
|
250
271
|
`host-load: the torrent thread costs ${(median * 1000).toFixed(1)}ms of the machine per downloaded MB ` +
|
|
251
272
|
`(median of ${readings.length}, latest ${(costPerMegabyte * 1000).toFixed(1)}ms over ${megabytes.toFixed(1)}MB, ` +
|
|
252
|
-
`its base draw ${((this.#observedThreadBaseDraw ?? 0) * 100).toFixed(1)}% of the machine taken off
|
|
273
|
+
`its base draw ${((this.#observedThreadBaseDraw ?? 0) * 100).toFixed(1)}% of the machine taken off; ` +
|
|
274
|
+
`for the log only, not the download limit or the quality offer)`
|
|
253
275
|
);
|
|
254
276
|
}
|
|
255
277
|
|
|
@@ -419,12 +441,13 @@ export class HostLoad {
|
|
|
419
441
|
}
|
|
420
442
|
|
|
421
443
|
/**
|
|
422
|
-
* The bytes every torrent has downloaded and the
|
|
423
|
-
*
|
|
444
|
+
* The bytes every torrent has downloaded and sent to peers, the bytes the
|
|
445
|
+
* torrent thread handed to readers, and that thread's processor seconds, read
|
|
446
|
+
* together by that thread, or null when it cannot be asked.
|
|
424
447
|
* Downloaded bytes only: every one is verified against the piece hash and
|
|
425
448
|
* written to the store; a byte sent back to the swarm is neither.
|
|
426
449
|
*
|
|
427
|
-
* @returns {Promise<{ downloaded: number | null, threadCpuSeconds: number | null } | null>}
|
|
450
|
+
* @returns {Promise<{ downloaded: number | null, uploaded: number | null, served: number | null, threadCpuSeconds: number | null } | null>}
|
|
428
451
|
*/
|
|
429
452
|
async #torrentTotals() {
|
|
430
453
|
if (typeof this.#host.getTorrentTotals !== "function") {
|
|
@@ -432,12 +455,8 @@ export class HostLoad {
|
|
|
432
455
|
}
|
|
433
456
|
try {
|
|
434
457
|
const totals = await this.#host.getTorrentTotals();
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
return {
|
|
438
|
-
downloaded: Number.isFinite(downloaded) ? downloaded : null,
|
|
439
|
-
threadCpuSeconds: Number.isFinite(threadCpuSeconds) ? threadCpuSeconds : null
|
|
440
|
-
};
|
|
458
|
+
const read = (key) => { const value = Number(totals?.[key]); return Number.isFinite(value) ? value : null; };
|
|
459
|
+
return { downloaded: read("downloaded"), uploaded: read("uploaded"), served: read("served"), threadCpuSeconds: read("threadCpuSeconds") };
|
|
441
460
|
} catch {
|
|
442
461
|
return null; // the pool is busy or gone; a reading missed is not a fault
|
|
443
462
|
}
|
|
@@ -650,9 +669,8 @@ export class HostLoad {
|
|
|
650
669
|
*
|
|
651
670
|
* A viewer consumes the file at its own byte rate, and every one of those
|
|
652
671
|
* bytes is downloaded, verified and pushed by this process. Priced per
|
|
653
|
-
* megabyte by
|
|
654
|
-
*
|
|
655
|
-
* work on other threads is not in it. Zero while either term is unmeasured
|
|
672
|
+
* megabyte by `observedTorrentCostPerMegabyte`, which is unset while no price
|
|
673
|
+
* is established. Zero while either term is unmeasured
|
|
656
674
|
* — a guess here would refuse rungs on arithmetic nobody performed.
|
|
657
675
|
*
|
|
658
676
|
* @param {HlsSession} session
|
|
@@ -98,6 +98,7 @@ import { contentOf, LocalObservations, PROXY_ROOT } from "../encode/LocalObserva
|
|
|
98
98
|
import { configurationKeyOf } from "../encode/fingerprint.js";
|
|
99
99
|
import { qualityStateOf } from "../encode/quality/OutputQualityState.js";
|
|
100
100
|
import { downloadShare } from "../storage/processor-share.js";
|
|
101
|
+
import { secondsToServe, secondsUntilPiece } from "../encode/input-deadline.js";
|
|
101
102
|
import { ENCODE_RUN_STATE, processCanBeSignalled } from "../encode/encode-run-state.js";
|
|
102
103
|
|
|
103
104
|
// Where a variant and an audio rendition live under a session — `v/<height>/…`
|
|
@@ -1013,6 +1014,19 @@ export function wireOutputs({
|
|
|
1013
1014
|
index >= window.from && index <= window.to && window.urgent === true),
|
|
1014
1015
|
// The piece's rank in the output's own map, said with an input's state.
|
|
1015
1016
|
rankAt: (output, index) => parts.encodeOrchestrator?.rankAt?.(output.outputKey ?? "", index) ?? null,
|
|
1017
|
+
// WHEN THE VIEWER REACHES A PIECE: the output map's zones state it for their
|
|
1018
|
+
// near edge, and a piece inside a zone is that much film further on.
|
|
1019
|
+
deadlineOf: (output, index) => secondsUntilPiece(parts.encodeOrchestrator?.wantedSegmentsOn(output.outputKey ?? "") ?? [],
|
|
1020
|
+
output.timeline?.published ?? output.timeline?.boundaries, index),
|
|
1021
|
+
// HOW LONG A RUN STARTED AT `from` TAKES TO SERVE `serves`: its open, the
|
|
1022
|
+
// pieces it encodes up to and including `serves`, and the next one where a
|
|
1023
|
+
// piece waits for the file after it.
|
|
1024
|
+
firstPieceIn: (output, from, serves) => secondsToServe({
|
|
1025
|
+
open: parts.encodeOrchestrator?.openOf?.(output.outputKey ?? "")?.seconds,
|
|
1026
|
+
speed: parts.encodeOrchestrator?.speedOn?.(output.outputKey ?? ""),
|
|
1027
|
+
grid: output.timeline?.published ?? output.timeline?.boundaries,
|
|
1028
|
+
from, serves, waitsForNext: !parts.encodeRuns.publishesAlone(output)
|
|
1029
|
+
}),
|
|
1016
1030
|
log: line => logger.info(line),
|
|
1017
1031
|
changed: (output, result) => {
|
|
1018
1032
|
if (result.kind === "terminal") {
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Setting WebTorrent's download limit so that lowering it cannot fail a
|
|
3
|
+
* peer's data (torrent-tv/meta#178).
|
|
4
|
+
*
|
|
5
|
+
* WebTorrent's limit is a `speed-limiter` ThrottleGroup over `limiter`'s
|
|
6
|
+
* TokenBucket. The group cuts what a peer sends into slices of `rate / 10`
|
|
7
|
+
* at the moment a slice is cut, and a slice that waits for tokens asks for
|
|
8
|
+
* its full length again when it retries. `setRate` makes the bucket as large
|
|
9
|
+
* as the new rate, and the bucket refuses any request larger than itself
|
|
10
|
+
* ("exceeds bucket size"), which ends that peer's stream. So a slice cut at
|
|
11
|
+
* 47 MB/s (4.7 MB) that is still waiting when the limit falls to 0.6 MB/s
|
|
12
|
+
* fails — checked in `test/no-torrent/download-throttle.test.js`.
|
|
13
|
+
*
|
|
14
|
+
* The slice is kept at one BitTorrent block, 16 KiB, the length a peer's
|
|
15
|
+
* request asks for, whatever the rate: no slice can then outgrow a bucket of
|
|
16
|
+
* at least one block. Below one block the slice is the limit itself, as the
|
|
17
|
+
* group requires; the processor share never sets a limit that low, since it
|
|
18
|
+
* never falls below what the watched films play.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** The length of one request in the BitTorrent wire protocol (BEP 3). */
|
|
22
|
+
const BLOCK_BYTES = 16 * 1024;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {{ throttleDownload: (rate: number) => unknown, throttleGroups?: { down?: { setChunksize: (bytes: number) => void } } }} client
|
|
26
|
+
* @param {number} rate - Bytes per second; -1 lifts the limit.
|
|
27
|
+
*/
|
|
28
|
+
export function applyDownloadRate(client, rate) {
|
|
29
|
+
client.throttleDownload(rate);
|
|
30
|
+
if (rate >= 1) client.throttleGroups?.down?.setChunksize(Math.min(BLOCK_BYTES, Math.floor(rate)));
|
|
31
|
+
}
|
|
@@ -19,6 +19,7 @@ import { pieceStoreOf } from "./piece-store-of.js";
|
|
|
19
19
|
import { demandProgress } from "./demand-progress.js";
|
|
20
20
|
import { Urgency, urgencyName } from "./demand/index.js";
|
|
21
21
|
import { demandFor, firstClassStatement, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
|
|
22
|
+
import { applyDownloadRate } from "./download/throttle.js";
|
|
22
23
|
import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
23
24
|
import { deadlineOfMapZone, levelOfMapZone, priorityOfMapZone } from "./map-zones.js";
|
|
24
25
|
import { deriveSourceKey } from "../../utils/torrent-source-key.js";
|
|
@@ -1273,7 +1274,8 @@ export class TorrentPool {
|
|
|
1273
1274
|
if (!this.client || this.client.destroyed || typeof this.client.throttleDownload !== "function") return;
|
|
1274
1275
|
const rate = Number.isFinite(bytesPerSecond) ? Math.max(1, Math.round(bytesPerSecond)) : -1;
|
|
1275
1276
|
if (rate === this.#downloadLimit) return;
|
|
1276
|
-
|
|
1277
|
+
// One block per slice, so a lower limit cannot fail a slice already waiting (download/throttle.js).
|
|
1278
|
+
applyDownloadRate(this.client, rate);
|
|
1277
1279
|
// Said when the limit is set or lifted, and when it moves by a factor of
|
|
1278
1280
|
// two either way — the scale on which a limit changes what the swarm gives.
|
|
1279
1281
|
const scale = rate < 0 ? null : Math.round(Math.log2(rate));
|
|
@@ -100,6 +100,14 @@ const readsById = new Map();
|
|
|
100
100
|
*/
|
|
101
101
|
const openReads = new Map();
|
|
102
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Bytes this thread has handed to readers — encoder inputs, container reads and
|
|
105
|
+
* `/stream` — across every torrent. Said with the torrent totals so that the
|
|
106
|
+
* thread's processor can be set against each kind of work it does, not only
|
|
107
|
+
* against the bytes it downloads (torrent-tv/meta#178).
|
|
108
|
+
*/
|
|
109
|
+
let servedBytes = 0;
|
|
110
|
+
|
|
103
111
|
/**
|
|
104
112
|
* Shorthand for this file. The same path as `logger.info` anywhere else in the
|
|
105
113
|
* thread — kept only because it reads better at the hundred call sites here.
|
|
@@ -261,6 +269,7 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
|
261
269
|
fragment.release();
|
|
262
270
|
break;
|
|
263
271
|
}
|
|
272
|
+
servedBytes += fragment.length;
|
|
264
273
|
// One fragment in flight at a time. Each one holds a piece pinned, and
|
|
265
274
|
// the store guarantees only two resident pieces at its smallest budget —
|
|
266
275
|
// holding two pins while asking for a third would deadlock it against
|
|
@@ -423,7 +432,7 @@ async function runCommand(command, params, id) {
|
|
|
423
432
|
// bytes, so the price of a downloaded byte is taken over one interval
|
|
424
433
|
// whether or not anything encodes (torrent-tv/meta#178).
|
|
425
434
|
const usage = process.threadCpuUsage();
|
|
426
|
-
return { downloaded, uploaded, threadCpuSeconds: (usage.user + usage.system) / 1e6 };
|
|
435
|
+
return { downloaded, uploaded, served: servedBytes, threadCpuSeconds: (usage.user + usage.system) / 1e6 };
|
|
427
436
|
}
|
|
428
437
|
|
|
429
438
|
|
|
@@ -521,6 +530,7 @@ async function runCommand(command, params, id) {
|
|
|
521
530
|
const torrent = isUsableTorrentHandle(known) ? known : null;
|
|
522
531
|
const bytes = await readWhileInUse(openReads, id, torrent, params.fileIndex,
|
|
523
532
|
() => readHeldBytes(torrent, params.fileIndex, params.start, params.end, logger));
|
|
533
|
+
servedBytes += bytes?.length ?? 0;
|
|
524
534
|
return { bytes };
|
|
525
535
|
}
|
|
526
536
|
|
|
@@ -529,6 +539,7 @@ async function runCommand(command, params, id) {
|
|
|
529
539
|
const torrent = isUsableTorrentHandle(known) ? known : null;
|
|
530
540
|
const buffers = await readWhileInUse(openReads, id, torrent, params.fileIndex,
|
|
531
541
|
() => readHeldRanges(torrent, params.fileIndex, params.ranges, params.maxBytes, logger));
|
|
542
|
+
for (const buffer of buffers ?? []) servedBytes += buffer?.length ?? 0;
|
|
532
543
|
return { buffers };
|
|
533
544
|
}
|
|
534
545
|
|