@torrent-tv/proxy 2.80.12 → 2.80.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/docs/download-architecture.md +65 -0
- package/docs/encode-architecture.md +52 -0
- package/package.json +1 -1
- package/routes/api/transcode-sessions/net-report/post.js +64 -53
- package/routes/stream/get.js +324 -300
- package/routes/transcode/session-file/get.js +22 -2
- package/services/download/SwarmSelection.js +20 -4
- package/services/encode/SegmentDemand.js +34 -0
- package/services/encode/SegmentStore.js +743 -718
- package/services/hls-session-manager.js +22 -29
- package/services/orchestrators/EncodeOrchestrator.js +735 -726
- package/services/output/LiveOutputs.js +36 -0
- package/services/output/playlists.js +86 -11
- package/services/output/rates.js +95 -0
- package/services/piece-store/piece-lru.js +9 -1
- package/services/priority/PriorityOrchestrator.js +284 -278
- package/services/priority/WaitLedger.js +142 -0
- package/services/supply-margin.js +40 -1
- package/services/torrent-pool.js +38 -1
- package/services/torrent-worker/client.js +27 -12
- package/services/torrent-worker/piece-reader.js +9 -4
- package/services/torrent-worker/worker.js +5 -0
- package/services/viewer/Viewer.js +345 -297
- package/test/declared-rates.test.js +116 -0
- package/test/piece-lru.test.js +5 -1
- package/test/supply-margin.test.js +25 -8
- package/test/wait-ledger.test.js +100 -0
|
@@ -16,6 +16,8 @@
|
|
|
16
16
|
|
|
17
17
|
import { variantHeightsFor } from "./ladder.js";
|
|
18
18
|
|
|
19
|
+
import { masterRateArgs } from "./rates.js";
|
|
20
|
+
|
|
19
21
|
export class LiveOutputs {
|
|
20
22
|
/**
|
|
21
23
|
* @param {object} params
|
|
@@ -264,4 +266,38 @@ export class LiveOutputs {
|
|
|
264
266
|
}
|
|
265
267
|
return this.splicableHeights(owner).length >= 2;
|
|
266
268
|
}
|
|
269
|
+
/**
|
|
270
|
+
* Everything the master playlist needs of a session except its soundtracks.
|
|
271
|
+
*
|
|
272
|
+
* The shape of the film — which heights can be spliced, how big the source
|
|
273
|
+
* is, how the pieces are packaged — and the rates it carries. All of it is a
|
|
274
|
+
* fact about the OUTPUT, and it was assembled at the call site in the session
|
|
275
|
+
* manager, which is the file that is being taken apart.
|
|
276
|
+
*
|
|
277
|
+
* The soundtracks are not here on purpose: which of them is marked default is
|
|
278
|
+
* the asking VIEWER'S business, and that belongs to whoever holds viewers.
|
|
279
|
+
*
|
|
280
|
+
* @param {object} session
|
|
281
|
+
* @param {(address: string) => { index: number, size: number }} largestPiece -
|
|
282
|
+
* The biggest piece an output has made, asked of whoever owns the disk. A
|
|
283
|
+
* plain function, so this layer holds no store.
|
|
284
|
+
* @returns {object}
|
|
285
|
+
*/
|
|
286
|
+
masterFactsOf(session, largestPiece) {
|
|
287
|
+
return {
|
|
288
|
+
playlistVersion: session.segmentFormat.playlistVersion,
|
|
289
|
+
heights: this.splicableHeights(session),
|
|
290
|
+
sourceWidth: Number(session.file?.width) || 0,
|
|
291
|
+
sourceHeight: Number(session.file?.height) || 0,
|
|
292
|
+
...masterRateArgs({
|
|
293
|
+
fileLength: Number(session.file?.length) || 0,
|
|
294
|
+
durationSeconds: Number(session.file?.durationSeconds) || 0,
|
|
295
|
+
largest: largestPiece(session.outputKey ?? ""),
|
|
296
|
+
boundaries: session.timeline?.published ?? session.timeline?.boundaries ?? null,
|
|
297
|
+
producedHeight: this.producedHeightOf(session),
|
|
298
|
+
capKbps: Number(session.rateCapKbps) || 0
|
|
299
|
+
})
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
267
303
|
}
|
|
@@ -58,19 +58,75 @@ export function escapeAttribute(value) {
|
|
|
58
58
|
return String(value ?? "").replace(/"/g, "'").replace(/[\u0000-\u001f\u007f]/g, " ").trim();
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
// The smallest rate the HLS specification tolerates in a `BANDWIDTH` attribute.
|
|
62
|
+
// It is a floor on what may be DECLARED, not a belief about any content: a file
|
|
63
|
+
// whose rate is not yet known is described by it, and so is a variant so small
|
|
64
|
+
// that the arithmetic below would go under it.
|
|
65
|
+
const MINIMUM_DECLARED_BITS_PER_SECOND = 400_000;
|
|
66
|
+
|
|
61
67
|
/**
|
|
62
|
-
*
|
|
68
|
+
* How many bits of film there are per second of playback, for one height.
|
|
69
|
+
*
|
|
70
|
+
* MEASURED, and the measurement is available before anything is encoded. The
|
|
71
|
+
* comment here used to say the opposite — "a measurement we do not have before
|
|
72
|
+
* encoding starts" — and gave `height * height * 3.2` instead, which for 1080
|
|
73
|
+
* is 3 732 480. Field 2026-09-08: that was declared for a file carrying
|
|
74
|
+
* 18.4 Mbit/s, five times more, and the arithmetic that reads it is not
|
|
75
|
+
* cosmetic.
|
|
76
|
+
*
|
|
77
|
+
* **What reads it.** The browser sizes its cushion in BYTES from this figure
|
|
78
|
+
* times the seconds it is asked to hold, so a figure five times low makes the
|
|
79
|
+
* cushion five times shallow: 120 s asked bought 56 MB, which is 26 s of that
|
|
80
|
+
* film, and the deepest the browser ever held was 17.1 s. And hls.js compares
|
|
81
|
+
* it against its own estimate of the link to decide a level is unplayable —
|
|
82
|
+
* which is why an inflated figure is not the answer either: its own recovery
|
|
83
|
+
* then moves level, and that path does not honour our pinning (measured, 2.59.3).
|
|
84
|
+
*
|
|
85
|
+
* **Where the figure comes from.** Two cases, both exact:
|
|
86
|
+
*
|
|
87
|
+
* - a height that is COPIED carries the source's own bits, so it is the file's
|
|
88
|
+
* length over its duration;
|
|
89
|
+
* - a height that is RE-ENCODED carries what the encoder is capped at, which we
|
|
90
|
+
* impose ourselves.
|
|
63
91
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
92
|
+
* The specification wants the PEAK per segment in `BANDWIDTH` and the average
|
|
93
|
+
* in `AVERAGE-BANDWIDTH`; both are emitted, and the peak is scaled from the
|
|
94
|
+
* average by the ratio the largest produced segment has actually shown — never
|
|
95
|
+
* a chosen multiplier, and equal to the average until a segment exists.
|
|
68
96
|
*
|
|
69
|
-
* @param {
|
|
97
|
+
* @param {object} params
|
|
98
|
+
* @param {number} params.averageBitsPerSecond - The film's own rate, measured.
|
|
99
|
+
* @param {number} params.height
|
|
100
|
+
* @param {number} params.sourceHeight
|
|
101
|
+
* @param {number} [params.capKbps] - What a re-encoded height is capped at.
|
|
70
102
|
* @returns {number}
|
|
71
103
|
*/
|
|
72
|
-
export function
|
|
73
|
-
|
|
104
|
+
export function bitrateFor({ averageBitsPerSecond, height, sourceHeight, capKbps = 0 }) {
|
|
105
|
+
// ONE EXPRESSION, and each term is a measured quantity or the absence of one
|
|
106
|
+
// written as the identity of its operation. There is no case analysis here
|
|
107
|
+
// because there are no cases: the rate a variant carries is what the source
|
|
108
|
+
// carries, shrunk by how much less picture there is, and never more than what
|
|
109
|
+
// we cap the encoder at.
|
|
110
|
+
//
|
|
111
|
+
// - what the source carries: `length * 8 / duration`, exact. Unknown is 0,
|
|
112
|
+
// and 0 falls to the floor below, which is what "not measured" means;
|
|
113
|
+
// - how much less picture: the ratio of pixel counts, and never above 1 —
|
|
114
|
+
// a variant at or above the source's height carries the source's bits.
|
|
115
|
+
// The pixel count is the one term of the relation that is a fact rather
|
|
116
|
+
// than an opinion about the encoder, and it errs HIGH for a small height,
|
|
117
|
+
// which is the safe direction: the cushion is sized generously and the
|
|
118
|
+
// player does not conclude the level is beyond its link;
|
|
119
|
+
// - what we cap it at: exact where we impose one, and `Infinity` where we
|
|
120
|
+
// do not, which is the identity of `min` and so states "no cap" without a
|
|
121
|
+
// branch;
|
|
122
|
+
// - the floor: the smallest figure the specification tolerates, and the
|
|
123
|
+
// identity of `max`.
|
|
124
|
+
const measured = Number(averageBitsPerSecond) > 0 ? Number(averageBitsPerSecond) : 0;
|
|
125
|
+
const shrink = height > 0 && sourceHeight > 0
|
|
126
|
+
? Math.min(1, (height * height) / (sourceHeight * sourceHeight))
|
|
127
|
+
: 1;
|
|
128
|
+
const cap = capKbps > 0 ? capKbps * 1000 : Number.POSITIVE_INFINITY;
|
|
129
|
+
return Math.round(Math.max(MINIMUM_DECLARED_BITS_PER_SECOND, Math.min(measured * shrink, cap)));
|
|
74
130
|
}
|
|
75
131
|
|
|
76
132
|
/**
|
|
@@ -134,7 +190,10 @@ export function mediaPlaylistText({ boundaries, segmentFormat }) {
|
|
|
134
190
|
* sourceWidth: number,
|
|
135
191
|
* sourceHeight: number,
|
|
136
192
|
* renditions?: Array<{ trackIndex: number, name: string, language: string, isDefault: boolean }>,
|
|
137
|
-
* playlistFileName: string
|
|
193
|
+
* playlistFileName: string,
|
|
194
|
+
* averageBitsPerSecond?: number,
|
|
195
|
+
* peakOverAverage?: number,
|
|
196
|
+
* capKbpsFor?: (height: number) => number
|
|
138
197
|
* }} params
|
|
139
198
|
* @returns {string}
|
|
140
199
|
*/
|
|
@@ -144,7 +203,10 @@ export function masterPlaylistText({
|
|
|
144
203
|
sourceWidth,
|
|
145
204
|
sourceHeight,
|
|
146
205
|
renditions = [],
|
|
147
|
-
playlistFileName
|
|
206
|
+
playlistFileName,
|
|
207
|
+
averageBitsPerSecond = 0,
|
|
208
|
+
peakOverAverage = 1,
|
|
209
|
+
capKbpsFor = () => 0
|
|
148
210
|
}) {
|
|
149
211
|
const lines = ["#EXTM3U", `#EXT-X-VERSION:${playlistVersion}`];
|
|
150
212
|
const audioGroup = renditions.length > 0 ? AUDIO_GROUP_ID : "";
|
|
@@ -160,8 +222,21 @@ export function masterPlaylistText({
|
|
|
160
222
|
const width = sourceHeight > 0 && sourceWidth > 0
|
|
161
223
|
? Math.round((sourceWidth / sourceHeight) * height / 2) * 2
|
|
162
224
|
: 0;
|
|
225
|
+
const average = bitrateFor({
|
|
226
|
+
averageBitsPerSecond,
|
|
227
|
+
height,
|
|
228
|
+
sourceHeight,
|
|
229
|
+
capKbps: capKbpsFor(height)
|
|
230
|
+
});
|
|
231
|
+
// BOTH, because they answer different questions and the specification has a
|
|
232
|
+
// name for each: the peak is what a link must carry at the worst moment,
|
|
233
|
+
// the average is what the whole variant costs. hls.js sizes its byte budget
|
|
234
|
+
// from BANDWIDTH, so the peak is what stops the cushion being sized for a
|
|
235
|
+
// quiet stretch and running dry on a loud one — measured on the field file,
|
|
236
|
+
// 17.1 Mbit/s median against 73 Mbit/s at its peak.
|
|
237
|
+
const peak = Math.round(average * Math.max(1, peakOverAverage));
|
|
163
238
|
lines.push(
|
|
164
|
-
`#EXT-X-STREAM-INF:BANDWIDTH=${
|
|
239
|
+
`#EXT-X-STREAM-INF:BANDWIDTH=${peak},AVERAGE-BANDWIDTH=${average}` +
|
|
165
240
|
(width > 0 ? `,RESOLUTION=${width}x${height}` : "") +
|
|
166
241
|
(audioGroup ? `,AUDIO="${audioGroup}"` : "")
|
|
167
242
|
);
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file How many bits of film there are per second of playback.
|
|
3
|
+
*
|
|
4
|
+
* A fact about the OUTPUT, and the two figures the HLS specification asks for:
|
|
5
|
+
* the average over the whole variant, and the peak one segment reaches. Both
|
|
6
|
+
* are measured — the average from the file's own length and duration, the peak
|
|
7
|
+
* from the biggest piece actually produced and the span it covers.
|
|
8
|
+
*
|
|
9
|
+
* It used to be `height * height * 3.2`, a rule of thumb, justified by a comment
|
|
10
|
+
* saying the measurement was not available before encoding starts. It is: the
|
|
11
|
+
* length and the duration are known when the session is created, and the peak
|
|
12
|
+
* refines itself as pieces appear. Field 2026-09-08: 3.73 Mbit/s declared for a
|
|
13
|
+
* file carrying 18.4, and the browser's cushion is sized in BYTES from that
|
|
14
|
+
* figure — 120 s asked bought 26 s of film.
|
|
15
|
+
*
|
|
16
|
+
* Pure: plain numbers in, plain numbers out. Nothing here knows what a session,
|
|
17
|
+
* a store or a torrent is.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The two rates to declare, from what has been measured of this file.
|
|
22
|
+
*
|
|
23
|
+
* @param {object} params
|
|
24
|
+
* @param {number} params.fileLength - Bytes of the source file.
|
|
25
|
+
* @param {number} params.durationSeconds
|
|
26
|
+
* @param {{ index: number, size: number }} [params.largest] - The biggest piece
|
|
27
|
+
* produced so far, with its number. An index of `-1` means none yet.
|
|
28
|
+
* @param {number[]} [params.boundaries] - Where the file is cut, so the biggest
|
|
29
|
+
* piece's own span is known. Bytes alone cannot give a rate.
|
|
30
|
+
* @returns {{ averageBitsPerSecond: number, peakOverAverage: number }}
|
|
31
|
+
*/
|
|
32
|
+
export function declaredRates({ fileLength, durationSeconds, largest = null, boundaries = null }) {
|
|
33
|
+
const length = Number(fileLength);
|
|
34
|
+
const duration = Number(durationSeconds);
|
|
35
|
+
if (!(length > 0) || !(duration > 0)) {
|
|
36
|
+
return { averageBitsPerSecond: 0, peakOverAverage: 1 };
|
|
37
|
+
}
|
|
38
|
+
const averageBitsPerSecond = (length * 8) / duration;
|
|
39
|
+
const index = Number(largest?.index);
|
|
40
|
+
const size = Number(largest?.size);
|
|
41
|
+
if (!Number.isInteger(index) || index < 0 || !(size > 0) || !Array.isArray(boundaries)) {
|
|
42
|
+
// Nothing has been produced, so the peak is not yet a measured quantity and
|
|
43
|
+
// is declared equal to the average. It rises as soon as one piece exists,
|
|
44
|
+
// and a ratio invented meanwhile would be exactly the fabrication this file
|
|
45
|
+
// replaced.
|
|
46
|
+
return { averageBitsPerSecond, peakOverAverage: 1 };
|
|
47
|
+
}
|
|
48
|
+
const span = Number(boundaries[index + 1]) - Number(boundaries[index]);
|
|
49
|
+
if (!(span > 0)) {
|
|
50
|
+
return { averageBitsPerSecond, peakOverAverage: 1 };
|
|
51
|
+
}
|
|
52
|
+
// Never below one: the peak cannot be under the average, and a piece that
|
|
53
|
+
// happens to be the smallest in a short session must not lower the figure the
|
|
54
|
+
// player sizes its cushion from.
|
|
55
|
+
return {
|
|
56
|
+
averageBitsPerSecond,
|
|
57
|
+
peakOverAverage: Math.max(1, ((size * 8) / span) / averageBitsPerSecond)
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The three arguments the master playlist needs to declare its rates.
|
|
63
|
+
*
|
|
64
|
+
* Assembled here rather than at the call site, because all three are facts
|
|
65
|
+
* about the OUTPUT and none of them is a fact about a session: the file's own
|
|
66
|
+
* length and duration, the biggest piece made of it, and the cap imposed on the
|
|
67
|
+
* one height being produced. The caller holds them and passes them as numbers.
|
|
68
|
+
*
|
|
69
|
+
* Only the height being produced has a cap — the other heights do not exist
|
|
70
|
+
* yet, and stating one for them would be a guess about a session nobody has
|
|
71
|
+
* made.
|
|
72
|
+
*
|
|
73
|
+
* @param {object} params
|
|
74
|
+
* @param {number} params.fileLength
|
|
75
|
+
* @param {number} params.durationSeconds
|
|
76
|
+
* @param {{ index: number, size: number } | null} [params.largest]
|
|
77
|
+
* @param {number[] | null} [params.boundaries]
|
|
78
|
+
* @param {number} [params.producedHeight] - The height this output encodes at.
|
|
79
|
+
* @param {number} [params.capKbps] - What that height is capped at, if it is.
|
|
80
|
+
* @returns {{ averageBitsPerSecond: number, peakOverAverage: number,
|
|
81
|
+
* capKbpsFor: (height: number) => number }}
|
|
82
|
+
*/
|
|
83
|
+
export function masterRateArgs({
|
|
84
|
+
fileLength,
|
|
85
|
+
durationSeconds,
|
|
86
|
+
largest = null,
|
|
87
|
+
boundaries = null,
|
|
88
|
+
producedHeight = 0,
|
|
89
|
+
capKbps = 0
|
|
90
|
+
}) {
|
|
91
|
+
return {
|
|
92
|
+
...declaredRates({ fileLength, durationSeconds, largest, boundaries }),
|
|
93
|
+
capKbpsFor: (height) => (height === producedHeight ? capKbps : 0)
|
|
94
|
+
};
|
|
95
|
+
}
|
|
@@ -268,9 +268,16 @@ export class PieceLru {
|
|
|
268
268
|
* a union wider than the capacity cannot be held however the eviction is
|
|
269
269
|
* ordered.
|
|
270
270
|
*
|
|
271
|
-
* @returns {{ readers: number, unionPieces: number, widestPieces: number, capacity: number }}
|
|
271
|
+
* @returns {{ readers: number, names: string[], unionPieces: number, widestPieces: number, capacity: number }}
|
|
272
272
|
*/
|
|
273
273
|
demand() {
|
|
274
|
+
// WHO THEY ARE, not only how many. A "reader" here is whoever declared a
|
|
275
|
+
// range, and on 2026-09-08 the field said `5 reader(s) want 24 piece(s) of
|
|
276
|
+
// 25` on a session with two encoders — because the priority map declares one
|
|
277
|
+
// range per zone and four of its zones were arriving as four readers. The
|
|
278
|
+
// count alone could not say that, and choosing between "narrow the windows"
|
|
279
|
+
// and "raise the allowance" was guesswork until the names were printed.
|
|
280
|
+
const names = [...this.#protected.keys()].map(String).sort();
|
|
274
281
|
const ranges = [...this.#protected.values()]
|
|
275
282
|
.map((range) => ({ from: range.from, to: range.to }))
|
|
276
283
|
.sort((left, right) => left.from - right.from);
|
|
@@ -288,6 +295,7 @@ export class PieceLru {
|
|
|
288
295
|
}
|
|
289
296
|
return {
|
|
290
297
|
readers: ranges.length,
|
|
298
|
+
names,
|
|
291
299
|
unionPieces,
|
|
292
300
|
widestPieces,
|
|
293
301
|
capacity: this.#capacity
|