@torrent-tv/proxy 2.82.0 → 2.83.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/CLAUDE.md +11 -0
- package/docs/disk-architecture.md +161 -0
- package/docs/encode-architecture.md +36 -7
- package/package.json +1 -1
- package/routes/api/sources/stats/get.js +11 -2
- package/routes/stream/get.js +74 -3
- package/services/delivery-probe.js +248 -43
- package/services/disk/keep.js +48 -0
- package/services/disk/returns.js +103 -0
- package/services/download/SwarmSelection.js +5 -5
- package/services/download/registry.js +20 -0
- package/services/encode/EncodeRun.js +1 -0
- package/services/encode/Encoder.js +15 -0
- package/services/encode/QsvEncoder.js +5 -0
- package/services/encode/SegmentStore.js +14 -0
- package/services/encode/VaapiEncoder.js +5 -0
- package/services/encode/encode-exit.js +17 -0
- package/services/encode/start-stop-cost.js +6 -2
- package/services/files/CompletedFiles.js +276 -0
- package/services/files/piece-from-whole-file.js +118 -0
- package/services/hls-session-manager.js +17 -1
- package/services/hwaccel.js +4 -0
- package/services/output/cut-grid.js +13 -3
- package/services/piece-store/piece-disk-store.js +143 -8
- package/services/piece-store/piece-lru.js +17 -0
- package/services/piece-store/shared-piece-store.js +1803 -1549
- package/services/torrent-pool.js +246 -26
- package/services/torrent-worker/client.js +21 -0
- package/services/torrent-worker/protocol.js +9 -1
- package/services/torrent-worker/worker.js +183 -2
- package/test/completed-files.test.js +115 -0
- package/test/cuts-follow-published-grid.test.js +35 -0
- package/test/delivery-probe.test.js +114 -1
- package/test/encode-exit.test.js +18 -0
- package/test/keeping-period.test.js +83 -0
- package/test/piece-disk-store.test.js +114 -0
- package/test/piece-from-whole-file.test.js +129 -0
- package/test/piece-store-eviction.test.js +28 -15
- package/test/piece-store-never-refuses.test.js +153 -0
- package/test/piece-store-reservations.test.js +16 -3
- package/test/probe-wedge-certainty.test.js +3 -3
- package/test/shared-piece-store.test.js +27 -13
- package/test/stream-route.test.js +41 -0
- package/test/swarm-follows-readers.test.js +126 -0
- package/test/swarm-reach.test.js +5 -0
- package/test/upload-hurry.test.js +27 -0
|
@@ -99,16 +99,46 @@ export function allowedGap({
|
|
|
99
99
|
if (!(bytesPerSecond > 0) || !(intervalMs > 0)) {
|
|
100
100
|
return null;
|
|
101
101
|
}
|
|
102
|
-
const drainMs = (Math.max(queuedBytes, 0) / bytesPerSecond) * 1000;
|
|
103
|
-
const waitMs =
|
|
104
|
-
drainMs +
|
|
105
|
-
Math.max(rttMs, 0) +
|
|
106
|
-
Math.max(echoIntervalMs, 0) +
|
|
107
|
-
Math.max(peerLoopLagMs, 0);
|
|
108
102
|
// At least one: a probe sent and not yet echoed is the ordinary state.
|
|
109
|
-
return Math.max(1, Math.ceil(
|
|
103
|
+
return Math.max(1, Math.ceil(allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs, peerLoopLagMs }) / intervalMs));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* How long a probe may legitimately take to be reported back, in milliseconds.
|
|
108
|
+
*
|
|
109
|
+
* THE QUANTITY {@link allowedGap} COMPUTES AND THEN THROWS AWAY by dividing it
|
|
110
|
+
* into probes. Probes are the wrong unit and always were: the same probe goes
|
|
111
|
+
* down every channel INCLUDING the one carrying the film, and SCTP schedules
|
|
112
|
+
* per association, so a probe waits behind queued video exactly as a segment
|
|
113
|
+
* does. What is behind is then not the peer's answer but the probe itself —
|
|
114
|
+
* and the honest measure of that is the age of the newest probe the peer has
|
|
115
|
+
* seen, which this is compared against.
|
|
116
|
+
*
|
|
117
|
+
* Four terms, every one measured: the queue's own drain time at the rate this
|
|
118
|
+
* connection is achieving, the crossing, the peer's reporting cadence, and how
|
|
119
|
+
* late the peer's event loop is running.
|
|
120
|
+
*
|
|
121
|
+
* @param {{ queuedBytes: number, bytesPerSecond: number, rttMs: number,
|
|
122
|
+
* echoIntervalMs?: number, peerLoopLagMs?: number }} state
|
|
123
|
+
* @returns {number}
|
|
124
|
+
*/
|
|
125
|
+
export function allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs = 0, peerLoopLagMs = 0 }) {
|
|
126
|
+
if (!(bytesPerSecond > 0)) {
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
const drainMs = (Math.max(queuedBytes, 0) / bytesPerSecond) * 1000;
|
|
130
|
+
return drainMs + Math.max(rttMs, 0) + Math.max(echoIntervalMs, 0) + Math.max(peerLoopLagMs, 0);
|
|
110
131
|
}
|
|
111
132
|
|
|
133
|
+
/**
|
|
134
|
+
* How far back the send times of probes are kept.
|
|
135
|
+
*
|
|
136
|
+
* A probe older than the point at which the reverse direction is called gone
|
|
137
|
+
* can say nothing further, and that bound is itself derived per connection —
|
|
138
|
+
* this is the outer edge of it, kept so the map cannot grow with the session.
|
|
139
|
+
*/
|
|
140
|
+
const PROBE_HISTORY_MS = 10 * 60 * 1000;
|
|
141
|
+
|
|
112
142
|
/** The label the browser gives the unordered, non-retransmitting channel. */
|
|
113
143
|
export const UNRELIABLE_LABEL = "proxy-fast";
|
|
114
144
|
|
|
@@ -144,12 +174,33 @@ const REPORT_INTERVAL_MS = 5_000;
|
|
|
144
174
|
* as {@link wedgeIsCertain} in `data-channel-handler.js`, applied to the
|
|
145
175
|
* probe's own counter instead of the transport's byte counter.
|
|
146
176
|
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
177
|
+
* THE FLOOR IS NOT ONE PROBE INTERVAL. "The longest gap this connection has
|
|
178
|
+
* ever shown" is nothing at all in its first minutes, and the floor under it
|
|
179
|
+
* was a single interval — 500 ms — so any two-second quiet stretch while the
|
|
180
|
+
* browser filled its cushion read as longer than anything healthy ever seen.
|
|
181
|
+
* Field 2026-09-11: two captures of 180 s each, triggered at `wedged 1s` and
|
|
182
|
+
* `wedged 2s`, on a connection with `rtt=5ms`, every queue at 0 B and the
|
|
183
|
+
* viewer watching; 65 MB of them were left on the addon's disk.
|
|
184
|
+
*
|
|
185
|
+
* What replaces it is the connection's own answer to how long a legitimate
|
|
186
|
+
* report may take: the peer reports twice a second, the report has to cross,
|
|
187
|
+
* and the peer's own event loop may be late — all three measured, all three
|
|
188
|
+
* already summed for `echoStaleMs`. A stretch shorter than that proves nothing
|
|
189
|
+
* whatever about the association.
|
|
190
|
+
*
|
|
191
|
+
* @param {{ stuckForMs: number, longestHealthySeenGapMs: number, intervalMs?: number,
|
|
192
|
+
* legitimateReportMs?: number }} state
|
|
193
|
+
* @returns {{ isCertain: boolean, needMs: number }}
|
|
149
194
|
*/
|
|
150
|
-
export function probeWedgeIsCertain({
|
|
151
|
-
|
|
152
|
-
|
|
195
|
+
export function probeWedgeIsCertain({
|
|
196
|
+
stuckForMs,
|
|
197
|
+
longestHealthySeenGapMs,
|
|
198
|
+
intervalMs = PROBE_INTERVAL_MS,
|
|
199
|
+
legitimateReportMs = 0
|
|
200
|
+
}) {
|
|
201
|
+
const floorMs = Math.max(intervalMs, Number.isFinite(legitimateReportMs) ? legitimateReportMs : 0);
|
|
202
|
+
const needMs = Math.max(longestHealthySeenGapMs, floorMs);
|
|
203
|
+
return { isCertain: stuckForMs >= needMs, needMs };
|
|
153
204
|
}
|
|
154
205
|
|
|
155
206
|
/**
|
|
@@ -218,47 +269,76 @@ export function probeWedgeIsCertain({ stuckForMs, longestHealthySeenGapMs, inter
|
|
|
218
269
|
export function readProbeState(state) {
|
|
219
270
|
const seenOf = (label) =>
|
|
220
271
|
state.seen instanceof Map ? state.seen.get(label) : state.seen?.[label];
|
|
272
|
+
/**
|
|
273
|
+
* @param {unknown} source
|
|
274
|
+
* @param {string} label
|
|
275
|
+
* @returns {number | null}
|
|
276
|
+
*/
|
|
277
|
+
const timeOf = (source, label) => {
|
|
278
|
+
const value = source instanceof Map ? source.get(label) : /** @type {any} */ (source)?.[label];
|
|
279
|
+
return Number.isFinite(value) ? Number(value) : null;
|
|
280
|
+
};
|
|
221
281
|
const allowedOf = (label) => {
|
|
222
282
|
const source = state.allowed;
|
|
223
283
|
const value = source instanceof Map ? source.get(label) : source?.[label];
|
|
224
284
|
return Number.isInteger(value) ? Number(value) : null;
|
|
225
285
|
};
|
|
226
286
|
const parts = [];
|
|
227
|
-
let
|
|
228
|
-
let
|
|
229
|
-
let
|
|
230
|
-
let
|
|
287
|
+
let isOrderedBehind = false;
|
|
288
|
+
let isUnreliableBehind = false;
|
|
289
|
+
let isUnreliableKnown = false;
|
|
290
|
+
let isJudgeable = false;
|
|
231
291
|
for (const label of state.labels) {
|
|
232
292
|
const seen = seenOf(label);
|
|
233
293
|
const gap = Number.isInteger(seen) ? state.seq - Number(seen) : null;
|
|
234
294
|
const allowance = allowedOf(label);
|
|
235
|
-
|
|
295
|
+
// BY TIME WHERE IT IS KNOWN. The count is what the line prints, because it
|
|
296
|
+
// is what a reader recognises; what decides is a time.
|
|
297
|
+
//
|
|
298
|
+
// The measured one-way delay first: it is the forward direction itself,
|
|
299
|
+
// with the two clocks reconciled from the exchange. The age of the newest
|
|
300
|
+
// reported probe is the fallback, and it is larger than the thing itself by
|
|
301
|
+
// the peer's reporting cadence and the way back — which is why its own
|
|
302
|
+
// allowance carries both of those and the one-way allowance carries
|
|
303
|
+
// neither.
|
|
304
|
+
const oneWay = timeOf(state.oneWayMs, label);
|
|
305
|
+
const oneWayAllowed = timeOf(state.allowedOneWayMs, label);
|
|
306
|
+
const lagMs = oneWay !== null && oneWayAllowed !== null ? oneWay : timeOf(state.behindMs, label);
|
|
307
|
+
const mayWaitMs = oneWay !== null && oneWayAllowed !== null
|
|
308
|
+
? oneWayAllowed
|
|
309
|
+
: timeOf(state.allowedWaitMs, label);
|
|
310
|
+
const lagText = lagMs === null || mayWaitMs === null
|
|
311
|
+
? ""
|
|
312
|
+
: ` ${Math.round(lagMs)}ms of ${Math.round(mayWaitMs)}ms${oneWay !== null ? " one way" : ""}`;
|
|
313
|
+
parts.push(`${label}=${seen ?? "?"}(gap ${gap ?? "?"} of ${allowance ?? "?"}${lagText})`);
|
|
236
314
|
if (allowance === null) {
|
|
237
315
|
continue;
|
|
238
316
|
}
|
|
239
|
-
|
|
240
|
-
const
|
|
317
|
+
isJudgeable = true;
|
|
318
|
+
const isBehind = lagMs !== null && mayWaitMs !== null
|
|
319
|
+
? lagMs > mayWaitMs
|
|
320
|
+
: gap === null || gap > allowance;
|
|
241
321
|
if (label === UNRELIABLE_LABEL) {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
} else if (
|
|
245
|
-
|
|
322
|
+
isUnreliableKnown = true;
|
|
323
|
+
isUnreliableBehind = isBehind;
|
|
324
|
+
} else if (isBehind) {
|
|
325
|
+
isOrderedBehind = true;
|
|
246
326
|
}
|
|
247
327
|
}
|
|
248
|
-
const
|
|
328
|
+
const isPeerAdvancing = state.peerBytesAdvancing === true;
|
|
249
329
|
const detail =
|
|
250
330
|
`sent=${state.seq} ${parts.join(" ")} ` +
|
|
251
331
|
`echoAge=${state.echoAgeMs === null ? "never" : `${state.echoAgeMs}ms`}` +
|
|
252
332
|
(state.peerBytesAdvancing === null || state.peerBytesAdvancing === undefined
|
|
253
333
|
? ""
|
|
254
|
-
: ` peerBytes=${
|
|
334
|
+
: ` peerBytes=${isPeerAdvancing ? "advancing" : "still"}`) +
|
|
255
335
|
(Number.isFinite(state.peerLoopLagMs) ? ` peerLoopLag=${Math.round(Number(state.peerLoopLagMs))}ms` : "") +
|
|
256
336
|
(state.peerVisibility ? ` peerTab=${state.peerVisibility}` : "");
|
|
257
337
|
|
|
258
338
|
if (state.echoes === 0) {
|
|
259
339
|
return { verdict: "no-echo-yet", detail };
|
|
260
340
|
}
|
|
261
|
-
if (!
|
|
341
|
+
if (!isJudgeable) {
|
|
262
342
|
return { verdict: "no-rate-yet", detail };
|
|
263
343
|
}
|
|
264
344
|
const staleAfterMs = Number.isFinite(state.echoStaleMs) && state.echoStaleMs > 0
|
|
@@ -267,21 +347,21 @@ export function readProbeState(state) {
|
|
|
267
347
|
if (state.echoAgeMs !== null && state.echoAgeMs > staleAfterMs) {
|
|
268
348
|
return { verdict: "reverse-direction-gone", detail };
|
|
269
349
|
}
|
|
270
|
-
if (!
|
|
350
|
+
if (!isOrderedBehind && !(isUnreliableKnown && isUnreliableBehind)) {
|
|
271
351
|
return { verdict: "flowing", detail };
|
|
272
352
|
}
|
|
273
353
|
// Bytes are still arriving at the far end. Whatever the probe gaps say, this
|
|
274
354
|
// association has not stopped — the probes are behind a backlog, which is
|
|
275
355
|
// what filling a cushion looks like from here.
|
|
276
|
-
if (
|
|
356
|
+
if (isPeerAdvancing) {
|
|
277
357
|
return { verdict: "flowing", detail };
|
|
278
358
|
}
|
|
279
|
-
if (
|
|
359
|
+
if (isOrderedBehind && isUnreliableKnown && !isUnreliableBehind) {
|
|
280
360
|
return { verdict: "stream-stuck", detail };
|
|
281
361
|
}
|
|
282
|
-
if (
|
|
362
|
+
if (isOrderedBehind) {
|
|
283
363
|
return {
|
|
284
|
-
verdict:
|
|
364
|
+
verdict: isUnreliableKnown ? "association-stopped" : "ordered-behind-no-comparison",
|
|
285
365
|
detail
|
|
286
366
|
};
|
|
287
367
|
}
|
|
@@ -327,6 +407,18 @@ export function createDeliveryProbe({
|
|
|
327
407
|
const now = Date.now();
|
|
328
408
|
connection.seq += 1;
|
|
329
409
|
connection.sentAt = now;
|
|
410
|
+
// WHEN each probe went out, so that what the peer reports can be read as a
|
|
411
|
+
// time rather than as a count of probes. Pruned to the oldest probe any
|
|
412
|
+
// judgement could still be about: once a probe is older than the point at
|
|
413
|
+
// which the reverse direction is called gone, its age says nothing further.
|
|
414
|
+
connection.sentAtBySeq.set(connection.seq, now);
|
|
415
|
+
for (const [probeNumber, sentAt] of connection.sentAtBySeq) {
|
|
416
|
+
if (now - sentAt > PROBE_HISTORY_MS) {
|
|
417
|
+
connection.sentAtBySeq.delete(probeNumber);
|
|
418
|
+
} else {
|
|
419
|
+
break;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
330
422
|
const message = JSON.stringify({ type: "probe", seq: connection.seq, sentAt: now });
|
|
331
423
|
for (const channel of connection.channels.keys()) {
|
|
332
424
|
try {
|
|
@@ -343,6 +435,10 @@ export function createDeliveryProbe({
|
|
|
343
435
|
const rttMs = Number(delivery?.rttMs) || 0;
|
|
344
436
|
/** @type {Map<string, number | null>} */
|
|
345
437
|
const allowed = new Map();
|
|
438
|
+
/** How long each channel's newest unreported probe may legitimately be. */
|
|
439
|
+
const allowedWait = new Map();
|
|
440
|
+
/** The same, for the measured one-way time where the clocks are reconciled. */
|
|
441
|
+
const allowedOneWay = new Map();
|
|
346
442
|
for (const [channel, label] of connection.channels) {
|
|
347
443
|
let queuedBytes = 0;
|
|
348
444
|
try {
|
|
@@ -358,6 +454,26 @@ export function createDeliveryProbe({
|
|
|
358
454
|
peerLoopLagMs: connection.peerLoopLagMs ?? 0,
|
|
359
455
|
intervalMs
|
|
360
456
|
});
|
|
457
|
+
const waitMs = allowedWaitMs({
|
|
458
|
+
queuedBytes,
|
|
459
|
+
bytesPerSecond,
|
|
460
|
+
rttMs,
|
|
461
|
+
echoIntervalMs: connection.echoIntervalMs,
|
|
462
|
+
peerLoopLagMs: connection.peerLoopLagMs ?? 0
|
|
463
|
+
});
|
|
464
|
+
const heldWait = allowedWait.get(label);
|
|
465
|
+
if (!Number.isFinite(heldWait) || waitMs > heldWait) {
|
|
466
|
+
allowedWait.set(label, waitMs);
|
|
467
|
+
}
|
|
468
|
+
// What a probe may take ONE WAY: the queue's own drain time and half the
|
|
469
|
+
// crossing. Neither the peer's reporting cadence nor its loop delay
|
|
470
|
+
// belongs here — with the clocks reconciled they are no longer in the
|
|
471
|
+
// measurement they would have to be allowed for.
|
|
472
|
+
const oneWayAllowed = allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs: rttMs / 2 });
|
|
473
|
+
const heldOneWay = allowedOneWay.get(label);
|
|
474
|
+
if (!Number.isFinite(heldOneWay) || oneWayAllowed > heldOneWay) {
|
|
475
|
+
allowedOneWay.set(label, oneWayAllowed);
|
|
476
|
+
}
|
|
361
477
|
// Several channels can carry one label only in malformed cases; the
|
|
362
478
|
// larger allowance is the safer of the two.
|
|
363
479
|
const held = allowed.get(label);
|
|
@@ -398,6 +514,25 @@ export function createDeliveryProbe({
|
|
|
398
514
|
peerLoopLagMs: connection.peerLoopLagMs,
|
|
399
515
|
peerVisibility: connection.peerVisibility,
|
|
400
516
|
allowed,
|
|
517
|
+
// WHAT IS BEHIND, IN TIME. For each channel: how old the newest probe the
|
|
518
|
+
// peer has reported seeing is. The same probe goes down every channel
|
|
519
|
+
// including the one carrying the film, and SCTP schedules per
|
|
520
|
+
// association, so a probe waits behind queued video exactly as a segment
|
|
521
|
+
// does — which means a count of outstanding probes measures the queue,
|
|
522
|
+
// not the association. This is the queue's own time, against the time the
|
|
523
|
+
// queue is allowed to take.
|
|
524
|
+
behindMs: Object.fromEntries(
|
|
525
|
+
[...connection.seen.entries()].map(([label, newestSeenNumber]) => {
|
|
526
|
+
const sentAt = connection.sentAtBySeq.get(newestSeenNumber);
|
|
527
|
+
return [label, Number.isFinite(sentAt) ? now - sentAt : null];
|
|
528
|
+
})
|
|
529
|
+
),
|
|
530
|
+
allowedWaitMs: Object.fromEntries(allowedWait.entries()),
|
|
531
|
+
// The measured forward delay, where the two clocks have been reconciled.
|
|
532
|
+
// Preferred over the age above because it is the thing itself: the age
|
|
533
|
+
// also carries the peer's reporting cadence and the way back.
|
|
534
|
+
oneWayMs: Object.fromEntries(connection.oneWayMs.entries()),
|
|
535
|
+
allowedOneWayMs: Object.fromEntries(allowedOneWay.entries()),
|
|
401
536
|
// Same arithmetic for the echo's own age: the peer cannot answer sooner
|
|
402
537
|
// than its own cadence allows, nor sooner than its own event loop runs.
|
|
403
538
|
echoStaleMs:
|
|
@@ -422,11 +557,24 @@ export function createDeliveryProbe({
|
|
|
422
557
|
// as a legitimate gap, is the wedge this exists to catch.
|
|
423
558
|
const stuckForMs = connection.lastSeenAdvanceAt === 0 ? 0 : now - connection.lastSeenAdvanceAt;
|
|
424
559
|
if (verdict === "association-stopped") {
|
|
425
|
-
const {
|
|
560
|
+
const { isCertain, needMs } = probeWedgeIsCertain({
|
|
426
561
|
stuckForMs,
|
|
427
|
-
longestHealthySeenGapMs: connection.longestHealthySeenGapMs
|
|
562
|
+
longestHealthySeenGapMs: connection.longestHealthySeenGapMs,
|
|
563
|
+
// The same sum the echo's own staleness is judged by: the peer reports
|
|
564
|
+
// twice a second, the report has to cross, and the peer's loop may be
|
|
565
|
+
// late. Nothing shorter than that says anything.
|
|
566
|
+
// BOTH cadences, because they beat against each other: a probe sent
|
|
567
|
+
// just after the peer composed a report shows up only in the NEXT one.
|
|
568
|
+
// So the longest legitimate gap between two advances is one probe
|
|
569
|
+
// interval plus one report interval plus the crossing plus whatever
|
|
570
|
+
// the peer's own loop is behind by.
|
|
571
|
+
legitimateReportMs:
|
|
572
|
+
intervalMs +
|
|
573
|
+
connection.echoIntervalMs +
|
|
574
|
+
rttMs +
|
|
575
|
+
Math.max(connection.peerLoopLagMs ?? 0, 0)
|
|
428
576
|
});
|
|
429
|
-
if (
|
|
577
|
+
if (isCertain && !connection.probeCaptureStarted) {
|
|
430
578
|
connection.probeCaptureStarted = true;
|
|
431
579
|
const reasonText =
|
|
432
580
|
`probe seen-counter unmoved ${Math.round(stuckForMs / 1000)}s against the ` +
|
|
@@ -497,6 +645,24 @@ export function createDeliveryProbe({
|
|
|
497
645
|
reportedAt: 0,
|
|
498
646
|
lastSeenAdvanceAt: 0,
|
|
499
647
|
longestHealthySeenGapMs: 0,
|
|
648
|
+
sentAtBySeq: new Map(),
|
|
649
|
+
// THE TWO CLOCKS, separated by the exchange rather than assumed to
|
|
650
|
+
// agree. Every report carries when each channel's newest probe
|
|
651
|
+
// arrived at the peer and when the report itself left; with the time
|
|
652
|
+
// this proxy stamped into that probe, and the time the report
|
|
653
|
+
// arrives, that is the four timestamps an offset is computed from.
|
|
654
|
+
//
|
|
655
|
+
// The estimate is kept from the report with the SMALLEST round trip
|
|
656
|
+
// seen, because that is the one that queued the least — and the two
|
|
657
|
+
// directions here are as unequal as they get, film one way and almost
|
|
658
|
+
// nothing the other, which is exactly where an offset taken from a
|
|
659
|
+
// busy moment is wrong by half the difference.
|
|
660
|
+
/** @type {number | null} */
|
|
661
|
+
clockOffsetMs: null,
|
|
662
|
+
/** @type {number} */
|
|
663
|
+
offsetFromRoundTripMs: Number.POSITIVE_INFINITY,
|
|
664
|
+
/** How long the newest probe took to reach the peer, by channel. @type {Map<string, number>} */
|
|
665
|
+
oneWayMs: new Map(),
|
|
500
666
|
probeCaptureStarted: false,
|
|
501
667
|
timer: null
|
|
502
668
|
};
|
|
@@ -565,6 +731,43 @@ export function createDeliveryProbe({
|
|
|
565
731
|
connection.lastSeenAdvanceAt = now;
|
|
566
732
|
}
|
|
567
733
|
}
|
|
734
|
+
// THE FOUR TIMESTAMPS. `t1` is when this proxy sent the probe the peer
|
|
735
|
+
// is reporting, `t2` when the peer received it, `t3` when the peer sent
|
|
736
|
+
// this report, `t4` is now. From them the round trip is
|
|
737
|
+
// `(t4 - t1) - (t3 - t2)` — the peer's own thinking time removed — and
|
|
738
|
+
// the difference between the two clocks is `((t2 - t1) + (t3 - t4)) / 2`.
|
|
739
|
+
//
|
|
740
|
+
// The offset is kept from the report whose round trip was SMALLEST: that
|
|
741
|
+
// arithmetic assumes the two directions are equally quick, which here
|
|
742
|
+
// they are not — film one way, almost nothing the other — and the least
|
|
743
|
+
// queued sample is the one where the assumption costs least. A fresh
|
|
744
|
+
// minimum replaces the estimate, so a clock that jumps is followed within
|
|
745
|
+
// a few reports rather than believed for ever.
|
|
746
|
+
const peerSentAt = Number(echo?.sentAt);
|
|
747
|
+
const seenAt = echo?.seenAt;
|
|
748
|
+
if (Number.isFinite(peerSentAt) && seenAt && typeof seenAt === "object") {
|
|
749
|
+
for (const [label, peerSawAtRaw] of Object.entries(seenAt)) {
|
|
750
|
+
const peerSawAt = Number(peerSawAtRaw);
|
|
751
|
+
const newestSeenNumber = connection.seen.get(label);
|
|
752
|
+
const weSentAt = Number.isInteger(newestSeenNumber)
|
|
753
|
+
? connection.sentAtBySeq.get(newestSeenNumber)
|
|
754
|
+
: undefined;
|
|
755
|
+
if (!Number.isFinite(peerSawAt) || !Number.isFinite(weSentAt)) {
|
|
756
|
+
continue;
|
|
757
|
+
}
|
|
758
|
+
const roundTrip = (now - weSentAt) - (peerSentAt - peerSawAt);
|
|
759
|
+
if (roundTrip >= 0 && roundTrip < connection.offsetFromRoundTripMs) {
|
|
760
|
+
connection.offsetFromRoundTripMs = roundTrip;
|
|
761
|
+
connection.clockOffsetMs = ((peerSawAt - weSentAt) + (peerSentAt - now)) / 2;
|
|
762
|
+
}
|
|
763
|
+
if (connection.clockOffsetMs !== null) {
|
|
764
|
+
// What the probe itself took, one way, with the clocks reconciled.
|
|
765
|
+
// Negative only if the offset is stale, and then it says so rather
|
|
766
|
+
// than being hidden.
|
|
767
|
+
connection.oneWayMs.set(label, peerSawAt - weSentAt - connection.clockOffsetMs);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
568
771
|
// What the far end says it has received at the transport level. It is the
|
|
569
772
|
// one figure that separates a backlog from a stopped association, and it
|
|
570
773
|
// arrives on the direction that goes on working through a freeze.
|
|
@@ -575,17 +778,19 @@ export function createDeliveryProbe({
|
|
|
575
778
|
// Sum of per-channel bytes. Transport's bytesReceived advances on SACKs
|
|
576
779
|
// (140 B/s on a wedge) and gives a false "advancing" every other tick —
|
|
577
780
|
// channel bytes are flat on a wedge, so they are the correct signal.
|
|
578
|
-
const
|
|
579
|
-
if (
|
|
580
|
-
let
|
|
581
|
-
let
|
|
582
|
-
for (const
|
|
583
|
-
if (
|
|
584
|
-
|
|
585
|
-
|
|
781
|
+
const peerChannels = echo?.report?.channels;
|
|
782
|
+
if (peerChannels && typeof peerChannels === "object") {
|
|
783
|
+
let bytesAcrossChannels = 0;
|
|
784
|
+
let hasChannelReport = false;
|
|
785
|
+
for (const channelReport of Object.values(peerChannels)) {
|
|
786
|
+
if (channelReport && Number.isFinite(channelReport.bytes)) {
|
|
787
|
+
bytesAcrossChannels += channelReport.bytes;
|
|
788
|
+
hasChannelReport = true;
|
|
586
789
|
}
|
|
587
790
|
}
|
|
588
|
-
if (
|
|
791
|
+
if (hasChannelReport) {
|
|
792
|
+
connection.peerChannelBytes = bytesAcrossChannels;
|
|
793
|
+
}
|
|
589
794
|
}
|
|
590
795
|
// How far behind the far end's own event loop is running. A browser that
|
|
591
796
|
// cannot run its timers cannot answer a probe, and every allowance here
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file How long material nobody is using is kept.
|
|
3
|
+
*
|
|
4
|
+
* ONE NUMBER, IN ONE PLACE, because everything it governs stands for the same
|
|
5
|
+
* unmeasured thing: whether the viewer comes back. Everything else in the
|
|
6
|
+
* decision is measured or measurable — re-downloading a piece from the swarm is
|
|
7
|
+
* ~1430 ms on the field host, re-making a segment is its own encode time, and
|
|
8
|
+
* the disk now has an owner that prices holding it. Only the return is unknown,
|
|
9
|
+
* and a period is what stands in for it.
|
|
10
|
+
*
|
|
11
|
+
* IT WAS THREE NUMBERS AND THEY CONTRADICTED EACH OTHER (read 2026-09-10):
|
|
12
|
+
*
|
|
13
|
+
* torrent and its downloaded bytes 15 minutes
|
|
14
|
+
* session 30 minutes
|
|
15
|
+
* produced segments 6 hours
|
|
16
|
+
*
|
|
17
|
+
* The torrent therefore went at fifteen minutes while the session it feeds
|
|
18
|
+
* lived to thirty, so between them there was a session with no source: a viewer
|
|
19
|
+
* returning at the twentieth minute got a session that could not make a single
|
|
20
|
+
* new segment until the torrent was added again. That is not two answers to one
|
|
21
|
+
* question, it is a contradiction — and it is what two independent guesses about
|
|
22
|
+
* one unknown produce.
|
|
23
|
+
*
|
|
24
|
+
* With one hour for both kinds of material the session dies first, which is the
|
|
25
|
+
* right order and needs no rule of its own to enforce.
|
|
26
|
+
*
|
|
27
|
+
* WHY AN HOUR, honestly: nothing derives it. It is a stand-in, chosen to be
|
|
28
|
+
* long enough that an interruption — a phone call, a meal — does not cost the
|
|
29
|
+
* film, and short enough that a household disk is not held for a day by
|
|
30
|
+
* somebody who is not coming back. It is the OWNER'S disk, and holding
|
|
31
|
+
* gigabytes on it because there happens to be room is taking something that is
|
|
32
|
+
* not ours.
|
|
33
|
+
*
|
|
34
|
+
* WHAT REPLACES IT, and the proxy is already in a position to measure it: every
|
|
35
|
+
* session opened on an output whose segments are still on disk IS a return, and
|
|
36
|
+
* its age is known. `services/disk/returns.js` records them. A week of those and
|
|
37
|
+
* the distribution answers this directly — keep material for as long as returns
|
|
38
|
+
* actually happen — and then the two kinds can have different numbers, since
|
|
39
|
+
* their costs of coming back differ.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* How long material nobody has read is kept, in milliseconds.
|
|
44
|
+
*
|
|
45
|
+
* Read by the torrent pool for a torrent and its downloaded bytes, and by the
|
|
46
|
+
* session manager for the segments an encoder produced.
|
|
47
|
+
*/
|
|
48
|
+
export const IDLE_KEEP_MS = 60 * 60 * 1000;
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file How long after material stops being read somebody asks for it again.
|
|
3
|
+
*
|
|
4
|
+
* The one term of "how long do we keep this" that nothing measures. Everything
|
|
5
|
+
* else in that decision is a measured quantity — re-downloading a piece from
|
|
6
|
+
* the swarm is ~1430 ms on the field host, re-making a segment is its own
|
|
7
|
+
* encode time, and the disk has an owner that prices holding it. Only the
|
|
8
|
+
* return is unknown, and `IDLE_KEEP_MS` is a guess standing in for it.
|
|
9
|
+
*
|
|
10
|
+
* IT IS MEASURABLE HERE AND NOWHERE ELSE. A session opened on an output whose
|
|
11
|
+
* segments are still on disk IS a return, and its age is known exactly: the
|
|
12
|
+
* store records when each output was last read. Nothing needs to be inferred.
|
|
13
|
+
*
|
|
14
|
+
* This changes no behaviour. It records, and says what it has seen once in a
|
|
15
|
+
* while, so that after a week of ordinary use the period can be derived from
|
|
16
|
+
* what viewers actually do instead of from what a period felt like.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** How many returns are kept. Enough to see a shape, few enough to say in a line. */
|
|
20
|
+
const KEPT = 200;
|
|
21
|
+
|
|
22
|
+
export class Returns {
|
|
23
|
+
/** Ages in milliseconds, newest last. @type {number[]} */
|
|
24
|
+
#ages = [];
|
|
25
|
+
|
|
26
|
+
/** Sessions opened on material this proxy no longer had. */
|
|
27
|
+
#cold = 0;
|
|
28
|
+
|
|
29
|
+
/** Sessions opened on material that was still there. */
|
|
30
|
+
#warm = 0;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Note a session being opened on an output.
|
|
34
|
+
*
|
|
35
|
+
* @param {object} params
|
|
36
|
+
* @param {number | null} params.lastReadAt - When that output was last read,
|
|
37
|
+
* or null where this proxy has never held it.
|
|
38
|
+
* @param {number} params.now
|
|
39
|
+
* @returns {void}
|
|
40
|
+
*/
|
|
41
|
+
note({ lastReadAt, now }) {
|
|
42
|
+
if (!Number.isFinite(lastReadAt) || lastReadAt === null || lastReadAt <= 0) {
|
|
43
|
+
this.#cold += 1;
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
this.#warm += 1;
|
|
47
|
+
this.#ages.push(Math.max(0, now - /** @type {number} */ (lastReadAt)));
|
|
48
|
+
while (this.#ages.length > KEPT) {
|
|
49
|
+
this.#ages.shift();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* What the returns look like, or null while there have been none.
|
|
55
|
+
*
|
|
56
|
+
* The MEDIAN and the LONGEST, because those are the two the period has to sit
|
|
57
|
+
* between: shorter than the median throws away material half the returns
|
|
58
|
+
* wanted, and longer than the longest keeps material no return has ever
|
|
59
|
+
* reached.
|
|
60
|
+
*
|
|
61
|
+
* @returns {{ warm: number, cold: number, medianMs: number, longestMs: number } | null}
|
|
62
|
+
*/
|
|
63
|
+
shape() {
|
|
64
|
+
if (this.#ages.length === 0) {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
const sorted = [...this.#ages].sort((left, right) => left - right);
|
|
68
|
+
const middle = Math.floor(sorted.length / 2);
|
|
69
|
+
return {
|
|
70
|
+
warm: this.#warm,
|
|
71
|
+
cold: this.#cold,
|
|
72
|
+
medianMs: sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle],
|
|
73
|
+
longestMs: sorted[sorted.length - 1]
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* One line, for the log, or null while there is nothing to say.
|
|
79
|
+
*
|
|
80
|
+
* @param {number} keepMs - What is being kept for now, so the reading and the
|
|
81
|
+
* guess it will replace stand side by side.
|
|
82
|
+
* @returns {string | null}
|
|
83
|
+
*/
|
|
84
|
+
describe(keepMs) {
|
|
85
|
+
const shape = this.shape();
|
|
86
|
+
if (shape === null) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
return (
|
|
90
|
+
`returns: ${shape.warm} session(s) opened on material still held, ` +
|
|
91
|
+
`${shape.cold} on material gone; median ${minutes(shape.medianMs)} after the last read, ` +
|
|
92
|
+
`longest ${minutes(shape.longestMs)} — kept for ${minutes(keepMs)}`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* @param {number} ms
|
|
99
|
+
* @returns {string}
|
|
100
|
+
*/
|
|
101
|
+
function minutes(ms) {
|
|
102
|
+
return `${Math.round(ms / 60000)}min`;
|
|
103
|
+
}
|
|
@@ -180,15 +180,15 @@ export class SwarmSelection {
|
|
|
180
180
|
this.#clearDisplacement();
|
|
181
181
|
return;
|
|
182
182
|
}
|
|
183
|
-
const
|
|
184
|
-
const
|
|
185
|
-
if (this.#displacing && this.#displacing.from ===
|
|
183
|
+
const nearest = Math.min(...blocked.map((range) => range.from));
|
|
184
|
+
const furthest = Math.max(...blocked.map((range) => range.to));
|
|
185
|
+
if (this.#displacing && this.#displacing.from === nearest && this.#displacing.to === furthest) {
|
|
186
186
|
return;
|
|
187
187
|
}
|
|
188
188
|
this.#clearDisplacement();
|
|
189
189
|
try {
|
|
190
|
-
this.#torrent.critical?.(
|
|
191
|
-
this.#displacing = { from, to };
|
|
190
|
+
this.#torrent.critical?.(nearest, furthest);
|
|
191
|
+
this.#displacing = { from: nearest, to: furthest };
|
|
192
192
|
} catch {
|
|
193
193
|
// silent-ok: displacement is an optimisation, and a torrent being torn
|
|
194
194
|
// down is not worth failing a read over.
|
|
@@ -41,6 +41,26 @@ export function demandFor(torrent) {
|
|
|
41
41
|
return entry;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* Whether this torrent is still short of anything anybody asked for.
|
|
46
|
+
*
|
|
47
|
+
* Read by the upload policy: while a reader is missing bytes, a little upload
|
|
48
|
+
* buys the reciprocity that gets them; once nothing declared is missing it buys
|
|
49
|
+
* nothing at all, and on 2026-09-11 the proxy went on offering 512 KB/s of a
|
|
50
|
+
* fully downloaded file to 596 peers for forty-eight minutes, reading a 4 MB
|
|
51
|
+
* piece off the disk for every 16 KB it sent.
|
|
52
|
+
*
|
|
53
|
+
* False for a torrent nothing has been stated about, which is the same answer
|
|
54
|
+
* and the right one: nobody is waiting for it.
|
|
55
|
+
*
|
|
56
|
+
* @param {object} torrent
|
|
57
|
+
* @returns {boolean}
|
|
58
|
+
*/
|
|
59
|
+
export function hasUnmetDemand(torrent) {
|
|
60
|
+
const held = byTorrent.get(torrent);
|
|
61
|
+
return held ? held.selection.hasUrgentMissing() : false;
|
|
62
|
+
}
|
|
63
|
+
|
|
44
64
|
/**
|
|
45
65
|
* Give up everything stated for a torrent that is going.
|
|
46
66
|
*
|
|
@@ -642,6 +642,7 @@ export class EncodeRun {
|
|
|
642
642
|
const outcome = classifyEncodeExit({
|
|
643
643
|
code,
|
|
644
644
|
producedThrough: this.#produced.size > 0 ? this.reached : null,
|
|
645
|
+
producedCount: this.#produced.size,
|
|
645
646
|
lastSegmentIndex: endOfWork,
|
|
646
647
|
inputUnavailable: this.inputUnavailable(this.lastError)
|
|
647
648
|
});
|
|
@@ -82,6 +82,21 @@ export class Encoder {
|
|
|
82
82
|
throw new Error(`${this.name} does not say how to build its video arguments.`);
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
/**
|
|
86
|
+
* What this kind needs BEFORE the input when it is being benchmarked.
|
|
87
|
+
*
|
|
88
|
+
* Not `inputArgs`: those set up hardware DECODING, and a benchmark is fed raw
|
|
89
|
+
* frames — there is nothing to decode. What a device-backed encoder still
|
|
90
|
+
* needs is the device itself, and without it `h264_vaapi` and `h264_qsv` fail
|
|
91
|
+
* to open at all, so the benchmark would answer "this host cannot encode"
|
|
92
|
+
* about a host that encodes perfectly well.
|
|
93
|
+
*
|
|
94
|
+
* @returns {string[]}
|
|
95
|
+
*/
|
|
96
|
+
benchmarkInputArgs() {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
|
|
85
100
|
/**
|
|
86
101
|
* How to encode raw frames at one rung of this kind's speed ladder, for the
|
|
87
102
|
* startup benchmark and nothing else.
|
|
@@ -35,6 +35,11 @@ export class QsvEncoder extends Encoder {
|
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/** @returns {string[]} */
|
|
39
|
+
benchmarkInputArgs() {
|
|
40
|
+
return this.device ? ["-qsv_device", this.device] : [];
|
|
41
|
+
}
|
|
42
|
+
|
|
38
43
|
/** @param {string | null} rung @returns {string[]} */
|
|
39
44
|
benchmarkArgs(rung = null) {
|
|
40
45
|
const preset = rung ? ["-preset", rung] : [];
|