@torrent-tv/proxy 2.97.24 → 2.97.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server.js +8 -2
- package/services/encode/EncodeInputs.js +17 -0
- package/services/encode/quality/HostLoad.js +12 -2
- package/services/server/ViewerRequests.js +22 -6
- package/services/server/wire-outputs.js +1 -0
- package/services/storage/memory-report.js +15 -1
- package/services/torrent/download/FutureDownload.js +5 -2
- package/services/torrent/download/SelectionLimit.js +161 -0
- package/services/torrent/download/registry.js +16 -1
- package/services/torrent/torrent-pool.js +4 -1
- package/services/torrent/worker/worker.js +7 -2
- package/services/viewer/DownloadMaps.js +6 -1
- package/services/viewer/target-input-expansion.js +88 -0
package/package.json
CHANGED
package/server.js
CHANGED
|
@@ -17,6 +17,7 @@ import { ProbeRequests } from "./services/media/ProbeRequests.js";
|
|
|
17
17
|
import { IndexMemory } from "./services/storage/IndexMemory.js";
|
|
18
18
|
import { probePackets } from "./services/media/probe-packets.js";
|
|
19
19
|
import { segmentDemands, sourceDeadline } from "./services/viewer/segment-demands.js";
|
|
20
|
+
import { expandToTargetInputs } from "./services/viewer/target-input-expansion.js";
|
|
20
21
|
import { SourcePreparation } from "./services/viewer/SourcePreparation.js";
|
|
21
22
|
import { handleHealthzGet } from "./routes/healthz/get.js";
|
|
22
23
|
import { handleApiDeliverySinkGet } from "./routes/api/delivery-sink/get.js";
|
|
@@ -490,11 +491,16 @@ export async function startProxyServer({
|
|
|
490
491
|
const pieceLength = Number(torrentPool.knownTorrent(map.sourceKey)?.pieceLength);
|
|
491
492
|
const converted = [];
|
|
492
493
|
const demands = [];
|
|
494
|
+
// What each output's next run would copy, asked of the swarm as one group
|
|
495
|
+
// with its first segment (`target-input-expansion.js`).
|
|
496
|
+
const targets = new Map();
|
|
497
|
+
const expanded = () => expandToTargetInputs(converted, outputKey => targets.get(outputKey) ?? null);
|
|
493
498
|
for (const output of outputParts.outputs.values()) {
|
|
494
499
|
if (output.file.sourceKey !== map.sourceKey) continue;
|
|
495
500
|
const consumers = [...outputParts.viewers.forOutput(output).values()].filter(viewer =>
|
|
496
501
|
viewer.isPresent() && !outputParts.outputs.supersededBy(output, viewer.activeVariantId ?? null));
|
|
497
502
|
if (consumers.length === 0) continue;
|
|
503
|
+
targets.set(output.outputKey, outputParts.encodeInputs?.targetInputBytes?.(output) ?? null);
|
|
498
504
|
const inventory = outputParts.segmentStore.sizesOf(output.outputKey);
|
|
499
505
|
let bytes = 0, seconds = 0;
|
|
500
506
|
for (const [index, size] of inventory) {
|
|
@@ -560,7 +566,7 @@ export async function startProxyServer({
|
|
|
560
566
|
if (waiting.has(zone.outputKey)) continue;
|
|
561
567
|
// What is resolved so far is stated before a lower zone is worked out
|
|
562
568
|
// (DownloadMaps, torrent-tv/meta#178).
|
|
563
|
-
map.workingOut?.(zone.priority,
|
|
569
|
+
map.workingOut?.(zone.priority, expanded);
|
|
564
570
|
if (direct) {
|
|
565
571
|
// Asked per zone too: a soundtrack of an AVI is read from its own
|
|
566
572
|
// packets, which the same index states (torrent-tv/meta#166).
|
|
@@ -606,7 +612,7 @@ export async function startProxyServer({
|
|
|
606
612
|
segmentSourceRanges.set(rangesKey, input.ranges);
|
|
607
613
|
convert(input.ranges);
|
|
608
614
|
}
|
|
609
|
-
return
|
|
615
|
+
return expanded();
|
|
610
616
|
}
|
|
611
617
|
});
|
|
612
618
|
const mediaReads = new MediaReadRequests({
|
|
@@ -303,6 +303,23 @@ export class EncodeInputs {
|
|
|
303
303
|
return this.stretchFor(output)?.bytes ?? null;
|
|
304
304
|
}
|
|
305
305
|
|
|
306
|
+
/**
|
|
307
|
+
* What the next run of this output would be given to copy, whatever has
|
|
308
|
+
* arrived: the stretch `stretchFor` sizes, within the memory encoder inputs
|
|
309
|
+
* may hold. The download map asks the swarm for that much as one group
|
|
310
|
+
* (`viewer/target-input-expansion.js`); a stretch shortened to what is here
|
|
311
|
+
* would ask only for what is here. Null while it is not known.
|
|
312
|
+
*
|
|
313
|
+
* @param {object} output
|
|
314
|
+
* @returns {number | null}
|
|
315
|
+
*/
|
|
316
|
+
targetInputBytes(output) {
|
|
317
|
+
const bytes = this.stretchBytesFor(output);
|
|
318
|
+
if (bytes === null) return null;
|
|
319
|
+
const capacity = this.#capacity();
|
|
320
|
+
return Number.isFinite(capacity) ? Math.min(bytes, capacity) : bytes;
|
|
321
|
+
}
|
|
322
|
+
|
|
306
323
|
#prepare(request) {
|
|
307
324
|
request.pending = true;
|
|
308
325
|
request.revision = this.#revision;
|
|
@@ -31,7 +31,15 @@ function threadSays(now, previous) {
|
|
|
31
31
|
const reads = Number.isFinite(now.readerReads) && Number.isFinite(previous.readerReads) && now.readerReads > previous.readerReads
|
|
32
32
|
&& Number.isFinite(now.readerMs) && Number.isFinite(previous.readerMs)
|
|
33
33
|
? `; ${now.readerReads - previous.readerReads} read(s) for readers took ${Math.round(now.readerMs - previous.readerMs)}ms in it` : "";
|
|
34
|
-
|
|
34
|
+
// Whether our own selection held the swarm back over the interval, and what
|
|
35
|
+
// it gave in the time it did not (`SelectionLimit`, torrent-tv/meta#178).
|
|
36
|
+
const ms = (now.selection?.ms ?? 0) - (previous.selection?.ms ?? 0);
|
|
37
|
+
const freeMs = (now.selection?.freeMs ?? 0) - (previous.selection?.freeMs ?? 0);
|
|
38
|
+
const freeBytes = (now.selection?.freeBytes ?? 0) - (previous.selection?.freeBytes ?? 0);
|
|
39
|
+
const held = ms > 0 && freeMs >= 0 && freeMs <= ms
|
|
40
|
+
? `; our selection held a peer back ${Math.round((1 - freeMs / ms) * 100)}% of the time` +
|
|
41
|
+
(freeMs > 0 ? `, the rest gave ${(freeBytes / freeMs / 1000).toFixed(2)} MB/s` : "") : "";
|
|
42
|
+
return loop + reads + held;
|
|
35
43
|
}
|
|
36
44
|
|
|
37
45
|
export class HostLoad {
|
|
@@ -211,6 +219,7 @@ export class HostLoad {
|
|
|
211
219
|
readerReads: totals?.readerReads ?? null,
|
|
212
220
|
loopP99Ms: totals?.loopP99Ms ?? null,
|
|
213
221
|
loopMaxMs: totals?.loopMaxMs ?? null,
|
|
222
|
+
selection: totals?.selection ?? null,
|
|
214
223
|
metadata: this.#host.readMetadataActivity?.()
|
|
215
224
|
};
|
|
216
225
|
this.#learnThreadCost(now);
|
|
@@ -482,7 +491,8 @@ export class HostLoad {
|
|
|
482
491
|
const totals = await this.#host.getTorrentTotals();
|
|
483
492
|
const read = (key) => { const value = Number(totals?.[key]); return Number.isFinite(value) ? value : null; };
|
|
484
493
|
return { downloaded: read("downloaded"), uploaded: read("uploaded"), served: read("served"), threadCpuSeconds: read("threadCpuSeconds"),
|
|
485
|
-
readerMs: read("readerMs"), readerReads: read("readerReads"), loopP99Ms: read("loopP99Ms"), loopMaxMs: read("loopMaxMs")
|
|
494
|
+
readerMs: read("readerMs"), readerReads: read("readerReads"), loopP99Ms: read("loopP99Ms"), loopMaxMs: read("loopMaxMs"),
|
|
495
|
+
selection: totals?.selection ?? null };
|
|
486
496
|
} catch {
|
|
487
497
|
return null; // the pool is busy or gone; a reading missed is not a fault
|
|
488
498
|
}
|
|
@@ -880,15 +880,31 @@ export class ViewerRequests {
|
|
|
880
880
|
return trend.snapshot();
|
|
881
881
|
}
|
|
882
882
|
|
|
883
|
+
/**
|
|
884
|
+
* What the swarm can give this source. A rate measured while our own
|
|
885
|
+
* selection held a connection back measures our asking: on a cold Matroska
|
|
886
|
+
* it read 0.4-1.3 MB/s while the same swarm later gave 12-14 MB/s, and the
|
|
887
|
+
* forecast swung from 0.9 s to 19 075 s (torrent-tv/meta#178). So it is read
|
|
888
|
+
* from the time no connection was held back (`SelectionLimit`), and a later
|
|
889
|
+
* such reading lowers it as readily as it raises it. Until there is one, the
|
|
890
|
+
* rate delivered stands, which is the least the swarm can give.
|
|
891
|
+
*/
|
|
883
892
|
#sourceRateReadings(state, id, stats, now) {
|
|
884
|
-
let
|
|
885
|
-
if (!
|
|
886
|
-
|
|
887
|
-
|
|
893
|
+
let source = state.sources.get(id);
|
|
894
|
+
if (!source) {
|
|
895
|
+
const { RateTrend } = this.#host.playbackReadiness;
|
|
896
|
+
source = { delivered: new RateTrend(this.#host.lookaheadSeconds), free: new RateTrend(this.#host.lookaheadSeconds), limit: null };
|
|
897
|
+
state.sources.set(id, source);
|
|
888
898
|
}
|
|
889
899
|
if (Number.isFinite(stats?.downloadSpeed) && stats.downloadSpeed >= 0) {
|
|
890
|
-
|
|
900
|
+
source.delivered.add(now, stats.downloadSpeed);
|
|
891
901
|
}
|
|
892
|
-
|
|
902
|
+
const limit = stats?.selectionLimit;
|
|
903
|
+
if (limit && source.limit && limit.freeMs > source.limit.freeMs) {
|
|
904
|
+
source.free.add(now, Math.max(0, limit.freeBytes - source.limit.freeBytes) * 1000 / (limit.freeMs - source.limit.freeMs));
|
|
905
|
+
}
|
|
906
|
+
// Totals that went back belong to a torrent added again: read on from them.
|
|
907
|
+
if (limit) source.limit = limit;
|
|
908
|
+
return source.free.snapshot() ?? source.delivered.snapshot();
|
|
893
909
|
}
|
|
894
910
|
}
|
|
@@ -254,6 +254,7 @@ export function wireOutputs({
|
|
|
254
254
|
directoryFor: (address, format) => parts.segmentStore.directoryFor(address, format),
|
|
255
255
|
publish: (address, makingName, format, read) => parts.segmentStore.publish(address, makingName, format, read),
|
|
256
256
|
closedBytesOf: (address, makingName) => parts.segmentStore.closedBytesOf(address, makingName),
|
|
257
|
+
writtenAtOf: (address, makingName) => parts.segmentStore.writtenAtOf(address, makingName),
|
|
257
258
|
mediaRangesOf: (address, index, where) => parts.segmentStore.mediaRangesOf(address, index, where),
|
|
258
259
|
remove: (address, index, because) => parts.segmentStore.remove(address, index, because)
|
|
259
260
|
};
|
|
@@ -468,6 +468,8 @@ export function startMemoryReport({
|
|
|
468
468
|
/** @type {Record<string, number>} */
|
|
469
469
|
let lastWritten = {};
|
|
470
470
|
let lastWrittenAt = 0;
|
|
471
|
+
/** The anonymous memory and the time the mappings were last read at. */
|
|
472
|
+
let mappingsReadAt = null;
|
|
471
473
|
const slug = (label || scope).replace(/[^a-z0-9]+/gi, "-").toLowerCase();
|
|
472
474
|
const directory = snapshotDir || os.tmpdir();
|
|
473
475
|
|
|
@@ -553,13 +555,25 @@ export function startMemoryReport({
|
|
|
553
555
|
// the reading now happens once a second rather than once a minute.
|
|
554
556
|
const available = availableMemory();
|
|
555
557
|
anonymousBytes = await readAnonymousMemory();
|
|
558
|
+
// THE MAPPINGS ONLY WHERE THE ANONYMOUS MEMORY THEY BREAK DOWN HAS
|
|
559
|
+
// MOVED, or once in a quiet interval. The line is written on every
|
|
560
|
+
// swing of resident memory, which array buffers taken and given back
|
|
561
|
+
// make every two or three seconds, and each walk of `smaps` cost the
|
|
562
|
+
// main thread about 70 ms: 15.6 s of it in ten minutes of a cold AVI
|
|
563
|
+
// on Home Assistant, while segments waited to be sent
|
|
564
|
+
// (torrent-tv/meta#178).
|
|
565
|
+
const mappingsDue = mappingsReadAt === null || now - mappingsReadAt.at >= quietMs ||
|
|
566
|
+
!Number.isFinite(anonymousBytes) || !Number.isFinite(mappingsReadAt.anonymousBytes) ||
|
|
567
|
+
Math.abs(anonymousBytes - mappingsReadAt.anonymousBytes) >= changeBytes;
|
|
568
|
+
const mappings = mappingsDue ? await readMappingSummary() : null;
|
|
569
|
+
if (mappingsDue) mappingsReadAt = { at: now, anonymousBytes };
|
|
556
570
|
log(describeMemory({
|
|
557
571
|
scope,
|
|
558
572
|
label,
|
|
559
573
|
process: processMemory,
|
|
560
574
|
available,
|
|
561
575
|
anonymousBytes,
|
|
562
|
-
mappings
|
|
576
|
+
mappings,
|
|
563
577
|
diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
|
|
564
578
|
stores,
|
|
565
579
|
extra
|
|
@@ -16,7 +16,7 @@ export async function futureDownload(entries, { now = Date.now(), findStore = pi
|
|
|
16
16
|
const forecasts = new Map();
|
|
17
17
|
const pending = [];
|
|
18
18
|
let steps = 0;
|
|
19
|
-
for (const { torrent, register } of entries) {
|
|
19
|
+
for (const { torrent, register, limit = null } of entries) {
|
|
20
20
|
if (torrent.destroyed || torrent.ready === false) continue;
|
|
21
21
|
const wanted = new Map();
|
|
22
22
|
for (const window of register.windows()) {
|
|
@@ -81,7 +81,10 @@ export async function futureDownload(entries, { now = Date.now(), findStore = pi
|
|
|
81
81
|
const remaining = piece?.missing;
|
|
82
82
|
const speed = (torrent.wires ?? []).reduce((sum, wire) => {
|
|
83
83
|
if (wire.destroyed || !wire.peerPieces?.get(index)) return sum;
|
|
84
|
-
|
|
84
|
+
// What it carried with its queue kept full, where it has been seen so:
|
|
85
|
+
// its speed while our selection gave it little measures our asking
|
|
86
|
+
// (`SelectionLimit`, torrent-tv/meta#178).
|
|
87
|
+
const rate = Number(limit?.capacityOf?.(wire) ?? wire.downloadSpeed?.());
|
|
85
88
|
return Number.isFinite(rate) && rate > 0 ? sum + rate : sum;
|
|
86
89
|
}, 0);
|
|
87
90
|
pending.push({ arrivals, index, ...band, seconds:
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Whether what this proxy let the swarm send was what held it back.
|
|
3
|
+
*
|
|
4
|
+
* A download rate measured while we asked for too little describes our asking,
|
|
5
|
+
* not the swarm: on a cold Matroska the forecast read 0.4-1.3 MB/s, swung from
|
|
6
|
+
* 0.9 s to 19 075 s, and the same swarm later gave 12-14 MB/s
|
|
7
|
+
* (torrent-tv/meta#178). So the rate is split by cause, here, on the torrent's
|
|
8
|
+
* thread, where the connections are.
|
|
9
|
+
*
|
|
10
|
+
* A connection was held back by our selection when three things held at once:
|
|
11
|
+
*
|
|
12
|
+
* 1. the peer let us download (it was not choking us);
|
|
13
|
+
* 2. its queue was short of what WebTorrent keeps on it — the blocks half a
|
|
14
|
+
* second of its speed carries, at most what the peer accepts (2.8.5
|
|
15
|
+
* `getBlockPipelineLength` with `PIPELINE_MIN_DURATION`), below which
|
|
16
|
+
* `_updateWire` asks for more;
|
|
17
|
+
* 3. the peer held a piece the priority map wants and we lack, outside what
|
|
18
|
+
* the selection states — something it could have sent had we asked.
|
|
19
|
+
*
|
|
20
|
+
* A request still queued does not make a connection busy; a queue topped up is
|
|
21
|
+
* what does. A peer that lacks every piece the map wants, or chokes us, or
|
|
22
|
+
* slows down, is the swarm's own limit and stays in the measurement.
|
|
23
|
+
*
|
|
24
|
+
* An interval of time takes the state found at its start. Totals only grow:
|
|
25
|
+
* whoever reads them takes differences over its own span, so this owner keeps
|
|
26
|
+
* no window of its own. Per connection, the rate it carried while its queue was
|
|
27
|
+
* kept full is kept as what it can carry, for the forecast of single pieces.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { piecesOf } from "../demand/pieces.js";
|
|
31
|
+
|
|
32
|
+
const BLOCK_BYTES = 16 * 1024;
|
|
33
|
+
/** WebTorrent 2.8.5 `PIPELINE_MIN_DURATION`: the queue is refilled below this many seconds of speed. */
|
|
34
|
+
const PIPELINE_MIN_SECONDS = 0.5;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Whether WebTorrent would ask this connection for more right now.
|
|
38
|
+
*
|
|
39
|
+
* @param {object} wire
|
|
40
|
+
* @returns {boolean}
|
|
41
|
+
*/
|
|
42
|
+
export function queueIsShort(wire) {
|
|
43
|
+
let length = 2 + Math.ceil(PIPELINE_MIN_SECONDS * (Number(wire.downloadSpeed?.()) || 0) / BLOCK_BYTES);
|
|
44
|
+
const reqq = wire.peerExtendedHandshake?.reqq;
|
|
45
|
+
if (typeof reqq === "number" && reqq > 0) length = Math.min(length, reqq);
|
|
46
|
+
return (wire.requests?.length ?? 0) < length;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class SelectionLimit {
|
|
50
|
+
#torrent;
|
|
51
|
+
#register;
|
|
52
|
+
#selection;
|
|
53
|
+
#now;
|
|
54
|
+
/** The last sample: when, the torrent's bytes then, and whether it was held back. */
|
|
55
|
+
#last = null;
|
|
56
|
+
#totals = { bytes: 0, ms: 0, freeBytes: 0, freeMs: 0, heldBackMs: 0 };
|
|
57
|
+
/** Per connection: its bytes at the last sample, whether its queue was full, what it carried full. */
|
|
58
|
+
#wires = new WeakMap();
|
|
59
|
+
/** The last sample's held-back connections, for the log. */
|
|
60
|
+
heldBack = 0;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param {object} params
|
|
64
|
+
* @param {object} params.torrent
|
|
65
|
+
* @param {import("../demand/index.js").DemandRegister} params.register
|
|
66
|
+
* @param {{ statedRanges: () => Array<{ from: number, to: number }> }} params.selection
|
|
67
|
+
* @param {() => number} [params.now]
|
|
68
|
+
*/
|
|
69
|
+
constructor({ torrent, register, selection, now = Date.now }) {
|
|
70
|
+
this.#torrent = torrent;
|
|
71
|
+
this.#register = register;
|
|
72
|
+
this.#selection = selection;
|
|
73
|
+
this.#now = now;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Look at every connection once, close the interval since the last look, and
|
|
78
|
+
* open the next with what is found.
|
|
79
|
+
*
|
|
80
|
+
* @returns {void}
|
|
81
|
+
*/
|
|
82
|
+
sample() {
|
|
83
|
+
const torrent = this.#torrent;
|
|
84
|
+
if (!torrent || torrent.destroyed) return;
|
|
85
|
+
const at = this.#now();
|
|
86
|
+
const downloaded = Number(torrent.downloaded) || 0;
|
|
87
|
+
if (this.#last !== null && at > this.#last.at) {
|
|
88
|
+
const ms = at - this.#last.at;
|
|
89
|
+
const bytes = Math.max(0, downloaded - this.#last.downloaded);
|
|
90
|
+
this.#totals.bytes += bytes;
|
|
91
|
+
this.#totals.ms += ms;
|
|
92
|
+
if (this.#last.heldBack) this.#totals.heldBackMs += ms;
|
|
93
|
+
else {
|
|
94
|
+
this.#totals.freeBytes += bytes;
|
|
95
|
+
this.#totals.freeMs += ms;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const unasked = this.#unaskedWants();
|
|
99
|
+
let heldBack = 0;
|
|
100
|
+
for (const wire of torrent.wires ?? []) {
|
|
101
|
+
if (wire.destroyed) continue;
|
|
102
|
+
const short = queueIsShort(wire);
|
|
103
|
+
const seen = this.#wires.get(wire);
|
|
104
|
+
const wireBytes = Number(wire.downloaded) || 0;
|
|
105
|
+
// What a connection carries while its queue stays topped up is what it
|
|
106
|
+
// can carry; a stretch it was short in measures our asking.
|
|
107
|
+
if (seen && seen.full && !short && at > seen.at) {
|
|
108
|
+
seen.capacity = Math.max(0, wireBytes - seen.bytes) * 1000 / (at - seen.at);
|
|
109
|
+
}
|
|
110
|
+
this.#wires.set(wire, { at, bytes: wireBytes, full: !wire.peerChoking && !short, capacity: seen?.capacity ?? null });
|
|
111
|
+
if (wire.peerChoking || !short || unasked.length === 0) continue;
|
|
112
|
+
if (unasked.some(piece => wire.peerPieces?.get?.(piece))) heldBack += 1;
|
|
113
|
+
}
|
|
114
|
+
this.heldBack = heldBack;
|
|
115
|
+
this.#last = { at, downloaded, heldBack: heldBack > 0 };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Grow-only totals: every byte and millisecond sampled, those of intervals
|
|
120
|
+
* no connection was held back in, and the time some was.
|
|
121
|
+
*
|
|
122
|
+
* @returns {{ at: number | null, bytes: number, ms: number, freeBytes: number, freeMs: number, heldBackMs: number }}
|
|
123
|
+
*/
|
|
124
|
+
totals() {
|
|
125
|
+
return { at: this.#last?.at ?? null, ...this.#totals };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* What a connection carried while its queue was kept full, in bytes per
|
|
130
|
+
* second; null until it has been seen so.
|
|
131
|
+
*
|
|
132
|
+
* @param {object} wire
|
|
133
|
+
* @returns {number | null}
|
|
134
|
+
*/
|
|
135
|
+
capacityOf(wire) {
|
|
136
|
+
return this.#wires.get(wire)?.capacity ?? null;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Pieces the map wants, missing here, that the selection does not state, in map order. */
|
|
140
|
+
#unaskedWants() {
|
|
141
|
+
const torrent = this.#torrent;
|
|
142
|
+
const stated = this.#selection.statedRanges();
|
|
143
|
+
const pieces = [];
|
|
144
|
+
const seen = new Set();
|
|
145
|
+
for (const window of this.#register.windows()) {
|
|
146
|
+
const file = torrent.files?.[window.fileIndex];
|
|
147
|
+
if (!file) continue;
|
|
148
|
+
const range = piecesOf({ fileOffset: Number(file.offset), byteStart: window.byteStart,
|
|
149
|
+
byteEnd: Math.min(window.byteEnd, Number(file.length) - 1), pieceLength: Number(torrent.pieceLength) });
|
|
150
|
+
if (!range) continue;
|
|
151
|
+
for (let piece = range.from; piece <= range.to; piece++) {
|
|
152
|
+
if (seen.has(piece)) continue;
|
|
153
|
+
seen.add(piece);
|
|
154
|
+
if (torrent.bitfield?.get?.(piece)) continue;
|
|
155
|
+
if (stated.some(one => piece >= one.from && piece <= one.to)) continue;
|
|
156
|
+
pieces.push(piece);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return pieces;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import { DemandRegister } from "../demand/DemandRegister.js";
|
|
19
19
|
import { SwarmSelection } from "./SwarmSelection.js";
|
|
20
|
+
import { SelectionLimit } from "./SelectionLimit.js";
|
|
20
21
|
import { futureDownload } from "./FutureDownload.js";
|
|
21
22
|
import { urgencyName } from "../demand/index.js";
|
|
22
23
|
|
|
@@ -56,7 +57,8 @@ export function demandFor(torrent) {
|
|
|
56
57
|
return held;
|
|
57
58
|
}
|
|
58
59
|
const register = new DemandRegister();
|
|
59
|
-
const
|
|
60
|
+
const selection = new SwarmSelection({ torrent, register });
|
|
61
|
+
const entry = { torrent, register, selection, limit: new SelectionLimit({ torrent, register, selection }), withdrawalRevision: 0 };
|
|
60
62
|
byTorrent.set(torrent, entry);
|
|
61
63
|
live.add(entry);
|
|
62
64
|
entry.onBytesChanged = () => reconcileAll();
|
|
@@ -105,6 +107,16 @@ export function forgetTorrent(torrent) {
|
|
|
105
107
|
live.delete(held);
|
|
106
108
|
}
|
|
107
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Whether this proxy's own selection held the swarm back, as grow-only totals
|
|
112
|
+
* (`SelectionLimit`); null for a torrent nothing has been stated about.
|
|
113
|
+
*
|
|
114
|
+
* @param {object} torrent
|
|
115
|
+
*/
|
|
116
|
+
export function selectionLimitOf(torrent) {
|
|
117
|
+
return byTorrent.get(torrent)?.limit.totals() ?? null;
|
|
118
|
+
}
|
|
119
|
+
|
|
108
120
|
/** What the swarm was last told to fetch first: the level, its priority and the pieces per torrent. */
|
|
109
121
|
export function firstClassStatement() {
|
|
110
122
|
return firstClassStated;
|
|
@@ -142,6 +154,9 @@ export function reconcileAll() {
|
|
|
142
154
|
} finally {
|
|
143
155
|
publishingSelections = false;
|
|
144
156
|
}
|
|
157
|
+
// Each torrent's connections looked at under what it has just been told:
|
|
158
|
+
// on every verified piece and on the pool's timer.
|
|
159
|
+
for (const entry of entries) entry.limit.sample();
|
|
145
160
|
// What the swarm is told to fetch FIRST, said when it changes. Lower classes
|
|
146
161
|
// are withheld because a wire with no first-class piece would fall through.
|
|
147
162
|
const said = entries.map(entry => `${String(entry.torrent.infoHash ?? "").slice(0, 8)}: ` +
|
|
@@ -18,7 +18,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
18
18
|
import { pieceStoreOf } from "./piece-store-of.js";
|
|
19
19
|
import { demandProgress } from "./demand-progress.js";
|
|
20
20
|
import { Urgency, urgencyName } from "./demand/index.js";
|
|
21
|
-
import { demandFor, firstClassStatement, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
|
|
21
|
+
import { demandFor, firstClassStatement, forgetTorrent, reconcileAll, hasUnmetDemand, selectionLimitOf } from "./download/registry.js";
|
|
22
22
|
import { applyDownloadRate } from "./download/throttle.js";
|
|
23
23
|
import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
24
24
|
import { deadlineOfMapZone, levelOfMapZone, priorityOfMapZone } from "./map-zones.js";
|
|
@@ -2057,6 +2057,9 @@ export class TorrentPool {
|
|
|
2057
2057
|
numPeers,
|
|
2058
2058
|
downloadSpeed,
|
|
2059
2059
|
uploadSpeed,
|
|
2060
|
+
// Bytes and time split by whether our own selection held a connection
|
|
2061
|
+
// back: what the swarm can give is read from the rest (`SelectionLimit`).
|
|
2062
|
+
selectionLimit: selectionLimitOf(torrent),
|
|
2060
2063
|
connectedPeers: reach.connectedPeers,
|
|
2061
2064
|
deliveringPeers: reach.deliveringPeers,
|
|
2062
2065
|
knownPeers: reach.knownPeers,
|
|
@@ -28,7 +28,7 @@ import { Readable } from "node:stream";
|
|
|
28
28
|
import { heldFileBytes } from "./held-file-bytes.js";
|
|
29
29
|
import { createSendStream } from "./channel.js";
|
|
30
30
|
import { readFragments, supplyFiguresFor } from "./piece-reader.js";
|
|
31
|
-
import { demandFor, forecastDownloads } from "../download/registry.js";
|
|
31
|
+
import { demandFor, forecastDownloads, selectionLimitOf } from "../download/registry.js";
|
|
32
32
|
import { heldRangesOf, ownsItsMemory, readHeldBytes, readHeldRanges } from "./held-bytes.js";
|
|
33
33
|
import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
|
|
34
34
|
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
|
|
@@ -435,11 +435,16 @@ async function runCommand(command, params, id) {
|
|
|
435
435
|
// be.
|
|
436
436
|
let downloaded = 0;
|
|
437
437
|
let uploaded = 0;
|
|
438
|
+
// Over every torrent: the time our own selection held a connection back,
|
|
439
|
+
// and the bytes and time of the rest (`SelectionLimit`).
|
|
440
|
+
const selection = { ms: 0, heldBackMs: 0, freeBytes: 0, freeMs: 0 };
|
|
438
441
|
for (const torrent of pool.client?.torrents ?? []) {
|
|
439
442
|
const gotBytes = Number(torrent?.downloaded);
|
|
440
443
|
const sentBytes = Number(torrent?.uploaded);
|
|
441
444
|
downloaded += Number.isFinite(gotBytes) ? gotBytes : 0;
|
|
442
445
|
uploaded += Number.isFinite(sentBytes) ? sentBytes : 0;
|
|
446
|
+
const limit = selectionLimitOf(torrent);
|
|
447
|
+
if (limit) for (const key of Object.keys(selection)) selection[key] += limit[key];
|
|
443
448
|
}
|
|
444
449
|
// This thread's own processor time, read in the same moment as the
|
|
445
450
|
// bytes, so the price of a downloaded byte is taken over one interval
|
|
@@ -450,7 +455,7 @@ async function runCommand(command, params, id) {
|
|
|
450
455
|
const loopMaxMs = loopDelay.count > 0 ? loopDelay.max / 1e6 : null;
|
|
451
456
|
loopDelay.reset();
|
|
452
457
|
return { downloaded, uploaded, served: servedBytes, threadCpuSeconds: (usage.user + usage.system) / 1e6,
|
|
453
|
-
readerMs, readerReads, loopP99Ms, loopMaxMs };
|
|
458
|
+
readerMs, readerReads, loopP99Ms, loopMaxMs, selection };
|
|
454
459
|
}
|
|
455
460
|
|
|
456
461
|
|
|
@@ -87,9 +87,14 @@ export class DownloadMaps {
|
|
|
87
87
|
// pass that works nothing out — every pass over a film already worked out
|
|
88
88
|
// — states nothing early.
|
|
89
89
|
let stated = Number.POSITIVE_INFINITY;
|
|
90
|
-
|
|
90
|
+
// The zones are asked for only when they are stated: worked out before
|
|
91
|
+
// every lower zone of a pass, they were the whole pass over again at each
|
|
92
|
+
// of its 4000 steps, and a cold AVI's start forecast waited 71 s for it
|
|
93
|
+
// (Home Assistant 2026-10-13, torrent-tv/meta#178).
|
|
94
|
+
const workingOut = (priority, zonesOf) => {
|
|
91
95
|
if (!(priority < stated)) return;
|
|
92
96
|
stated = priority;
|
|
97
|
+
const zones = typeof zonesOf === "function" ? zonesOf() : zonesOf;
|
|
93
98
|
if (zones.length === 0 || !isCurrent()) return;
|
|
94
99
|
this.#setZones(file, withRetained(zones, retained));
|
|
95
100
|
this.#emit(file).catch(() => undefined);
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The next run's whole input, asked of the swarm as one group.
|
|
3
|
+
*
|
|
4
|
+
* While anything urgent is missing, the swarm is told only the first class
|
|
5
|
+
* (`torrent/download/SwarmSelection.js`), and at a cold start the map's first
|
|
6
|
+
* class is one segment's input. A peer holding none of its pieces then has
|
|
7
|
+
* nothing to send, and a run is admitted only as far as its input has arrived
|
|
8
|
+
* (`encode/EncodeInputs.js`), so it opens again a few seconds of film later.
|
|
9
|
+
*
|
|
10
|
+
* WebTorrent 2.8.5 keeps no order between several non-zero selections — it
|
|
11
|
+
* rotates them (`shufflePriority`) — so the run's input is not stated as a
|
|
12
|
+
* second group behind the first: the segments it covers take the first
|
|
13
|
+
* segment's place in the map, and the swarm is told them as one group. How
|
|
14
|
+
* far that reaches is what `EncodeInputs` would copy for the run, from its
|
|
15
|
+
* copy rate, its open and the memory allowed; it is not worked out again here.
|
|
16
|
+
* A read stopped on its bytes is stated at a level of its own above any of
|
|
17
|
+
* this, and a seek, which moves the map's first segment, moves this with it
|
|
18
|
+
* (torrent-tv/meta#178).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Bytes covered by a list of inclusive ranges, each byte counted once: the
|
|
23
|
+
* declarations and indexes every segment of an original file names are held
|
|
24
|
+
* once by the run that reads them.
|
|
25
|
+
*
|
|
26
|
+
* @param {Array<[number, number]>} ranges
|
|
27
|
+
* @returns {number}
|
|
28
|
+
*/
|
|
29
|
+
function coveredBytes(ranges) {
|
|
30
|
+
let total = 0;
|
|
31
|
+
let end = -1;
|
|
32
|
+
for (const [start, last] of [...ranges].sort((left, right) => left[0] - right[0])) {
|
|
33
|
+
if (last <= end) continue;
|
|
34
|
+
total += last - Math.max(start, end + 1) + 1;
|
|
35
|
+
end = last;
|
|
36
|
+
}
|
|
37
|
+
return total;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The map's byte zones with each output's next run input raised to its first
|
|
42
|
+
* segment's place.
|
|
43
|
+
*
|
|
44
|
+
* @template {{ outputKey: string, index: number, priority: number, urgent?: boolean, behind?: boolean, deferred?: boolean, byteStart: number, byteEnd: number }} Zone
|
|
45
|
+
* @param {Zone[]} zones - One or more per segment, as the map's pass made them.
|
|
46
|
+
* @param {(outputKey: string) => number | null} targetBytesOf - What the
|
|
47
|
+
* output's next run would be given to copy; nothing is raised without it.
|
|
48
|
+
* @returns {Zone[]} In their order; raised zones are copies.
|
|
49
|
+
*/
|
|
50
|
+
export function expandToTargetInputs(zones, targetBytesOf) {
|
|
51
|
+
/** @type {Map<string, Map<number, Zone[]>>} */
|
|
52
|
+
const byOutput = new Map();
|
|
53
|
+
for (const zone of zones) {
|
|
54
|
+
let segments = byOutput.get(zone.outputKey);
|
|
55
|
+
if (!segments) byOutput.set(zone.outputKey, segments = new Map());
|
|
56
|
+
const list = segments.get(zone.index);
|
|
57
|
+
if (list) list.push(zone);
|
|
58
|
+
else segments.set(zone.index, [zone]);
|
|
59
|
+
}
|
|
60
|
+
/** @type {Map<Zone, Zone>} */
|
|
61
|
+
const raised = new Map();
|
|
62
|
+
for (const [outputKey, segments] of byOutput) {
|
|
63
|
+
const target = targetBytesOf(outputKey);
|
|
64
|
+
if (!(target > 0)) continue;
|
|
65
|
+
let first = null;
|
|
66
|
+
for (const [index, list] of segments) {
|
|
67
|
+
const priority = Math.max(...list.map(zone => zone.priority));
|
|
68
|
+
if (first === null || priority > first.priority || (priority === first.priority && index < first.index)) {
|
|
69
|
+
first = { index, priority, zone: list.find(zone => zone.priority === priority) };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (first === null) continue;
|
|
73
|
+
const held = segments.get(first.index).map(zone => [zone.byteStart, zone.byteEnd]);
|
|
74
|
+
// The run's input is a stretch of consecutive segments: one already made,
|
|
75
|
+
// or not yet worked out, ends it.
|
|
76
|
+
for (let index = first.index + 1; segments.has(index); index++) {
|
|
77
|
+
const list = segments.get(index);
|
|
78
|
+
const ranges = [...held, ...list.map(zone => [zone.byteStart, zone.byteEnd])];
|
|
79
|
+
if (coveredBytes(ranges) > target) break;
|
|
80
|
+
held.push(...list.map(zone => [zone.byteStart, zone.byteEnd]));
|
|
81
|
+
for (const zone of list) {
|
|
82
|
+
raised.set(zone, { ...zone, priority: first.priority, urgent: first.zone.urgent,
|
|
83
|
+
behind: first.zone.behind, deferred: first.zone.deferred });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return raised.size === 0 ? zones : zones.map(zone => raised.get(zone) ?? zone);
|
|
88
|
+
}
|