@torrent-tv/proxy 2.98.13 → 2.98.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/services/media/ContainerOrchestrator.js +13 -2
- package/services/media/container/AviContainer.js +34 -0
- package/services/torrent/download/SwarmSelection.js +19 -1
- package/services/torrent/piece-timing.js +120 -0
- package/services/torrent/torrent-pool.js +45 -8
- package/services/torrent/worker/fastest-wires.js +62 -31
- package/services/torrent/worker/tail-repeat.js +154 -0
package/package.json
CHANGED
|
@@ -136,10 +136,15 @@ export class ContainerOrchestrator {
|
|
|
136
136
|
let lifetime = this.#lifetimes.get(key);
|
|
137
137
|
if (!lifetime) this.#lifetimes.set(key, lifetime = {});
|
|
138
138
|
const isCurrent = () => this.#lifetimes.get(key) === lifetime && params.isCurrent?.() !== false;
|
|
139
|
-
const previous = this.#reads.get(key) ?? Promise.resolve();
|
|
140
139
|
// Reads of one file run one after another, so an answer can wait for every
|
|
141
140
|
// read asked before it. Told to whoever asked, apart from the read itself,
|
|
142
141
|
// because the two call for different remedies (torrent-tv/meta#166).
|
|
142
|
+
// A statement answered from what was already read reads and parses
|
|
143
|
+
// nothing, so it is answered beside the queue and holds nobody up: the
|
|
144
|
+
// picture's first input of an AVI waited about 4 s behind the sound's
|
|
145
|
+
// table of the whole film it does not use (r38, torrent-tv/meta#178).
|
|
146
|
+
const aside = this.#answersWithoutReading(key, statement);
|
|
147
|
+
const previous = aside ? Promise.resolve() : this.#reads.get(key) ?? Promise.resolve();
|
|
143
148
|
const askedAt = Date.now();
|
|
144
149
|
const reading = previous.catch(() => undefined).then(async () => {
|
|
145
150
|
if (!isCurrent()) return { result: { kind: "terminal", reason: "request-obsolete", requestId: params.requestId } };
|
|
@@ -149,7 +154,7 @@ export class ContainerOrchestrator {
|
|
|
149
154
|
params.onTimed?.(statement, startedAt - askedAt, Date.now() - startedAt);
|
|
150
155
|
return { result, revision };
|
|
151
156
|
});
|
|
152
|
-
this.#reads.set(key, reading);
|
|
157
|
+
if (!aside) this.#reads.set(key, reading);
|
|
153
158
|
try {
|
|
154
159
|
const { result, revision } = await reading;
|
|
155
160
|
if (!isCurrent()) return { kind: "terminal", reason: "request-obsolete", requestId: params.requestId };
|
|
@@ -162,6 +167,12 @@ export class ContainerOrchestrator {
|
|
|
162
167
|
}
|
|
163
168
|
}
|
|
164
169
|
|
|
170
|
+
/** Whether a statement of this file is answered from what was already read: its track table kept here, or the container's word. */
|
|
171
|
+
#answersWithoutReading(key, statement) {
|
|
172
|
+
if (statement === "tracks" && this.tracks.has(key)) return true;
|
|
173
|
+
return this.cache.get(key)?.answersWithoutReading?.(statement) === true;
|
|
174
|
+
}
|
|
175
|
+
|
|
165
176
|
async #inspectRead(params, statement) {
|
|
166
177
|
const requestId = params.requestId ?? `${params.sourceKey}:${params.fileIndex}:${statement}`;
|
|
167
178
|
this.#activityEpoch++;
|
|
@@ -42,6 +42,8 @@ export class AviContainer extends Container {
|
|
|
42
42
|
#workTags = null;
|
|
43
43
|
/** The index alone: undefined until read, null when the file states none. */
|
|
44
44
|
#index = undefined;
|
|
45
|
+
/** The index while it is being read, so two asks read it once. */
|
|
46
|
+
#indexReading = null;
|
|
45
47
|
#layout = null;
|
|
46
48
|
/** The first RIFF's `idx1`: undefined until looked for, null when there is none. */
|
|
47
49
|
#idx1Chunk = undefined;
|
|
@@ -167,6 +169,38 @@ export class AviContainer extends Container {
|
|
|
167
169
|
*/
|
|
168
170
|
async #readIndex() {
|
|
169
171
|
if (this.#index !== undefined) return this.#index;
|
|
172
|
+
// One reading for every asker: a read of the picture's ranges may now be
|
|
173
|
+
// answered beside the sound's table (`answersWithoutReading`), and two
|
|
174
|
+
// asks at once would each read the index and each hold its memory.
|
|
175
|
+
this.#indexReading ??= this.#readIndexOnce().finally(() => { this.#indexReading = null; });
|
|
176
|
+
return this.#indexReading;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Whether `statement` is answered from what this container has already
|
|
181
|
+
* read, without reading or parsing anything more — so it may be answered
|
|
182
|
+
* beside a read of the same file still in progress
|
|
183
|
+
* (`ContainerOrchestrator.inspect`). Field r38 (Home Assistant, 2026-10-14):
|
|
184
|
+
* the picture's first input waited about 4 s behind the sound's packet table
|
|
185
|
+
* of the whole film, which it does not use — its ranges are named from the
|
|
186
|
+
* index alone (`readSourceRanges`), while the table is the sound's
|
|
187
|
+
* (`readPacketIndex`), built in portions that give the thread back
|
|
188
|
+
* (torrent-tv/meta#178).
|
|
189
|
+
*
|
|
190
|
+
* Once the index has been read, so have the tracks and the place of `movi`
|
|
191
|
+
* it was read with, and the picture's statements read nothing else.
|
|
192
|
+
*
|
|
193
|
+
* @param {string} statement
|
|
194
|
+
* @returns {boolean}
|
|
195
|
+
*/
|
|
196
|
+
answersWithoutReading(statement) {
|
|
197
|
+
if (statement === "media-info") return Boolean(this.mediaInfo);
|
|
198
|
+
if (statement === "source-navigation" || statement === "source-ranges") return this.#index !== undefined;
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The index, read from the file; see `#readIndex`. */
|
|
203
|
+
async #readIndexOnce() {
|
|
170
204
|
const tracks = await this.readTracks();
|
|
171
205
|
const { streams } = await this.#readHeaders();
|
|
172
206
|
const { movi } = await this.#topLevel();
|
|
@@ -46,6 +46,8 @@ export class SwarmSelection {
|
|
|
46
46
|
#stated = new Map();
|
|
47
47
|
/** Claimants whose windows are currently protected in memory. */
|
|
48
48
|
#protectedInMemory = new Set();
|
|
49
|
+
/** The pieces of the urgent first class, as last stated: what somebody waits for now. */
|
|
50
|
+
#required = [];
|
|
49
51
|
#findStore;
|
|
50
52
|
|
|
51
53
|
/**
|
|
@@ -110,7 +112,10 @@ export class SwarmSelection {
|
|
|
110
112
|
else union.push({ from: range.from, to: range.to, priority });
|
|
111
113
|
}
|
|
112
114
|
for (const range of union) wanted.set(`${range.from}-${range.to}-${priority}`, range);
|
|
113
|
-
if (priority === 1)
|
|
115
|
+
if (priority === 1) {
|
|
116
|
+
this.firstClassPieces = union.map(range => range.from === range.to ? `${range.from}` : `${range.from}-${range.to}`);
|
|
117
|
+
this.#required = strictFirstClass ? union.map(({ from, to }) => ({ from, to })) : [];
|
|
118
|
+
}
|
|
114
119
|
}
|
|
115
120
|
this.firstClass = first;
|
|
116
121
|
|
|
@@ -150,8 +155,21 @@ export class SwarmSelection {
|
|
|
150
155
|
return { stated, withdrawn };
|
|
151
156
|
}
|
|
152
157
|
|
|
158
|
+
/**
|
|
159
|
+
* The pieces of the urgent first class as last stated, or none while the
|
|
160
|
+
* first class is conditional: what somebody is waiting for now, and the
|
|
161
|
+
* only pieces whose last blocks are worth asking of a second peer
|
|
162
|
+
* (`TailRepeat`).
|
|
163
|
+
*
|
|
164
|
+
* @returns {Array<{ from: number, to: number }>}
|
|
165
|
+
*/
|
|
166
|
+
requiredPieces() {
|
|
167
|
+
return this.#required;
|
|
168
|
+
}
|
|
169
|
+
|
|
153
170
|
/** Take everything back. The torrent is going, or nobody wants anything. */
|
|
154
171
|
releaseAll() {
|
|
172
|
+
this.#required = [];
|
|
155
173
|
for (const range of this.#stated.values()) {
|
|
156
174
|
this.#deselect(range);
|
|
157
175
|
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file When each piece was asked of the swarm, and how long its blocks were
|
|
3
|
+
* in flight (torrent-tv/meta#178).
|
|
4
|
+
*
|
|
5
|
+
* The order in which pieces were verified cannot say why one came last: it
|
|
6
|
+
* may have been asked for late, or asked for early with its last blocks held
|
|
7
|
+
* by slow peers, and the two need different fixes. Field 2026-10-14: the
|
|
8
|
+
* first input of a 1080p MKV waited 18.7 s for pieces 1 and 2, verified after
|
|
9
|
+
* pieces 3-33. WebTorrent states no outgoing request, so the torrent pool
|
|
10
|
+
* tells this record each block it asks and each block that arrives; the
|
|
11
|
+
* record answers once, when the piece is verified, and then forgets it.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @typedef {object} PieceTiming
|
|
16
|
+
* @property {number} askedAgoMs - From the first block asked to the verification.
|
|
17
|
+
* @property {number} asks - Blocks asked, duplicates included.
|
|
18
|
+
* @property {number} peers - Distinct peers asked.
|
|
19
|
+
* @property {number} slowestMs - The longest a block was in flight, from its first ask to its arrival.
|
|
20
|
+
* @property {string | null} slowestPeer - Who sent that block.
|
|
21
|
+
* @property {number} lastInFlightMs - How long the block that completed the piece was in flight.
|
|
22
|
+
* @property {string | null} lastPeer - Who sent it.
|
|
23
|
+
* @property {number} askedAgain - Asks of a block already asked of another peer.
|
|
24
|
+
* @property {number} arrivedTwice - Blocks that arrived after a copy of them had, and were dropped.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
export class PieceTimings {
|
|
28
|
+
/** @type {Map<number, { firstAskAt: number, asks: number, peers: Set<string>, blocks: Map<number, number>, arrived: Set<number>, askedAgain: number, arrivedTwice: number, slowestMs: number, slowestPeer: string | null, lastInFlightMs: number, lastPeer: string | null }>} */
|
|
29
|
+
#pieces = new Map();
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A block of `piece` at `offset` was asked of `peer`.
|
|
33
|
+
*
|
|
34
|
+
* @param {number} piece
|
|
35
|
+
* @param {number} offset
|
|
36
|
+
* @param {string} peer
|
|
37
|
+
* @param {number} now
|
|
38
|
+
*/
|
|
39
|
+
asked(piece, offset, peer, now) {
|
|
40
|
+
let record = this.#pieces.get(piece);
|
|
41
|
+
if (!record) {
|
|
42
|
+
record = { firstAskAt: now, asks: 0, peers: new Set(), blocks: new Map(), arrived: new Set(), askedAgain: 0, arrivedTwice: 0,
|
|
43
|
+
slowestMs: 0, slowestPeer: null, lastInFlightMs: 0, lastPeer: null };
|
|
44
|
+
this.#pieces.set(piece, record);
|
|
45
|
+
}
|
|
46
|
+
record.asks += 1;
|
|
47
|
+
record.peers.add(peer);
|
|
48
|
+
if (!record.blocks.has(offset)) record.blocks.set(offset, now);
|
|
49
|
+
else record.askedAgain += 1;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A block of `piece` at `offset` arrived from `peer`. A block never asked
|
|
54
|
+
* through this record — it is not one of ours — says nothing.
|
|
55
|
+
*
|
|
56
|
+
* @param {number} piece
|
|
57
|
+
* @param {number} offset
|
|
58
|
+
* @param {string} peer
|
|
59
|
+
* @param {number} now
|
|
60
|
+
*/
|
|
61
|
+
arrived(piece, offset, peer, now) {
|
|
62
|
+
const record = this.#pieces.get(piece);
|
|
63
|
+
const askedAt = record?.blocks.get(offset);
|
|
64
|
+
if (askedAt === undefined) return;
|
|
65
|
+
// A copy of a block that had already arrived is dropped by the piece; its
|
|
66
|
+
// bytes are what asking a second peer cost.
|
|
67
|
+
if (record.arrived.has(offset)) {
|
|
68
|
+
record.arrivedTwice += 1;
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
record.arrived.add(offset);
|
|
72
|
+
const inFlightMs = now - askedAt;
|
|
73
|
+
if (inFlightMs >= record.slowestMs) {
|
|
74
|
+
record.slowestMs = inFlightMs;
|
|
75
|
+
record.slowestPeer = peer;
|
|
76
|
+
}
|
|
77
|
+
record.lastInFlightMs = inFlightMs;
|
|
78
|
+
record.lastPeer = peer;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The timing of `piece`, verified at `now`, or null when no block of it was
|
|
83
|
+
* asked through this record; forgotten either way.
|
|
84
|
+
*
|
|
85
|
+
* @param {number} piece
|
|
86
|
+
* @param {number} now
|
|
87
|
+
* @returns {PieceTiming | null}
|
|
88
|
+
*/
|
|
89
|
+
verified(piece, now) {
|
|
90
|
+
const record = this.#pieces.get(piece);
|
|
91
|
+
this.#pieces.delete(piece);
|
|
92
|
+
if (!record) return null;
|
|
93
|
+
return {
|
|
94
|
+
askedAgoMs: now - record.firstAskAt,
|
|
95
|
+
asks: record.asks,
|
|
96
|
+
peers: record.peers.size,
|
|
97
|
+
slowestMs: record.slowestMs,
|
|
98
|
+
slowestPeer: record.slowestPeer,
|
|
99
|
+
lastInFlightMs: record.lastInFlightMs,
|
|
100
|
+
lastPeer: record.lastPeer,
|
|
101
|
+
askedAgain: record.askedAgain,
|
|
102
|
+
arrivedTwice: record.arrivedTwice
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The timing said after a verified piece.
|
|
109
|
+
*
|
|
110
|
+
* @param {PieceTiming | null} timing
|
|
111
|
+
* @returns {string}
|
|
112
|
+
*/
|
|
113
|
+
export function describePieceTiming(timing) {
|
|
114
|
+
if (!timing) return "";
|
|
115
|
+
return `; first block asked ${(timing.askedAgoMs / 1000).toFixed(1)}s before, ${timing.asks} block ask(s) to ${timing.peers} peer(s), ` +
|
|
116
|
+
`slowest block ${(timing.slowestMs / 1000).toFixed(1)}s in flight from ${timing.slowestPeer ?? "?"}, ` +
|
|
117
|
+
`the last one ${(timing.lastInFlightMs / 1000).toFixed(1)}s from ${timing.lastPeer ?? "?"}` +
|
|
118
|
+
(timing.askedAgain > 0 || timing.arrivedTwice > 0
|
|
119
|
+
? `, ${timing.askedAgain} block(s) asked again, ${timing.arrivedTwice} arrived twice` : "");
|
|
120
|
+
}
|
|
@@ -24,6 +24,23 @@ import { withdrawClaim } from "./download/withdraw-claim.js";
|
|
|
24
24
|
import { deadlineOfMapZone, levelOfMapZone, priorityOfMapZone } from "./map-zones.js";
|
|
25
25
|
import { deriveSourceKey } from "../../utils/torrent-source-key.js";
|
|
26
26
|
import { noteTorrentDestroyed } from "./destroyed-torrents.js";
|
|
27
|
+
import { PieceTimings, describePieceTiming } from "./piece-timing.js";
|
|
28
|
+
import { TailRepeat } from "./worker/tail-repeat.js";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The second-peer requests for the last blocks of a piece somebody waits for,
|
|
32
|
+
* one per torrent, made when the torrent is first watched and stopped when it
|
|
33
|
+
* is forgotten (`worker/tail-repeat.js`).
|
|
34
|
+
*
|
|
35
|
+
* @type {WeakMap<object, TailRepeat>}
|
|
36
|
+
*/
|
|
37
|
+
const tailRepeats = new WeakMap();
|
|
38
|
+
|
|
39
|
+
/** Stop asking second peers for a torrent that is going. */
|
|
40
|
+
function dropTailRepeat(torrent) {
|
|
41
|
+
tailRepeats.get(torrent)?.dispose();
|
|
42
|
+
tailRepeats.delete(torrent);
|
|
43
|
+
}
|
|
27
44
|
|
|
28
45
|
/** How a window stated from the priority map names itself. */
|
|
29
46
|
const MAP_CLAIMANT = "priority-map";
|
|
@@ -1352,6 +1369,8 @@ export class TorrentPool {
|
|
|
1352
1369
|
// swarm went on fetching the old first class for 1.16 s, until the next
|
|
1353
1370
|
// piece happened to be verified (r34, torrent-tv/meta#178).
|
|
1354
1371
|
reconcileAll();
|
|
1372
|
+
// And a new map may make other pieces the ones somebody waits for.
|
|
1373
|
+
tailRepeats.get(torrent)?.poke();
|
|
1355
1374
|
// WHAT THE SWARM WAS ACTUALLY TOLD, said on change and never on a timer.
|
|
1356
1375
|
// The map was applied in silence: that it had been BUILT was visible in the
|
|
1357
1376
|
// encoding's own line, and that the download had received it was visible
|
|
@@ -1502,21 +1521,37 @@ export class TorrentPool {
|
|
|
1502
1521
|
const addedAt = Date.now();
|
|
1503
1522
|
this.#swarmTimingByTorrent.set(torrent, { addedAt, firstPeerAt: null });
|
|
1504
1523
|
|
|
1505
|
-
torrent.on("download-requested", ({ piece, deadlineAt, requestId, peer, queued }) => {
|
|
1506
|
-
logger.info(`download request=${requestId} piece=${piece} deadline=${deadlineAt} peer=${peer} outstanding=${queued}`);
|
|
1507
|
-
});
|
|
1508
1524
|
torrent.on("download-request-cancelled", ({ piece, offset, length, reason, requestId }) => {
|
|
1509
1525
|
logger.info(`download request=${requestId ?? `${torrent.infoHash ?? "unknown"}:${piece}:${offset}`} cancel bytes=${length} reason=${reason}`);
|
|
1510
1526
|
});
|
|
1511
1527
|
torrent.on("download-request-cancel-failed", ({ piece, offset, length, error }) => {
|
|
1512
1528
|
logger.warn(`download request=${torrent.infoHash ?? "unknown"}:${piece}:${offset} cancel bytes=${length} failed: ${error}`);
|
|
1513
1529
|
});
|
|
1514
|
-
// WHEN each piece arrived
|
|
1515
|
-
//
|
|
1516
|
-
//
|
|
1517
|
-
//
|
|
1530
|
+
// WHEN each piece arrived, which decides how far an encoder reading the
|
|
1531
|
+
// file can go before it meets a piece still missing (torrent-tv/meta#151),
|
|
1532
|
+
// and WHY it came when it did: when its first block was asked and how long
|
|
1533
|
+
// its blocks were in flight, and from whom (`piece-timing.js`). One line
|
|
1534
|
+
// per piece: a 2.3 GB film is 563.
|
|
1535
|
+
const timings = new PieceTimings();
|
|
1536
|
+
tailRepeats.set(torrent, new TailRepeat({ torrent, requiredPieces: () => demandFor(torrent).selection.requiredPieces(),
|
|
1537
|
+
log: (line) => logger.info(`download ${label()} ${line}`) }));
|
|
1538
|
+
torrent.on("wire", (wire) => {
|
|
1539
|
+
// WebTorrent asks a wire for a block by calling `request` and states
|
|
1540
|
+
// nothing; a request the wire refuses (choked, closed) is not queued and
|
|
1541
|
+
// not counted.
|
|
1542
|
+
if (typeof wire?.request !== "function") return;
|
|
1543
|
+
const peer = String(wire.remoteAddress ?? "?");
|
|
1544
|
+
const request = wire.request;
|
|
1545
|
+
wire.request = function (index, offset, length, callback) {
|
|
1546
|
+
const queued = this.requests.length;
|
|
1547
|
+
const answer = request.call(this, index, offset, length, callback);
|
|
1548
|
+
if (this.requests.length > queued) timings.asked(index, offset, peer, Date.now());
|
|
1549
|
+
return answer;
|
|
1550
|
+
};
|
|
1551
|
+
wire.on("piece", (index, offset) => timings.arrived(index, offset, peer, Date.now()));
|
|
1552
|
+
});
|
|
1518
1553
|
torrent.on("verified", (piece) => {
|
|
1519
|
-
logger.info(`download ${label()} verified piece=${piece}`);
|
|
1554
|
+
logger.info(`download ${label()} verified piece=${piece}${describePieceTiming(timings.verified(piece, Date.now()))}`);
|
|
1520
1555
|
});
|
|
1521
1556
|
|
|
1522
1557
|
// The first connected peer, said once, because the wait for it can BE the
|
|
@@ -1948,6 +1983,7 @@ export class TorrentPool {
|
|
|
1948
1983
|
return;
|
|
1949
1984
|
}
|
|
1950
1985
|
forgetTorrent(torrent);
|
|
1986
|
+
dropTailRepeat(torrent);
|
|
1951
1987
|
this.#forgetBookkeeping(torrent);
|
|
1952
1988
|
const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
|
|
1953
1989
|
logger.warn(
|
|
@@ -1993,6 +2029,7 @@ export class TorrentPool {
|
|
|
1993
2029
|
#removeTorrent(torrent, reason = "unknown") {
|
|
1994
2030
|
// Everything anybody stated for this torrent goes with it.
|
|
1995
2031
|
forgetTorrent(torrent);
|
|
2032
|
+
dropTailRepeat(torrent);
|
|
1996
2033
|
if (!torrent) {
|
|
1997
2034
|
return;
|
|
1998
2035
|
}
|
|
@@ -37,6 +37,9 @@
|
|
|
37
37
|
*/
|
|
38
38
|
const SHORT_TAIL_BLOCKS = 16;
|
|
39
39
|
|
|
40
|
+
/** A block of a piece, as `torrent-piece` cuts it (`Piece.BLOCK_LENGTH`): a request's offset over this is its block. */
|
|
41
|
+
const BLOCK_BYTES = 1 << 14;
|
|
42
|
+
|
|
40
43
|
/**
|
|
41
44
|
* The library's own request entry. Internal, so its absence must be noticed
|
|
42
45
|
* rather than swallowed: without it this lever silently does nothing.
|
|
@@ -248,10 +251,10 @@ export function describePieceTail(torrent, pieceIndex) {
|
|
|
248
251
|
* work, which no average speed can show.
|
|
249
252
|
*
|
|
250
253
|
* The cost is bounded by the same measurement: the tails were 2 to 14 blocks of
|
|
251
|
-
* 512, so 32 to 224 KB against the 8 MiB piece they hold up.
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
*
|
|
254
|
+
* 512, so 32 to 224 KB against the 8 MiB piece they hold up. A block already in
|
|
255
|
+
* flight on `maxCopies` wires is not asked again, however often this is called:
|
|
256
|
+
* it is called on every block that arrives (`TailRepeat`), and without that
|
|
257
|
+
* check each call asked one more copy of the same block.
|
|
255
258
|
*
|
|
256
259
|
* Safe against the completion path: a block that arrives twice is dropped by
|
|
257
260
|
* `Piece.set`, which writes only into an empty slot, and a piece already
|
|
@@ -260,17 +263,26 @@ export function describePieceTail(torrent, pieceIndex) {
|
|
|
260
263
|
*
|
|
261
264
|
* @param {object} torrent
|
|
262
265
|
* @param {number} pieceIndex
|
|
263
|
-
* @
|
|
266
|
+
* @param {object} [options]
|
|
267
|
+
* @param {number} [options.maxCopies] - How many wires a missing block may be
|
|
268
|
+
* in flight on at once: the first request and one copy beside it.
|
|
269
|
+
* @returns {{ duplicated: number, missing: number, wires: number, unplaced: number }}
|
|
264
270
|
* `duplicated` counts requests the library actually placed for a block that
|
|
265
|
-
* was already outstanding elsewhere
|
|
271
|
+
* was already outstanding elsewhere; `unplaced` the blocks in flight on one
|
|
272
|
+
* wire alone that no other wire could be asked for now.
|
|
266
273
|
*/
|
|
267
|
-
export function duplicateTailFor(torrent, pieceIndex) {
|
|
274
|
+
export function duplicateTailFor(torrent, pieceIndex, { maxCopies = 2 } = {}) {
|
|
268
275
|
const piece = torrent?.pieces?.[pieceIndex];
|
|
269
276
|
const buffer = Array.isArray(piece?._buffer) ? piece._buffer : null;
|
|
270
|
-
if (!piece || !buffer || typeof piece.cancel !== "function") {
|
|
271
|
-
return { duplicated: 0, missing: 0, wires: 0 };
|
|
277
|
+
if (!piece || !buffer || typeof piece.cancel !== "function" || !canPlaceRequests(torrent)) {
|
|
278
|
+
return { duplicated: 0, missing: 0, wires: 0, unplaced: 0 };
|
|
272
279
|
}
|
|
273
280
|
const chunks = Number.isFinite(piece._chunks) ? piece._chunks : buffer.length;
|
|
281
|
+
// Most calls are for a piece still arriving in bulk; its own count says so
|
|
282
|
+
// without walking its blocks or every wire's queue.
|
|
283
|
+
if (Number.isFinite(piece._buffered) && chunks - piece._buffered > SHORT_TAIL_BLOCKS) {
|
|
284
|
+
return { duplicated: 0, missing: chunks - piece._buffered, wires: 0, unplaced: 0 };
|
|
285
|
+
}
|
|
274
286
|
const missing = [];
|
|
275
287
|
for (let index = 0; index < chunks; index += 1) {
|
|
276
288
|
if (!buffer[index]) {
|
|
@@ -279,7 +291,7 @@ export function duplicateTailFor(torrent, pieceIndex) {
|
|
|
279
291
|
}
|
|
280
292
|
const candidates = wiresForPiece(torrent, pieceIndex);
|
|
281
293
|
if (missing.length === 0 || candidates.length === 0) {
|
|
282
|
-
return { duplicated: 0, missing: missing.length, wires: candidates.length };
|
|
294
|
+
return { duplicated: 0, missing: missing.length, wires: candidates.length, unplaced: 0 };
|
|
283
295
|
}
|
|
284
296
|
|
|
285
297
|
// A tail this short is what a blocked read is actually waiting on, and a
|
|
@@ -288,31 +300,50 @@ export function duplicateTailFor(torrent, pieceIndex) {
|
|
|
288
300
|
// bytes that are already coming — measured 2026-08-19, the tails a reader
|
|
289
301
|
// waits on are 2 to 14 blocks of 512.
|
|
290
302
|
if (missing.length > SHORT_TAIL_BLOCKS) {
|
|
291
|
-
return { duplicated: 0, missing: missing.length, wires: candidates.length };
|
|
303
|
+
return { duplicated: 0, missing: missing.length, wires: candidates.length, unplaced: 0 };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Which wires each missing block is in flight on, read from the wires' own
|
|
307
|
+
// queues: the library keeps a block's reservation for one wire only, and a
|
|
308
|
+
// copy asked here is visible nowhere else.
|
|
309
|
+
const holders = new Map(missing.map(block => [block, new Set()]));
|
|
310
|
+
for (const wire of Array.isArray(torrent.wires) ? torrent.wires : []) {
|
|
311
|
+
for (const request of Array.isArray(wire?.requests) ? wire.requests : []) {
|
|
312
|
+
if (request?.piece !== pieceIndex) continue;
|
|
313
|
+
holders.get(Math.floor(request.offset / BLOCK_BYTES))?.add(wire);
|
|
314
|
+
}
|
|
292
315
|
}
|
|
293
316
|
|
|
294
|
-
let duplicated = 0;
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
//
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
317
|
+
let duplicated = 0, unplaced = 0;
|
|
318
|
+
for (const block of missing) {
|
|
319
|
+
const held = holders.get(block);
|
|
320
|
+
// Not in flight anywhere: the library's own picker asks for it; a block
|
|
321
|
+
// nobody has been asked for is not a tail held by a slow peer.
|
|
322
|
+
if (held.size === 0 || held.size >= maxCopies) continue;
|
|
323
|
+
let placed = false;
|
|
324
|
+
for (const wire of candidates) {
|
|
325
|
+
if (held.has(wire)) continue;
|
|
326
|
+
// Freeing the reservation is what makes the block askable again; the
|
|
327
|
+
// request already in flight is deliberately left alone, because the
|
|
328
|
+
// point is to have two of them.
|
|
329
|
+
piece.cancel(block);
|
|
330
|
+
// `false` is hotswap: the library's own swap must not run on top of
|
|
331
|
+
// this, or it would free a THIRD wire's block as well.
|
|
332
|
+
if (torrent._request(wire, pieceIndex, false) === true) {
|
|
333
|
+
held.add(wire);
|
|
334
|
+
duplicated += 1;
|
|
335
|
+
placed = true;
|
|
336
|
+
break;
|
|
337
|
+
}
|
|
338
|
+
// Refused before anything was reserved (that wire's pipeline is full):
|
|
339
|
+
// the freed block is taken back, or the library would hand it to the
|
|
340
|
+
// next wire that asks, a copy nobody counted. The next wire's pipeline
|
|
341
|
+
// is its own, so it is still asked.
|
|
342
|
+
piece.reserve();
|
|
306
343
|
}
|
|
307
|
-
|
|
308
|
-
// next one's — pipelines are per wire. This used to stop the whole pass on
|
|
309
|
-
// the first refusal, on the stated reasoning that the remaining wires were
|
|
310
|
-
// "no emptier", which is an assumption about other peers' queues that
|
|
311
|
-
// nothing here measures. The block whose reservation was freed stays in the
|
|
312
|
-
// piece's cancellation stack and is handed to whoever asks next, which is
|
|
313
|
-
// harmless: it is already in flight elsewhere.
|
|
344
|
+
if (!placed) unplaced += 1;
|
|
314
345
|
}
|
|
315
|
-
return { duplicated, missing: missing.length, wires: candidates.length };
|
|
346
|
+
return { duplicated, missing: missing.length, wires: candidates.length, unplaced };
|
|
316
347
|
}
|
|
317
348
|
|
|
318
349
|
/**
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Ask a second peer for the last blocks of a piece somebody is waiting
|
|
3
|
+
* for, when one slow peer holds them (torrent-tv/meta#178).
|
|
4
|
+
*
|
|
5
|
+
* Field r40 (Home Assistant, 2026-10-14, a cold 1080p MKV): the first run's
|
|
6
|
+
* input waited 10.0 s, last for piece 8, whose slowest block was 4.5 s in
|
|
7
|
+
* flight from one peer; pieces 14-16 waited 10.0-11.5 s on one peer each. A
|
|
8
|
+
* block is reserved for one wire, and the piece completes when its slowest
|
|
9
|
+
* holder delivers (`duplicateTailFor`). Nothing had asked a second peer since
|
|
10
|
+
* the blocked-read path that did so was removed (`0e38ec3`, torrent-tv/meta#95).
|
|
11
|
+
*
|
|
12
|
+
* Which pieces: those of the urgent first class the swarm was last told
|
|
13
|
+
* (`SwarmSelection.requiredPieces`) — what somebody waits for now. A map that
|
|
14
|
+
* withdraws them, or an input that is ready, takes them out, and nothing more
|
|
15
|
+
* is asked for them.
|
|
16
|
+
*
|
|
17
|
+
* When: a tail changes when a block arrives, a wire's queue moves on, or a
|
|
18
|
+
* peer becomes able or unable to give the piece; so it is looked at on those —
|
|
19
|
+
* a block arriving, a wire unchoking, a peer stating a piece, a wire joining
|
|
20
|
+
* or closing, a piece verified, a new map. Several in one turn of the event
|
|
21
|
+
* loop are one look. A tail no other wire could be asked for now is looked at
|
|
22
|
+
* once more after the time a wire's queue is sized to empty in — WebTorrent
|
|
23
|
+
* sizes it to one second of the wire's speed (`PIPELINE_MAX_DURATION`) — in
|
|
24
|
+
* case nothing else happens in between.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { duplicateTailFor } from "./fastest-wires.js";
|
|
28
|
+
|
|
29
|
+
/** WebTorrent's `PIPELINE_MAX_DURATION`: a wire is given one second of its speed to fetch. */
|
|
30
|
+
const WIRE_QUEUE_SECONDS = 1;
|
|
31
|
+
|
|
32
|
+
/** What a wire says, besides a block arriving, that can change who could be asked for a piece. */
|
|
33
|
+
const WIRE_EVENTS = ["unchoke", "have", "bitfield"];
|
|
34
|
+
|
|
35
|
+
export class TailRepeat {
|
|
36
|
+
#torrent;
|
|
37
|
+
#requiredPieces;
|
|
38
|
+
#duplicate;
|
|
39
|
+
#schedule;
|
|
40
|
+
#wake;
|
|
41
|
+
#clear;
|
|
42
|
+
#scheduled = false;
|
|
43
|
+
#timer = null;
|
|
44
|
+
#disposed = false;
|
|
45
|
+
/** @type {Map<object, { onBlock: (index: number) => void, onPeer: () => void }>} */
|
|
46
|
+
#wires = new Map();
|
|
47
|
+
#onWire;
|
|
48
|
+
#onPoke;
|
|
49
|
+
#log;
|
|
50
|
+
/**
|
|
51
|
+
* Grow-only, for the log: block copies asked, looks taken, and blocks that
|
|
52
|
+
* arrived after their piece was complete — what the copies cost.
|
|
53
|
+
*/
|
|
54
|
+
totals = { duplicated: 0, looks: 0, lateBlocks: 0 };
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {object} params
|
|
58
|
+
* @param {object} params.torrent
|
|
59
|
+
* @param {() => Array<{ from: number, to: number }>} params.requiredPieces
|
|
60
|
+
* @param {typeof duplicateTailFor} [params.duplicate]
|
|
61
|
+
* @param {(work: () => void) => void} [params.schedule] - Runs work once this turn of the event loop is over.
|
|
62
|
+
* @param {(work: () => void, ms: number) => unknown} [params.wake]
|
|
63
|
+
* @param {(handle: unknown) => void} [params.clear]
|
|
64
|
+
* @param {(line: string) => void} [params.log]
|
|
65
|
+
*/
|
|
66
|
+
constructor({ torrent, requiredPieces, duplicate = duplicateTailFor, schedule = setImmediate, wake = setTimeout, clear = clearTimeout, log = null }) {
|
|
67
|
+
this.#log = log;
|
|
68
|
+
this.#torrent = torrent;
|
|
69
|
+
this.#requiredPieces = requiredPieces;
|
|
70
|
+
this.#duplicate = duplicate;
|
|
71
|
+
this.#schedule = schedule;
|
|
72
|
+
this.#wake = wake;
|
|
73
|
+
this.#clear = clear;
|
|
74
|
+
this.#onWire = (wire) => this.#watch(wire);
|
|
75
|
+
this.#onPoke = () => this.poke();
|
|
76
|
+
torrent.on?.("wire", this.#onWire);
|
|
77
|
+
torrent.on?.("verified", this.#onPoke);
|
|
78
|
+
for (const wire of Array.isArray(torrent.wires) ? torrent.wires : []) this.#watch(wire);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Look at the tails once this turn of the event loop is over. */
|
|
82
|
+
poke() {
|
|
83
|
+
if (this.#scheduled || this.#disposed) return;
|
|
84
|
+
this.#scheduled = true;
|
|
85
|
+
this.#schedule(() => this.#look());
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Stop watching: the torrent is going. */
|
|
89
|
+
dispose() {
|
|
90
|
+
this.#disposed = true;
|
|
91
|
+
this.#torrent.removeListener?.("wire", this.#onWire);
|
|
92
|
+
this.#torrent.removeListener?.("verified", this.#onPoke);
|
|
93
|
+
for (const wire of [...this.#wires.keys()]) this.#forget(wire);
|
|
94
|
+
if (this.#timer !== null) this.#clear(this.#timer);
|
|
95
|
+
this.#timer = null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
#watch(wire) {
|
|
99
|
+
if (!wire || this.#wires.has(wire) || typeof wire.on !== "function") return;
|
|
100
|
+
const listeners = {
|
|
101
|
+
onBlock: (index) => {
|
|
102
|
+
// A block of a piece already verified is a copy that came second; the
|
|
103
|
+
// block that completes a piece arrives before the piece is verified.
|
|
104
|
+
if (Number.isInteger(index) && this.#torrent.bitfield?.get?.(index)) this.totals.lateBlocks += 1;
|
|
105
|
+
this.poke();
|
|
106
|
+
},
|
|
107
|
+
onPeer: () => this.poke()
|
|
108
|
+
};
|
|
109
|
+
this.#wires.set(wire, listeners);
|
|
110
|
+
wire.on("piece", listeners.onBlock);
|
|
111
|
+
for (const event of WIRE_EVENTS) wire.on(event, listeners.onPeer);
|
|
112
|
+
wire.once?.("close", () => {
|
|
113
|
+
this.#forget(wire);
|
|
114
|
+
this.poke();
|
|
115
|
+
});
|
|
116
|
+
this.poke();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
#forget(wire) {
|
|
120
|
+
const listeners = this.#wires.get(wire);
|
|
121
|
+
if (!listeners) return;
|
|
122
|
+
wire.removeListener?.("piece", listeners.onBlock);
|
|
123
|
+
for (const event of WIRE_EVENTS) wire.removeListener?.(event, listeners.onPeer);
|
|
124
|
+
this.#wires.delete(wire);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
#look() {
|
|
128
|
+
this.#scheduled = false;
|
|
129
|
+
if (this.#disposed) return;
|
|
130
|
+
this.totals.looks += 1;
|
|
131
|
+
let unplaced = 0;
|
|
132
|
+
const asked = [];
|
|
133
|
+
for (const { from, to } of this.#requiredPieces()) {
|
|
134
|
+
for (let index = from; index <= to; index += 1) {
|
|
135
|
+
if (this.#torrent.bitfield?.get?.(index)) continue;
|
|
136
|
+
const result = this.#duplicate(this.#torrent, index);
|
|
137
|
+
this.totals.duplicated += result.duplicated;
|
|
138
|
+
unplaced += result.unplaced;
|
|
139
|
+
if (result.duplicated > 0) asked.push(`${index}:${result.duplicated}/${result.missing}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
if (asked.length > 0) {
|
|
143
|
+
this.#log?.(`tail: asked a second peer for the last blocks of piece:blocks/missing ${asked.join(" ")}; ` +
|
|
144
|
+
`${this.totals.duplicated} block copies asked so far, ${this.totals.lateBlocks} block(s) arrived after their piece was complete`);
|
|
145
|
+
}
|
|
146
|
+
if (unplaced > 0 && this.#timer === null) {
|
|
147
|
+
this.#timer = this.#wake(() => {
|
|
148
|
+
this.#timer = null;
|
|
149
|
+
this.poke();
|
|
150
|
+
}, WIRE_QUEUE_SECONDS * 1000);
|
|
151
|
+
this.#timer?.unref?.();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|