@torrent-tv/proxy 2.69.2 → 2.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/CLAUDE.md +12 -0
- package/docs/download-architecture.md +175 -0
- package/docs/logs.md +8 -0
- package/package.json +1 -1
- package/services/demand/DemandRegister.js +182 -0
- package/services/demand/Urgency.js +137 -0
- package/services/demand/Window.js +118 -0
- package/services/demand/index.js +11 -0
- package/services/demand/pieces.js +140 -0
- package/services/download/SwarmSelection.js +313 -0
- package/services/download/index.js +8 -0
- package/services/download/registry.js +96 -0
- package/services/piece-store/piece-lru.js +30 -0
- package/services/piece-store/shared-piece-store.js +494 -34
- package/services/torrent-pool.js +64 -197
- package/services/torrent-worker/fastest-wires.js +319 -279
- package/services/torrent-worker/piece-reader.js +146 -178
- package/services/torrent-worker/worker.js +75 -14
- package/test/demand-register.test.js +195 -0
- package/test/fastest-wires.test.js +23 -1
- package/test/memory-budget.test.js +73 -29
- package/test/piece-store-eviction.test.js +220 -0
- package/test/read-bands.test.js +17 -10
- package/test/read-window.test.js +20 -8
- package/test/swarm-selection.test.js +191 -0
- package/utils/logger.js +62 -20
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What is wanted, by whom, and how urgently.
|
|
3
|
+
*
|
|
4
|
+
* These are the rules the download layer and the memory store both read, so a
|
|
5
|
+
* defect here is a defect in two places at once. Everything is numbers: no
|
|
6
|
+
* torrent, no library, no piece store.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import test from "node:test";
|
|
10
|
+
import assert from "node:assert/strict";
|
|
11
|
+
|
|
12
|
+
import { DemandRegister } from "../services/demand/DemandRegister.js";
|
|
13
|
+
import { unionOf, Window } from "../services/demand/Window.js";
|
|
14
|
+
import {
|
|
15
|
+
isConditional,
|
|
16
|
+
mayDisplaceSlowPeer,
|
|
17
|
+
selectionPriority,
|
|
18
|
+
Urgency
|
|
19
|
+
} from "../services/demand/Urgency.js";
|
|
20
|
+
import { nearestFirst, piecesNeededFor, piecesOf, piecesWithin } from "../services/demand/pieces.js";
|
|
21
|
+
|
|
22
|
+
const MEGABYTE = 1024 * 1024;
|
|
23
|
+
|
|
24
|
+
test("a window states bytes and refuses what it cannot mean", () => {
|
|
25
|
+
const window = new Window({
|
|
26
|
+
claimant: "video", fileIndex: 0, byteStart: 100, byteEnd: 199, urgency: Urgency.NEAR
|
|
27
|
+
});
|
|
28
|
+
assert.equal(window.byteLength, 100, "inclusive at both ends");
|
|
29
|
+
|
|
30
|
+
assert.throws(() => new Window({
|
|
31
|
+
claimant: "", fileIndex: 0, byteStart: 0, byteEnd: 1, urgency: 0
|
|
32
|
+
}), /claimant/, "a window nobody can release is a leak with a name missing");
|
|
33
|
+
assert.throws(() => new Window({
|
|
34
|
+
claimant: "video", fileIndex: 0, byteStart: 200, byteEnd: 100, urgency: 0
|
|
35
|
+
}), /Byte end/);
|
|
36
|
+
assert.throws(() => new Window({
|
|
37
|
+
claimant: "video", fileIndex: -1, byteStart: 0, byteEnd: 1, urgency: 0
|
|
38
|
+
}), /File index/);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("windows of different files are never merged", () => {
|
|
42
|
+
const register = new DemandRegister();
|
|
43
|
+
register.state({ claimant: "a", fileIndex: 0, byteStart: 0, byteEnd: 99, urgency: Urgency.NEAR });
|
|
44
|
+
register.state({ claimant: "b", fileIndex: 1, byteStart: 0, byteEnd: 99, urgency: Urgency.NEAR });
|
|
45
|
+
|
|
46
|
+
// Byte 50 of file 0 and byte 50 of file 1 are different bytes. Two viewers on
|
|
47
|
+
// two episodes of one release is the ordinary case, not an edge one.
|
|
48
|
+
assert.deepEqual(register.union(), [
|
|
49
|
+
{ byteStart: 0, byteEnd: 99 },
|
|
50
|
+
{ byteStart: 0, byteEnd: 99 }
|
|
51
|
+
]);
|
|
52
|
+
assert.deepEqual(register.files(), [0, 1]);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("the union of overlapping windows is their union, not their sum", () => {
|
|
56
|
+
// Picture and sound are two readers of one file and their windows overlap by
|
|
57
|
+
// construction. Adding them would report a demand that is not there.
|
|
58
|
+
assert.deepEqual(
|
|
59
|
+
unionOf([
|
|
60
|
+
{ byteStart: 0, byteEnd: 99 },
|
|
61
|
+
{ byteStart: 50, byteEnd: 149 }
|
|
62
|
+
]),
|
|
63
|
+
[{ byteStart: 0, byteEnd: 149 }]
|
|
64
|
+
);
|
|
65
|
+
// Touching ranges are one run: 0-99 and 100-199 have nothing between them.
|
|
66
|
+
assert.deepEqual(
|
|
67
|
+
unionOf([
|
|
68
|
+
{ byteStart: 100, byteEnd: 199 },
|
|
69
|
+
{ byteStart: 0, byteEnd: 99 }
|
|
70
|
+
]),
|
|
71
|
+
[{ byteStart: 0, byteEnd: 199 }]
|
|
72
|
+
);
|
|
73
|
+
assert.deepEqual(
|
|
74
|
+
unionOf([
|
|
75
|
+
{ byteStart: 0, byteEnd: 99 },
|
|
76
|
+
{ byteStart: 200, byteEnd: 299 }
|
|
77
|
+
]),
|
|
78
|
+
[{ byteStart: 0, byteEnd: 99 }, { byteStart: 200, byteEnd: 299 }],
|
|
79
|
+
"a gap between them is a gap"
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("a claimant states one window, and restating replaces it", () => {
|
|
84
|
+
const register = new DemandRegister();
|
|
85
|
+
register.state({ claimant: "video", fileIndex: 0, byteStart: 0, byteEnd: 99, urgency: Urgency.NEAR });
|
|
86
|
+
register.state({ claimant: "video", fileIndex: 0, byteStart: 100, byteEnd: 199, urgency: Urgency.NEAR });
|
|
87
|
+
|
|
88
|
+
// A reader walking a film restates its window many times a second. If those
|
|
89
|
+
// accumulated, the download set would be the whole file within a minute.
|
|
90
|
+
assert.equal(register.size, 1);
|
|
91
|
+
assert.deepEqual(register.union(0), [{ byteStart: 100, byteEnd: 199 }]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("a withdrawal names its own claimant and nothing else", () => {
|
|
95
|
+
const register = new DemandRegister();
|
|
96
|
+
register.state({ claimant: "video", fileIndex: 0, byteStart: 0, byteEnd: 99, urgency: Urgency.NEAR });
|
|
97
|
+
register.state({ claimant: "audio", fileIndex: 0, byteStart: 0, byteEnd: 99, urgency: Urgency.NEAR });
|
|
98
|
+
|
|
99
|
+
assert.equal(register.withdraw("audio"), true);
|
|
100
|
+
assert.equal(register.withdraw("audio"), false, "a second release withdraws nothing");
|
|
101
|
+
assert.equal(register.withdraw("nobody"), false, "a stray release matches nothing");
|
|
102
|
+
assert.equal(register.size, 1, "and video still wants what it wanted");
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("the speculative levels are stated only while nothing urgent is missing", () => {
|
|
106
|
+
const register = new DemandRegister();
|
|
107
|
+
register.state({ claimant: "video", fileIndex: 0, byteStart: 0, byteEnd: 99, urgency: Urgency.BLOCKED });
|
|
108
|
+
register.state({ claimant: "fill", fileIndex: 0, byteStart: 1000, byteEnd: 9999, urgency: Urgency.TAIL });
|
|
109
|
+
register.state({ claimant: "back", fileIndex: 0, byteStart: 0, byteEnd: 999, urgency: Urgency.BEHIND });
|
|
110
|
+
|
|
111
|
+
// Something urgent is missing: the speculative levels are not stated at all.
|
|
112
|
+
// Withdrawn, not lowered — a withdrawn window is not in the download set, so
|
|
113
|
+
// a peer with nothing urgent to give cannot fall through to it and spend the
|
|
114
|
+
// shared link on a piece nobody is waiting for.
|
|
115
|
+
assert.deepEqual(
|
|
116
|
+
register.levelsToState((window) => window.urgency !== Urgency.BLOCKED),
|
|
117
|
+
[Urgency.BLOCKED, Urgency.NEAR, Urgency.AHEAD]
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
// Everything urgent has arrived: they are stated, in order.
|
|
121
|
+
assert.deepEqual(
|
|
122
|
+
register.levelsToState(() => true),
|
|
123
|
+
[Urgency.BLOCKED, Urgency.NEAR, Urgency.AHEAD, Urgency.TAIL, Urgency.BEHIND]
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
// The tail is complete but the gap behind is not: the gap stays stated and
|
|
127
|
+
// nothing below it exists to withhold.
|
|
128
|
+
assert.deepEqual(
|
|
129
|
+
register.levelsToState((window) => window.urgency !== Urgency.BEHIND),
|
|
130
|
+
[Urgency.BLOCKED, Urgency.NEAR, Urgency.AHEAD, Urgency.TAIL, Urgency.BEHIND]
|
|
131
|
+
);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("only the level being waited on may take a block from a slow peer", () => {
|
|
135
|
+
assert.equal(mayDisplaceSlowPeer(Urgency.BLOCKED), true);
|
|
136
|
+
for (const urgency of [Urgency.NEAR, Urgency.AHEAD, Urgency.TAIL, Urgency.BEHIND]) {
|
|
137
|
+
assert.equal(mayDisplaceSlowPeer(urgency), false);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("the library is given the only distinction it keeps", () => {
|
|
142
|
+
// Measured against the vendored 2.8.5: distinct non-zero priorities order the
|
|
143
|
+
// selection list once and are round-robined afterwards by `shufflePriority`.
|
|
144
|
+
// Zero is the only value that stays put, always last.
|
|
145
|
+
assert.equal(selectionPriority(Urgency.BLOCKED), 1);
|
|
146
|
+
assert.equal(selectionPriority(Urgency.NEAR), 1);
|
|
147
|
+
assert.equal(selectionPriority(Urgency.AHEAD), 1);
|
|
148
|
+
assert.equal(selectionPriority(Urgency.TAIL), 0);
|
|
149
|
+
assert.equal(selectionPriority(Urgency.BEHIND), 0);
|
|
150
|
+
|
|
151
|
+
assert.equal(isConditional(Urgency.AHEAD), false);
|
|
152
|
+
assert.equal(isConditional(Urgency.TAIL), true);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("bytes become pieces in one place, and a partial piece is a whole piece", () => {
|
|
156
|
+
const pieceLength = 4 * MEGABYTE;
|
|
157
|
+
// A file starting one piece into the torrent, wanting its first byte.
|
|
158
|
+
assert.deepEqual(
|
|
159
|
+
piecesOf({ fileOffset: pieceLength, byteStart: 0, byteEnd: 0, pieceLength }),
|
|
160
|
+
{ from: 1, to: 1 }
|
|
161
|
+
);
|
|
162
|
+
// A range ending one byte into the next piece still needs that whole piece:
|
|
163
|
+
// the protocol delivers and verifies nothing smaller.
|
|
164
|
+
assert.deepEqual(
|
|
165
|
+
piecesOf({ fileOffset: 0, byteStart: 0, byteEnd: pieceLength, pieceLength }),
|
|
166
|
+
{ from: 0, to: 1 }
|
|
167
|
+
);
|
|
168
|
+
assert.equal(piecesOf({ fileOffset: 0, byteStart: 10, byteEnd: 5, pieceLength }), null);
|
|
169
|
+
assert.equal(piecesOf({ fileOffset: 0, byteStart: 0, byteEnd: 1, pieceLength: 0 }), null);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("a budget buys whole places, and a window needs one more than it measures", () => {
|
|
173
|
+
const pieceLength = 16 * MEGABYTE;
|
|
174
|
+
// The failure this rounding hid: 64 MB of allowance is four places, and two
|
|
175
|
+
// readers asking for 96 MB each want six. Counted in bytes the shortage is
|
|
176
|
+
// plain; counted in floored pieces it is not.
|
|
177
|
+
assert.equal(piecesWithin(64 * MEGABYTE, pieceLength), 4);
|
|
178
|
+
assert.equal(piecesNeededFor(96 * MEGABYTE, pieceLength), 7, "six, plus the one it can straddle");
|
|
179
|
+
assert.equal(piecesWithin(0, pieceLength), 0);
|
|
180
|
+
assert.equal(piecesNeededFor(0, pieceLength), 0);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("the gap behind the playhead is stated nearest first", () => {
|
|
184
|
+
// WebTorrent walks a selection from its start upwards, so one claim over
|
|
185
|
+
// everything behind the viewer would be fetched from the beginning of the
|
|
186
|
+
// file — the end furthest from where a backward seek lands.
|
|
187
|
+
const ranges = nearestFirst({ byteStart: 0, byteEnd: 999, parts: 4 });
|
|
188
|
+
assert.equal(ranges.length, 4);
|
|
189
|
+
assert.equal(ranges[0].byteEnd, 999, "the part nearest the playhead comes first");
|
|
190
|
+
assert.equal(ranges[ranges.length - 1].byteStart, 0, "and the furthest comes last");
|
|
191
|
+
for (const range of ranges) {
|
|
192
|
+
assert.ok(range.byteStart <= range.byteEnd);
|
|
193
|
+
}
|
|
194
|
+
assert.deepEqual(nearestFirst({ byteStart: 10, byteEnd: 5, parts: 2 }), []);
|
|
195
|
+
});
|
|
@@ -91,13 +91,35 @@ test("a refusal is counted as a refusal", () => {
|
|
|
91
91
|
const result = askFastestWiresFor(torrent, 11);
|
|
92
92
|
assert.equal(result.asked, 0);
|
|
93
93
|
assert.equal(result.considered, 2);
|
|
94
|
+
// Refused, but not because every block was spoken for — this stub has no
|
|
95
|
+
// reservations at all. The two reasons are different and only the second is
|
|
96
|
+
// the one WebTorrent's displacement thresholds decide.
|
|
97
|
+
assert.equal(result.refusedWhileReserved, 0);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("a refusal with every block already reserved is counted separately", () => {
|
|
101
|
+
// The interesting refusal: a fast peer we picked was turned away because
|
|
102
|
+
// every block belongs to somebody else and displacement did not happen. The
|
|
103
|
+
// thresholds that decide it are constants in the library — the asker above
|
|
104
|
+
// 16 KB/s, the holder below 48 KB/s and twice as slow — so a holder at
|
|
105
|
+
// 50 KB/s is never displaced however long the piece has waited. This number
|
|
106
|
+
// is what would justify replacing that rule.
|
|
107
|
+
const { torrent } = torrentWith([wire({ speed: 500 })], { refuse: true });
|
|
108
|
+
torrent.pieces = [];
|
|
109
|
+
torrent.pieces[11] = { reserve: () => -1, cancel: () => undefined };
|
|
110
|
+
const result = askFastestWiresFor(torrent, 11);
|
|
111
|
+
assert.equal(result.asked, 0);
|
|
112
|
+
assert.equal(result.refusedWhileReserved, 1);
|
|
94
113
|
});
|
|
95
114
|
|
|
96
115
|
test("a build without the request entry is reported, not silently skipped", () => {
|
|
97
116
|
assert.equal(canPlaceRequests({ wires: [] }), false);
|
|
98
117
|
assert.equal(canPlaceRequests({ wires: [], _request: () => true }), true);
|
|
99
118
|
const result = askFastestWiresFor({ wires: [wire({ speed: 1 })] }, 1);
|
|
100
|
-
assert.deepEqual(
|
|
119
|
+
assert.deepEqual(
|
|
120
|
+
result,
|
|
121
|
+
{ asked: 0, refusedWhileReserved: 0, attempted: 0, considered: 0, fastestBytesPerSecond: 0 }
|
|
122
|
+
);
|
|
101
123
|
});
|
|
102
124
|
|
|
103
125
|
test("a wire that cannot say how fast it is ranks last rather than throwing", () => {
|
|
@@ -8,9 +8,12 @@ import {
|
|
|
8
8
|
watchedFigures
|
|
9
9
|
} from "../services/memory-report.js";
|
|
10
10
|
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
divideAllowance,
|
|
12
|
+
forgetMachineMemory,
|
|
13
|
+
machineAllowanceBytes,
|
|
14
|
+
machineReserveBytes,
|
|
15
|
+
noteMachineMemory,
|
|
16
|
+
SharedPieceStore
|
|
14
17
|
} from "../services/piece-store/shared-piece-store.js";
|
|
15
18
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
16
19
|
import os from "node:os";
|
|
@@ -19,31 +22,69 @@ import path from "node:path";
|
|
|
19
22
|
const MEGABYTE = 1024 * 1024;
|
|
20
23
|
const GIGABYTE = 1024 * MEGABYTE;
|
|
21
24
|
|
|
22
|
-
test("the
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
assert.equal(
|
|
29
|
-
|
|
30
|
-
|
|
25
|
+
test("the stores are allowed what the machine has, less what others were seen to need", () => {
|
|
26
|
+
// Not a share of what is free. A share is a number chosen out of nothing, and
|
|
27
|
+
// the three that were here until 2026-09-02 — a quarter, a 64 MB floor, a
|
|
28
|
+
// 512 MB ceiling — all came from one observation of one host on 2026-08-03.
|
|
29
|
+
// `MemAvailable` is what can be taken ON TOP of what is held, so what the
|
|
30
|
+
// stores already hold is added back to get the ceiling they could reach.
|
|
31
|
+
assert.equal(
|
|
32
|
+
machineAllowanceBytes(2 * GIGABYTE, 300 * MEGABYTE, 0),
|
|
33
|
+
2 * GIGABYTE + 300 * MEGABYTE
|
|
34
|
+
);
|
|
35
|
+
assert.equal(
|
|
36
|
+
machineAllowanceBytes(2 * GIGABYTE, 300 * MEGABYTE, 500 * MEGABYTE),
|
|
37
|
+
2 * GIGABYTE - 200 * MEGABYTE
|
|
38
|
+
);
|
|
39
|
+
assert.equal(
|
|
40
|
+
machineAllowanceBytes(100 * MEGABYTE, 0, 4 * GIGABYTE),
|
|
41
|
+
0,
|
|
42
|
+
"a reserve larger than everything leaves nothing, and says so rather than going negative"
|
|
43
|
+
);
|
|
31
44
|
});
|
|
32
45
|
|
|
33
|
-
test("
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
//
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
assert.equal(
|
|
46
|
+
test("what to leave for everyone else is measured, starts at nothing, and ages out", () => {
|
|
47
|
+
// Field 2026-09-02: while this proxy held 76-133 MB the machine's available
|
|
48
|
+
// memory fell from 2378 MB to 306 MB and came back. That fall was somebody
|
|
49
|
+
// else's, and it is the quantity to leave room for. On a quiet host the same
|
|
50
|
+
// reading stays near zero, which is the right answer there.
|
|
51
|
+
forgetMachineMemory();
|
|
52
|
+
assert.equal(noteMachineMemory(2378 * MEGABYTE, 100 * MEGABYTE), 0, "nothing is reserved on faith");
|
|
53
|
+
|
|
54
|
+
// A fall of 2072 MB while the stores grew by 20 MB: 2052 MB of it was theirs.
|
|
55
|
+
assert.equal(noteMachineMemory(306 * MEGABYTE, 120 * MEGABYTE), 2052 * MEGABYTE);
|
|
56
|
+
|
|
57
|
+
// Memory coming back does not lower what has been seen to be needed — the
|
|
58
|
+
// spike can happen again, and that is what there has to be room for.
|
|
59
|
+
assert.equal(noteMachineMemory(3567 * MEGABYTE, 120 * MEGABYTE), 2052 * MEGABYTE);
|
|
60
|
+
|
|
61
|
+
// But it does not stand for ever either. A window of observations, not a
|
|
62
|
+
// high-water: one spike hours ago must stop squeezing the stores, which is
|
|
63
|
+
// the same mistake the block re-use gap would make with an all-time maximum.
|
|
64
|
+
for (let quiet = 0; quiet < 60; quiet += 1) {
|
|
65
|
+
noteMachineMemory(3567 * MEGABYTE, 120 * MEGABYTE);
|
|
66
|
+
}
|
|
67
|
+
assert.equal(machineReserveBytes(), 0, "an hour of quiet leaves the spike behind");
|
|
40
68
|
});
|
|
41
69
|
|
|
42
|
-
test("
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
|
|
46
|
-
assert.
|
|
70
|
+
test("everyone gets what they asked for while the asks fit", () => {
|
|
71
|
+
// The usual case by a wide margin: two readers of one film declare 32-192 MB
|
|
72
|
+
// between them against gigabytes of free memory, so the machine's limit never
|
|
73
|
+
// binds and the demand is what decides.
|
|
74
|
+
assert.deepEqual(
|
|
75
|
+
divideAllowance([100 * MEGABYTE, 50 * MEGABYTE], GIGABYTE),
|
|
76
|
+
[100 * MEGABYTE, 50 * MEGABYTE]
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
// When they do not fit, each is cut in proportion to what it asked, so a
|
|
80
|
+
// store wanting little is not cut to make room for one wanting much.
|
|
81
|
+
const cut = divideAllowance([300 * MEGABYTE, 100 * MEGABYTE], 200 * MEGABYTE);
|
|
82
|
+
assert.equal(cut[0], 150 * MEGABYTE);
|
|
83
|
+
assert.equal(cut[1], 50 * MEGABYTE);
|
|
84
|
+
assert.equal(cut[0] + cut[1], 200 * MEGABYTE);
|
|
85
|
+
|
|
86
|
+
assert.deepEqual(divideAllowance([], GIGABYTE), []);
|
|
87
|
+
assert.deepEqual(divideAllowance([0, 0], 0), [0, 0], "nobody asking takes nothing");
|
|
47
88
|
});
|
|
48
89
|
|
|
49
90
|
test("the memory line says bytes, and names what it could not measure", () => {
|
|
@@ -131,7 +172,7 @@ test("a thread's reading leaves out what belongs to the process", () => {
|
|
|
131
172
|
assert.doesNotMatch(line, /machine has/);
|
|
132
173
|
});
|
|
133
174
|
|
|
134
|
-
test("a store's allowance follows the machine,
|
|
175
|
+
test("a store's allowance follows the machine, in both directions", async () => {
|
|
135
176
|
// The defect: a store created on an idle machine kept an idle machine's
|
|
136
177
|
// allowance for life and went on growing into memory the host no longer had.
|
|
137
178
|
const directory = await mkdtemp(path.join(os.tmpdir(), "budget-revision-"));
|
|
@@ -149,11 +190,14 @@ test("a store's allowance follows the machine, and never passes its reservation"
|
|
|
149
190
|
assert.ok(lowered.ceilingBytes < born, "a busier machine buys fewer slots");
|
|
150
191
|
assert.equal(store.stats().budgetBytes, lowered.ceilingBytes, "the line says what is allowed now");
|
|
151
192
|
|
|
152
|
-
// The machine empties again: the ceiling
|
|
153
|
-
//
|
|
154
|
-
//
|
|
193
|
+
// The machine empties again: the ceiling rises with it, PAST where this
|
|
194
|
+
// store started. Until 2026-09-02 it could not — the ceiling was capped at
|
|
195
|
+
// a figure computed once in the constructor, so a torrent opened while the
|
|
196
|
+
// machine was full kept a small allowance for its whole life however much
|
|
197
|
+
// memory was freed afterwards.
|
|
155
198
|
const raised = store.reviseGrowthCeiling(1024 * 1024 * 1024);
|
|
156
|
-
assert.
|
|
199
|
+
assert.ok(raised.ceilingBytes > born, "an emptier machine buys more slots than it started with");
|
|
200
|
+
assert.equal(store.stats().budgetBytes, raised.ceilingBytes);
|
|
157
201
|
} finally {
|
|
158
202
|
await new Promise((resolve) => store.destroy(resolve));
|
|
159
203
|
await rm(directory, { recursive: true, force: true });
|
|
@@ -181,3 +181,223 @@ test("the store says why it spills: what is asked of it, what it had to take, ho
|
|
|
181
181
|
await fs.rm(directory, { recursive: true, force: true });
|
|
182
182
|
}
|
|
183
183
|
});
|
|
184
|
+
|
|
185
|
+
test("a piece nobody has declared does not push out one that is being read", async () => {
|
|
186
|
+
const capacity = 4;
|
|
187
|
+
const { store, directory } = await makeStore(capacity);
|
|
188
|
+
try {
|
|
189
|
+
// A reader declares the pieces it will need, and they are put in memory.
|
|
190
|
+
store.protectRange("video", 0, 3);
|
|
191
|
+
for (let index = 0; index < 4; index += 1) {
|
|
192
|
+
await put(store, index, pieceOf(index));
|
|
193
|
+
}
|
|
194
|
+
assert.equal(store.stats().resident, capacity, "the declared window fills the store");
|
|
195
|
+
|
|
196
|
+
// Now pieces arrive that the download fetched ahead of every reader. Before
|
|
197
|
+
// 2026-09-02 each of them claimed a slot and evicted one of the four above,
|
|
198
|
+
// which was then read back from disk moments later: 6565 spills and 7575
|
|
199
|
+
// revivals in 44 minutes, 53.6% of reads served from memory.
|
|
200
|
+
for (let index = 100; index < 110; index += 1) {
|
|
201
|
+
await put(store, index, pieceOf(index));
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const stats = store.stats();
|
|
205
|
+
assert.equal(stats.admittedToDisk, 10, "every undeclared arrival went straight to disk");
|
|
206
|
+
assert.equal(stats.admittedOutsideWindow, 10);
|
|
207
|
+
assert.equal(stats.admittedInsideWindow, 4);
|
|
208
|
+
assert.equal(stats.spills, 0, "and nothing had to be pushed out to make room");
|
|
209
|
+
|
|
210
|
+
// The declared pieces are still in memory, and every piece reads back as
|
|
211
|
+
// itself from whichever tier holds it.
|
|
212
|
+
assert.equal(stats.resident, capacity);
|
|
213
|
+
for (const index of [0, 1, 2, 3, 100, 105, 109]) {
|
|
214
|
+
const bytes = await get(store, index);
|
|
215
|
+
assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
|
|
216
|
+
}
|
|
217
|
+
} finally {
|
|
218
|
+
store.destroy(() => undefined);
|
|
219
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
test("before any reader has declared anything, an arriving piece still goes to memory", async () => {
|
|
224
|
+
const { store, directory } = await makeStore(4);
|
|
225
|
+
try {
|
|
226
|
+
// No window declared: there is no basis for calling a piece unwanted, so
|
|
227
|
+
// the store behaves as it always did. This is the initial download, and the
|
|
228
|
+
// warm-up fetches of the header and the tail.
|
|
229
|
+
for (let index = 0; index < 8; index += 1) {
|
|
230
|
+
await put(store, index, pieceOf(index));
|
|
231
|
+
}
|
|
232
|
+
const stats = store.stats();
|
|
233
|
+
assert.equal(stats.admittedToDisk, 0, "nothing was refused memory on a guess");
|
|
234
|
+
assert.equal(stats.admittedOutsideWindow, 8);
|
|
235
|
+
assert.ok(stats.spills > 0, "the store filled and evicted, as it did before");
|
|
236
|
+
} finally {
|
|
237
|
+
store.destroy(() => undefined);
|
|
238
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
test("the store asks for what its readers declared, and for a whole window at least", async () => {
|
|
243
|
+
const { store, directory } = await makeStore(64);
|
|
244
|
+
try {
|
|
245
|
+
// With nobody reading there is no demand to speak of, so the store asks for
|
|
246
|
+
// what it is already allowed and the first revision after a read begins
|
|
247
|
+
// brings it down.
|
|
248
|
+
const idle = store.wantedBytes;
|
|
249
|
+
assert.equal(idle, store.stats().budgetBytes);
|
|
250
|
+
|
|
251
|
+
// Two readers of one file — picture and sound — overlapping by
|
|
252
|
+
// construction. The ask is their union, not their sum.
|
|
253
|
+
store.protectRange("video", 10, 29);
|
|
254
|
+
store.protectRange("audio", 25, 44);
|
|
255
|
+
assert.equal(store.wantedBytes, 35 * PIECE, "10..44 is thirty-five pieces, not forty");
|
|
256
|
+
|
|
257
|
+
// One reader with a window wider than the union of nothing else: the ask
|
|
258
|
+
// never falls below a single window, or that reader's read cannot complete
|
|
259
|
+
// at all — every resident piece ends up pinned and the read returns zero
|
|
260
|
+
// bytes.
|
|
261
|
+
store.releaseProtection("audio");
|
|
262
|
+
store.releaseProtection("video");
|
|
263
|
+
store.protectRange("video", 0, 49);
|
|
264
|
+
assert.equal(store.wantedBytes, 50 * PIECE);
|
|
265
|
+
} finally {
|
|
266
|
+
store.destroy(() => undefined);
|
|
267
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("a block is re-used instead of a new one being allocated for every piece", async () => {
|
|
272
|
+
const capacity = 4;
|
|
273
|
+
const { store, directory } = await makeStore(capacity);
|
|
274
|
+
try {
|
|
275
|
+
// Twenty pieces through a store that may hold four. Before 2026-09-02 that
|
|
276
|
+
// was twenty allocations of a piece each, every one of them released only
|
|
277
|
+
// when the collector got to it — 7575 of them in 44 minutes in the field,
|
|
278
|
+
// and 1.86 GB held while the store's own accounting said 352 MB.
|
|
279
|
+
for (let index = 0; index < 20; index += 1) {
|
|
280
|
+
await put(store, index, pieceOf(index));
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const stats = store.stats();
|
|
284
|
+
assert.ok(
|
|
285
|
+
stats.blocksAllocated <= capacity,
|
|
286
|
+
`the pool never exceeds the allowance: ${stats.blocksAllocated} blocks for ${capacity} slots`
|
|
287
|
+
);
|
|
288
|
+
assert.equal(stats.committedBytes, stats.blocksAllocated * PIECE);
|
|
289
|
+
assert.equal(stats.returnedWhilePinned, 0);
|
|
290
|
+
assert.ok(stats.reuseGapMs !== null, "blocks were taken from the free list, not freshly made");
|
|
291
|
+
|
|
292
|
+
// And every piece still reads back as itself: a re-used block must not
|
|
293
|
+
// carry the last piece's bytes into the next one.
|
|
294
|
+
for (let index = 0; index < 20; index += 1) {
|
|
295
|
+
const bytes = await get(store, index);
|
|
296
|
+
assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
|
|
297
|
+
}
|
|
298
|
+
} finally {
|
|
299
|
+
store.destroy(() => undefined);
|
|
300
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
301
|
+
}
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
test("a spare block is given up once it has sat longer than the store's own working rhythm", async () => {
|
|
305
|
+
const { store, directory } = await makeStore(8);
|
|
306
|
+
try {
|
|
307
|
+
for (let index = 0; index < 12; index += 1) {
|
|
308
|
+
await put(store, index, pieceOf(index));
|
|
309
|
+
}
|
|
310
|
+
// The allowance falls: pieces are written out and their blocks fall spare.
|
|
311
|
+
store.reviseGrowthCeiling(2 * PIECE);
|
|
312
|
+
const spare = store.stats().blocksFree;
|
|
313
|
+
|
|
314
|
+
// Nothing is given up while the blocks are younger than the longest wait
|
|
315
|
+
// this store has actually seen between a block falling free and being
|
|
316
|
+
// wanted again.
|
|
317
|
+
assert.equal(store.sweepFreeBlocks(Date.now()), 0, "a block in use moments ago is not spare");
|
|
318
|
+
assert.equal(store.stats().blocksFree, spare);
|
|
319
|
+
|
|
320
|
+
// An hour later they plainly are.
|
|
321
|
+
const released = store.sweepFreeBlocks(Date.now() + 3_600_000);
|
|
322
|
+
assert.equal(released, spare);
|
|
323
|
+
assert.equal(store.stats().blocksFree, 0);
|
|
324
|
+
assert.equal(store.stats().blocksReleased, released);
|
|
325
|
+
} finally {
|
|
326
|
+
store.destroy(() => undefined);
|
|
327
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("evicting a piece the disk already holds costs no second write", async () => {
|
|
332
|
+
const { store, directory } = await makeStore(2);
|
|
333
|
+
try {
|
|
334
|
+
// Three pieces through two slots: piece 0 is written out.
|
|
335
|
+
for (let index = 0; index < 3; index += 1) {
|
|
336
|
+
await put(store, index, pieceOf(index));
|
|
337
|
+
}
|
|
338
|
+
const written = store.stats().spills;
|
|
339
|
+
assert.ok(written > 0);
|
|
340
|
+
assert.equal(store.stats().spillsSkipped, 0, "the first write of a piece is a real one");
|
|
341
|
+
|
|
342
|
+
// Read it back — it returns to memory and the copy stays on disk. Evicting
|
|
343
|
+
// it again writes bytes that are already there, byte for byte, because only
|
|
344
|
+
// `put` removes the disk copy and no `put` has happened.
|
|
345
|
+
assert.ok((await get(store, 0)).equals(pieceOf(0)));
|
|
346
|
+
for (let index = 10; index < 13; index += 1) {
|
|
347
|
+
await put(store, index, pieceOf(index));
|
|
348
|
+
}
|
|
349
|
+
assert.ok(store.stats().spillsSkipped > 0, "the second write of the same bytes is skipped");
|
|
350
|
+
|
|
351
|
+
// And the piece still comes back correctly from the disk copy.
|
|
352
|
+
assert.ok((await get(store, 0)).equals(pieceOf(0)));
|
|
353
|
+
} finally {
|
|
354
|
+
store.destroy(() => undefined);
|
|
355
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
356
|
+
}
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
test("a store whose readers have gone asks for nothing, one that never had them keeps its opening", async () => {
|
|
360
|
+
const { store, directory } = await makeStore(16);
|
|
361
|
+
try {
|
|
362
|
+
// Never had a reader: this is the initial download and the warm-up fetches
|
|
363
|
+
// of the header and the tail, with a read on its way.
|
|
364
|
+
const opening = store.wantedBytes;
|
|
365
|
+
assert.equal(opening, store.stats().budgetBytes);
|
|
366
|
+
|
|
367
|
+
store.protectRange("video", 0, 9);
|
|
368
|
+
assert.equal(store.wantedBytes, 10 * PIECE);
|
|
369
|
+
|
|
370
|
+
// The read ends. Its torrent sits until the pool's idle timer removes it,
|
|
371
|
+
// and that timer needs a refcount of zero and can be a quarter of an hour
|
|
372
|
+
// away. Holding the pieces for a reader that has gone is memory taken from
|
|
373
|
+
// the machine for nothing.
|
|
374
|
+
store.releaseProtection("video");
|
|
375
|
+
assert.ok(store.wantedBytes < opening, "a store with no readers left asks for nothing");
|
|
376
|
+
} finally {
|
|
377
|
+
store.destroy(() => undefined);
|
|
378
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test("the allowance is never cut below one reader's whole window", async () => {
|
|
383
|
+
const { store, directory } = await makeStore(16);
|
|
384
|
+
try {
|
|
385
|
+
store.protectRange("video", 0, 9);
|
|
386
|
+
// The machine says this store may have two pieces. Obeying that would leave
|
|
387
|
+
// it unable to finish the read it is serving: every resident piece pinned,
|
|
388
|
+
// zero bytes returned, and ffmpeg taking that for the end of the file —
|
|
389
|
+
// which killed every encoder on that file in the field on 2026-08-15.
|
|
390
|
+
const revised = store.reviseGrowthCeiling(2 * PIECE);
|
|
391
|
+
assert.equal(revised.ceilingBytes, 10 * PIECE, "one whole window is the floor");
|
|
392
|
+
assert.equal(revised.belowAWindow, true, "and the store says the share was smaller than that");
|
|
393
|
+
|
|
394
|
+
// With no reader there is no window to protect and the share is obeyed.
|
|
395
|
+
store.releaseProtection("video");
|
|
396
|
+
const obeyed = store.reviseGrowthCeiling(2 * PIECE);
|
|
397
|
+
assert.equal(obeyed.ceilingBytes, 2 * PIECE);
|
|
398
|
+
assert.equal(obeyed.belowAWindow, false);
|
|
399
|
+
} finally {
|
|
400
|
+
store.destroy(() => undefined);
|
|
401
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
402
|
+
}
|
|
403
|
+
});
|
package/test/read-bands.test.js
CHANGED
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
import test from "node:test";
|
|
12
12
|
import assert from "node:assert/strict";
|
|
13
13
|
import { bandWidthsFrom, bandsFrom, sameBands } from "../services/torrent-worker/piece-reader.js";
|
|
14
|
+
import { Urgency } from "../services/demand/Urgency.js";
|
|
14
15
|
|
|
15
16
|
const PIECE = 8 * 1024 * 1024;
|
|
16
17
|
|
|
17
|
-
test("the urgent band
|
|
18
|
+
test("the urgent band is the one being read, and the lead bands come behind it in order", () => {
|
|
18
19
|
const bands = bandsFrom({
|
|
19
20
|
urgent: { from: 100, to: 103 },
|
|
20
21
|
pieceIndex: 100,
|
|
@@ -26,15 +27,15 @@ test("the urgent band keeps the highest priority, and the lead bands come behind
|
|
|
26
27
|
assert.deepEqual(
|
|
27
28
|
bands,
|
|
28
29
|
[
|
|
29
|
-
{ from: 100, to: 103,
|
|
30
|
-
{ from: 104, to: 108,
|
|
31
|
-
{ from: 109, to: 128,
|
|
30
|
+
{ from: 100, to: 103, urgency: Urgency.NEAR },
|
|
31
|
+
{ from: 104, to: 108, urgency: Urgency.AHEAD },
|
|
32
|
+
{ from: 109, to: 128, urgency: Urgency.AHEAD }
|
|
32
33
|
],
|
|
33
34
|
"no gap, no overlap, and each band strictly less urgent than the one in front"
|
|
34
35
|
);
|
|
35
36
|
});
|
|
36
37
|
|
|
37
|
-
test("no
|
|
38
|
+
test("no band is speculative while the reader is still walking the file", () => {
|
|
38
39
|
const bands = bandsFrom({
|
|
39
40
|
urgent: { from: 0, to: 1 },
|
|
40
41
|
pieceIndex: 0,
|
|
@@ -44,7 +45,7 @@ test("no priority is zero, because the library's own whole-file fill sits there"
|
|
|
44
45
|
});
|
|
45
46
|
|
|
46
47
|
assert.ok(
|
|
47
|
-
bands.every((band) => band.
|
|
48
|
+
bands.every((band) => band.urgency <= Urgency.AHEAD),
|
|
48
49
|
"a band at 0 would be indistinguishable from the background fill of the whole torrent"
|
|
49
50
|
);
|
|
50
51
|
});
|
|
@@ -72,7 +73,7 @@ test("what was never downloaded behind the viewer is not asked for until the lea
|
|
|
72
73
|
});
|
|
73
74
|
assert.deepEqual(
|
|
74
75
|
covered[covered.length - 1],
|
|
75
|
-
{ from: 0, to: 49,
|
|
76
|
+
{ from: 0, to: 49, urgency: Urgency.BEHIND },
|
|
76
77
|
"once there is nothing left ahead, the gap behind the viewer is worth filling"
|
|
77
78
|
);
|
|
78
79
|
});
|
|
@@ -121,12 +122,18 @@ test("with nothing measured yet both lead bands fall back to the reader's own wi
|
|
|
121
122
|
});
|
|
122
123
|
|
|
123
124
|
test("an unchanged claim is recognised, so it is not released and re-made on every read", () => {
|
|
124
|
-
const bands = [
|
|
125
|
+
const bands = [
|
|
126
|
+
{ from: 1, to: 2, urgency: Urgency.NEAR },
|
|
127
|
+
{ from: 3, to: 9, urgency: Urgency.AHEAD }
|
|
128
|
+
];
|
|
125
129
|
|
|
126
130
|
assert.equal(sameBands(bands, [...bands.map((band) => ({ ...band }))]), true);
|
|
127
|
-
assert.equal(sameBands(bands, [{ from: 1, to: 2,
|
|
131
|
+
assert.equal(sameBands(bands, [{ from: 1, to: 2, urgency: Urgency.NEAR }]), false);
|
|
128
132
|
assert.equal(
|
|
129
|
-
sameBands(bands, [
|
|
133
|
+
sameBands(bands, [
|
|
134
|
+
{ from: 1, to: 2, urgency: Urgency.NEAR },
|
|
135
|
+
{ from: 3, to: 9, urgency: Urgency.BEHIND }
|
|
136
|
+
]),
|
|
130
137
|
false,
|
|
131
138
|
"the same range at another urgency is a different claim"
|
|
132
139
|
);
|