@torrent-tv/proxy 2.82.0 → 2.83.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/CLAUDE.md +11 -0
- package/docs/disk-architecture.md +161 -0
- package/docs/encode-architecture.md +36 -7
- package/package.json +1 -1
- package/routes/api/sources/stats/get.js +11 -2
- package/routes/stream/get.js +74 -3
- package/services/delivery-probe.js +248 -43
- package/services/disk/keep.js +48 -0
- package/services/disk/returns.js +103 -0
- package/services/download/SwarmSelection.js +5 -5
- package/services/download/registry.js +20 -0
- package/services/encode/EncodeRun.js +1 -0
- package/services/encode/Encoder.js +15 -0
- package/services/encode/QsvEncoder.js +5 -0
- package/services/encode/SegmentStore.js +14 -0
- package/services/encode/VaapiEncoder.js +5 -0
- package/services/encode/encode-exit.js +17 -0
- package/services/encode/start-stop-cost.js +6 -2
- package/services/files/CompletedFiles.js +276 -0
- package/services/files/piece-from-whole-file.js +118 -0
- package/services/hls-session-manager.js +17 -1
- package/services/hwaccel.js +4 -0
- package/services/output/cut-grid.js +13 -3
- package/services/piece-store/piece-disk-store.js +143 -8
- package/services/piece-store/piece-lru.js +17 -0
- package/services/piece-store/shared-piece-store.js +1803 -1549
- package/services/torrent-pool.js +246 -26
- package/services/torrent-worker/client.js +21 -0
- package/services/torrent-worker/protocol.js +9 -1
- package/services/torrent-worker/worker.js +183 -2
- package/test/completed-files.test.js +115 -0
- package/test/cuts-follow-published-grid.test.js +35 -0
- package/test/delivery-probe.test.js +114 -1
- package/test/encode-exit.test.js +18 -0
- package/test/keeping-period.test.js +83 -0
- package/test/piece-disk-store.test.js +114 -0
- package/test/piece-from-whole-file.test.js +129 -0
- package/test/piece-store-eviction.test.js +28 -15
- package/test/piece-store-never-refuses.test.js +153 -0
- package/test/piece-store-reservations.test.js +16 -3
- package/test/probe-wedge-certainty.test.js +3 -3
- package/test/shared-piece-store.test.js +27 -13
- package/test/stream-route.test.js +41 -0
- package/test/swarm-follows-readers.test.js +126 -0
- package/test/swarm-reach.test.js +5 -0
- package/test/upload-hurry.test.js +27 -0
|
@@ -265,3 +265,117 @@ test("the spill ceiling is what the disk's owner said, divided between the store
|
|
|
265
265
|
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
266
266
|
}
|
|
267
267
|
});
|
|
268
|
+
|
|
269
|
+
test("what lies behind every reader goes without waiting for the disk to be short", async () => {
|
|
270
|
+
// THE SECOND RULE. The ceiling is a share of free space, and on a roomy host
|
|
271
|
+
// that is tens of gigabytes against a measured growth of 14 400 MB in one
|
|
272
|
+
// viewing — so the ceiling alone never binds and nothing is removed until the
|
|
273
|
+
// torrent itself goes. A piece behind every read head has been read.
|
|
274
|
+
const { store, directory } = await makeStore(1000 * PIECE);
|
|
275
|
+
try {
|
|
276
|
+
for (const index of [0, 1, 2, 3, 4, 5]) {
|
|
277
|
+
await store.write(index, pieceOf(index));
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const removed = store.forgetBehind([3, 4]);
|
|
281
|
+
await store.settled();
|
|
282
|
+
|
|
283
|
+
assert.equal(removed, 3, "the three behind the earliest reader should have gone");
|
|
284
|
+
assert.deepEqual([0, 1, 2].map((index) => store.has(index)), [false, false, false]);
|
|
285
|
+
assert.deepEqual([3, 4, 5].map((index) => store.has(index)), [true, true, true]);
|
|
286
|
+
assert.equal(store.stats().behind, 3);
|
|
287
|
+
} finally {
|
|
288
|
+
await store.destroy();
|
|
289
|
+
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test("with no reader at all, nothing is thrown away", async () => {
|
|
294
|
+
// A store between reads is not a store nobody wants. What empties it whole is
|
|
295
|
+
// the torrent going idle, which removes the store and its directory.
|
|
296
|
+
const { store, directory } = await makeStore(1000 * PIECE);
|
|
297
|
+
try {
|
|
298
|
+
for (const index of [0, 1, 2]) {
|
|
299
|
+
await store.write(index, pieceOf(index));
|
|
300
|
+
}
|
|
301
|
+
assert.equal(store.forgetBehind([]), 0);
|
|
302
|
+
assert.equal(store.size, 3);
|
|
303
|
+
} finally {
|
|
304
|
+
await store.destroy();
|
|
305
|
+
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test("a piece being read is not taken even when it is behind everybody", async () => {
|
|
310
|
+
const { store, directory } = await makeStore(1000 * PIECE);
|
|
311
|
+
try {
|
|
312
|
+
await store.write(0, pieceOf(0));
|
|
313
|
+
await store.write(5, pieceOf(5));
|
|
314
|
+
const reading = store.read(0, Buffer.alloc(PIECE));
|
|
315
|
+
const removed = store.forgetBehind([5]);
|
|
316
|
+
await reading;
|
|
317
|
+
await store.settled();
|
|
318
|
+
|
|
319
|
+
assert.equal(removed, 0, "a piece under a reader was thrown away");
|
|
320
|
+
assert.equal(store.has(0), true);
|
|
321
|
+
} finally {
|
|
322
|
+
await store.destroy();
|
|
323
|
+
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test("when there is no room, what is behind the readers goes before what is ahead", async () => {
|
|
328
|
+
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "piece-disk-heads-"));
|
|
329
|
+
const clock = { at: 1000 };
|
|
330
|
+
// The reader stands on #5. #4 is behind it and was touched LAST, so under the
|
|
331
|
+
// old rule — least recently used — it would have been the safest piece there.
|
|
332
|
+
const store = new PieceDiskStore({
|
|
333
|
+
directory,
|
|
334
|
+
name: "pieces",
|
|
335
|
+
chunkLength: PIECE,
|
|
336
|
+
allowanceBytes: 3 * PIECE,
|
|
337
|
+
now: () => clock.at,
|
|
338
|
+
readHeads: () => [5]
|
|
339
|
+
});
|
|
340
|
+
try {
|
|
341
|
+
for (const index of [8, 7, 4]) {
|
|
342
|
+
clock.at += 100;
|
|
343
|
+
await store.write(index, pieceOf(index));
|
|
344
|
+
}
|
|
345
|
+
clock.at += 100;
|
|
346
|
+
await store.write(9, pieceOf(9));
|
|
347
|
+
await store.settled();
|
|
348
|
+
|
|
349
|
+
assert.equal(store.has(4), false, "the piece behind the reader was kept because it was touched last");
|
|
350
|
+
assert.deepEqual([7, 8, 9].map((index) => store.has(index)), [true, true, true]);
|
|
351
|
+
} finally {
|
|
352
|
+
await store.destroy();
|
|
353
|
+
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
test("a store takes up the pieces a previous life left in its directory", async () => {
|
|
358
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "adopt-"));
|
|
359
|
+
try {
|
|
360
|
+
const first = new PieceDiskStore({ directory: root, name: "film.pieces", chunkLength: 1024 });
|
|
361
|
+
await first.write(7, Buffer.alloc(1024, 7));
|
|
362
|
+
await first.write(9, Buffer.alloc(1024, 9));
|
|
363
|
+
await first.close();
|
|
364
|
+
|
|
365
|
+
// A torrent torn down and added again gets a NEW store over the same
|
|
366
|
+
// directory. Before 2026-09-11 its index started empty, so every piece it
|
|
367
|
+
// in fact had read as missing and the film was downloaded a second time
|
|
368
|
+
// while the first copy sat beside it.
|
|
369
|
+
const second = new PieceDiskStore({ directory: root, name: "film.pieces", chunkLength: 1024 });
|
|
370
|
+
assert.equal(second.size, 2, "the pieces already on disk were not taken up");
|
|
371
|
+
assert.equal(second.bytes, 2048, "nor were their bytes counted");
|
|
372
|
+
assert.ok(second.has(7) && second.has(9));
|
|
373
|
+
|
|
374
|
+
const target = Buffer.alloc(1024);
|
|
375
|
+
await second.read(7, target);
|
|
376
|
+
assert.ok(target.equals(Buffer.alloc(1024, 7)), "and they read back as themselves");
|
|
377
|
+
await second.destroy();
|
|
378
|
+
} finally {
|
|
379
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
380
|
+
}
|
|
381
|
+
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file A piece read out of the files it was assembled into.
|
|
3
|
+
*
|
|
4
|
+
* Without this a whole file is a second copy of bytes the piece store also
|
|
5
|
+
* holds, and neither copy can be dropped. With it the spilled copy is a
|
|
6
|
+
* duplicate and can go — one episode on the field host of 2026-09-11 was
|
|
7
|
+
* 1417 MB of segments beside 1424 MB of spilled pieces — and a torrent can be
|
|
8
|
+
* destroyed and added again without fetching a byte.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import test from "node:test";
|
|
12
|
+
import assert from "node:assert/strict";
|
|
13
|
+
import fs from "node:fs/promises";
|
|
14
|
+
import os from "node:os";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../services/files/piece-from-whole-file.js";
|
|
17
|
+
|
|
18
|
+
const PIECE = 16;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Two files laid end to end, as a torrent lays them out, with a piece straddling
|
|
22
|
+
* the boundary between them.
|
|
23
|
+
*
|
|
24
|
+
* @returns {Promise<{ root: string, files: object[], length: number, bytes: Buffer }>}
|
|
25
|
+
*/
|
|
26
|
+
async function twoFiles() {
|
|
27
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "piece-of-whole-"));
|
|
28
|
+
const first = Buffer.alloc(20, 1);
|
|
29
|
+
const second = Buffer.alloc(24, 2);
|
|
30
|
+
await fs.writeFile(path.join(root, "0"), first);
|
|
31
|
+
await fs.writeFile(path.join(root, "1"), second);
|
|
32
|
+
return {
|
|
33
|
+
root,
|
|
34
|
+
files: [
|
|
35
|
+
{ offset: 0, length: first.length },
|
|
36
|
+
{ offset: first.length, length: second.length }
|
|
37
|
+
],
|
|
38
|
+
length: first.length + second.length,
|
|
39
|
+
bytes: Buffer.concat([first, second])
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @param {string} root
|
|
45
|
+
* @param {number[]} whole - Which file indexes this proxy holds whole.
|
|
46
|
+
* @returns {(fileIndex: number) => { path: string, length: number } | null}
|
|
47
|
+
*/
|
|
48
|
+
const holds = (root, whole) => (fileIndex) =>
|
|
49
|
+
whole.includes(fileIndex) ? { path: path.join(root, String(fileIndex)), length: 0 } : null;
|
|
50
|
+
|
|
51
|
+
test("a piece inside one file comes back byte for byte", async () => {
|
|
52
|
+
const { root, files, length, bytes } = await twoFiles();
|
|
53
|
+
try {
|
|
54
|
+
const piece = await pieceFromWholeFiles({
|
|
55
|
+
index: 0,
|
|
56
|
+
pieceLength: PIECE,
|
|
57
|
+
length,
|
|
58
|
+
files,
|
|
59
|
+
wholeFileAt: holds(root, [0, 1])
|
|
60
|
+
});
|
|
61
|
+
assert.deepEqual(piece, bytes.subarray(0, PIECE));
|
|
62
|
+
} finally {
|
|
63
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("a piece straddling two files is stitched from both", async () => {
|
|
68
|
+
const { root, files, length, bytes } = await twoFiles();
|
|
69
|
+
try {
|
|
70
|
+
// Piece 1 covers bytes 16..31: four from the first file, twelve from the
|
|
71
|
+
// second.
|
|
72
|
+
const piece = await pieceFromWholeFiles({
|
|
73
|
+
index: 1,
|
|
74
|
+
pieceLength: PIECE,
|
|
75
|
+
length,
|
|
76
|
+
files,
|
|
77
|
+
wholeFileAt: holds(root, [0, 1])
|
|
78
|
+
});
|
|
79
|
+
assert.deepEqual(piece, bytes.subarray(PIECE, PIECE * 2));
|
|
80
|
+
} finally {
|
|
81
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("the last piece is read to the end of the torrent and no further", async () => {
|
|
86
|
+
const { root, files, length, bytes } = await twoFiles();
|
|
87
|
+
try {
|
|
88
|
+
// 44 bytes in pieces of 16: the last piece is 12 long.
|
|
89
|
+
const piece = await pieceFromWholeFiles({
|
|
90
|
+
index: 2,
|
|
91
|
+
pieceLength: PIECE,
|
|
92
|
+
length,
|
|
93
|
+
files,
|
|
94
|
+
wholeFileAt: holds(root, [0, 1])
|
|
95
|
+
});
|
|
96
|
+
assert.equal(piece.length, 12);
|
|
97
|
+
assert.deepEqual(piece, bytes.subarray(32));
|
|
98
|
+
} finally {
|
|
99
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("a piece any part of which is not held whole is refused, not half read", async () => {
|
|
104
|
+
const { root, files, length } = await twoFiles();
|
|
105
|
+
try {
|
|
106
|
+
// Only the first file is here; piece 1 straddles both. Half a piece is
|
|
107
|
+
// worse than none: the layer above would hash it and mark it bad.
|
|
108
|
+
const piece = await pieceFromWholeFiles({
|
|
109
|
+
index: 1,
|
|
110
|
+
pieceLength: PIECE,
|
|
111
|
+
length,
|
|
112
|
+
files,
|
|
113
|
+
wholeFileAt: holds(root, [0])
|
|
114
|
+
});
|
|
115
|
+
assert.equal(piece, null);
|
|
116
|
+
assert.equal(
|
|
117
|
+
pieceIsInWholeFiles({ index: 1, pieceLength: PIECE, length, files, wholeFileAt: holds(root, [0]) }),
|
|
118
|
+
false
|
|
119
|
+
);
|
|
120
|
+
// And one wholly inside the file that IS held is both readable and known to
|
|
121
|
+
// be a duplicate of what is on the spill.
|
|
122
|
+
assert.equal(
|
|
123
|
+
pieceIsInWholeFiles({ index: 0, pieceLength: PIECE, length, files, wholeFileAt: holds(root, [0]) }),
|
|
124
|
+
true
|
|
125
|
+
);
|
|
126
|
+
} finally {
|
|
127
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
128
|
+
}
|
|
129
|
+
});
|
|
@@ -177,7 +177,13 @@ test("the store says why it spills: what is asked of it, what it had to take, ho
|
|
|
177
177
|
// eviction on ADMISSION — `asked.spills` above is zero, where before it all
|
|
178
178
|
// six arrivals displaced a nearer piece and were read back moments later.
|
|
179
179
|
assert.ok(after.fromDisk > 0, "the pieces that went to disk were read back from it");
|
|
180
|
-
|
|
180
|
+
// Through the reader's own entry point, which is the one that populates
|
|
181
|
+
// memory. `get` is the torrent client's, and since 2026-09-11 it answers
|
|
182
|
+
// from disk without taking a block: a peer asks for kilobytes of a piece
|
|
183
|
+
// that is megabytes, and reviving the whole of it for that was how an
|
|
184
|
+
// upload capped at 512 KB/s produced 49 696 revivals in one session.
|
|
185
|
+
await store.reside(9);
|
|
186
|
+
assert.ok(store.stats().revivals > 0, "a read for playback brings the piece into memory");
|
|
181
187
|
// No age to report, and that is right rather than missing: an age measures
|
|
182
188
|
// how long an EVICTED piece stayed away, and these were never resident —
|
|
183
189
|
// they were written on arrival and read back once.
|
|
@@ -252,11 +258,12 @@ test("before any reader has declared anything, an arriving piece still goes to m
|
|
|
252
258
|
test("the store asks for what its readers declared, and for a whole window at least", async () => {
|
|
253
259
|
const { store, directory } = await makeStore(64);
|
|
254
260
|
try {
|
|
255
|
-
// With nobody reading
|
|
256
|
-
//
|
|
257
|
-
//
|
|
261
|
+
// With nobody reading, the store asks for a floor — one window's worth,
|
|
262
|
+
// and until it has been asked for anything, the minimum. It does NOT ask
|
|
263
|
+
// for everything it is allowed: pieces arriving for a reader still on its
|
|
264
|
+
// way have the disk.
|
|
258
265
|
const idle = store.wantedBytes;
|
|
259
|
-
assert.
|
|
266
|
+
assert.ok(idle > 0 && idle < store.stats().budgetBytes, "an idle store asks for a floor, not its whole allowance");
|
|
260
267
|
|
|
261
268
|
// Two readers of one file — picture and sound — overlapping by
|
|
262
269
|
// construction. The ask is their union, not their sum.
|
|
@@ -375,10 +382,11 @@ test("evicting a piece the disk already holds costs no second write", async () =
|
|
|
375
382
|
assert.ok(written > 0);
|
|
376
383
|
assert.equal(store.stats().spillsSkipped, 0, "the first write of a piece is a real one");
|
|
377
384
|
|
|
378
|
-
// Read it back — it returns to memory and the copy stays on
|
|
379
|
-
// it again writes bytes that are already there, byte for
|
|
380
|
-
// `put` removes the disk copy and no `put` has happened.
|
|
385
|
+
// Read it back for playback — it returns to memory and the copy stays on
|
|
386
|
+
// disk. Evicting it again writes bytes that are already there, byte for
|
|
387
|
+
// byte, because only `put` removes the disk copy and no `put` has happened.
|
|
381
388
|
assert.ok((await get(store, 0)).equals(pieceOf(0)));
|
|
389
|
+
await store.reside(0);
|
|
382
390
|
for (let index = 10; index < 13; index += 1) {
|
|
383
391
|
await put(store, index, pieceOf(index));
|
|
384
392
|
}
|
|
@@ -396,19 +404,24 @@ test("a store whose readers have gone asks for nothing, one that never had them
|
|
|
396
404
|
const { store, directory } = await makeStore(16);
|
|
397
405
|
try {
|
|
398
406
|
// Never had a reader: this is the initial download and the warm-up fetches
|
|
399
|
-
// of the header and the tail, with a read on its way.
|
|
407
|
+
// of the header and the tail, with a read on its way. Those pieces have the
|
|
408
|
+
// disk, so the store asks for the minimum rather than for its opening
|
|
409
|
+
// allowance — a torrent nobody has read yet held 4180 MB of a machine's
|
|
410
|
+
// memory on 2026-09-11 while the film being watched was allowed 12 MB.
|
|
400
411
|
const opening = store.wantedBytes;
|
|
401
|
-
assert.
|
|
412
|
+
assert.ok(opening < store.stats().budgetBytes, "a store nobody has read keeps the minimum");
|
|
402
413
|
|
|
403
414
|
store.protectRange("video", 0, 9);
|
|
404
415
|
assert.equal(store.wantedBytes, 10 * PIECE);
|
|
405
416
|
|
|
406
|
-
// The read ends
|
|
407
|
-
//
|
|
408
|
-
//
|
|
409
|
-
// the
|
|
417
|
+
// The read ends, and the store keeps room for ONE window — what the next
|
|
418
|
+
// read will ask for within seconds. Falling to the minimum here is what
|
|
419
|
+
// left a store at three blocks when the viewer switched files on
|
|
420
|
+
// 2026-09-11, with the allowance re-derived a minute later and a claim
|
|
421
|
+
// giving up after five seconds.
|
|
410
422
|
store.releaseProtection("video");
|
|
411
|
-
assert.
|
|
423
|
+
assert.equal(store.wantedBytes, 10 * PIECE, "the floor between readers is the widest window seen");
|
|
424
|
+
assert.ok(store.wantedBytes > opening, "which is more than a store nobody has read keeps");
|
|
412
425
|
} finally {
|
|
413
426
|
store.destroy(() => undefined);
|
|
414
427
|
await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file A store that is short of memory must inconvenience a read, never end a
|
|
3
|
+
* torrent.
|
|
4
|
+
*
|
|
5
|
+
* The field failure of 2026-09-11, in one sentence: the store could not hand
|
|
6
|
+
* out a block, threw, the throw travelled out through the torrent client's own
|
|
7
|
+
* write callback, and the client destroyed the torrent. For the rest of the
|
|
8
|
+
* process every read of that film answered `File 1 not found in
|
|
9
|
+
* torrent:d4022ff4…`, `/stats` reported `peers=0 connected of 1186 known`, and
|
|
10
|
+
* the viewer could not open anything until the addon was restarted.
|
|
11
|
+
*
|
|
12
|
+
* Three properties hold it shut, and each is checked here on its own:
|
|
13
|
+
*
|
|
14
|
+
* 1. an arriving piece is never refused — it has the disk;
|
|
15
|
+
* 2. a read on the torrent client's own path takes no block at all, so an
|
|
16
|
+
* upload can neither wait for memory nor be refused it;
|
|
17
|
+
* 3. a store between readers keeps room for one window, because that is what
|
|
18
|
+
* the next read asks for and the allowance is otherwise re-derived a
|
|
19
|
+
* minute later — twelve times slower than a claim gives up.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import test from "node:test";
|
|
23
|
+
import assert from "node:assert/strict";
|
|
24
|
+
import fs from "node:fs/promises";
|
|
25
|
+
import os from "node:os";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
|
|
28
|
+
|
|
29
|
+
const PIECE = 1024;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {SharedPieceStore} store
|
|
33
|
+
* @param {number} index
|
|
34
|
+
* @returns {Promise<void>}
|
|
35
|
+
*/
|
|
36
|
+
const put = (store, index) =>
|
|
37
|
+
new Promise((resolve, reject) => {
|
|
38
|
+
store.put(index, Buffer.alloc(PIECE, index % 251), (error) => (error ? reject(error) : resolve()));
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {SharedPieceStore} store
|
|
43
|
+
* @param {number} index
|
|
44
|
+
* @param {{ offset: number, length: number }} range
|
|
45
|
+
* @returns {Promise<Buffer>}
|
|
46
|
+
*/
|
|
47
|
+
const get = (store, index, range) =>
|
|
48
|
+
new Promise((resolve, reject) => {
|
|
49
|
+
store.get(index, range, (error, bytes) => (error ? reject(error) : resolve(bytes)));
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* @returns {Promise<string>}
|
|
54
|
+
*/
|
|
55
|
+
const directory = () => fs.mkdtemp(path.join(os.tmpdir(), "never-refuses-"));
|
|
56
|
+
|
|
57
|
+
test("a piece is never refused for want of memory", async () => {
|
|
58
|
+
const root = await directory();
|
|
59
|
+
// Every block held by a piece that may not leave. The ceiling never falls
|
|
60
|
+
// below two, so this is how a store with nothing to give is reached: what
|
|
61
|
+
// held the blocks in the field was two writes on their way to disk, and a pin
|
|
62
|
+
// reaches the same state deterministically.
|
|
63
|
+
//
|
|
64
|
+
// This one takes the store's own patience to run — it must be seen to give
|
|
65
|
+
// up and keep the piece anyway.
|
|
66
|
+
const store = new SharedPieceStore(PIECE, {
|
|
67
|
+
length: PIECE * 8,
|
|
68
|
+
memoryBytes: PIECE * 2,
|
|
69
|
+
path: root,
|
|
70
|
+
name: "no-block-to-be-had"
|
|
71
|
+
});
|
|
72
|
+
try {
|
|
73
|
+
await put(store, 0);
|
|
74
|
+
await put(store, 1);
|
|
75
|
+
store.pin(0);
|
|
76
|
+
store.pin(1);
|
|
77
|
+
|
|
78
|
+
await put(store, 3);
|
|
79
|
+
const stats = store.stats();
|
|
80
|
+
assert.equal(stats.spilled, 1, "the piece is on disk");
|
|
81
|
+
assert.equal(store.locate(3), null, "and not in memory, since no block could be had for it");
|
|
82
|
+
assert.equal(stats.resident, 2, "the pinned pieces were not taken from under their reader");
|
|
83
|
+
assert.ok(
|
|
84
|
+
stats.admittedWithoutSlot >= 1,
|
|
85
|
+
"and the store says so, rather than the torrent client saying it with a destroyed torrent"
|
|
86
|
+
);
|
|
87
|
+
const bytes = await get(store, 3, { offset: 0, length: PIECE });
|
|
88
|
+
assert.equal(bytes.length, PIECE, "and it reads back");
|
|
89
|
+
assert.equal(bytes[0], 3 % 251, "with its own contents");
|
|
90
|
+
} finally {
|
|
91
|
+
await new Promise((resolve) => store.destroy(resolve));
|
|
92
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("a read for the torrent client takes no block and only its own range", async () => {
|
|
97
|
+
const root = await directory();
|
|
98
|
+
const store = new SharedPieceStore(PIECE, {
|
|
99
|
+
length: PIECE * 8,
|
|
100
|
+
memoryBytes: PIECE,
|
|
101
|
+
path: root,
|
|
102
|
+
name: "ranged-read"
|
|
103
|
+
});
|
|
104
|
+
try {
|
|
105
|
+
// Two fit; the third displaces the oldest, which goes to disk.
|
|
106
|
+
await put(store, 0);
|
|
107
|
+
await put(store, 1);
|
|
108
|
+
await put(store, 2);
|
|
109
|
+
const before = store.stats();
|
|
110
|
+
assert.equal(before.spilled, 1, "the first piece is on disk");
|
|
111
|
+
|
|
112
|
+
const wanted = 16;
|
|
113
|
+
const bytes = await get(store, 0, { offset: PIECE - wanted, length: wanted });
|
|
114
|
+
assert.equal(bytes.length, wanted, "only what was asked for comes back");
|
|
115
|
+
assert.equal(bytes[0], 0, "and it is that piece's own bytes");
|
|
116
|
+
|
|
117
|
+
const after = store.stats();
|
|
118
|
+
assert.equal(
|
|
119
|
+
after.blocksAllocated,
|
|
120
|
+
before.blocksAllocated,
|
|
121
|
+
"and answering it took no block: a peer asks for kilobytes and a piece here is megabytes"
|
|
122
|
+
);
|
|
123
|
+
assert.equal(after.resident, before.resident, "nothing was revived for it");
|
|
124
|
+
assert.equal(after.fromDisk, before.fromDisk + 1, "and the read is counted as coming from disk");
|
|
125
|
+
} finally {
|
|
126
|
+
await new Promise((resolve) => store.destroy(resolve));
|
|
127
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("a store between readers keeps room for one window", async () => {
|
|
132
|
+
const root = await directory();
|
|
133
|
+
const store = new SharedPieceStore(PIECE, {
|
|
134
|
+
length: PIECE * 200,
|
|
135
|
+
memoryBytes: PIECE * 64,
|
|
136
|
+
path: root,
|
|
137
|
+
name: "between-readers"
|
|
138
|
+
});
|
|
139
|
+
try {
|
|
140
|
+
store.protectRange("read-1", 10, 21, 100);
|
|
141
|
+
const asked = store.wantedBytes;
|
|
142
|
+
assert.ok(asked >= PIECE * 12, "with a reader, the window it declared is asked for");
|
|
143
|
+
|
|
144
|
+
store.releaseProtection("read-1");
|
|
145
|
+
assert.ok(
|
|
146
|
+
store.wantedBytes >= PIECE * 12,
|
|
147
|
+
"and with the reader gone the floor is still one window — the next read asks for the same again"
|
|
148
|
+
);
|
|
149
|
+
} finally {
|
|
150
|
+
await new Promise((resolve) => store.destroy(resolve));
|
|
151
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
152
|
+
}
|
|
153
|
+
});
|
|
@@ -129,10 +129,12 @@ const get = (store, index) =>
|
|
|
129
129
|
*
|
|
130
130
|
* @param {() => boolean} holds
|
|
131
131
|
* @param {string} what
|
|
132
|
+
* @param {number} [limit] - A backstop, never the measurement: one of the
|
|
133
|
+
* conditions here is the store's own patience, which is itself five seconds.
|
|
132
134
|
* @returns {Promise<void>}
|
|
133
135
|
*/
|
|
134
|
-
async function until(holds, what) {
|
|
135
|
-
const deadline = Date.now() +
|
|
136
|
+
async function until(holds, what, limit = 5_000) {
|
|
137
|
+
const deadline = Date.now() + limit;
|
|
136
138
|
while (!holds()) {
|
|
137
139
|
if (Date.now() > deadline) {
|
|
138
140
|
throw new Error(`timed out waiting until ${what}`);
|
|
@@ -196,10 +198,21 @@ test("a claim that cannot be met ends in an error, not in waiting for ever", { t
|
|
|
196
198
|
// other is pinned. The old rule waited while anything was nominally in
|
|
197
199
|
// flight, which here is for ever.
|
|
198
200
|
store.pin(1);
|
|
199
|
-
|
|
201
|
+
// The claim gives up within its own patience — that is the property. What
|
|
202
|
+
// the caller does with the refusal is its own business, and since
|
|
203
|
+
// 2026-09-11 an arriving piece answers it by taking the disk instead of
|
|
204
|
+
// failing the torrent client's write.
|
|
205
|
+
const arriving = put(store, 3);
|
|
206
|
+
await until(
|
|
207
|
+
() => store.stats().blockedByPins > 0,
|
|
208
|
+
"the claim gave up rather than waiting for ever",
|
|
209
|
+
20_000
|
|
210
|
+
);
|
|
200
211
|
|
|
201
212
|
store.unpin(1);
|
|
202
213
|
disk.releaseWrites();
|
|
214
|
+
await arriving;
|
|
215
|
+
disk.releaseWrites();
|
|
203
216
|
await spilling;
|
|
204
217
|
} finally {
|
|
205
218
|
disk.releaseWrites();
|
|
@@ -12,7 +12,7 @@ test("a seen-counter bounded lag is not a wedge, however long it lasts", () => {
|
|
|
12
12
|
stuckForMs: 3400,
|
|
13
13
|
longestHealthySeenGapMs: 3500
|
|
14
14
|
});
|
|
15
|
-
assert.equal(verdict.
|
|
15
|
+
assert.equal(verdict.isCertain, false);
|
|
16
16
|
});
|
|
17
17
|
|
|
18
18
|
test("a seen-counter frozen past this connection's own worst legitimate gap is a wedge", () => {
|
|
@@ -23,7 +23,7 @@ test("a seen-counter frozen past this connection's own worst legitimate gap is a
|
|
|
23
23
|
stuckForMs: 90_000,
|
|
24
24
|
longestHealthySeenGapMs: 3500
|
|
25
25
|
});
|
|
26
|
-
assert.equal(verdict.
|
|
26
|
+
assert.equal(verdict.isCertain, true);
|
|
27
27
|
});
|
|
28
28
|
|
|
29
29
|
test("with no healthy history yet, one probe interval is still required", () => {
|
|
@@ -31,7 +31,7 @@ test("with no healthy history yet, one probe interval is still required", () =>
|
|
|
31
31
|
stuckForMs: PROBE_INTERVAL_MS - 1,
|
|
32
32
|
longestHealthySeenGapMs: 0
|
|
33
33
|
});
|
|
34
|
-
assert.equal(verdict.
|
|
34
|
+
assert.equal(verdict.isCertain, false);
|
|
35
35
|
assert.equal(verdict.needMs, PROBE_INTERVAL_MS);
|
|
36
36
|
});
|
|
37
37
|
|
|
@@ -151,11 +151,16 @@ test("refuses to make room when every resident piece is being read", async () =>
|
|
|
151
151
|
store.pin(0);
|
|
152
152
|
store.pin(1);
|
|
153
153
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
)
|
|
154
|
+
// The pinned pieces stay where they are, and the arrival is kept anyway —
|
|
155
|
+
// on the disk, which is what it has. Refusing it was how a store short of
|
|
156
|
+
// memory ended a torrent: the refusal travelled out through the torrent
|
|
157
|
+
// client's own write callback and the client destroyed the torrent
|
|
158
|
+
// (field 2026-09-11).
|
|
159
|
+
await put(store, 2, piece(2));
|
|
160
|
+
assert.ok(store.locate(0), "the pinned piece was taken from under its reader");
|
|
161
|
+
assert.ok(store.locate(1), "the pinned piece was taken from under its reader");
|
|
162
|
+
assert.equal(store.locate(2), null, "and the arrival did not displace either of them");
|
|
163
|
+
assert.ok((await get(store, 2)).equals(piece(2)), "the arrival is kept, and reads back");
|
|
159
164
|
} finally {
|
|
160
165
|
store.unpin(0);
|
|
161
166
|
store.unpin(1);
|
|
@@ -172,7 +177,7 @@ test("a piece revived from disk is readable by offset again", async () => {
|
|
|
172
177
|
await put(store, 2, piece(2)); // pushes piece 0 out to disk
|
|
173
178
|
assert.equal(store.locate(0), null, "piece 0 should have left memory");
|
|
174
179
|
|
|
175
|
-
await
|
|
180
|
+
await store.reside(0); // brings it back, which is what a playback read does
|
|
176
181
|
const located = store.locate(0);
|
|
177
182
|
assert.ok(located, "piece 0 was not brought back into memory");
|
|
178
183
|
const view = Buffer.from(located.buffer, located.offset, located.length);
|
|
@@ -248,11 +253,18 @@ test("counts where reads were served from, so the budget can be judged", async (
|
|
|
248
253
|
const stats = store.stats();
|
|
249
254
|
assert.equal(stats.fromMemory, 2, "memory reads miscounted");
|
|
250
255
|
assert.equal(stats.fromDisk, 1, "disk reads miscounted");
|
|
251
|
-
//
|
|
252
|
-
//
|
|
253
|
-
//
|
|
254
|
-
|
|
255
|
-
|
|
256
|
+
// One: piece 0 goes out to make room for piece 2. The read that follows is
|
|
257
|
+
// the torrent client's, and since 2026-09-11 it is answered from the disk
|
|
258
|
+
// copy without taking a block — so nothing has to be written out for it and
|
|
259
|
+
// nothing is revived. A read for PLAYBACK still populates memory, and pays
|
|
260
|
+
// for it with a spill; that is the next assertion.
|
|
261
|
+
assert.equal(stats.spills, 1, "spills miscounted");
|
|
262
|
+
assert.equal(stats.revivals, 0, "the client's own read does not revive");
|
|
263
|
+
|
|
264
|
+
await store.reside(0);
|
|
265
|
+
const afterPlayback = store.stats();
|
|
266
|
+
assert.equal(afterPlayback.revivals, 1, "a read for playback revives");
|
|
267
|
+
assert.equal(afterPlayback.spills, 2, "and pays for its block with a spill");
|
|
256
268
|
assert.equal(stats.blockedByPins, 0);
|
|
257
269
|
assert.equal(stats.capacity, 2);
|
|
258
270
|
} finally {
|
|
@@ -268,9 +280,11 @@ test("counts a refusal caused by pinned pieces", async () => {
|
|
|
268
280
|
await put(store, 1, piece(1));
|
|
269
281
|
store.pin(0);
|
|
270
282
|
store.pin(1);
|
|
271
|
-
await
|
|
283
|
+
await put(store, 2, piece(2));
|
|
272
284
|
|
|
273
|
-
|
|
285
|
+
const stats = store.stats();
|
|
286
|
+
assert.equal(stats.blockedByPins, 1, "the store not being able to give a block went unrecorded");
|
|
287
|
+
assert.equal(stats.admittedWithoutSlot, 1, "and what it did instead went unrecorded");
|
|
274
288
|
} finally {
|
|
275
289
|
store.unpin(0);
|
|
276
290
|
store.unpin(1);
|
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
|
|
11
11
|
import test from "node:test";
|
|
12
12
|
import assert from "node:assert/strict";
|
|
13
|
+
import fs from "node:fs/promises";
|
|
14
|
+
import os from "node:os";
|
|
15
|
+
import path from "node:path";
|
|
13
16
|
import { handleStreamGet } from "../routes/stream/get.js";
|
|
14
17
|
|
|
15
18
|
/**
|
|
@@ -143,3 +146,41 @@ test("a ranged GET is reported as a real read position", async () => {
|
|
|
143
146
|
|
|
144
147
|
assert.deepEqual(state.prioritized, [{ byteStart: 4_390_000_000, wholeFileRead: false }]);
|
|
145
148
|
});
|
|
149
|
+
|
|
150
|
+
test("a file downloaded whole is served from disk without touching the torrent", async () => {
|
|
151
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "stream-whole-"));
|
|
152
|
+
const where = path.join(root, "0");
|
|
153
|
+
const bytes = Buffer.from("a film that is a file now", "utf8");
|
|
154
|
+
await fs.writeFile(where, bytes);
|
|
155
|
+
try {
|
|
156
|
+
const { req, reply, sent, deps } = harness({ method: "GET", range: "bytes=2-6" });
|
|
157
|
+
let asked = false;
|
|
158
|
+
deps.torrentPool.getTorrent = async () => {
|
|
159
|
+
asked = true;
|
|
160
|
+
throw new Error("the torrent was asked for, which is what this avoids");
|
|
161
|
+
};
|
|
162
|
+
deps.torrentPool.wholeFiles = new Map([
|
|
163
|
+
["abc/0", { path: where, length: bytes.length, name: "film.mkv" }]
|
|
164
|
+
]);
|
|
165
|
+
// The source key IS the identity — `torrent:<infohash>` — and it is all the
|
|
166
|
+
// route needs to find the file.
|
|
167
|
+
deps.sourceRegistry = { get: () => ({ sourceType: "torrent", source: "magnet:?xt=urn:btih:abc" }) };
|
|
168
|
+
req.query = { sourceKey: "torrent:abc", fileIndex: "0" };
|
|
169
|
+
|
|
170
|
+
await handleStreamGet(req, reply, deps);
|
|
171
|
+
|
|
172
|
+
assert.equal(asked, false, "the torrent was asked for");
|
|
173
|
+
assert.equal(sent.code, 206);
|
|
174
|
+
assert.equal(sent.headers["content-range"], `bytes 2-6/${bytes.length}`);
|
|
175
|
+
assert.equal(sent.headers["content-length"], "5");
|
|
176
|
+
} finally {
|
|
177
|
+
await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("a file this proxy does not have whole still goes to the torrent", async () => {
|
|
182
|
+
const { req, reply, state, deps } = harness({ method: "GET", range: "bytes=0-99" });
|
|
183
|
+
deps.torrentPool.wholeFiles = new Map([["other/7", { path: "/nowhere", length: 1, name: "x" }]]);
|
|
184
|
+
await handleStreamGet(req, reply, deps);
|
|
185
|
+
assert.equal(state.claims, 1, "the ordinary path was not taken");
|
|
186
|
+
});
|