@torrent-tv/proxy 2.9.76 → 2.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +1 -1
- package/routes/stream/get.js +141 -116
- package/services/piece-store/shared-piece-store.js +488 -416
- package/services/torrent-worker/client.js +306 -275
- package/services/torrent-worker/file-claims.js +91 -0
- package/services/torrent-worker/piece-reader.js +175 -0
- package/services/torrent-worker/pool-adapter.js +188 -179
- package/services/torrent-worker/protocol.js +12 -0
- package/services/torrent-worker/worker.js +153 -68
- package/test/file-claims.test.js +64 -0
- package/test/piece-reader.test.js +162 -0
- package/test/stream-route.test.js +121 -0
- package/test/worker-source-race.test.js +76 -0
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
import "./install-webrtc-shim.js";
|
|
25
25
|
import { parentPort, workerData } from "node:worker_threads";
|
|
26
26
|
import { createSendStream } from "./channel.js";
|
|
27
|
-
import {
|
|
27
|
+
import { createFileClaims } from "./file-claims.js";
|
|
28
|
+
import { readFragments } from "./piece-reader.js";
|
|
29
|
+
import { Command, Event } from "./protocol.js";
|
|
28
30
|
|
|
29
31
|
// Imported dynamically, and that is load-bearing: static imports are RESOLVED
|
|
30
32
|
// during linking, before any module body runs, so a statically imported pool
|
|
@@ -32,7 +34,7 @@ import { Command, Event, STREAM_CHUNK_BYTES } from "./protocol.js";
|
|
|
32
34
|
// the hook above had a chance to register. Verified the hard way: with a static
|
|
33
35
|
// import the process still aborted, and the stack named the genuine polyfill.
|
|
34
36
|
const { TorrentPool } = await import("../torrent-pool.js");
|
|
35
|
-
const { collectStoreStats } = await import("../piece-store/shared-piece-store.js");
|
|
37
|
+
const { collectStoreStats, findSharedStore } = await import("../piece-store/shared-piece-store.js");
|
|
36
38
|
|
|
37
39
|
const pool = new TorrentPool({
|
|
38
40
|
maxDiskBytes: workerData?.maxDiskBytes,
|
|
@@ -41,8 +43,8 @@ const pool = new TorrentPool({
|
|
|
41
43
|
|
|
42
44
|
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
43
45
|
const torrentsByKey = new Map();
|
|
44
|
-
/** File
|
|
45
|
-
const
|
|
46
|
+
/** File claims, each with its own identity — see `file-claims.js`. */
|
|
47
|
+
const fileClaims = createFileClaims();
|
|
46
48
|
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
47
49
|
const readsById = new Map();
|
|
48
50
|
|
|
@@ -58,17 +60,80 @@ function log(message) {
|
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
/**
|
|
61
|
-
* The torrent for a sourceKey,
|
|
63
|
+
* The torrent for a sourceKey, waiting for it if it is still being added.
|
|
64
|
+
*
|
|
65
|
+
* The map holds a PROMISE, registered the moment the add begins rather than
|
|
66
|
+
* when it finishes. That distinction is the whole fix: adding a magnet takes as
|
|
67
|
+
* long as its metadata does — seconds to tens of seconds — and until 2.9.77
|
|
68
|
+
* everything naming that source in the meantime was told `Unknown source`,
|
|
69
|
+
* which is false. The source exists; it is not ready. Reproduced with a magnet
|
|
70
|
+
* nobody seeds: stats, the file listing and a read all failed instantly while
|
|
71
|
+
* the add was still in flight, which on the loading screen shows up as no
|
|
72
|
+
* peers, no progress, and a plan request that fails before the torrent has had
|
|
73
|
+
* a chance to start.
|
|
74
|
+
*
|
|
75
|
+
* A source that was never added still throws, which is the honest answer.
|
|
62
76
|
*
|
|
63
77
|
* @param {string} sourceKey
|
|
64
|
-
* @returns {import("webtorrent").Torrent}
|
|
78
|
+
* @returns {Promise<import("webtorrent").Torrent>}
|
|
65
79
|
*/
|
|
66
|
-
function requireTorrent(sourceKey) {
|
|
67
|
-
const
|
|
68
|
-
if (!
|
|
80
|
+
async function requireTorrent(sourceKey) {
|
|
81
|
+
const pending = torrentsByKey.get(sourceKey);
|
|
82
|
+
if (!pending) {
|
|
69
83
|
throw new Error(`Unknown source ${sourceKey}.`);
|
|
70
84
|
}
|
|
71
|
-
return
|
|
85
|
+
return pending;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Fragments waiting for the main thread to say it has finished reading them,
|
|
90
|
+
* keyed by request id. One per read, because only one fragment is in flight.
|
|
91
|
+
*
|
|
92
|
+
* @type {Map<number, () => void>}
|
|
93
|
+
*/
|
|
94
|
+
const fragmentWaiters = new Map();
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Wake a read that is waiting for a fragment to be confirmed.
|
|
98
|
+
*
|
|
99
|
+
* Used both by the confirmation itself and by cancellation — a cancelled read
|
|
100
|
+
* will never be confirmed, and without this it would wait forever holding a pin.
|
|
101
|
+
*
|
|
102
|
+
* @param {number} id
|
|
103
|
+
* @returns {void}
|
|
104
|
+
*/
|
|
105
|
+
function settleFragment(id) {
|
|
106
|
+
const done = fragmentWaiters.get(id);
|
|
107
|
+
if (done) {
|
|
108
|
+
fragmentWaiters.delete(id);
|
|
109
|
+
done();
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Send one fragment's position and wait until the main thread is done with it.
|
|
115
|
+
*
|
|
116
|
+
* The pin is dropped only after the confirmation, because until then the other
|
|
117
|
+
* thread may still be reading those exact bytes.
|
|
118
|
+
*
|
|
119
|
+
* @param {number} id
|
|
120
|
+
* @param {import("./piece-reader.js").PieceFragment} fragment
|
|
121
|
+
* @returns {Promise<void>}
|
|
122
|
+
*/
|
|
123
|
+
function sendFragment(id, fragment) {
|
|
124
|
+
return new Promise((resolve) => {
|
|
125
|
+
fragmentWaiters.set(id, () => {
|
|
126
|
+
fragment.release();
|
|
127
|
+
resolve();
|
|
128
|
+
});
|
|
129
|
+
parentPort.postMessage({
|
|
130
|
+
type: Event.FRAGMENT,
|
|
131
|
+
id,
|
|
132
|
+
pieceIndex: fragment.pieceIndex,
|
|
133
|
+
offset: fragment.offset,
|
|
134
|
+
length: fragment.length
|
|
135
|
+
});
|
|
136
|
+
});
|
|
72
137
|
}
|
|
73
138
|
|
|
74
139
|
/**
|
|
@@ -89,7 +154,7 @@ function requireTorrent(sourceKey) {
|
|
|
89
154
|
* @returns {Promise<void>}
|
|
90
155
|
*/
|
|
91
156
|
async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
92
|
-
const torrent = requireTorrent(sourceKey);
|
|
157
|
+
const torrent = await requireTorrent(sourceKey);
|
|
93
158
|
const file = torrent.files?.[fileIndex];
|
|
94
159
|
if (!file) {
|
|
95
160
|
throw new Error(`File ${fileIndex} not found in ${sourceKey}.`);
|
|
@@ -107,39 +172,31 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
|
107
172
|
// an empty input.
|
|
108
173
|
const releaseRead = pool.acquireFile(torrent, fileIndex);
|
|
109
174
|
|
|
110
|
-
const
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
// Coalesce WebTorrent's own chunking (piece-sized, often much smaller) up to
|
|
114
|
-
// our chunk size: a round trip costs ~100 µs, so sending its native pieces
|
|
115
|
-
// straight through would multiply the crossings for no benefit.
|
|
116
|
-
let pendingParts = [];
|
|
117
|
-
let pendingBytes = 0;
|
|
118
|
-
|
|
119
|
-
const flush = async () => {
|
|
120
|
-
if (pendingBytes === 0) {
|
|
121
|
-
return;
|
|
122
|
-
}
|
|
123
|
-
const merged = pendingParts.length === 1 ? pendingParts[0] : Buffer.concat(pendingParts, pendingBytes);
|
|
124
|
-
pendingParts = [];
|
|
125
|
-
pendingBytes = 0;
|
|
126
|
-
await sender.send(merged);
|
|
127
|
-
};
|
|
175
|
+
const rangeStart = start ?? 0;
|
|
176
|
+
const rangeEnd = end ?? file.length - 1;
|
|
128
177
|
|
|
129
178
|
let failed = false;
|
|
130
179
|
try {
|
|
131
|
-
|
|
180
|
+
// Positions in shared memory, not bytes: the main thread maps the same pool
|
|
181
|
+
// and reads each fragment in place, so nothing is copied and nothing is
|
|
182
|
+
// transferred. See `piece-reader.js`.
|
|
183
|
+
for await (const fragment of readFragments({
|
|
184
|
+
torrent,
|
|
185
|
+
fileIndex,
|
|
186
|
+
start: rangeStart,
|
|
187
|
+
end: rangeEnd,
|
|
188
|
+
cancellation: sender
|
|
189
|
+
})) {
|
|
132
190
|
if (sender.isCancelled()) {
|
|
191
|
+
fragment.release();
|
|
133
192
|
break;
|
|
134
193
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
if (!sender.isCancelled()) {
|
|
142
|
-
await flush();
|
|
194
|
+
// One fragment in flight at a time. Each one holds a piece pinned, and
|
|
195
|
+
// the store guarantees only two resident pieces at its smallest budget —
|
|
196
|
+
// holding two pins while asking for a third would deadlock it against
|
|
197
|
+
// itself. The round trip costs ~100 µs against a piece worth megabytes,
|
|
198
|
+
// so there is nothing to win by overlapping them.
|
|
199
|
+
await sendFragment(id, fragment);
|
|
143
200
|
}
|
|
144
201
|
} catch (error) {
|
|
145
202
|
// The end-of-read marker means "the body is complete". Sending it after a
|
|
@@ -151,15 +208,16 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
|
151
208
|
throw error;
|
|
152
209
|
} finally {
|
|
153
210
|
readsById.delete(id);
|
|
211
|
+
// Any fragment still awaiting confirmation will never get one now; settling
|
|
212
|
+
// it here releases its pin rather than leaking a held slot.
|
|
213
|
+
settleFragment(id);
|
|
154
214
|
releaseRead();
|
|
155
215
|
if (!failed) {
|
|
156
216
|
sender.end();
|
|
157
217
|
}
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
|
|
161
|
-
source.destroy();
|
|
162
|
-
}
|
|
218
|
+
// Nothing else to tear down: the reader owns no stream of its own, and a
|
|
219
|
+
// cancelled read stops at its next fragment boundary because it polls the
|
|
220
|
+
// same `sender` for cancellation.
|
|
163
221
|
}
|
|
164
222
|
}
|
|
165
223
|
|
|
@@ -174,11 +232,31 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
|
|
|
174
232
|
async function runCommand(command, params, id) {
|
|
175
233
|
switch (command) {
|
|
176
234
|
case Command.ADD_SOURCE: {
|
|
177
|
-
|
|
178
|
-
|
|
235
|
+
// Registered before it resolves, so anything naming this source while it
|
|
236
|
+
// is being added waits for it instead of being told it does not exist.
|
|
237
|
+
// Reusing the same promise for a repeated add also collapses two callers
|
|
238
|
+
// racing to open the same torrent into one.
|
|
239
|
+
let pending = torrentsByKey.get(params.sourceKey);
|
|
240
|
+
if (!pending) {
|
|
241
|
+
pending = pool.getTorrent(params.sourceType, params.source);
|
|
242
|
+
torrentsByKey.set(params.sourceKey, pending);
|
|
243
|
+
// A failed add must not be remembered, or every later attempt at this
|
|
244
|
+
// source replays the same failure. The handler also marks the rejection
|
|
245
|
+
// as observed, so it cannot surface as an unhandled one.
|
|
246
|
+
pending.catch(() => {
|
|
247
|
+
if (torrentsByKey.get(params.sourceKey) === pending) {
|
|
248
|
+
torrentsByKey.delete(params.sourceKey);
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
const torrent = await pending;
|
|
179
253
|
return {
|
|
180
254
|
infoHash: torrent.infoHash,
|
|
181
255
|
name: torrent.name,
|
|
256
|
+
// The piece pool itself. A `SharedArrayBuffer` crosses as a reference to
|
|
257
|
+
// the same memory rather than a copy, which is what lets the main thread
|
|
258
|
+
// read a piece where it already lies instead of being sent its bytes.
|
|
259
|
+
sharedBuffer: findSharedStore(torrent)?.sharedBuffer ?? null,
|
|
182
260
|
// Files cross as plain data; the objects stay here.
|
|
183
261
|
files: (torrent.files ?? []).map((file, index) => ({
|
|
184
262
|
index,
|
|
@@ -190,7 +268,7 @@ async function runCommand(command, params, id) {
|
|
|
190
268
|
}
|
|
191
269
|
|
|
192
270
|
case Command.LIST_FILES: {
|
|
193
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
271
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
194
272
|
return (torrent.files ?? []).map((file, index) => ({
|
|
195
273
|
index,
|
|
196
274
|
name: file.name,
|
|
@@ -200,42 +278,42 @@ async function runCommand(command, params, id) {
|
|
|
200
278
|
}
|
|
201
279
|
|
|
202
280
|
case Command.ACQUIRE_FILE: {
|
|
203
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
204
|
-
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
281
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
282
|
+
// Every acquire is its own claim. Sharing one per file meant the first
|
|
283
|
+
// reader to finish released the hold while others were still reading.
|
|
284
|
+
return fileClaims.open(
|
|
285
|
+
params.sourceKey,
|
|
286
|
+
params.fileIndex,
|
|
287
|
+
pool.acquireFile(torrent, params.fileIndex)
|
|
288
|
+
);
|
|
211
289
|
}
|
|
212
290
|
|
|
213
291
|
case Command.RELEASE_FILE: {
|
|
214
|
-
const
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
release
|
|
292
|
+
const released = fileClaims.close(params.claimId);
|
|
293
|
+
if (!released) {
|
|
294
|
+
// Not fatal — but it means a release arrived twice or after teardown,
|
|
295
|
+
// and silence here is what let the previous scheme look healthy.
|
|
296
|
+
log(`release for unknown file claim ${params.claimId}`);
|
|
219
297
|
}
|
|
220
|
-
return
|
|
298
|
+
return released;
|
|
221
299
|
}
|
|
222
300
|
|
|
223
301
|
case Command.FILE_STATS: {
|
|
224
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
302
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
225
303
|
return pool.getFileStats(torrent, params.fileIndex, {
|
|
226
304
|
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
227
305
|
});
|
|
228
306
|
}
|
|
229
307
|
|
|
230
308
|
case Command.PRIORITIZE: {
|
|
231
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
309
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
232
310
|
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes);
|
|
233
311
|
return true;
|
|
234
312
|
}
|
|
235
313
|
|
|
236
314
|
case Command.PREFETCH_EDGES: {
|
|
237
|
-
const torrent = requireTorrent(params.sourceKey);
|
|
238
|
-
return pool.prefetchFileEdges(torrent, params.fileIndex, params.
|
|
315
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
316
|
+
return pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {});
|
|
239
317
|
}
|
|
240
318
|
|
|
241
319
|
case Command.READ_RANGE: {
|
|
@@ -253,14 +331,14 @@ async function runCommand(command, params, id) {
|
|
|
253
331
|
|
|
254
332
|
case Command.CANCEL_READ: {
|
|
255
333
|
readsById.get(params.readId)?.cancel();
|
|
334
|
+
// A cancelled read will never have its outstanding fragment confirmed, so
|
|
335
|
+
// wake it here — otherwise it waits forever with a piece pinned.
|
|
336
|
+
settleFragment(params.readId);
|
|
256
337
|
return true;
|
|
257
338
|
}
|
|
258
339
|
|
|
259
340
|
case Command.DESTROY_ALL: {
|
|
260
|
-
|
|
261
|
-
release();
|
|
262
|
-
}
|
|
263
|
-
releaseByClaim.clear();
|
|
341
|
+
fileClaims.closeAll();
|
|
264
342
|
torrentsByKey.clear();
|
|
265
343
|
await pool.destroyAll();
|
|
266
344
|
return true;
|
|
@@ -279,6 +357,13 @@ parentPort.on("message", async (message) => {
|
|
|
279
357
|
return;
|
|
280
358
|
}
|
|
281
359
|
|
|
360
|
+
// The main thread has finished reading a fragment out of shared memory, so
|
|
361
|
+
// its piece may be unpinned and the read may continue.
|
|
362
|
+
if (message?.type === Event.FRAGMENT_DONE) {
|
|
363
|
+
settleFragment(message.id);
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
|
|
282
367
|
const { command, id, params } = message ?? {};
|
|
283
368
|
try {
|
|
284
369
|
const result = await runCommand(command, params ?? {}, id);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file File claims must be held per reader, not per file.
|
|
3
|
+
*
|
|
4
|
+
* The proxy reads one file from several places at once — ffmpeg's input, the
|
|
5
|
+
* keyframe index, the codec probe, a second viewer. Keying claims by file made
|
|
6
|
+
* them shared, so the first reader to finish released the hold while the others
|
|
7
|
+
* were still reading, and the file's data could then be removed under them.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import test from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { createFileClaims } from "../services/torrent-worker/file-claims.js";
|
|
13
|
+
|
|
14
|
+
test("two readers of one file hold two claims", () => {
|
|
15
|
+
const claims = createFileClaims();
|
|
16
|
+
let released = 0;
|
|
17
|
+
|
|
18
|
+
const first = claims.open("source", 0, () => (released += 1));
|
|
19
|
+
const second = claims.open("source", 0, () => (released += 1));
|
|
20
|
+
|
|
21
|
+
assert.notEqual(first, second, "the second reader reused the first one's claim");
|
|
22
|
+
assert.equal(claims.size, 2);
|
|
23
|
+
|
|
24
|
+
claims.close(first);
|
|
25
|
+
assert.equal(released, 1, "closing one claim released more than one hold");
|
|
26
|
+
assert.equal(claims.size, 1, "the second reader's claim went with the first");
|
|
27
|
+
|
|
28
|
+
claims.close(second);
|
|
29
|
+
assert.equal(released, 2);
|
|
30
|
+
assert.equal(claims.size, 0);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("a repeated release affects nothing and reports itself", () => {
|
|
34
|
+
const claims = createFileClaims();
|
|
35
|
+
let released = 0;
|
|
36
|
+
const claimId = claims.open("source", 3, () => (released += 1));
|
|
37
|
+
|
|
38
|
+
assert.equal(claims.close(claimId), true);
|
|
39
|
+
assert.equal(claims.close(claimId), false, "a second release was accepted as valid");
|
|
40
|
+
assert.equal(released, 1, "the hold was released twice");
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("a release naming nothing is rejected rather than guessed at", () => {
|
|
44
|
+
const claims = createFileClaims();
|
|
45
|
+
let released = 0;
|
|
46
|
+
claims.open("source", 0, () => (released += 1));
|
|
47
|
+
|
|
48
|
+
assert.equal(claims.close("source:0:999"), false);
|
|
49
|
+
assert.equal(released, 0, "an unknown claim released a real hold");
|
|
50
|
+
assert.equal(claims.size, 1);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("teardown releases every outstanding claim", () => {
|
|
54
|
+
const claims = createFileClaims();
|
|
55
|
+
let released = 0;
|
|
56
|
+
claims.open("a", 0, () => (released += 1));
|
|
57
|
+
claims.open("a", 1, () => (released += 1));
|
|
58
|
+
claims.open("b", 0, () => (released += 1));
|
|
59
|
+
|
|
60
|
+
claims.closeAll();
|
|
61
|
+
|
|
62
|
+
assert.equal(released, 3);
|
|
63
|
+
assert.equal(claims.size, 0);
|
|
64
|
+
});
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Turning a byte range into positions in shared memory.
|
|
3
|
+
*
|
|
4
|
+
* This arithmetic fails silently when it is wrong: the response comes back the
|
|
5
|
+
* right length and full of the wrong bytes. Piece numbers are torrent-wide
|
|
6
|
+
* while a read is expressed in file coordinates, and the first and last pieces
|
|
7
|
+
* of a range are almost always partial — so every case here is a boundary.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import test from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { EventEmitter } from "node:events";
|
|
13
|
+
import { readFragments } from "../services/torrent-worker/piece-reader.js";
|
|
14
|
+
import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
|
|
15
|
+
import os from "node:os";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
import fs from "node:fs/promises";
|
|
18
|
+
|
|
19
|
+
const PIECE = 1024;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A torrent whose pieces are all present, backed by a real store so that
|
|
23
|
+
* `locate`/`reside`/`pin` behave as they do in production.
|
|
24
|
+
*
|
|
25
|
+
* @param {{ fileOffset: number, fileLength: number, totalLength: number }} shape
|
|
26
|
+
*/
|
|
27
|
+
async function fakeTorrent({ fileOffset, fileLength, totalLength }) {
|
|
28
|
+
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "piece-reader-test-"));
|
|
29
|
+
const store = new SharedPieceStore(PIECE, {
|
|
30
|
+
length: totalLength,
|
|
31
|
+
memoryBytes: 64 * PIECE,
|
|
32
|
+
path: directory,
|
|
33
|
+
name: "test"
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const pieceCount = Math.ceil(totalLength / PIECE);
|
|
37
|
+
for (let index = 0; index < pieceCount; index += 1) {
|
|
38
|
+
const length = index === pieceCount - 1 ? totalLength - index * PIECE : PIECE;
|
|
39
|
+
const piece = Buffer.alloc(length);
|
|
40
|
+
// Each byte encodes its own absolute position, so a misplaced offset is
|
|
41
|
+
// visible in the value itself rather than only in the length.
|
|
42
|
+
for (let at = 0; at < length; at += 1) {
|
|
43
|
+
piece[at] = (index * PIECE + at) % 251;
|
|
44
|
+
}
|
|
45
|
+
await new Promise((resolve, reject) => {
|
|
46
|
+
store.put(index, piece, (error) => (error ? reject(error) : resolve()));
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const torrent = Object.assign(new EventEmitter(), {
|
|
51
|
+
pieceLength: PIECE,
|
|
52
|
+
store,
|
|
53
|
+
bitfield: { get: () => true },
|
|
54
|
+
files: [{ offset: fileOffset, length: fileLength, name: "file.bin" }],
|
|
55
|
+
select() {},
|
|
56
|
+
critical() {}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
return { torrent, store, directory };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Collect a range through the reader, as the worker does. */
|
|
63
|
+
async function readRange(torrent, start, end) {
|
|
64
|
+
const collected = [];
|
|
65
|
+
const positions = [];
|
|
66
|
+
const pool = Buffer.from(torrent.store.sharedBuffer);
|
|
67
|
+
for await (const fragment of readFragments({
|
|
68
|
+
torrent,
|
|
69
|
+
fileIndex: 0,
|
|
70
|
+
start,
|
|
71
|
+
end,
|
|
72
|
+
cancellation: { isCancelled: () => false }
|
|
73
|
+
})) {
|
|
74
|
+
collected.push(Buffer.from(pool.subarray(fragment.offset, fragment.offset + fragment.length)));
|
|
75
|
+
positions.push({ piece: fragment.pieceIndex, length: fragment.length });
|
|
76
|
+
fragment.release();
|
|
77
|
+
}
|
|
78
|
+
return { bytes: Buffer.concat(collected), positions };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** What the bytes at an absolute torrent offset should be. */
|
|
82
|
+
function expectedBytes(absoluteStart, length) {
|
|
83
|
+
const expected = Buffer.alloc(length);
|
|
84
|
+
for (let at = 0; at < length; at += 1) {
|
|
85
|
+
expected[at] = (absoluteStart + at) % 251;
|
|
86
|
+
}
|
|
87
|
+
return expected;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
test("a range inside one piece is read from that piece only", async () => {
|
|
91
|
+
const { torrent, store, directory } = await fakeTorrent({
|
|
92
|
+
fileOffset: 0,
|
|
93
|
+
fileLength: 4 * PIECE,
|
|
94
|
+
totalLength: 4 * PIECE
|
|
95
|
+
});
|
|
96
|
+
try {
|
|
97
|
+
const { bytes, positions } = await readRange(torrent, 100, 199);
|
|
98
|
+
assert.equal(positions.length, 1);
|
|
99
|
+
assert.deepEqual(bytes, expectedBytes(100, 100));
|
|
100
|
+
} finally {
|
|
101
|
+
store.destroy(() => undefined);
|
|
102
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("a range spanning pieces reassembles in order", async () => {
|
|
107
|
+
const { torrent, store, directory } = await fakeTorrent({
|
|
108
|
+
fileOffset: 0,
|
|
109
|
+
fileLength: 4 * PIECE,
|
|
110
|
+
totalLength: 4 * PIECE
|
|
111
|
+
});
|
|
112
|
+
try {
|
|
113
|
+
const start = PIECE - 10;
|
|
114
|
+
const end = 2 * PIECE + 9;
|
|
115
|
+
const { bytes, positions } = await readRange(torrent, start, end);
|
|
116
|
+
assert.deepEqual(positions.map((entry) => entry.piece), [0, 1, 2]);
|
|
117
|
+
assert.deepEqual(bytes, expectedBytes(start, end - start + 1));
|
|
118
|
+
} finally {
|
|
119
|
+
store.destroy(() => undefined);
|
|
120
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("a file that does not start at a piece boundary is still read correctly", async () => {
|
|
125
|
+
// The usual case in a multi-file torrent, and the one where using file
|
|
126
|
+
// offsets as if they were torrent offsets returns the wrong bytes at the
|
|
127
|
+
// right length.
|
|
128
|
+
const fileOffset = PIECE + 300;
|
|
129
|
+
const { torrent, store, directory } = await fakeTorrent({
|
|
130
|
+
fileOffset,
|
|
131
|
+
fileLength: 2 * PIECE,
|
|
132
|
+
totalLength: 5 * PIECE
|
|
133
|
+
});
|
|
134
|
+
try {
|
|
135
|
+
const { bytes } = await readRange(torrent, 0, 1499);
|
|
136
|
+
assert.deepEqual(bytes, expectedBytes(fileOffset, 1500));
|
|
137
|
+
} finally {
|
|
138
|
+
store.destroy(() => undefined);
|
|
139
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("every fragment releases its pin, so nothing stays held", async () => {
|
|
144
|
+
const { torrent, store, directory } = await fakeTorrent({
|
|
145
|
+
fileOffset: 0,
|
|
146
|
+
fileLength: 4 * PIECE,
|
|
147
|
+
totalLength: 4 * PIECE
|
|
148
|
+
});
|
|
149
|
+
try {
|
|
150
|
+
await readRange(torrent, 0, 4 * PIECE - 1);
|
|
151
|
+
// With every piece unpinned the store can still make room; if a pin leaked
|
|
152
|
+
// it would eventually refuse.
|
|
153
|
+
const before = store.stats().blockedByPins;
|
|
154
|
+
for (let index = 0; index < 200; index += 1) {
|
|
155
|
+
await store.reside(index % 4);
|
|
156
|
+
}
|
|
157
|
+
assert.equal(store.stats().blockedByPins, before, "a pin was left behind");
|
|
158
|
+
} finally {
|
|
159
|
+
store.destroy(() => undefined);
|
|
160
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
161
|
+
}
|
|
162
|
+
});
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What `/stream` does with a HEAD request.
|
|
3
|
+
*
|
|
4
|
+
* Fastify answers HEAD from the GET handler, so without an explicit branch a
|
|
5
|
+
* HEAD started a read of the entire file. Node discards the body, but the read
|
|
6
|
+
* itself runs on and the response never completes, which blocks the next
|
|
7
|
+
* request on that keep-alive connection. The keyframe index asks for the file
|
|
8
|
+
* size with exactly such a HEAD before every transcode session.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import test from "node:test";
|
|
12
|
+
import assert from "node:assert/strict";
|
|
13
|
+
import { handleStreamGet } from "../routes/stream/get.js";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Minimal stand-ins for the parts of Fastify and the pool this route touches.
|
|
17
|
+
*
|
|
18
|
+
* @param {{ method: string, range?: string }} request
|
|
19
|
+
* @returns {{ req: object, reply: object, sent: object, opened: string[], claims: number }}
|
|
20
|
+
*/
|
|
21
|
+
function harness({ method, range }) {
|
|
22
|
+
const opened = [];
|
|
23
|
+
const state = { claims: 0 };
|
|
24
|
+
|
|
25
|
+
const sent = { code: 200, headers: {}, body: undefined, called: false };
|
|
26
|
+
const reply = {
|
|
27
|
+
code(value) {
|
|
28
|
+
sent.code = value;
|
|
29
|
+
return reply;
|
|
30
|
+
},
|
|
31
|
+
header(name, value) {
|
|
32
|
+
sent.headers[name.toLowerCase()] = value;
|
|
33
|
+
return reply;
|
|
34
|
+
},
|
|
35
|
+
send(body) {
|
|
36
|
+
sent.called = true;
|
|
37
|
+
sent.body = body;
|
|
38
|
+
return reply;
|
|
39
|
+
},
|
|
40
|
+
hijack() {
|
|
41
|
+
sent.hijacked = true;
|
|
42
|
+
},
|
|
43
|
+
raw: {
|
|
44
|
+
// `bindRelease` listens here; HEAD writes its response here.
|
|
45
|
+
once() {},
|
|
46
|
+
writeHead(code, headers) {
|
|
47
|
+
sent.code = code;
|
|
48
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
49
|
+
sent.headers[name.toLowerCase()] = value;
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
end() {
|
|
53
|
+
sent.called = true;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const file = {
|
|
59
|
+
name: "movie.mkv",
|
|
60
|
+
length: 5_869_669_065,
|
|
61
|
+
createReadStream(options = {}) {
|
|
62
|
+
opened.push(`${options.start ?? 0}-${options.end ?? "end"}`);
|
|
63
|
+
return { on() {} };
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
const torrentPool = {
|
|
68
|
+
async getTorrent() {
|
|
69
|
+
return { files: [file], sourceKey: "key" };
|
|
70
|
+
},
|
|
71
|
+
acquireFile() {
|
|
72
|
+
state.claims += 1;
|
|
73
|
+
return () => undefined;
|
|
74
|
+
},
|
|
75
|
+
prioritizeByteRange() {}
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const req = {
|
|
79
|
+
method,
|
|
80
|
+
query: { sourceType: "magnet", source: "magnet:?xt=urn:btih:abc", fileIndex: "0" },
|
|
81
|
+
headers: range ? { range } : {},
|
|
82
|
+
raw: { once() {} }
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
return { req, reply, sent, opened, state, deps: { sourceRegistry: { get: () => null }, torrentPool } };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
test("HEAD reports the size without opening a read", async () => {
|
|
89
|
+
const { req, reply, sent, opened, state, deps } = harness({ method: "HEAD" });
|
|
90
|
+
|
|
91
|
+
await handleStreamGet(req, reply, deps);
|
|
92
|
+
|
|
93
|
+
assert.deepEqual(opened, [], "HEAD started a read of the file");
|
|
94
|
+
assert.equal(state.claims, 0, "HEAD claimed the file it never read");
|
|
95
|
+
// The real size, not the zero Fastify substitutes for an empty payload — the
|
|
96
|
+
// keyframe index reads this header and treats 0 as "no index".
|
|
97
|
+
assert.equal(sent.headers["content-length"], "5869669065");
|
|
98
|
+
assert.equal(sent.headers["accept-ranges"], "bytes");
|
|
99
|
+
assert.equal(sent.called, true, "HEAD never completed its response");
|
|
100
|
+
assert.equal(sent.body, undefined, "HEAD answered with a body");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("GET still streams the bytes", async () => {
|
|
104
|
+
const { req, reply, sent, opened, deps } = harness({ method: "GET" });
|
|
105
|
+
|
|
106
|
+
await handleStreamGet(req, reply, deps);
|
|
107
|
+
|
|
108
|
+
assert.equal(opened.length, 1, "GET did not open a read");
|
|
109
|
+
assert.equal(sent.headers["content-length"], "5869669065");
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("GET with a range streams only that range", async () => {
|
|
113
|
+
const { req, reply, sent, opened, deps } = harness({ method: "GET", range: "bytes=100-199" });
|
|
114
|
+
|
|
115
|
+
await handleStreamGet(req, reply, deps);
|
|
116
|
+
|
|
117
|
+
assert.deepEqual(opened, ["100-199"]);
|
|
118
|
+
assert.equal(sent.code, 206);
|
|
119
|
+
assert.equal(sent.headers["content-range"], "bytes 100-199/5869669065");
|
|
120
|
+
assert.equal(sent.headers["content-length"], "100");
|
|
121
|
+
});
|