@torrent-tv/proxy 2.9.76 → 2.9.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,416 +1,488 @@
1
- /**
2
- * @file Torrent pieces in shared memory, with disk as the second tier.
3
- *
4
- * Replaces the chunk store WebTorrent would otherwise build for itself. Two
5
- * reasons, one forced and one chosen.
6
- *
7
- * **Forced.** WebTorrent's own piece cache hands out the buffer it keeps using
8
- * and slices again on the next read. The torrent client now runs on its own
9
- * thread, and moving a piece to the main thread by transferring ownership
10
- * detached the cache's memory: proxy 2.9.71-2.9.73 answered every read with an
11
- * empty body (`Stream ends prematurely at 0`) and the failure was invisible,
12
- * because the error never reached the reader. Owning the memory ourselves
13
- * removes the question of whose it was.
14
- *
15
- * **Chosen.** Memory this side of the thread boundary can be *shared* memory,
16
- * which the main thread reads by offset instead of receiving as bytes — see
17
- * {@link SharedPieceStore#locate}. Measured on the field host: a piece copy
18
- * costs 3.64 ms, a read from the page cache into a buffer we own 7.63 ms, and
19
- * re-downloading a piece from the swarm ~1430 ms. So memory first, disk under
20
- * it, and never the swarm twice.
21
- *
22
- * What this deliberately does NOT do is manage the disk as a cache of its own.
23
- * Pieces evicted from memory are written once and read back on demand; the file
24
- * is discarded whole when the torrent goes away. libtorrent 2.0 and webtor's
25
- * seeder both concluded that a hand-rolled disk cache earns less than it costs,
26
- * and nothing here disagrees.
27
- */
28
-
29
- import os from "node:os";
30
- import { PieceLru } from "./piece-lru.js";
31
- import { DiskTier } from "./disk-tier.js";
32
-
33
- /**
34
- * Live stores, so the worker can report on them.
35
- *
36
- * WebTorrent constructs the store itself, deep inside its own wrappers, so
37
- * there is no handle to reach for from outside. Registering here is what makes
38
- * the store's behaviour visible in the field at all — without it the first
39
- * strange case has nothing to go on.
40
- *
41
- * @type {Set<SharedPieceStore>}
42
- */
43
- const liveStores = new Set();
44
-
45
- /**
46
- * A snapshot of every live store, for logging.
47
- *
48
- * @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }[]}
49
- */
50
- export function collectStoreStats() {
51
- return [...liveStores].map((store) => store.stats());
52
- }
53
-
54
- /**
55
- * Ceiling for the automatic budget, and the share of free memory it will take.
56
- *
57
- * A flat default would be a guess dressed as a decision: the proxy runs on
58
- * whatever the owner has, from a Pi to a rented box, and the budget is **per
59
- * torrent** several viewers mean several of these. Measured on the field host
60
- * after one session: the proxy container sat at 796 MB with 4.1 GB free and 1.3
61
- * GB already in swap, so a fixed half-gigabyte per torrent is not something to
62
- * hand out blindly. Hence: a quarter of what is free, capped.
63
- */
64
- const MEMORY_BUDGET_CEILING_BYTES = 512 * 1024 * 1024;
65
- const FREE_MEMORY_SHARE = 0.25;
66
-
67
- /**
68
- * Budget for one torrent's resident pieces when the caller names none.
69
- *
70
- * @returns {number}
71
- */
72
- function defaultMemoryBytes() {
73
- const share = Math.floor(os.freemem() * FREE_MEMORY_SHARE);
74
- return Math.max(MIN_BUDGET_BYTES, Math.min(MEMORY_BUDGET_CEILING_BYTES, share));
75
- }
76
-
77
- /** Floor for the automatic budget — below this the store thrashes to disk. */
78
- const MIN_BUDGET_BYTES = 64 * 1024 * 1024;
79
- /**
80
- * Never keep fewer than this many pieces resident, whatever the budget says.
81
- *
82
- * Two is the smallest workable number rather than a round one: a piece being
83
- * read holds its slot, so a second slot must exist for the next piece to land
84
- * in. With one, a single reader would deadlock the store against itself.
85
- */
86
- const MIN_RESIDENT_PIECES = 2;
87
-
88
- /**
89
- * A chunk store holding pieces in a `SharedArrayBuffer`, spilling to disk.
90
- *
91
- * Implements the `abstract-chunk-store` shape WebTorrent expects — `put`,
92
- * `get`, `close`, `destroy` — plus {@link locate}, {@link pin} and
93
- * {@link unpin}, which is how the main thread reads a piece without it being
94
- * copied or moved.
95
- */
96
- export class SharedPieceStore {
97
- #chunkLength;
98
- #lastChunkLength;
99
- #lastChunkIndex;
100
- #capacity;
101
- /** @type {SharedArrayBuffer} */
102
- #shared;
103
- /** @type {Buffer} A view over the whole pool, for slot arithmetic. */
104
- #pool;
105
- /** Piece index slot number. */
106
- #slotOf = new Map();
107
- /** Slot numbers not currently holding a piece. */
108
- #freeSlots = [];
109
- #lru;
110
- #disk;
111
- /** Slots backed by memory right now; grows towards {@link capacity}. */
112
- #allocatedSlots = 0;
113
- #closed = false;
114
- #name;
115
- /**
116
- * What the store has actually been doing. Reported, not just kept: the
117
- * balance between memory and disk reads is the number that says whether the
118
- * budget is right, and it cannot be guessed from outside.
119
- */
120
- #counters = {
121
- fromMemory: 0,
122
- fromDisk: 0,
123
- spills: 0,
124
- revivals: 0,
125
- blockedByPins: 0
126
- };
127
-
128
- /**
129
- * @param {number} chunkLength - Piece length, and therefore the slot size.
130
- * @param {object} [options]
131
- * @param {number} [options.length] - Total torrent length, so the short last piece is sized correctly.
132
- * @param {number} [options.memoryBytes] - Budget for resident pieces.
133
- * @param {string} [options.path] - Directory for the spill file.
134
- * @param {string} [options.name] - Spill file name; must be unique per torrent.
135
- */
136
- constructor(chunkLength, options = {}) {
137
- if (!Number.isInteger(chunkLength) || chunkLength < 1) {
138
- throw new Error(`Chunk length must be a positive integer, got ${chunkLength}.`);
139
- }
140
- this.#chunkLength = chunkLength;
141
-
142
- const totalLength = Number.isFinite(options.length) ? options.length : 0;
143
- this.#lastChunkIndex = totalLength > 0 ? Math.ceil(totalLength / chunkLength) - 1 : -1;
144
- const remainder = totalLength % chunkLength;
145
- this.#lastChunkLength = remainder === 0 ? chunkLength : remainder;
146
-
147
- const memoryBytes = Number.isFinite(options.memoryBytes) && options.memoryBytes > 0
148
- ? options.memoryBytes
149
- : defaultMemoryBytes();
150
- this.#capacity = Math.max(MIN_RESIDENT_PIECES, Math.floor(memoryBytes / chunkLength));
151
-
152
- // Grows into the budget instead of taking it up front. The budget is per
153
- // torrent, so claiming all of it on `add` would charge a host for pieces
154
- // nobody has asked for and a torrent that is merely open, or one being
155
- // probed for its codecs, needs a handful of slots, not the ceiling.
156
- this.#shared = new SharedArrayBuffer(MIN_RESIDENT_PIECES * chunkLength, {
157
- maxByteLength: this.#capacity * chunkLength
158
- });
159
- this.#pool = Buffer.from(this.#shared);
160
- for (let slot = 0; slot < MIN_RESIDENT_PIECES; slot += 1) {
161
- this.#freeSlots.push(slot);
162
- }
163
- this.#allocatedSlots = MIN_RESIDENT_PIECES;
164
- this.#lru = new PieceLru(this.#capacity);
165
- this.#name = options.name ?? "pieces";
166
- this.#disk = new DiskTier({
167
- directory: options.path ?? ".",
168
- name: `${this.#name}.pieces`,
169
- chunkLength
170
- });
171
- liveStores.add(this);
172
- }
173
-
174
- /**
175
- * What this store has been doing, for the periodic report.
176
- *
177
- * @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }}
178
- */
179
- stats() {
180
- return {
181
- name: this.#name,
182
- resident: this.#slotOf.size,
183
- capacity: this.#capacity,
184
- spilled: this.#disk.size,
185
- ...this.#counters
186
- };
187
- }
188
-
189
- /** `abstract-chunk-store` exposes the piece size under this name. */
190
- get chunkLength() {
191
- return this.#chunkLength;
192
- }
193
-
194
- /**
195
- * The pool itself, so another thread can map the same memory and read a piece
196
- * by the offset {@link locate} reports.
197
- *
198
- * @returns {SharedArrayBuffer}
199
- */
200
- get sharedBuffer() {
201
- return this.#shared;
202
- }
203
-
204
- /** How many pieces fit in memory at once. */
205
- get capacity() {
206
- return this.#capacity;
207
- }
208
-
209
- /** How many pieces are resident right now. */
210
- get residentCount() {
211
- return this.#slotOf.size;
212
- }
213
-
214
- /** How many pieces have been spilled to disk. */
215
- get spilledCount() {
216
- return this.#disk.size;
217
- }
218
-
219
- /**
220
- * Length of a given piece the last one is usually short.
221
- *
222
- * @param {number} index
223
- * @returns {number}
224
- */
225
- #lengthOf(index) {
226
- return index === this.#lastChunkIndex ? this.#lastChunkLength : this.#chunkLength;
227
- }
228
-
229
- /**
230
- * Where a resident piece sits in the shared pool, or `null` if it is not
231
- * resident.
232
- *
233
- * The main thread reads straight from those bytes, so callers MUST hold a pin
234
- * across the read see {@link pin}.
235
- *
236
- * @param {number} index
237
- * @returns {{ offset: number, length: number } | null}
238
- */
239
- locate(index) {
240
- const slot = this.#slotOf.get(index);
241
- if (slot === undefined) {
242
- return null;
243
- }
244
- return { offset: slot * this.#chunkLength, length: this.#lengthOf(index) };
245
- }
246
-
247
- /**
248
- * Hold a piece in memory across a read. Nested; release with {@link unpin}.
249
- *
250
- * @param {number} index
251
- * @returns {void}
252
- */
253
- pin(index) {
254
- this.#lru.pin(index);
255
- }
256
-
257
- /**
258
- * @param {number} index
259
- * @returns {void}
260
- */
261
- unpin(index) {
262
- this.#lru.unpin(index);
263
- }
264
-
265
- /**
266
- * Make a slot available, spilling the least recently used piece if need be.
267
- *
268
- * @returns {Promise<number>} Slot number.
269
- */
270
- async #claimSlot() {
271
- const free = this.#freeSlots.pop();
272
- if (free !== undefined) {
273
- return free;
274
- }
275
-
276
- // Room left in the budget: take more memory rather than evicting. Growing
277
- // replaces the view over the pool, so every slot offset stays valid — the
278
- // bytes do not move.
279
- if (this.#allocatedSlots < this.#capacity) {
280
- this.#allocatedSlots += 1;
281
- this.#shared.grow(this.#allocatedSlots * this.#chunkLength);
282
- this.#pool = Buffer.from(this.#shared);
283
- return this.#allocatedSlots - 1;
284
- }
285
-
286
- const victim = this.#lru.evictionCandidate();
287
- if (victim === null) {
288
- // Every resident piece is being read. Taking one anyway is precisely the
289
- // failure this store exists to make impossible.
290
- this.#counters.blockedByPins += 1;
291
- throw new Error("Every resident piece is pinned; no slot can be freed.");
292
- }
293
-
294
- const slot = this.#slotOf.get(victim);
295
- const bytes = this.#pool.subarray(slot * this.#chunkLength, slot * this.#chunkLength + this.#lengthOf(victim));
296
- await this.#disk.write(victim, bytes);
297
- this.#counters.spills += 1;
298
-
299
- this.#slotOf.delete(victim);
300
- this.#lru.remove(victim);
301
- return slot;
302
- }
303
-
304
- /**
305
- * Store a piece.
306
- *
307
- * @param {number} index
308
- * @param {Uint8Array} bytes
309
- * @param {(error?: Error | null) => void} [callback]
310
- * @returns {void}
311
- */
312
- put(index, bytes, callback = () => undefined) {
313
- if (this.#closed) {
314
- queueMicrotask(() => callback(new Error("Piece store is closed.")));
315
- return;
316
- }
317
-
318
- const existing = this.#slotOf.get(index);
319
- const write = async () => {
320
- const slot = existing ?? (await this.#claimSlot());
321
- bytes.copy
322
- ? bytes.copy(this.#pool, slot * this.#chunkLength)
323
- : this.#pool.set(bytes, slot * this.#chunkLength);
324
- this.#slotOf.set(index, slot);
325
- this.#lru.touch(index);
326
- // A newer copy is in memory; whatever is on disk is stale.
327
- this.#disk.forget(index);
328
- };
329
-
330
- write().then(() => callback(null), (error) => callback(error));
331
- }
332
-
333
- /**
334
- * Fetch a piece, or a range within it.
335
- *
336
- * Returns a buffer of its own rather than a view into the pool: WebTorrent
337
- * keeps what it is given — to verify a hash, to serve a peer and the slot
338
- * underneath may be reused meanwhile. The thread-crossing path avoids this
339
- * copy entirely by going through {@link locate}.
340
- *
341
- * @param {number} index
342
- * @param {{ offset?: number, length?: number } | ((error: Error | null, bytes?: Buffer) => void)} [options]
343
- * @param {(error: Error | null, bytes?: Buffer) => void} [callback]
344
- * @returns {void}
345
- */
346
- get(index, options, callback) {
347
- if (typeof options === "function") {
348
- return this.get(index, undefined, options);
349
- }
350
- const done = callback ?? (() => undefined);
351
- if (this.#closed) {
352
- queueMicrotask(() => done(new Error("Piece store is closed.")));
353
- return;
354
- }
355
-
356
- const pieceLength = this.#lengthOf(index);
357
- const offset = options?.offset ?? 0;
358
- const length = options?.length ?? pieceLength - offset;
359
-
360
- const fetch = async () => {
361
- const slot = this.#slotOf.get(index);
362
- if (slot !== undefined) {
363
- this.#lru.touch(index);
364
- this.#counters.fromMemory += 1;
365
- const start = slot * this.#chunkLength + offset;
366
- return Buffer.from(this.#pool.subarray(start, start + length));
367
- }
368
-
369
- if (!this.#disk.has(index)) {
370
- throw new Error(`Piece ${index} is not in the store.`);
371
- }
372
-
373
- // Bring it back into memory: it was just asked for, so it is likely to be
374
- // asked for again, and the caller may follow up with `locate`.
375
- const revived = await this.#claimSlot();
376
- const target = this.#pool.subarray(
377
- revived * this.#chunkLength,
378
- revived * this.#chunkLength + pieceLength
379
- );
380
- await this.#disk.read(index, target);
381
- this.#slotOf.set(index, revived);
382
- this.#lru.touch(index);
383
- this.#counters.fromDisk += 1;
384
- this.#counters.revivals += 1;
385
- const start = revived * this.#chunkLength + offset;
386
- return Buffer.from(this.#pool.subarray(start, start + length));
387
- };
388
-
389
- fetch().then((bytes) => done(null, bytes), (error) => done(error));
390
- }
391
-
392
- /**
393
- * Close the store, keeping the spill file.
394
- *
395
- * @param {(error?: Error | null) => void} [callback]
396
- * @returns {void}
397
- */
398
- close(callback = () => undefined) {
399
- this.#closed = true;
400
- liveStores.delete(this);
401
- this.#disk.close().then(() => callback(null), (error) => callback(error));
402
- }
403
-
404
- /**
405
- * Close the store and delete everything it wrote.
406
- *
407
- * @param {(error?: Error | null) => void} [callback]
408
- * @returns {void}
409
- */
410
- destroy(callback = () => undefined) {
411
- this.#closed = true;
412
- liveStores.delete(this);
413
- this.#slotOf.clear();
414
- this.#disk.destroy().then(() => callback(null), (error) => callback(error));
415
- }
416
- }
1
+ /**
2
+ * @file Torrent pieces in shared memory, with disk as the second tier.
3
+ *
4
+ * Replaces the chunk store WebTorrent would otherwise build for itself. Two
5
+ * reasons, one forced and one chosen.
6
+ *
7
+ * **Forced.** WebTorrent's own piece cache hands out the buffer it keeps using
8
+ * and slices again on the next read. The torrent client now runs on its own
9
+ * thread, and moving a piece to the main thread by transferring ownership
10
+ * detached the cache's memory: proxy 2.9.71-2.9.73 answered every read with an
11
+ * empty body (`Stream ends prematurely at 0`) and the failure was invisible,
12
+ * because the error never reached the reader. Owning the memory ourselves
13
+ * removes the question of whose it was.
14
+ *
15
+ * **Chosen.** Memory this side of the thread boundary can be *shared* memory,
16
+ * which the main thread reads by offset instead of receiving as bytes — see
17
+ * {@link SharedPieceStore#locate}. Measured on the field host: a piece copy
18
+ * costs 3.64 ms, a read from the page cache into a buffer we own 7.63 ms, and
19
+ * re-downloading a piece from the swarm ~1430 ms. So memory first, disk under
20
+ * it, and never the swarm twice.
21
+ *
22
+ * What this deliberately does NOT do is manage the disk as a cache of its own.
23
+ * Pieces evicted from memory are written once and read back on demand; the file
24
+ * is discarded whole when the torrent goes away. libtorrent 2.0 and webtor's
25
+ * seeder both concluded that a hand-rolled disk cache earns less than it costs,
26
+ * and nothing here disagrees.
27
+ */
28
+
29
+ import os from "node:os";
30
+ import { PieceLru } from "./piece-lru.js";
31
+ import { DiskTier } from "./disk-tier.js";
32
+
33
+ /**
34
+ * Live stores, so the worker can report on them.
35
+ *
36
+ * WebTorrent constructs the store itself, deep inside its own wrappers, so
37
+ * there is no handle to reach for from outside. Registering here is what makes
38
+ * the store's behaviour visible in the field at all — without it the first
39
+ * strange case has nothing to go on.
40
+ *
41
+ * @type {Set<SharedPieceStore>}
42
+ */
43
+ const liveStores = new Set();
44
+
45
+ /**
46
+ * A snapshot of every live store, for logging.
47
+ *
48
+ * @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }[]}
49
+ */
50
+ export function collectStoreStats() {
51
+ return [...liveStores].map((store) => store.stats());
52
+ }
53
+
54
+ /**
55
+ * The shared store behind a torrent, or `null` if it is not one of ours.
56
+ *
57
+ * WebTorrent wraps whatever store it is given today in `ImmediateChunkStore`,
58
+ * and historically in a piece cache as well and offers no way to ask for the
59
+ * innermost one. Walking the `store` chain finds it regardless of how many
60
+ * wrappers there are or what order they sit in, which is sturdier than reaching
61
+ * for a fixed `torrent.store.store`.
62
+ *
63
+ * @param {{ store?: object } | null | undefined} torrent
64
+ * @returns {SharedPieceStore | null}
65
+ */
66
+ export function findSharedStore(torrent) {
67
+ let candidate = torrent?.store;
68
+ // Bounded rather than `while (candidate)`: a store that referenced itself
69
+ // would otherwise hang the thread instead of failing.
70
+ for (let depth = 0; candidate && depth < 8; depth += 1) {
71
+ if (candidate instanceof SharedPieceStore) {
72
+ return candidate;
73
+ }
74
+ candidate = candidate.store;
75
+ }
76
+ return null;
77
+ }
78
+
79
+ /**
80
+ * Ceiling for the automatic budget, and the share of free memory it will take.
81
+ *
82
+ * A flat default would be a guess dressed as a decision: the proxy runs on
83
+ * whatever the owner has, from a Pi to a rented box, and the budget is **per
84
+ * torrent** several viewers mean several of these. Measured on the field host
85
+ * after one session: the proxy container sat at 796 MB with 4.1 GB free and 1.3
86
+ * GB already in swap, so a fixed half-gigabyte per torrent is not something to
87
+ * hand out blindly. Hence: a quarter of what is free, capped.
88
+ */
89
+ const MEMORY_BUDGET_CEILING_BYTES = 512 * 1024 * 1024;
90
+ const FREE_MEMORY_SHARE = 0.25;
91
+
92
+ /**
93
+ * Budget for one torrent's resident pieces when the caller names none.
94
+ *
95
+ * @returns {number}
96
+ */
97
+ function defaultMemoryBytes() {
98
+ const share = Math.floor(os.freemem() * FREE_MEMORY_SHARE);
99
+ return Math.max(MIN_BUDGET_BYTES, Math.min(MEMORY_BUDGET_CEILING_BYTES, share));
100
+ }
101
+
102
+ /** Floor for the automatic budget — below this the store thrashes to disk. */
103
+ const MIN_BUDGET_BYTES = 64 * 1024 * 1024;
104
+ /**
105
+ * Never keep fewer than this many pieces resident, whatever the budget says.
106
+ *
107
+ * Two is the smallest workable number rather than a round one: a piece being
108
+ * read holds its slot, so a second slot must exist for the next piece to land
109
+ * in. With one, a single reader would deadlock the store against itself.
110
+ */
111
+ const MIN_RESIDENT_PIECES = 2;
112
+
113
+ /**
114
+ * A chunk store holding pieces in a `SharedArrayBuffer`, spilling to disk.
115
+ *
116
+ * Implements the `abstract-chunk-store` shape WebTorrent expects `put`,
117
+ * `get`, `close`, `destroy` plus {@link locate}, {@link pin} and
118
+ * {@link unpin}, which is how the main thread reads a piece without it being
119
+ * copied or moved.
120
+ */
121
+ export class SharedPieceStore {
122
+ #chunkLength;
123
+ #lastChunkLength;
124
+ #lastChunkIndex;
125
+ #capacity;
126
+ /** @type {SharedArrayBuffer} */
127
+ #shared;
128
+ /** @type {Buffer} A view over the whole pool, for slot arithmetic. */
129
+ #pool;
130
+ /** Piece index → slot number. */
131
+ #slotOf = new Map();
132
+ /** Slot numbers not currently holding a piece. */
133
+ #freeSlots = [];
134
+ #lru;
135
+ #disk;
136
+ /** Slots backed by memory right now; grows towards {@link capacity}. */
137
+ #allocatedSlots = 0;
138
+ #closed = false;
139
+ #name;
140
+ /**
141
+ * What the store has actually been doing. Reported, not just kept: the
142
+ * balance between memory and disk reads is the number that says whether the
143
+ * budget is right, and it cannot be guessed from outside.
144
+ */
145
+ #counters = {
146
+ fromMemory: 0,
147
+ fromDisk: 0,
148
+ spills: 0,
149
+ revivals: 0,
150
+ blockedByPins: 0
151
+ };
152
+
153
+ /**
154
+ * @param {number} chunkLength - Piece length, and therefore the slot size.
155
+ * @param {object} [options]
156
+ * @param {number} [options.length] - Total torrent length, so the short last piece is sized correctly.
157
+ * @param {number} [options.memoryBytes] - Budget for resident pieces.
158
+ * @param {string} [options.path] - Directory for the spill file.
159
+ * @param {string} [options.name] - Spill file name; must be unique per torrent.
160
+ */
161
+ constructor(chunkLength, options = {}) {
162
+ if (!Number.isInteger(chunkLength) || chunkLength < 1) {
163
+ throw new Error(`Chunk length must be a positive integer, got ${chunkLength}.`);
164
+ }
165
+ this.#chunkLength = chunkLength;
166
+
167
+ const totalLength = Number.isFinite(options.length) ? options.length : 0;
168
+ this.#lastChunkIndex = totalLength > 0 ? Math.ceil(totalLength / chunkLength) - 1 : -1;
169
+ const remainder = totalLength % chunkLength;
170
+ this.#lastChunkLength = remainder === 0 ? chunkLength : remainder;
171
+
172
+ const memoryBytes = Number.isFinite(options.memoryBytes) && options.memoryBytes > 0
173
+ ? options.memoryBytes
174
+ : defaultMemoryBytes();
175
+ this.#capacity = Math.max(MIN_RESIDENT_PIECES, Math.floor(memoryBytes / chunkLength));
176
+
177
+ // Grows into the budget instead of taking it up front. The budget is per
178
+ // torrent, so claiming all of it on `add` would charge a host for pieces
179
+ // nobody has asked for — and a torrent that is merely open, or one being
180
+ // probed for its codecs, needs a handful of slots, not the ceiling.
181
+ this.#shared = new SharedArrayBuffer(MIN_RESIDENT_PIECES * chunkLength, {
182
+ maxByteLength: this.#capacity * chunkLength
183
+ });
184
+ this.#pool = Buffer.from(this.#shared);
185
+ for (let slot = 0; slot < MIN_RESIDENT_PIECES; slot += 1) {
186
+ this.#freeSlots.push(slot);
187
+ }
188
+ this.#allocatedSlots = MIN_RESIDENT_PIECES;
189
+ this.#lru = new PieceLru(this.#capacity);
190
+ this.#name = options.name ?? "pieces";
191
+ this.#disk = new DiskTier({
192
+ directory: options.path ?? ".",
193
+ name: `${this.#name}.pieces`,
194
+ chunkLength
195
+ });
196
+ liveStores.add(this);
197
+ }
198
+
199
+ /**
200
+ * What this store has been doing, for the periodic report.
201
+ *
202
+ * @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }}
203
+ */
204
+ stats() {
205
+ return {
206
+ name: this.#name,
207
+ resident: this.#slotOf.size,
208
+ capacity: this.#capacity,
209
+ spilled: this.#disk.size,
210
+ ...this.#counters
211
+ };
212
+ }
213
+
214
+ /** `abstract-chunk-store` exposes the piece size under this name. */
215
+ get chunkLength() {
216
+ return this.#chunkLength;
217
+ }
218
+
219
+ /**
220
+ * The pool itself, so another thread can map the same memory and read a piece
221
+ * by the offset {@link locate} reports.
222
+ *
223
+ * @returns {SharedArrayBuffer}
224
+ */
225
+ get sharedBuffer() {
226
+ return this.#shared;
227
+ }
228
+
229
+ /** How many pieces fit in memory at once. */
230
+ get capacity() {
231
+ return this.#capacity;
232
+ }
233
+
234
+ /** How many pieces are resident right now. */
235
+ get residentCount() {
236
+ return this.#slotOf.size;
237
+ }
238
+
239
+ /** How many pieces have been spilled to disk. */
240
+ get spilledCount() {
241
+ return this.#disk.size;
242
+ }
243
+
244
+ /**
245
+ * Length of a given piece — the last one is usually short.
246
+ *
247
+ * @param {number} index
248
+ * @returns {number}
249
+ */
250
+ #lengthOf(index) {
251
+ return index === this.#lastChunkIndex ? this.#lastChunkLength : this.#chunkLength;
252
+ }
253
+
254
+ /**
255
+ * Where a resident piece sits in the shared pool, or `null` if it is not
256
+ * resident.
257
+ *
258
+ * The main thread reads straight from those bytes, so callers MUST hold a pin
259
+ * across the read — see {@link pin}.
260
+ *
261
+ * @param {number} index
262
+ * @returns {{ offset: number, length: number } | null}
263
+ */
264
+ locate(index) {
265
+ const slot = this.#slotOf.get(index);
266
+ if (slot === undefined) {
267
+ return null;
268
+ }
269
+ return { offset: slot * this.#chunkLength, length: this.#lengthOf(index) };
270
+ }
271
+
272
+ /**
273
+ * Hold a piece in memory across a read. Nested; release with {@link unpin}.
274
+ *
275
+ * @param {number} index
276
+ * @returns {void}
277
+ */
278
+ pin(index) {
279
+ this.#lru.pin(index);
280
+ }
281
+
282
+ /**
283
+ * @param {number} index
284
+ * @returns {void}
285
+ */
286
+ unpin(index) {
287
+ this.#lru.unpin(index);
288
+ }
289
+
290
+ /**
291
+ * Make a slot available, spilling the least recently used piece if need be.
292
+ *
293
+ * @returns {Promise<number>} Slot number.
294
+ */
295
+ async #claimSlot() {
296
+ const free = this.#freeSlots.pop();
297
+ if (free !== undefined) {
298
+ return free;
299
+ }
300
+
301
+ // Room left in the budget: take more memory rather than evicting. Growing
302
+ // replaces the view over the pool, so every slot offset stays valid — the
303
+ // bytes do not move.
304
+ if (this.#allocatedSlots < this.#capacity) {
305
+ this.#allocatedSlots += 1;
306
+ this.#shared.grow(this.#allocatedSlots * this.#chunkLength);
307
+ this.#pool = Buffer.from(this.#shared);
308
+ return this.#allocatedSlots - 1;
309
+ }
310
+
311
+ const victim = this.#lru.evictionCandidate();
312
+ if (victim === null) {
313
+ // Every resident piece is being read. Taking one anyway is precisely the
314
+ // failure this store exists to make impossible.
315
+ this.#counters.blockedByPins += 1;
316
+ throw new Error("Every resident piece is pinned; no slot can be freed.");
317
+ }
318
+
319
+ const slot = this.#slotOf.get(victim);
320
+ const bytes = this.#pool.subarray(slot * this.#chunkLength, slot * this.#chunkLength + this.#lengthOf(victim));
321
+ await this.#disk.write(victim, bytes);
322
+ this.#counters.spills += 1;
323
+
324
+ this.#slotOf.delete(victim);
325
+ this.#lru.remove(victim);
326
+ return slot;
327
+ }
328
+
329
+ /**
330
+ * Store a piece.
331
+ *
332
+ * @param {number} index
333
+ * @param {Uint8Array} bytes
334
+ * @param {(error?: Error | null) => void} [callback]
335
+ * @returns {void}
336
+ */
337
+ put(index, bytes, callback = () => undefined) {
338
+ if (this.#closed) {
339
+ queueMicrotask(() => callback(new Error("Piece store is closed.")));
340
+ return;
341
+ }
342
+
343
+ const existing = this.#slotOf.get(index);
344
+ const write = async () => {
345
+ const slot = existing ?? (await this.#claimSlot());
346
+ bytes.copy
347
+ ? bytes.copy(this.#pool, slot * this.#chunkLength)
348
+ : this.#pool.set(bytes, slot * this.#chunkLength);
349
+ this.#slotOf.set(index, slot);
350
+ this.#lru.touch(index);
351
+ // A newer copy is in memory; whatever is on disk is stale.
352
+ this.#disk.forget(index);
353
+ };
354
+
355
+ write().then(() => callback(null), (error) => callback(error));
356
+ }
357
+
358
+ /**
359
+ * Fetch a piece, or a range within it.
360
+ *
361
+ * Returns a buffer of its own rather than a view into the pool: WebTorrent
362
+ * keeps what it is given — to verify a hash, to serve a peer — and the slot
363
+ * underneath may be reused meanwhile. The thread-crossing path avoids this
364
+ * copy entirely by going through {@link locate}.
365
+ *
366
+ * @param {number} index
367
+ * @param {{ offset?: number, length?: number } | ((error: Error | null, bytes?: Buffer) => void)} [options]
368
+ * @param {(error: Error | null, bytes?: Buffer) => void} [callback]
369
+ * @returns {void}
370
+ */
371
+ get(index, options, callback) {
372
+ if (typeof options === "function") {
373
+ return this.get(index, undefined, options);
374
+ }
375
+ const done = callback ?? (() => undefined);
376
+ if (this.#closed) {
377
+ queueMicrotask(() => done(new Error("Piece store is closed.")));
378
+ return;
379
+ }
380
+
381
+ const pieceLength = this.#lengthOf(index);
382
+ const offset = options?.offset ?? 0;
383
+ const length = options?.length ?? pieceLength - offset;
384
+
385
+ const fetch = async () => {
386
+ const slot = this.#slotOf.get(index);
387
+ if (slot !== undefined) {
388
+ this.#lru.touch(index);
389
+ this.#counters.fromMemory += 1;
390
+ const start = slot * this.#chunkLength + offset;
391
+ return Buffer.from(this.#pool.subarray(start, start + length));
392
+ }
393
+
394
+ if (!this.#disk.has(index)) {
395
+ throw new Error(`Piece ${index} is not in the store.`);
396
+ }
397
+
398
+ // Bring it back into memory: it was just asked for, so it is likely to be
399
+ // asked for again, and the caller may follow up with `locate`.
400
+ const revived = await this.#claimSlot();
401
+ const target = this.#pool.subarray(
402
+ revived * this.#chunkLength,
403
+ revived * this.#chunkLength + pieceLength
404
+ );
405
+ await this.#disk.read(index, target);
406
+ this.#slotOf.set(index, revived);
407
+ this.#lru.touch(index);
408
+ this.#counters.fromDisk += 1;
409
+ this.#counters.revivals += 1;
410
+ const start = revived * this.#chunkLength + offset;
411
+ return Buffer.from(this.#pool.subarray(start, start + length));
412
+ };
413
+
414
+ fetch().then((bytes) => done(null, bytes), (error) => done(error));
415
+ }
416
+
417
+ /**
418
+ * Ensure a piece is in memory and say where it sits — without copying it.
419
+ *
420
+ * This is {@link get} minus its final copy, and it exists for exactly one
421
+ * caller: the reader that hands pieces to the other thread. That thread maps
422
+ * the same {@link sharedBuffer}, so an offset and a length are all it needs,
423
+ * and the bytes never move. `get` cannot serve that purpose because
424
+ * WebTorrent keeps what `get` returns while the slot underneath may be
425
+ * reused.
426
+ *
427
+ * The caller MUST hold a pin across the whole read — the returned offset
428
+ * stays valid only while the piece is pinned.
429
+ *
430
+ * @param {number} index
431
+ * @returns {Promise<{ offset: number, length: number } | null>} `null` when
432
+ * the store holds no such piece, in memory or on disk.
433
+ */
434
+ async reside(index) {
435
+ if (this.#closed) {
436
+ throw new Error("Piece store is closed.");
437
+ }
438
+
439
+ const slot = this.#slotOf.get(index);
440
+ if (slot !== undefined) {
441
+ this.#lru.touch(index);
442
+ this.#counters.fromMemory += 1;
443
+ return this.locate(index);
444
+ }
445
+
446
+ if (!this.#disk.has(index)) {
447
+ return null;
448
+ }
449
+
450
+ const pieceLength = this.#lengthOf(index);
451
+ const revived = await this.#claimSlot();
452
+ const target = this.#pool.subarray(
453
+ revived * this.#chunkLength,
454
+ revived * this.#chunkLength + pieceLength
455
+ );
456
+ await this.#disk.read(index, target);
457
+ this.#slotOf.set(index, revived);
458
+ this.#lru.touch(index);
459
+ this.#counters.fromDisk += 1;
460
+ this.#counters.revivals += 1;
461
+ return this.locate(index);
462
+ }
463
+
464
+ /**
465
+ * Close the store, keeping the spill file.
466
+ *
467
+ * @param {(error?: Error | null) => void} [callback]
468
+ * @returns {void}
469
+ */
470
+ close(callback = () => undefined) {
471
+ this.#closed = true;
472
+ liveStores.delete(this);
473
+ this.#disk.close().then(() => callback(null), (error) => callback(error));
474
+ }
475
+
476
+ /**
477
+ * Close the store and delete everything it wrote.
478
+ *
479
+ * @param {(error?: Error | null) => void} [callback]
480
+ * @returns {void}
481
+ */
482
+ destroy(callback = () => undefined) {
483
+ this.#closed = true;
484
+ liveStores.delete(this);
485
+ this.#slotOf.clear();
486
+ this.#disk.destroy().then(() => callback(null), (error) => callback(error));
487
+ }
488
+ }