@torrent-tv/proxy 2.9.76 → 2.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +1 -1
- package/routes/stream/get.js +141 -116
- package/services/piece-store/shared-piece-store.js +488 -416
- package/services/torrent-worker/client.js +306 -275
- package/services/torrent-worker/file-claims.js +91 -0
- package/services/torrent-worker/piece-reader.js +175 -0
- package/services/torrent-worker/pool-adapter.js +188 -179
- package/services/torrent-worker/protocol.js +12 -0
- package/services/torrent-worker/worker.js +153 -68
- package/test/file-claims.test.js +64 -0
- package/test/piece-reader.test.js +162 -0
- package/test/stream-route.test.js +121 -0
- package/test/worker-source-race.test.js +76 -0
|
@@ -1,416 +1,488 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Torrent pieces in shared memory, with disk as the second tier.
|
|
3
|
-
*
|
|
4
|
-
* Replaces the chunk store WebTorrent would otherwise build for itself. Two
|
|
5
|
-
* reasons, one forced and one chosen.
|
|
6
|
-
*
|
|
7
|
-
* **Forced.** WebTorrent's own piece cache hands out the buffer it keeps using
|
|
8
|
-
* and slices again on the next read. The torrent client now runs on its own
|
|
9
|
-
* thread, and moving a piece to the main thread by transferring ownership
|
|
10
|
-
* detached the cache's memory: proxy 2.9.71-2.9.73 answered every read with an
|
|
11
|
-
* empty body (`Stream ends prematurely at 0`) and the failure was invisible,
|
|
12
|
-
* because the error never reached the reader. Owning the memory ourselves
|
|
13
|
-
* removes the question of whose it was.
|
|
14
|
-
*
|
|
15
|
-
* **Chosen.** Memory this side of the thread boundary can be *shared* memory,
|
|
16
|
-
* which the main thread reads by offset instead of receiving as bytes — see
|
|
17
|
-
* {@link SharedPieceStore#locate}. Measured on the field host: a piece copy
|
|
18
|
-
* costs 3.64 ms, a read from the page cache into a buffer we own 7.63 ms, and
|
|
19
|
-
* re-downloading a piece from the swarm ~1430 ms. So memory first, disk under
|
|
20
|
-
* it, and never the swarm twice.
|
|
21
|
-
*
|
|
22
|
-
* What this deliberately does NOT do is manage the disk as a cache of its own.
|
|
23
|
-
* Pieces evicted from memory are written once and read back on demand; the file
|
|
24
|
-
* is discarded whole when the torrent goes away. libtorrent 2.0 and webtor's
|
|
25
|
-
* seeder both concluded that a hand-rolled disk cache earns less than it costs,
|
|
26
|
-
* and nothing here disagrees.
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
import os from "node:os";
|
|
30
|
-
import { PieceLru } from "./piece-lru.js";
|
|
31
|
-
import { DiskTier } from "./disk-tier.js";
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Live stores, so the worker can report on them.
|
|
35
|
-
*
|
|
36
|
-
* WebTorrent constructs the store itself, deep inside its own wrappers, so
|
|
37
|
-
* there is no handle to reach for from outside. Registering here is what makes
|
|
38
|
-
* the store's behaviour visible in the field at all — without it the first
|
|
39
|
-
* strange case has nothing to go on.
|
|
40
|
-
*
|
|
41
|
-
* @type {Set<SharedPieceStore>}
|
|
42
|
-
*/
|
|
43
|
-
const liveStores = new Set();
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* A snapshot of every live store, for logging.
|
|
47
|
-
*
|
|
48
|
-
* @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }[]}
|
|
49
|
-
*/
|
|
50
|
-
export function collectStoreStats() {
|
|
51
|
-
return [...liveStores].map((store) => store.stats());
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
/**
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
this.#
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
/**
|
|
215
|
-
get
|
|
216
|
-
return this.#
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
/**
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
* @returns {
|
|
224
|
-
*/
|
|
225
|
-
|
|
226
|
-
return
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
*
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
*
|
|
259
|
-
* @
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
}
|
|
416
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file Torrent pieces in shared memory, with disk as the second tier.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the chunk store WebTorrent would otherwise build for itself. Two
|
|
5
|
+
* reasons, one forced and one chosen.
|
|
6
|
+
*
|
|
7
|
+
* **Forced.** WebTorrent's own piece cache hands out the buffer it keeps using
|
|
8
|
+
* and slices again on the next read. The torrent client now runs on its own
|
|
9
|
+
* thread, and moving a piece to the main thread by transferring ownership
|
|
10
|
+
* detached the cache's memory: proxy 2.9.71-2.9.73 answered every read with an
|
|
11
|
+
* empty body (`Stream ends prematurely at 0`) and the failure was invisible,
|
|
12
|
+
* because the error never reached the reader. Owning the memory ourselves
|
|
13
|
+
* removes the question of whose it was.
|
|
14
|
+
*
|
|
15
|
+
* **Chosen.** Memory this side of the thread boundary can be *shared* memory,
|
|
16
|
+
* which the main thread reads by offset instead of receiving as bytes — see
|
|
17
|
+
* {@link SharedPieceStore#locate}. Measured on the field host: a piece copy
|
|
18
|
+
* costs 3.64 ms, a read from the page cache into a buffer we own 7.63 ms, and
|
|
19
|
+
* re-downloading a piece from the swarm ~1430 ms. So memory first, disk under
|
|
20
|
+
* it, and never the swarm twice.
|
|
21
|
+
*
|
|
22
|
+
* What this deliberately does NOT do is manage the disk as a cache of its own.
|
|
23
|
+
* Pieces evicted from memory are written once and read back on demand; the file
|
|
24
|
+
* is discarded whole when the torrent goes away. libtorrent 2.0 and webtor's
|
|
25
|
+
* seeder both concluded that a hand-rolled disk cache earns less than it costs,
|
|
26
|
+
* and nothing here disagrees.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import os from "node:os";
|
|
30
|
+
import { PieceLru } from "./piece-lru.js";
|
|
31
|
+
import { DiskTier } from "./disk-tier.js";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Live stores, so the worker can report on them.
|
|
35
|
+
*
|
|
36
|
+
* WebTorrent constructs the store itself, deep inside its own wrappers, so
|
|
37
|
+
* there is no handle to reach for from outside. Registering here is what makes
|
|
38
|
+
* the store's behaviour visible in the field at all — without it the first
|
|
39
|
+
* strange case has nothing to go on.
|
|
40
|
+
*
|
|
41
|
+
* @type {Set<SharedPieceStore>}
|
|
42
|
+
*/
|
|
43
|
+
const liveStores = new Set();
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A snapshot of every live store, for logging.
|
|
47
|
+
*
|
|
48
|
+
* @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }[]}
|
|
49
|
+
*/
|
|
50
|
+
export function collectStoreStats() {
|
|
51
|
+
return [...liveStores].map((store) => store.stats());
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The shared store behind a torrent, or `null` if it is not one of ours.
|
|
56
|
+
*
|
|
57
|
+
* WebTorrent wraps whatever store it is given — today in `ImmediateChunkStore`,
|
|
58
|
+
* and historically in a piece cache as well — and offers no way to ask for the
|
|
59
|
+
* innermost one. Walking the `store` chain finds it regardless of how many
|
|
60
|
+
* wrappers there are or what order they sit in, which is sturdier than reaching
|
|
61
|
+
* for a fixed `torrent.store.store`.
|
|
62
|
+
*
|
|
63
|
+
* @param {{ store?: object } | null | undefined} torrent
|
|
64
|
+
* @returns {SharedPieceStore | null}
|
|
65
|
+
*/
|
|
66
|
+
export function findSharedStore(torrent) {
|
|
67
|
+
let candidate = torrent?.store;
|
|
68
|
+
// Bounded rather than `while (candidate)`: a store that referenced itself
|
|
69
|
+
// would otherwise hang the thread instead of failing.
|
|
70
|
+
for (let depth = 0; candidate && depth < 8; depth += 1) {
|
|
71
|
+
if (candidate instanceof SharedPieceStore) {
|
|
72
|
+
return candidate;
|
|
73
|
+
}
|
|
74
|
+
candidate = candidate.store;
|
|
75
|
+
}
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Ceiling for the automatic budget, and the share of free memory it will take.
|
|
81
|
+
*
|
|
82
|
+
* A flat default would be a guess dressed as a decision: the proxy runs on
|
|
83
|
+
* whatever the owner has, from a Pi to a rented box, and the budget is **per
|
|
84
|
+
* torrent** — several viewers mean several of these. Measured on the field host
|
|
85
|
+
* after one session: the proxy container sat at 796 MB with 4.1 GB free and 1.3
|
|
86
|
+
* GB already in swap, so a fixed half-gigabyte per torrent is not something to
|
|
87
|
+
* hand out blindly. Hence: a quarter of what is free, capped.
|
|
88
|
+
*/
|
|
89
|
+
const MEMORY_BUDGET_CEILING_BYTES = 512 * 1024 * 1024;
|
|
90
|
+
const FREE_MEMORY_SHARE = 0.25;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Budget for one torrent's resident pieces when the caller names none.
|
|
94
|
+
*
|
|
95
|
+
* @returns {number}
|
|
96
|
+
*/
|
|
97
|
+
function defaultMemoryBytes() {
|
|
98
|
+
const share = Math.floor(os.freemem() * FREE_MEMORY_SHARE);
|
|
99
|
+
return Math.max(MIN_BUDGET_BYTES, Math.min(MEMORY_BUDGET_CEILING_BYTES, share));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Floor for the automatic budget — below this the store thrashes to disk. */
|
|
103
|
+
const MIN_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
104
|
+
/**
|
|
105
|
+
* Never keep fewer than this many pieces resident, whatever the budget says.
|
|
106
|
+
*
|
|
107
|
+
* Two is the smallest workable number rather than a round one: a piece being
|
|
108
|
+
* read holds its slot, so a second slot must exist for the next piece to land
|
|
109
|
+
* in. With one, a single reader would deadlock the store against itself.
|
|
110
|
+
*/
|
|
111
|
+
const MIN_RESIDENT_PIECES = 2;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* A chunk store holding pieces in a `SharedArrayBuffer`, spilling to disk.
|
|
115
|
+
*
|
|
116
|
+
* Implements the `abstract-chunk-store` shape WebTorrent expects — `put`,
|
|
117
|
+
* `get`, `close`, `destroy` — plus {@link locate}, {@link pin} and
|
|
118
|
+
* {@link unpin}, which is how the main thread reads a piece without it being
|
|
119
|
+
* copied or moved.
|
|
120
|
+
*/
|
|
121
|
+
export class SharedPieceStore {
|
|
122
|
+
#chunkLength;
|
|
123
|
+
#lastChunkLength;
|
|
124
|
+
#lastChunkIndex;
|
|
125
|
+
#capacity;
|
|
126
|
+
/** @type {SharedArrayBuffer} */
|
|
127
|
+
#shared;
|
|
128
|
+
/** @type {Buffer} A view over the whole pool, for slot arithmetic. */
|
|
129
|
+
#pool;
|
|
130
|
+
/** Piece index → slot number. */
|
|
131
|
+
#slotOf = new Map();
|
|
132
|
+
/** Slot numbers not currently holding a piece. */
|
|
133
|
+
#freeSlots = [];
|
|
134
|
+
#lru;
|
|
135
|
+
#disk;
|
|
136
|
+
/** Slots backed by memory right now; grows towards {@link capacity}. */
|
|
137
|
+
#allocatedSlots = 0;
|
|
138
|
+
#closed = false;
|
|
139
|
+
#name;
|
|
140
|
+
/**
|
|
141
|
+
* What the store has actually been doing. Reported, not just kept: the
|
|
142
|
+
* balance between memory and disk reads is the number that says whether the
|
|
143
|
+
* budget is right, and it cannot be guessed from outside.
|
|
144
|
+
*/
|
|
145
|
+
#counters = {
|
|
146
|
+
fromMemory: 0,
|
|
147
|
+
fromDisk: 0,
|
|
148
|
+
spills: 0,
|
|
149
|
+
revivals: 0,
|
|
150
|
+
blockedByPins: 0
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* @param {number} chunkLength - Piece length, and therefore the slot size.
|
|
155
|
+
* @param {object} [options]
|
|
156
|
+
* @param {number} [options.length] - Total torrent length, so the short last piece is sized correctly.
|
|
157
|
+
* @param {number} [options.memoryBytes] - Budget for resident pieces.
|
|
158
|
+
* @param {string} [options.path] - Directory for the spill file.
|
|
159
|
+
* @param {string} [options.name] - Spill file name; must be unique per torrent.
|
|
160
|
+
*/
|
|
161
|
+
constructor(chunkLength, options = {}) {
|
|
162
|
+
if (!Number.isInteger(chunkLength) || chunkLength < 1) {
|
|
163
|
+
throw new Error(`Chunk length must be a positive integer, got ${chunkLength}.`);
|
|
164
|
+
}
|
|
165
|
+
this.#chunkLength = chunkLength;
|
|
166
|
+
|
|
167
|
+
const totalLength = Number.isFinite(options.length) ? options.length : 0;
|
|
168
|
+
this.#lastChunkIndex = totalLength > 0 ? Math.ceil(totalLength / chunkLength) - 1 : -1;
|
|
169
|
+
const remainder = totalLength % chunkLength;
|
|
170
|
+
this.#lastChunkLength = remainder === 0 ? chunkLength : remainder;
|
|
171
|
+
|
|
172
|
+
const memoryBytes = Number.isFinite(options.memoryBytes) && options.memoryBytes > 0
|
|
173
|
+
? options.memoryBytes
|
|
174
|
+
: defaultMemoryBytes();
|
|
175
|
+
this.#capacity = Math.max(MIN_RESIDENT_PIECES, Math.floor(memoryBytes / chunkLength));
|
|
176
|
+
|
|
177
|
+
// Grows into the budget instead of taking it up front. The budget is per
|
|
178
|
+
// torrent, so claiming all of it on `add` would charge a host for pieces
|
|
179
|
+
// nobody has asked for — and a torrent that is merely open, or one being
|
|
180
|
+
// probed for its codecs, needs a handful of slots, not the ceiling.
|
|
181
|
+
this.#shared = new SharedArrayBuffer(MIN_RESIDENT_PIECES * chunkLength, {
|
|
182
|
+
maxByteLength: this.#capacity * chunkLength
|
|
183
|
+
});
|
|
184
|
+
this.#pool = Buffer.from(this.#shared);
|
|
185
|
+
for (let slot = 0; slot < MIN_RESIDENT_PIECES; slot += 1) {
|
|
186
|
+
this.#freeSlots.push(slot);
|
|
187
|
+
}
|
|
188
|
+
this.#allocatedSlots = MIN_RESIDENT_PIECES;
|
|
189
|
+
this.#lru = new PieceLru(this.#capacity);
|
|
190
|
+
this.#name = options.name ?? "pieces";
|
|
191
|
+
this.#disk = new DiskTier({
|
|
192
|
+
directory: options.path ?? ".",
|
|
193
|
+
name: `${this.#name}.pieces`,
|
|
194
|
+
chunkLength
|
|
195
|
+
});
|
|
196
|
+
liveStores.add(this);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* What this store has been doing, for the periodic report.
|
|
201
|
+
*
|
|
202
|
+
* @returns {{ name: string, resident: number, capacity: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }}
|
|
203
|
+
*/
|
|
204
|
+
stats() {
|
|
205
|
+
return {
|
|
206
|
+
name: this.#name,
|
|
207
|
+
resident: this.#slotOf.size,
|
|
208
|
+
capacity: this.#capacity,
|
|
209
|
+
spilled: this.#disk.size,
|
|
210
|
+
...this.#counters
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** `abstract-chunk-store` exposes the piece size under this name. */
|
|
215
|
+
get chunkLength() {
|
|
216
|
+
return this.#chunkLength;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The pool itself, so another thread can map the same memory and read a piece
|
|
221
|
+
* by the offset {@link locate} reports.
|
|
222
|
+
*
|
|
223
|
+
* @returns {SharedArrayBuffer}
|
|
224
|
+
*/
|
|
225
|
+
get sharedBuffer() {
|
|
226
|
+
return this.#shared;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** How many pieces fit in memory at once. */
|
|
230
|
+
get capacity() {
|
|
231
|
+
return this.#capacity;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** How many pieces are resident right now. */
|
|
235
|
+
get residentCount() {
|
|
236
|
+
return this.#slotOf.size;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** How many pieces have been spilled to disk. */
|
|
240
|
+
get spilledCount() {
|
|
241
|
+
return this.#disk.size;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Length of a given piece — the last one is usually short.
|
|
246
|
+
*
|
|
247
|
+
* @param {number} index
|
|
248
|
+
* @returns {number}
|
|
249
|
+
*/
|
|
250
|
+
#lengthOf(index) {
|
|
251
|
+
return index === this.#lastChunkIndex ? this.#lastChunkLength : this.#chunkLength;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Where a resident piece sits in the shared pool, or `null` if it is not
|
|
256
|
+
* resident.
|
|
257
|
+
*
|
|
258
|
+
* The main thread reads straight from those bytes, so callers MUST hold a pin
|
|
259
|
+
* across the read — see {@link pin}.
|
|
260
|
+
*
|
|
261
|
+
* @param {number} index
|
|
262
|
+
* @returns {{ offset: number, length: number } | null}
|
|
263
|
+
*/
|
|
264
|
+
locate(index) {
|
|
265
|
+
const slot = this.#slotOf.get(index);
|
|
266
|
+
if (slot === undefined) {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
return { offset: slot * this.#chunkLength, length: this.#lengthOf(index) };
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Hold a piece in memory across a read. Nested; release with {@link unpin}.
|
|
274
|
+
*
|
|
275
|
+
* @param {number} index
|
|
276
|
+
* @returns {void}
|
|
277
|
+
*/
|
|
278
|
+
pin(index) {
|
|
279
|
+
this.#lru.pin(index);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* @param {number} index
|
|
284
|
+
* @returns {void}
|
|
285
|
+
*/
|
|
286
|
+
unpin(index) {
|
|
287
|
+
this.#lru.unpin(index);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Make a slot available, spilling the least recently used piece if need be.
|
|
292
|
+
*
|
|
293
|
+
* @returns {Promise<number>} Slot number.
|
|
294
|
+
*/
|
|
295
|
+
async #claimSlot() {
|
|
296
|
+
const free = this.#freeSlots.pop();
|
|
297
|
+
if (free !== undefined) {
|
|
298
|
+
return free;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// Room left in the budget: take more memory rather than evicting. Growing
|
|
302
|
+
// replaces the view over the pool, so every slot offset stays valid — the
|
|
303
|
+
// bytes do not move.
|
|
304
|
+
if (this.#allocatedSlots < this.#capacity) {
|
|
305
|
+
this.#allocatedSlots += 1;
|
|
306
|
+
this.#shared.grow(this.#allocatedSlots * this.#chunkLength);
|
|
307
|
+
this.#pool = Buffer.from(this.#shared);
|
|
308
|
+
return this.#allocatedSlots - 1;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const victim = this.#lru.evictionCandidate();
|
|
312
|
+
if (victim === null) {
|
|
313
|
+
// Every resident piece is being read. Taking one anyway is precisely the
|
|
314
|
+
// failure this store exists to make impossible.
|
|
315
|
+
this.#counters.blockedByPins += 1;
|
|
316
|
+
throw new Error("Every resident piece is pinned; no slot can be freed.");
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
const slot = this.#slotOf.get(victim);
|
|
320
|
+
const bytes = this.#pool.subarray(slot * this.#chunkLength, slot * this.#chunkLength + this.#lengthOf(victim));
|
|
321
|
+
await this.#disk.write(victim, bytes);
|
|
322
|
+
this.#counters.spills += 1;
|
|
323
|
+
|
|
324
|
+
this.#slotOf.delete(victim);
|
|
325
|
+
this.#lru.remove(victim);
|
|
326
|
+
return slot;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Store a piece.
|
|
331
|
+
*
|
|
332
|
+
* @param {number} index
|
|
333
|
+
* @param {Uint8Array} bytes
|
|
334
|
+
* @param {(error?: Error | null) => void} [callback]
|
|
335
|
+
* @returns {void}
|
|
336
|
+
*/
|
|
337
|
+
put(index, bytes, callback = () => undefined) {
|
|
338
|
+
if (this.#closed) {
|
|
339
|
+
queueMicrotask(() => callback(new Error("Piece store is closed.")));
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const existing = this.#slotOf.get(index);
|
|
344
|
+
const write = async () => {
|
|
345
|
+
const slot = existing ?? (await this.#claimSlot());
|
|
346
|
+
bytes.copy
|
|
347
|
+
? bytes.copy(this.#pool, slot * this.#chunkLength)
|
|
348
|
+
: this.#pool.set(bytes, slot * this.#chunkLength);
|
|
349
|
+
this.#slotOf.set(index, slot);
|
|
350
|
+
this.#lru.touch(index);
|
|
351
|
+
// A newer copy is in memory; whatever is on disk is stale.
|
|
352
|
+
this.#disk.forget(index);
|
|
353
|
+
};
|
|
354
|
+
|
|
355
|
+
write().then(() => callback(null), (error) => callback(error));
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Fetch a piece, or a range within it.
|
|
360
|
+
*
|
|
361
|
+
* Returns a buffer of its own rather than a view into the pool: WebTorrent
|
|
362
|
+
* keeps what it is given — to verify a hash, to serve a peer — and the slot
|
|
363
|
+
* underneath may be reused meanwhile. The thread-crossing path avoids this
|
|
364
|
+
* copy entirely by going through {@link locate}.
|
|
365
|
+
*
|
|
366
|
+
* @param {number} index
|
|
367
|
+
* @param {{ offset?: number, length?: number } | ((error: Error | null, bytes?: Buffer) => void)} [options]
|
|
368
|
+
* @param {(error: Error | null, bytes?: Buffer) => void} [callback]
|
|
369
|
+
* @returns {void}
|
|
370
|
+
*/
|
|
371
|
+
get(index, options, callback) {
|
|
372
|
+
if (typeof options === "function") {
|
|
373
|
+
return this.get(index, undefined, options);
|
|
374
|
+
}
|
|
375
|
+
const done = callback ?? (() => undefined);
|
|
376
|
+
if (this.#closed) {
|
|
377
|
+
queueMicrotask(() => done(new Error("Piece store is closed.")));
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
const pieceLength = this.#lengthOf(index);
|
|
382
|
+
const offset = options?.offset ?? 0;
|
|
383
|
+
const length = options?.length ?? pieceLength - offset;
|
|
384
|
+
|
|
385
|
+
const fetch = async () => {
|
|
386
|
+
const slot = this.#slotOf.get(index);
|
|
387
|
+
if (slot !== undefined) {
|
|
388
|
+
this.#lru.touch(index);
|
|
389
|
+
this.#counters.fromMemory += 1;
|
|
390
|
+
const start = slot * this.#chunkLength + offset;
|
|
391
|
+
return Buffer.from(this.#pool.subarray(start, start + length));
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
if (!this.#disk.has(index)) {
|
|
395
|
+
throw new Error(`Piece ${index} is not in the store.`);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Bring it back into memory: it was just asked for, so it is likely to be
|
|
399
|
+
// asked for again, and the caller may follow up with `locate`.
|
|
400
|
+
const revived = await this.#claimSlot();
|
|
401
|
+
const target = this.#pool.subarray(
|
|
402
|
+
revived * this.#chunkLength,
|
|
403
|
+
revived * this.#chunkLength + pieceLength
|
|
404
|
+
);
|
|
405
|
+
await this.#disk.read(index, target);
|
|
406
|
+
this.#slotOf.set(index, revived);
|
|
407
|
+
this.#lru.touch(index);
|
|
408
|
+
this.#counters.fromDisk += 1;
|
|
409
|
+
this.#counters.revivals += 1;
|
|
410
|
+
const start = revived * this.#chunkLength + offset;
|
|
411
|
+
return Buffer.from(this.#pool.subarray(start, start + length));
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
fetch().then((bytes) => done(null, bytes), (error) => done(error));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Ensure a piece is in memory and say where it sits — without copying it.
|
|
419
|
+
*
|
|
420
|
+
* This is {@link get} minus its final copy, and it exists for exactly one
|
|
421
|
+
* caller: the reader that hands pieces to the other thread. That thread maps
|
|
422
|
+
* the same {@link sharedBuffer}, so an offset and a length are all it needs,
|
|
423
|
+
* and the bytes never move. `get` cannot serve that purpose because
|
|
424
|
+
* WebTorrent keeps what `get` returns while the slot underneath may be
|
|
425
|
+
* reused.
|
|
426
|
+
*
|
|
427
|
+
* The caller MUST hold a pin across the whole read — the returned offset
|
|
428
|
+
* stays valid only while the piece is pinned.
|
|
429
|
+
*
|
|
430
|
+
* @param {number} index
|
|
431
|
+
* @returns {Promise<{ offset: number, length: number } | null>} `null` when
|
|
432
|
+
* the store holds no such piece, in memory or on disk.
|
|
433
|
+
*/
|
|
434
|
+
async reside(index) {
|
|
435
|
+
if (this.#closed) {
|
|
436
|
+
throw new Error("Piece store is closed.");
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const slot = this.#slotOf.get(index);
|
|
440
|
+
if (slot !== undefined) {
|
|
441
|
+
this.#lru.touch(index);
|
|
442
|
+
this.#counters.fromMemory += 1;
|
|
443
|
+
return this.locate(index);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
if (!this.#disk.has(index)) {
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const pieceLength = this.#lengthOf(index);
|
|
451
|
+
const revived = await this.#claimSlot();
|
|
452
|
+
const target = this.#pool.subarray(
|
|
453
|
+
revived * this.#chunkLength,
|
|
454
|
+
revived * this.#chunkLength + pieceLength
|
|
455
|
+
);
|
|
456
|
+
await this.#disk.read(index, target);
|
|
457
|
+
this.#slotOf.set(index, revived);
|
|
458
|
+
this.#lru.touch(index);
|
|
459
|
+
this.#counters.fromDisk += 1;
|
|
460
|
+
this.#counters.revivals += 1;
|
|
461
|
+
return this.locate(index);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* Close the store, keeping the spill file.
|
|
466
|
+
*
|
|
467
|
+
* @param {(error?: Error | null) => void} [callback]
|
|
468
|
+
* @returns {void}
|
|
469
|
+
*/
|
|
470
|
+
close(callback = () => undefined) {
|
|
471
|
+
this.#closed = true;
|
|
472
|
+
liveStores.delete(this);
|
|
473
|
+
this.#disk.close().then(() => callback(null), (error) => callback(error));
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Close the store and delete everything it wrote.
|
|
478
|
+
*
|
|
479
|
+
* @param {(error?: Error | null) => void} [callback]
|
|
480
|
+
* @returns {void}
|
|
481
|
+
*/
|
|
482
|
+
destroy(callback = () => undefined) {
|
|
483
|
+
this.#closed = true;
|
|
484
|
+
liveStores.delete(this);
|
|
485
|
+
this.#slotOf.clear();
|
|
486
|
+
this.#disk.destroy().then(() => callback(null), (error) => callback(error));
|
|
487
|
+
}
|
|
488
|
+
}
|