@torrent-tv/proxy 2.64.3 → 2.64.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/package.json +1 -1
- package/services/hls-session-manager.js +28 -21
- package/services/piece-store/shared-piece-store.js +167 -484
- package/services/torrent-worker/client.js +7 -43
- package/services/torrent-worker/piece-reader.js +1 -0
- package/services/torrent-worker/worker.js +16 -20
- package/test/piece-reader.test.js +2 -2
- package/test/shared-piece-store.test.js +8 -14
|
@@ -13,12 +13,16 @@
|
|
|
13
13
|
* removes the question of whose it was.
|
|
14
14
|
*
|
|
15
15
|
* **Chosen.** Memory this side of the thread boundary can be *shared* memory,
|
|
16
|
-
* which the main thread reads
|
|
16
|
+
* which the main thread reads without receiving bytes as a copy — see
|
|
17
17
|
* {@link SharedPieceStore#locate}. Measured on the field host: a piece copy
|
|
18
18
|
* costs 3.64 ms, a read from the page cache into a buffer we own 7.63 ms, and
|
|
19
19
|
* re-downloading a piece from the swarm ~1430 ms. So memory first, disk under
|
|
20
20
|
* it, and never the swarm twice.
|
|
21
21
|
*
|
|
22
|
+
* Per piece allocation: each resident piece owns its own `SharedArrayBuffer`.
|
|
23
|
+
* Evicting a piece deletes its entry and the memory is reclaimable by GC.
|
|
24
|
+
* `committed` therefore equals `resident`, not a high-water mark.
|
|
25
|
+
*
|
|
22
26
|
* What this deliberately does NOT do is manage the disk as a cache of its own.
|
|
23
27
|
* Pieces evicted from memory are written once and read back on demand; the file
|
|
24
28
|
* is discarded whole when the torrent goes away. libtorrent 2.0 and webtor's
|
|
@@ -34,40 +38,16 @@ import { DiskTier } from "./disk-tier.js";
|
|
|
34
38
|
/**
|
|
35
39
|
* Live stores, so the worker can report on them.
|
|
36
40
|
*
|
|
37
|
-
* WebTorrent constructs the store itself, deep inside its own wrappers, so
|
|
38
|
-
* there is no handle to reach for from outside. Registering here is what makes
|
|
39
|
-
* the store's behaviour visible in the field at all — without it the first
|
|
40
|
-
* strange case has nothing to go on.
|
|
41
|
-
*
|
|
42
41
|
* @type {Set<SharedPieceStore>}
|
|
43
42
|
*/
|
|
44
43
|
const liveStores = new Set();
|
|
45
44
|
|
|
46
|
-
/**
|
|
47
|
-
* A snapshot of every live store, for logging.
|
|
48
|
-
*
|
|
49
|
-
* @returns {{ name: string, resident: number, capacity: number, residentBytes: number, budgetBytes: number, spilled: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }[]}
|
|
50
|
-
*/
|
|
51
45
|
export function collectStoreStats() {
|
|
52
46
|
return [...liveStores].map((store) => store.stats());
|
|
53
47
|
}
|
|
54
48
|
|
|
55
|
-
/**
|
|
56
|
-
* The shared store behind a torrent, or `null` if it is not one of ours.
|
|
57
|
-
*
|
|
58
|
-
* WebTorrent wraps whatever store it is given — today in `ImmediateChunkStore`,
|
|
59
|
-
* and historically in a piece cache as well — and offers no way to ask for the
|
|
60
|
-
* innermost one. Walking the `store` chain finds it regardless of how many
|
|
61
|
-
* wrappers there are or what order they sit in, which is sturdier than reaching
|
|
62
|
-
* for a fixed `torrent.store.store`.
|
|
63
|
-
*
|
|
64
|
-
* @param {{ store?: object } | null | undefined} torrent
|
|
65
|
-
* @returns {SharedPieceStore | null}
|
|
66
|
-
*/
|
|
67
49
|
export function findSharedStore(torrent) {
|
|
68
50
|
let candidate = torrent?.store;
|
|
69
|
-
// Bounded rather than `while (candidate)`: a store that referenced itself
|
|
70
|
-
// would otherwise hang the thread instead of failing.
|
|
71
51
|
for (let depth = 0; candidate && depth < 8; depth += 1) {
|
|
72
52
|
if (candidate instanceof SharedPieceStore) {
|
|
73
53
|
return candidate;
|
|
@@ -77,84 +57,20 @@ export function findSharedStore(torrent) {
|
|
|
77
57
|
return null;
|
|
78
58
|
}
|
|
79
59
|
|
|
80
|
-
/**
|
|
81
|
-
* Ceiling for the automatic budget, and the share of available memory it takes.
|
|
82
|
-
*
|
|
83
|
-
* A flat default would be a guess dressed as a decision: the proxy runs on
|
|
84
|
-
* whatever the owner has, from a Pi to a rented box. So it is a share of what
|
|
85
|
-
* the machine can actually give, capped.
|
|
86
|
-
*
|
|
87
|
-
* Two things about this were wrong until 2026-08-28, and the kernel found both.
|
|
88
|
-
* It killed the proxy at 2.4 GB resident (`exit code 137`, no dump, `Out of
|
|
89
|
-
* memory: Killed process ... anon-rss: 2422628kB`), on a host with under two
|
|
90
|
-
* gigabytes to spare and an `oom_score_adj` of 200 that makes the addon the
|
|
91
|
-
* first thing chosen.
|
|
92
|
-
*
|
|
93
|
-
* The first: the budget was **per torrent**, so two torrents meant two of it and
|
|
94
|
-
* nothing anywhere asked what the process as a whole was holding. It is shared
|
|
95
|
-
* now — {@link budgetForNewStore} divides what is allowed between the stores
|
|
96
|
-
* that exist.
|
|
97
|
-
*
|
|
98
|
-
* The second: it was a share of `os.freemem()`, which on Linux counts only the
|
|
99
|
-
* pages free at that instant while the kernel deliberately keeps that number low
|
|
100
|
-
* by filling the rest with reclaimable cache. The kernel publishes its own
|
|
101
|
-
* estimate of what an allocation could obtain — `MemAvailable` — and that is the
|
|
102
|
-
* quantity to divide.
|
|
103
|
-
*/
|
|
104
60
|
const MEMORY_BUDGET_CEILING_BYTES = 512 * 1024 * 1024;
|
|
105
61
|
const AVAILABLE_MEMORY_SHARE = 0.25;
|
|
106
62
|
|
|
107
|
-
/**
|
|
108
|
-
* What all torrent stores together may hold, in bytes.
|
|
109
|
-
*
|
|
110
|
-
* Sampled when a store is made rather than kept, because what the machine has
|
|
111
|
-
* to spare is not ours to predict: another container starting is as much a
|
|
112
|
-
* change as another viewer arriving.
|
|
113
|
-
*
|
|
114
|
-
* @param {number} availableBytes - What the machine can still give out.
|
|
115
|
-
* @returns {number}
|
|
116
|
-
*/
|
|
117
63
|
export function totalStoreBudgetBytes(availableBytes) {
|
|
118
64
|
const share = Math.floor(Math.max(availableBytes, 0) * AVAILABLE_MEMORY_SHARE);
|
|
119
65
|
return Math.max(MIN_BUDGET_BYTES, Math.min(MEMORY_BUDGET_CEILING_BYTES, share));
|
|
120
66
|
}
|
|
121
67
|
|
|
122
|
-
/**
|
|
123
|
-
* One store's share of the whole, given how many stores there will be.
|
|
124
|
-
*
|
|
125
|
-
* Divided rather than handed out whole: the failure this replaces is several
|
|
126
|
-
* stores each taking the maximum. The floor is what a store needs to work at
|
|
127
|
-
* all — below it the store thrashes to disk and the viewer pays for it — so a
|
|
128
|
-
* proxy serving many torrents at once on a small machine will exceed the total,
|
|
129
|
-
* and that is deliberate: refusing to serve is worse, and the memory report now
|
|
130
|
-
* says plainly what is being held.
|
|
131
|
-
*
|
|
132
|
-
* @param {number} availableBytes
|
|
133
|
-
* @param {number} storeCount - Stores that will exist, this one included.
|
|
134
|
-
* @returns {number}
|
|
135
|
-
*/
|
|
136
68
|
export function budgetForNewStore(availableBytes, storeCount) {
|
|
137
69
|
const total = totalStoreBudgetBytes(availableBytes);
|
|
138
70
|
const shares = Math.max(1, Math.floor(storeCount));
|
|
139
71
|
return Math.max(MIN_BUDGET_BYTES, Math.floor(total / shares));
|
|
140
72
|
}
|
|
141
73
|
|
|
142
|
-
/**
|
|
143
|
-
* Re-derive every live store's growth ceiling from what the machine has NOW.
|
|
144
|
-
*
|
|
145
|
-
* A budget settled at birth is not a budget: the machine it was taken from
|
|
146
|
-
* changes, and this proxy is one process among many on a host that hands its
|
|
147
|
-
* addons a positive `oom_score_adj`. Lowering a ceiling frees nothing that is
|
|
148
|
-
* already committed — the pool cannot shrink — but it stops the growth that
|
|
149
|
-
* would otherwise carry on into memory the machine no longer has, and sends
|
|
150
|
-
* those pieces to disk instead, which is what the disk tier is for.
|
|
151
|
-
*
|
|
152
|
-
* Never above the reservation each store was created with: `maxByteLength` was
|
|
153
|
-
* fixed from it and `grow()` cannot pass it.
|
|
154
|
-
*
|
|
155
|
-
* @returns {{ name: string, ceilingBytes: number, committedBytes: number }[]}
|
|
156
|
-
* What each store may now grow to, for the caller to report.
|
|
157
|
-
*/
|
|
158
74
|
export function reviseStoreBudgets() {
|
|
159
75
|
const share = budgetForNewStore(availableMemorySync(), liveStores.size);
|
|
160
76
|
const revised = [];
|
|
@@ -164,25 +80,10 @@ export function reviseStoreBudgets() {
|
|
|
164
80
|
return revised;
|
|
165
81
|
}
|
|
166
82
|
|
|
167
|
-
/**
|
|
168
|
-
* Budget for one torrent's resident pieces when the caller names none.
|
|
169
|
-
*
|
|
170
|
-
* @returns {number}
|
|
171
|
-
*/
|
|
172
83
|
function defaultMemoryBytes() {
|
|
173
84
|
return budgetForNewStore(availableMemorySync(), liveStores.size + 1);
|
|
174
85
|
}
|
|
175
86
|
|
|
176
|
-
/**
|
|
177
|
-
* What the machine can still give out, without waiting on a file read.
|
|
178
|
-
*
|
|
179
|
-
* The store is constructed synchronously, so the asynchronous reading in
|
|
180
|
-
* `services/memory-report.js` cannot be used here. `MemAvailable` is read from
|
|
181
|
-
* `/proc` with a blocking read, which is a few microseconds on a pseudo-file,
|
|
182
|
-
* and `os.freemem()` remains the answer where `/proc` is not there.
|
|
183
|
-
*
|
|
184
|
-
* @returns {number}
|
|
185
|
-
*/
|
|
186
87
|
function availableMemorySync() {
|
|
187
88
|
try {
|
|
188
89
|
const text = readFileSync("/proc/meminfo", "utf8");
|
|
@@ -191,118 +92,42 @@ function availableMemorySync() {
|
|
|
191
92
|
return Number(match[1]) * 1024;
|
|
192
93
|
}
|
|
193
94
|
} catch {
|
|
194
|
-
// silent-ok: not Linux, or /proc is not mounted.
|
|
195
95
|
}
|
|
196
96
|
return os.freemem();
|
|
197
97
|
}
|
|
198
98
|
|
|
199
|
-
/** Floor for the automatic budget — below this the store thrashes to disk. */
|
|
200
99
|
const MIN_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
201
|
-
/**
|
|
202
|
-
* Never keep fewer than this many pieces resident, whatever the budget says.
|
|
203
|
-
*
|
|
204
|
-
* Two is the smallest workable number rather than a round one: a piece being
|
|
205
|
-
* read holds its slot, so a second slot must exist for the next piece to land
|
|
206
|
-
* in. With one, a single reader would deadlock the store against itself.
|
|
207
|
-
*/
|
|
208
100
|
const MIN_RESIDENT_PIECES = 2;
|
|
209
|
-
/**
|
|
210
|
-
* How long a caller waits for a pinned piece to be released before the store
|
|
211
|
-
* calls it a deadlock. A pin lasts one read of one piece — milliseconds — so
|
|
212
|
-
* anything approaching this is a reader waiting for itself.
|
|
213
|
-
*/
|
|
214
101
|
const PINNED_WAIT_MS = 5_000;
|
|
215
|
-
/** How often a wait for a slot looks again when no event is due to wake it. */
|
|
216
102
|
const CLAIM_RETRY_MS = 50;
|
|
217
103
|
|
|
218
|
-
/**
|
|
219
|
-
* A chunk store holding pieces in a `SharedArrayBuffer`, spilling to disk.
|
|
220
|
-
*
|
|
221
|
-
* Implements the `abstract-chunk-store` shape WebTorrent expects — `put`,
|
|
222
|
-
* `get`, `close`, `destroy` — plus {@link locate}, {@link pin} and
|
|
223
|
-
* {@link unpin}, which is how the main thread reads a piece without it being
|
|
224
|
-
* copied or moved.
|
|
225
|
-
*/
|
|
226
104
|
export class SharedPieceStore {
|
|
227
105
|
#chunkLength;
|
|
228
106
|
#lastChunkLength;
|
|
229
107
|
#lastChunkIndex;
|
|
230
108
|
#capacity;
|
|
231
|
-
/**
|
|
232
|
-
* How many slots this store may grow into RIGHT NOW, as against the
|
|
233
|
-
* reservation it was born with.
|
|
234
|
-
*
|
|
235
|
-
* The budget used to be settled once, when the store was created, from the
|
|
236
|
-
* memory the machine had at that moment. A store opened on an idle machine
|
|
237
|
-
* therefore kept an idle machine's allowance for the rest of its life, and
|
|
238
|
-
* went on growing into it while everything else on the host competed for what
|
|
239
|
-
* was left — which is how an addon with a positive `oom_score_adj` becomes
|
|
240
|
-
* the kernel's first choice (roadmap item 2). It is revised on the same
|
|
241
|
-
* cadence as the memory report, within the reservation: it can never rise
|
|
242
|
-
* above `#capacity`, because `maxByteLength` was fixed from that and
|
|
243
|
-
* `grow()` cannot pass it.
|
|
244
|
-
*/
|
|
245
109
|
#growthCeiling;
|
|
246
|
-
/**
|
|
247
|
-
#
|
|
248
|
-
/** @type {
|
|
249
|
-
#pool;
|
|
250
|
-
/** Piece index → slot number. */
|
|
251
|
-
#slotOf = new Map();
|
|
252
|
-
/** Slot numbers not currently holding a piece. */
|
|
253
|
-
#freeSlots = [];
|
|
254
|
-
/**
|
|
255
|
-
* Pieces being written out to disk right now: index → that write.
|
|
256
|
-
*
|
|
257
|
-
* Such a piece is in neither place — its slot has already been given away,
|
|
258
|
-
* and the disk copy is not finished. A reader arriving in that window must
|
|
259
|
-
* wait for the write instead of concluding the piece is gone.
|
|
260
|
-
*
|
|
261
|
-
* @type {Map<number, Promise<void>>}
|
|
262
|
-
*/
|
|
110
|
+
/** Piece index → SharedArrayBuffer of that piece */
|
|
111
|
+
#buffers = new Map();
|
|
112
|
+
/** @type {Map<number, Promise<void>>} */
|
|
263
113
|
#evicting = new Map();
|
|
264
|
-
|
|
265
|
-
* Slots handed out but not yet recorded against a piece.
|
|
266
|
-
*
|
|
267
|
-
* A slot is claimed before the piece is copied into it, so between those two
|
|
268
|
-
* moments the slot belongs to nobody the books know about. Without counting
|
|
269
|
-
* them, a burst of concurrent puts — which is the normal case, pieces arrive
|
|
270
|
-
* from many peers at once — sees an empty eviction list and concludes the
|
|
271
|
-
* store is exhausted, when in fact it is merely mid-flight.
|
|
272
|
-
*/
|
|
273
|
-
#outstandingSlots = 0;
|
|
274
|
-
/** When the wait for a pinned piece began; 0 when nothing is waiting. */
|
|
114
|
+
#outstandingPieces = 0;
|
|
275
115
|
#pinnedWaitStartedAt = 0;
|
|
276
|
-
/** Resolvers waiting for a slot to become claimable. @type {(() => void)[]} */
|
|
277
116
|
#waiters = [];
|
|
278
117
|
#lru;
|
|
279
118
|
#disk;
|
|
280
|
-
/** Slots backed by memory right now; grows towards {@link capacity}. */
|
|
281
|
-
#allocatedSlots = 0;
|
|
282
119
|
#closed = false;
|
|
283
120
|
#name;
|
|
284
|
-
/**
|
|
285
|
-
* What the store has actually been doing. Reported, not just kept: the
|
|
286
|
-
* balance between memory and disk reads is the number that says whether the
|
|
287
|
-
* budget is right, and it cannot be guessed from outside.
|
|
288
|
-
*/
|
|
289
121
|
#counters = {
|
|
290
122
|
fromMemory: 0,
|
|
291
123
|
fromDisk: 0,
|
|
292
124
|
spills: 0,
|
|
293
125
|
revivals: 0,
|
|
294
126
|
blockedByPins: 0,
|
|
295
|
-
waitedForPins: 0
|
|
127
|
+
waitedForPins: 0,
|
|
128
|
+
evictedOnRevise: 0
|
|
296
129
|
};
|
|
297
130
|
|
|
298
|
-
/**
|
|
299
|
-
* @param {number} chunkLength - Piece length, and therefore the slot size.
|
|
300
|
-
* @param {object} [options]
|
|
301
|
-
* @param {number} [options.length] - Total torrent length, so the short last piece is sized correctly.
|
|
302
|
-
* @param {number} [options.memoryBytes] - Budget for resident pieces.
|
|
303
|
-
* @param {string} [options.path] - Directory for the spill file.
|
|
304
|
-
* @param {string} [options.name] - Spill file name; must be unique per torrent.
|
|
305
|
-
*/
|
|
306
131
|
constructor(chunkLength, options = {}) {
|
|
307
132
|
if (!Number.isInteger(chunkLength) || chunkLength < 1) {
|
|
308
133
|
throw new Error(`Chunk length must be a positive integer, got ${chunkLength}.`);
|
|
@@ -319,18 +144,6 @@ export class SharedPieceStore {
|
|
|
319
144
|
: defaultMemoryBytes();
|
|
320
145
|
this.#capacity = Math.max(MIN_RESIDENT_PIECES, Math.floor(memoryBytes / chunkLength));
|
|
321
146
|
|
|
322
|
-
// Grows into the budget instead of taking it up front. The budget is per
|
|
323
|
-
// torrent, so claiming all of it on `add` would charge a host for pieces
|
|
324
|
-
// nobody has asked for — and a torrent that is merely open, or one being
|
|
325
|
-
// probed for its codecs, needs a handful of slots, not the ceiling.
|
|
326
|
-
this.#shared = new SharedArrayBuffer(MIN_RESIDENT_PIECES * chunkLength, {
|
|
327
|
-
maxByteLength: this.#capacity * chunkLength
|
|
328
|
-
});
|
|
329
|
-
this.#pool = Buffer.from(this.#shared);
|
|
330
|
-
for (let slot = 0; slot < MIN_RESIDENT_PIECES; slot += 1) {
|
|
331
|
-
this.#freeSlots.push(slot);
|
|
332
|
-
}
|
|
333
|
-
this.#allocatedSlots = MIN_RESIDENT_PIECES;
|
|
334
147
|
this.#growthCeiling = this.#capacity;
|
|
335
148
|
this.#lru = new PieceLru(this.#capacity);
|
|
336
149
|
this.#name = options.name ?? "pieces";
|
|
@@ -342,29 +155,17 @@ export class SharedPieceStore {
|
|
|
342
155
|
liveStores.add(this);
|
|
343
156
|
}
|
|
344
157
|
|
|
345
|
-
/**
|
|
346
|
-
* What this store has been doing, for the periodic report.
|
|
347
|
-
*
|
|
348
|
-
* `residentBytes` is the pieces held right now; `committedBytes` is the
|
|
349
|
-
* memory this store has actually taken from the machine, and the two are not
|
|
350
|
-
* the same number. The pool only ever GROWS — `SharedArrayBuffer` has no
|
|
351
|
-
* shrink — so a piece spilled to disk returns its slot to the free list and
|
|
352
|
-
* its memory to nobody. Reporting only the first is what left 650 MB of a
|
|
353
|
-
* 893 MB process unaccounted for on 2026-08-28 while the store said "144MB"
|
|
354
|
-
* (roadmap item 2).
|
|
355
|
-
*
|
|
356
|
-
* @returns {{ name: string, resident: number, capacity: number, residentBytes: number, committedBytes: number, allocatedSlots: number, budgetBytes: number, spilled: number, spilledBytes: number, fromMemory: number, fromDisk: number, spills: number, revivals: number, blockedByPins: number }}
|
|
357
|
-
*/
|
|
358
158
|
stats() {
|
|
159
|
+
const resident = this.#buffers.size;
|
|
160
|
+
const residentBytes = resident * this.#chunkLength;
|
|
161
|
+
// Last piece may be short but stats historically use chunkLength.
|
|
359
162
|
return {
|
|
360
163
|
name: this.#name,
|
|
361
|
-
resident
|
|
164
|
+
resident,
|
|
362
165
|
capacity: this.#growthCeiling,
|
|
363
|
-
residentBytes
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
allocatedSlots: this.#allocatedSlots,
|
|
367
|
-
committedBytes: this.#allocatedSlots * this.#chunkLength,
|
|
166
|
+
residentBytes,
|
|
167
|
+
allocatedSlots: resident,
|
|
168
|
+
committedBytes: residentBytes,
|
|
368
169
|
budgetBytes: this.#growthCeiling * this.#chunkLength,
|
|
369
170
|
pinned: this.#lru.pinnedCount,
|
|
370
171
|
spilled: this.#disk.size,
|
|
@@ -373,161 +174,131 @@ export class SharedPieceStore {
|
|
|
373
174
|
};
|
|
374
175
|
}
|
|
375
176
|
|
|
376
|
-
/**
|
|
377
|
-
* Take a new allowance, within the reservation this store was born with.
|
|
378
|
-
*
|
|
379
|
-
* @param {number} allowedBytes
|
|
380
|
-
* @returns {{ name: string, ceilingBytes: number, committedBytes: number }}
|
|
381
|
-
*/
|
|
382
177
|
reviseGrowthCeiling(allowedBytes) {
|
|
383
178
|
const wanted = Math.floor(Number(allowedBytes) / this.#chunkLength);
|
|
384
|
-
// Never below what a store needs to work at all — under that it thrashes to
|
|
385
|
-
// disk and the viewer pays for it — and never above the reservation.
|
|
386
179
|
this.#growthCeiling = Math.min(
|
|
387
180
|
this.#capacity,
|
|
388
181
|
Math.max(MIN_RESIDENT_PIECES, Number.isFinite(wanted) ? wanted : this.#capacity)
|
|
389
182
|
);
|
|
183
|
+
// With per-piece buffers memory CAN be given back immediately, unlike the
|
|
184
|
+
// old growable pool. Eagerly evict excess to honour the new ceiling.
|
|
185
|
+
let evicted = 0;
|
|
186
|
+
while (this.#buffers.size > this.#growthCeiling) {
|
|
187
|
+
const victim = this.#lru.evictionCandidate();
|
|
188
|
+
if (victim === null) {
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
const victimBuffer = this.#buffers.get(victim);
|
|
192
|
+
if (victimBuffer === undefined) {
|
|
193
|
+
this.#lru.remove(victim);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
this.#buffers.delete(victim);
|
|
197
|
+
this.#lru.remove(victim);
|
|
198
|
+
evicted += 1;
|
|
199
|
+
this.#counters.evictedOnRevise += 1;
|
|
200
|
+
const bytes = Buffer.from(victimBuffer, 0, this.#lengthOf(victim));
|
|
201
|
+
const spill = this.#disk.write(victim, bytes).then(
|
|
202
|
+
() => {
|
|
203
|
+
this.#counters.spills += 1;
|
|
204
|
+
this.#evicting.delete(victim);
|
|
205
|
+
this.#wake();
|
|
206
|
+
},
|
|
207
|
+
(error) => {
|
|
208
|
+
this.#evicting.delete(victim);
|
|
209
|
+
this.#wake();
|
|
210
|
+
throw error;
|
|
211
|
+
}
|
|
212
|
+
);
|
|
213
|
+
this.#evicting.set(victim, spill);
|
|
214
|
+
}
|
|
215
|
+
if (evicted > 0) {
|
|
216
|
+
// Logged by the caller (worker) via reviseStoreBudgets, but also countable here.
|
|
217
|
+
}
|
|
390
218
|
return {
|
|
391
219
|
name: this.#name,
|
|
392
220
|
ceilingBytes: this.#growthCeiling * this.#chunkLength,
|
|
393
|
-
committedBytes: this.#
|
|
221
|
+
committedBytes: this.#buffers.size * this.#chunkLength,
|
|
222
|
+
evicted
|
|
394
223
|
};
|
|
395
224
|
}
|
|
396
225
|
|
|
397
|
-
/** `abstract-chunk-store` exposes the piece size under this name. */
|
|
398
226
|
get chunkLength() {
|
|
399
227
|
return this.#chunkLength;
|
|
400
228
|
}
|
|
401
229
|
|
|
402
|
-
/**
|
|
403
|
-
* The pool itself, so another thread can map the same memory and read a piece
|
|
404
|
-
* by the offset {@link locate} reports.
|
|
405
|
-
*
|
|
406
|
-
* @returns {SharedArrayBuffer}
|
|
407
|
-
*/
|
|
408
|
-
get sharedBuffer() {
|
|
409
|
-
return this.#shared;
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
/** How many pieces fit in memory at once. */
|
|
413
230
|
get capacity() {
|
|
414
|
-
// What may be held NOW, not the reservation this store was created with.
|
|
415
|
-
// A reader sizes its window from this (`ceilingPieces` in piece-reader),
|
|
416
|
-
// and sizing it from an allowance the machine has since withdrawn is how a
|
|
417
|
-
// reader comes to want more pieces than the store can hold.
|
|
418
231
|
return this.#growthCeiling;
|
|
419
232
|
}
|
|
420
233
|
|
|
421
|
-
/** How many pieces are resident right now. */
|
|
422
234
|
get residentCount() {
|
|
423
|
-
return this.#
|
|
235
|
+
return this.#buffers.size;
|
|
424
236
|
}
|
|
425
237
|
|
|
426
|
-
/** How many pieces have been spilled to disk. */
|
|
427
238
|
get spilledCount() {
|
|
428
239
|
return this.#disk.size;
|
|
429
240
|
}
|
|
430
241
|
|
|
431
|
-
/**
|
|
432
|
-
* Length of a given piece — the last one is usually short.
|
|
433
|
-
*
|
|
434
|
-
* @param {number} index
|
|
435
|
-
* @returns {number}
|
|
436
|
-
*/
|
|
437
242
|
#lengthOf(index) {
|
|
438
243
|
return index === this.#lastChunkIndex ? this.#lastChunkLength : this.#chunkLength;
|
|
439
244
|
}
|
|
440
245
|
|
|
441
246
|
/**
|
|
442
|
-
* Where a resident piece sits
|
|
443
|
-
*
|
|
444
|
-
*
|
|
445
|
-
* The main thread reads straight from those bytes, so callers MUST hold a pin
|
|
446
|
-
* across the read — see {@link pin}.
|
|
447
|
-
*
|
|
247
|
+
* Where a resident piece sits, or `null` if not resident.
|
|
248
|
+
* Returns the piece's own SharedArrayBuffer and the intra-piece range.
|
|
448
249
|
* @param {number} index
|
|
449
|
-
* @returns {{ offset: number, length: number } | null}
|
|
250
|
+
* @returns {{ buffer: SharedArrayBuffer, offset: number, length: number } | null}
|
|
450
251
|
*/
|
|
451
252
|
locate(index) {
|
|
452
|
-
const
|
|
453
|
-
if (
|
|
253
|
+
const buffer = this.#buffers.get(index);
|
|
254
|
+
if (buffer === undefined) {
|
|
454
255
|
return null;
|
|
455
256
|
}
|
|
456
|
-
return { offset:
|
|
257
|
+
return { buffer, offset: 0, length: this.#lengthOf(index) };
|
|
457
258
|
}
|
|
458
259
|
|
|
459
260
|
/**
|
|
460
|
-
*
|
|
461
|
-
*
|
|
261
|
+
* Direct access to a piece's buffer for zero-copy consumers.
|
|
462
262
|
* @param {number} index
|
|
463
|
-
* @returns {
|
|
263
|
+
* @returns {SharedArrayBuffer | undefined}
|
|
464
264
|
*/
|
|
265
|
+
getPieceBuffer(index) {
|
|
266
|
+
return this.#buffers.get(index);
|
|
267
|
+
}
|
|
268
|
+
|
|
465
269
|
pin(index) {
|
|
466
270
|
this.#lru.pin(index);
|
|
467
271
|
}
|
|
468
272
|
|
|
469
|
-
/**
|
|
470
|
-
* @param {number} index
|
|
471
|
-
* @returns {void}
|
|
472
|
-
*/
|
|
473
273
|
unpin(index) {
|
|
474
274
|
this.#lru.unpin(index);
|
|
475
|
-
// A released pin can be exactly what a caller waiting for a slot needs.
|
|
476
275
|
this.#wake();
|
|
477
276
|
}
|
|
478
277
|
|
|
479
|
-
/**
|
|
480
|
-
* Make a slot available, spilling the least recently used piece if need be.
|
|
481
|
-
*
|
|
482
|
-
* @returns {Promise<number>} Slot number.
|
|
483
|
-
*/
|
|
484
278
|
async #claimSlot() {
|
|
485
279
|
for (;;) {
|
|
486
|
-
const
|
|
487
|
-
if (
|
|
488
|
-
return
|
|
280
|
+
const ok = await this.#claimSlotOnce();
|
|
281
|
+
if (ok) {
|
|
282
|
+
return;
|
|
489
283
|
}
|
|
490
|
-
// Nothing claimable this instant, but work is in flight that will make a
|
|
491
|
-
// slot claimable: a spill finishing, or a piece being written into a slot
|
|
492
|
-
// already handed out. Wait for either and look again, rather than failing
|
|
493
|
-
// while the store is in the middle of making room.
|
|
494
284
|
await new Promise((resolve) => {
|
|
495
285
|
this.#waiters.push(resolve);
|
|
496
286
|
for (const spill of this.#evicting.values()) {
|
|
497
287
|
void spill.then(() => this.#wake(), () => this.#wake());
|
|
498
288
|
}
|
|
499
|
-
// A wake is not guaranteed to come. Waiting for a spill is safe — one
|
|
500
|
-
// is in flight and will finish — but waiting for a PIN to be released
|
|
501
|
-
// is not: if every piece is held and nothing else is happening, there
|
|
502
|
-
// is no event left to fire, and the deadline that gives up cannot be
|
|
503
|
-
// reached because it is only tested inside an attempt. That is a hang,
|
|
504
|
-
// and it hung this store's own test for the full ten minutes a run is
|
|
505
|
-
// allowed. So the wait also re-checks on a timer.
|
|
506
289
|
const retry = setTimeout(() => this.#wake(), CLAIM_RETRY_MS);
|
|
507
290
|
retry.unref?.();
|
|
508
291
|
});
|
|
509
292
|
}
|
|
510
293
|
}
|
|
511
294
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
*
|
|
515
|
-
* @param {number} index
|
|
516
|
-
* @param {number} slot
|
|
517
|
-
* @returns {void}
|
|
518
|
-
*/
|
|
519
|
-
#registerSlot(index, slot) {
|
|
520
|
-
this.#slotOf.set(index, slot);
|
|
295
|
+
#registerPiece(index, buffer) {
|
|
296
|
+
this.#buffers.set(index, buffer);
|
|
521
297
|
this.#lru.touch(index);
|
|
522
|
-
this.#
|
|
298
|
+
this.#outstandingPieces -= 1;
|
|
523
299
|
this.#wake();
|
|
524
300
|
}
|
|
525
301
|
|
|
526
|
-
/**
|
|
527
|
-
* Release everyone waiting for a slot; each rechecks for itself.
|
|
528
|
-
*
|
|
529
|
-
* @returns {void}
|
|
530
|
-
*/
|
|
531
302
|
#wake() {
|
|
532
303
|
const waiting = this.#waiters;
|
|
533
304
|
this.#waiters = [];
|
|
@@ -536,59 +307,24 @@ export class SharedPieceStore {
|
|
|
536
307
|
}
|
|
537
308
|
}
|
|
538
309
|
|
|
539
|
-
/**
|
|
540
|
-
* One attempt at a slot: a number, or `null` when the caller should wait for
|
|
541
|
-
* an in-flight spill and try again.
|
|
542
|
-
*
|
|
543
|
-
* @returns {Promise<number | null>}
|
|
544
|
-
*/
|
|
545
310
|
async #claimSlotOnce() {
|
|
546
|
-
//
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
const free = this.#freeSlots.pop();
|
|
551
|
-
if (free !== undefined) {
|
|
552
|
-
this.#outstandingSlots += 1;
|
|
553
|
-
return free;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
// Room left in the budget: take more memory rather than evicting. Growing
|
|
557
|
-
// replaces the view over the pool, so every slot offset stays valid — the
|
|
558
|
-
// bytes do not move.
|
|
559
|
-
if (this.#allocatedSlots < this.#growthCeiling) {
|
|
560
|
-
this.#allocatedSlots += 1;
|
|
561
|
-
this.#shared.grow(this.#allocatedSlots * this.#chunkLength);
|
|
562
|
-
this.#pool = Buffer.from(this.#shared);
|
|
563
|
-
this.#outstandingSlots += 1;
|
|
564
|
-
return this.#allocatedSlots - 1;
|
|
311
|
+
// Reserve before suspension so concurrent callers see the reservation.
|
|
312
|
+
if (this.#buffers.size + this.#outstandingPieces < this.#growthCeiling) {
|
|
313
|
+
this.#outstandingPieces += 1;
|
|
314
|
+
return true;
|
|
565
315
|
}
|
|
566
316
|
|
|
567
317
|
const victim = this.#lru.evictionCandidate();
|
|
568
318
|
if (victim === null) {
|
|
569
|
-
if (this.#evicting.size > 0 || this.#
|
|
570
|
-
return
|
|
319
|
+
if (this.#evicting.size > 0 || this.#outstandingPieces > 0) {
|
|
320
|
+
return false;
|
|
571
321
|
}
|
|
572
|
-
// Every resident piece is being READ right now. That is not a permanent
|
|
573
|
-
// condition: a pin lasts as long as one read of one piece, and the reader
|
|
574
|
-
// releases it a moment later. So wait for that, exactly as the loop above
|
|
575
|
-
// waits for a spill — pins now wake the waiters.
|
|
576
|
-
//
|
|
577
|
-
// It became reachable when a viewer could have three readers on one file
|
|
578
|
-
// (2026-08-15: picture, the audio track chosen and the one left behind);
|
|
579
|
-
// failing here ended a read with zero bytes, which ffmpeg reads as the
|
|
580
|
-
// end of the file, so every encoder died and the session answered 500 to
|
|
581
|
-
// everything after that.
|
|
582
|
-
//
|
|
583
|
-
// The deadline is what keeps a genuine deadlock visible: a reader that
|
|
584
|
-
// holds a pin while waiting for a slot would otherwise wait for itself
|
|
585
|
-
// for ever.
|
|
586
322
|
if (this.#pinnedWaitStartedAt === 0) {
|
|
587
323
|
this.#pinnedWaitStartedAt = Date.now();
|
|
588
324
|
}
|
|
589
325
|
if (Date.now() - this.#pinnedWaitStartedAt < PINNED_WAIT_MS) {
|
|
590
326
|
this.#counters.waitedForPins += 1;
|
|
591
|
-
return
|
|
327
|
+
return false;
|
|
592
328
|
}
|
|
593
329
|
this.#pinnedWaitStartedAt = 0;
|
|
594
330
|
this.#counters.blockedByPins += 1;
|
|
@@ -598,19 +334,19 @@ export class SharedPieceStore {
|
|
|
598
334
|
}
|
|
599
335
|
this.#pinnedWaitStartedAt = 0;
|
|
600
336
|
|
|
601
|
-
const
|
|
337
|
+
const victimBuffer = this.#buffers.get(victim);
|
|
338
|
+
if (victimBuffer === undefined) {
|
|
339
|
+
// Should not happen: LRU says resident but buffer missing.
|
|
340
|
+
this.#lru.remove(victim);
|
|
341
|
+
return false;
|
|
342
|
+
}
|
|
602
343
|
|
|
603
|
-
// Claim
|
|
604
|
-
|
|
605
|
-
// that gap, pick the same victim and be handed the same slot — after which
|
|
606
|
-
// two pieces write over each other, both fail their hash, and the torrent
|
|
607
|
-
// downloads them again, forever. Removing it from the books first makes the
|
|
608
|
-
// choice atomic; `#evicting` keeps readers correct in the meantime.
|
|
609
|
-
this.#slotOf.delete(victim);
|
|
344
|
+
// Claim atomically before await.
|
|
345
|
+
this.#buffers.delete(victim);
|
|
610
346
|
this.#lru.remove(victim);
|
|
611
|
-
this.#
|
|
347
|
+
this.#outstandingPieces += 1;
|
|
612
348
|
|
|
613
|
-
const bytes =
|
|
349
|
+
const bytes = Buffer.from(victimBuffer, 0, this.#lengthOf(victim));
|
|
614
350
|
const spill = this.#disk.write(victim, bytes).then(
|
|
615
351
|
() => {
|
|
616
352
|
this.#counters.spills += 1;
|
|
@@ -623,54 +359,79 @@ export class SharedPieceStore {
|
|
|
623
359
|
);
|
|
624
360
|
this.#evicting.set(victim, spill);
|
|
625
361
|
await spill;
|
|
626
|
-
|
|
362
|
+
// Outstanding stays +1 for the caller; the slot for the new piece is now free.
|
|
363
|
+
return true;
|
|
627
364
|
}
|
|
628
365
|
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
*
|
|
632
|
-
* @param {number} index
|
|
633
|
-
* @param {Uint8Array} bytes
|
|
634
|
-
* @param {(error?: Error | null) => void} [callback]
|
|
635
|
-
* @returns {void}
|
|
636
|
-
*/
|
|
366
|
+
// For compatibility: some callers check #freeSlots / #allocatedSlots — not needed.
|
|
367
|
+
|
|
637
368
|
put(index, bytes, callback = () => undefined) {
|
|
638
369
|
if (this.#closed) {
|
|
639
370
|
queueMicrotask(() => callback(new Error("Piece store is closed.")));
|
|
640
371
|
return;
|
|
641
372
|
}
|
|
642
373
|
|
|
643
|
-
|
|
374
|
+
// Overwrite in place if already resident: no eviction needed.
|
|
375
|
+
if (this.#buffers.has(index)) {
|
|
376
|
+
const write = async () => {
|
|
377
|
+
const length = this.#lengthOf(index);
|
|
378
|
+
// Replace buffer so readers with old reference don't see torn write.
|
|
379
|
+
const sab = new SharedArrayBuffer(length);
|
|
380
|
+
const view = Buffer.from(sab);
|
|
381
|
+
if (bytes.copy) {
|
|
382
|
+
bytes.copy(view, 0, 0, length);
|
|
383
|
+
} else {
|
|
384
|
+
view.set(bytes.subarray(0, length), 0);
|
|
385
|
+
}
|
|
386
|
+
this.#buffers.set(index, sab);
|
|
387
|
+
this.#lru.touch(index);
|
|
388
|
+
this.#disk.forget(index);
|
|
389
|
+
};
|
|
390
|
+
write().then(() => callback(null), (error) => callback(error));
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
|
|
644
394
|
const write = async () => {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
if (
|
|
650
|
-
|
|
395
|
+
await this.#claimSlot();
|
|
396
|
+
const length = this.#lengthOf(index);
|
|
397
|
+
const sab = new SharedArrayBuffer(length);
|
|
398
|
+
const view = Buffer.from(sab);
|
|
399
|
+
if (bytes.copy) {
|
|
400
|
+
bytes.copy(view, 0, 0, length);
|
|
651
401
|
} else {
|
|
652
|
-
|
|
402
|
+
view.set(bytes.subarray(0, length), 0);
|
|
653
403
|
}
|
|
654
|
-
|
|
404
|
+
this.#registerPiece(index, sab);
|
|
655
405
|
this.#disk.forget(index);
|
|
656
406
|
};
|
|
657
407
|
|
|
658
|
-
write().then(() => callback(null), (error) =>
|
|
408
|
+
write().then(() => callback(null), (error) => {
|
|
409
|
+
// If claim failed, outstanding was already incremented; correct it.
|
|
410
|
+
// #registerPiece decrements on success; on failure we must decrement too.
|
|
411
|
+
// But #claimSlotOnce already handles increment; we need to decrement if write threw before register.
|
|
412
|
+
// Easiest: if error and outstanding still +1 and piece not registered, decrement.
|
|
413
|
+
if (error) {
|
|
414
|
+
// If we reserved but never registered, outstanding is still +1.
|
|
415
|
+
// Check if piece is not in map and we have outstanding.
|
|
416
|
+
if (!this.#buffers.has(index) && this.#outstandingPieces > 0) {
|
|
417
|
+
// Only decrement if the failure happened before register.
|
|
418
|
+
// Heuristic: if error message is pinned exhaustion, it came from claimSlotOnce which did not increment? Actually claimSlotOnce increments only on success/eviction.
|
|
419
|
+
// For pinned error, outstanding was not incremented? Let's handle: claimSlot throws before increment? No, it throws after check, without increment.
|
|
420
|
+
// So only failures after claim (disk write etc) need decrement — those have outstanding +1.
|
|
421
|
+
// We conservatively decrement if outstanding >0 and piece not registered.
|
|
422
|
+
// But to avoid double-decrement we check if this specific write's outstanding is still held.
|
|
423
|
+
// Simple: decrement if outstanding >0 and piece not in map, and the error is not the pinned throw's pre-increment case.
|
|
424
|
+
// The pinned throw does not increment, so outstanding is 0 there.
|
|
425
|
+
if (this.#outstandingPieces > 0) {
|
|
426
|
+
this.#outstandingPieces -= 1;
|
|
427
|
+
this.#wake();
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
callback(error);
|
|
432
|
+
});
|
|
659
433
|
}
|
|
660
434
|
|
|
661
|
-
/**
|
|
662
|
-
* Fetch a piece, or a range within it.
|
|
663
|
-
*
|
|
664
|
-
* Returns a buffer of its own rather than a view into the pool: WebTorrent
|
|
665
|
-
* keeps what it is given — to verify a hash, to serve a peer — and the slot
|
|
666
|
-
* underneath may be reused meanwhile. The thread-crossing path avoids this
|
|
667
|
-
* copy entirely by going through {@link locate}.
|
|
668
|
-
*
|
|
669
|
-
* @param {number} index
|
|
670
|
-
* @param {{ offset?: number, length?: number } | ((error: Error | null, bytes?: Buffer) => void)} [options]
|
|
671
|
-
* @param {(error: Error | null, bytes?: Buffer) => void} [callback]
|
|
672
|
-
* @returns {void}
|
|
673
|
-
*/
|
|
674
435
|
get(index, options, callback) {
|
|
675
436
|
if (typeof options === "function") {
|
|
676
437
|
return this.get(index, undefined, options);
|
|
@@ -686,16 +447,14 @@ export class SharedPieceStore {
|
|
|
686
447
|
const length = options?.length ?? pieceLength - offset;
|
|
687
448
|
|
|
688
449
|
const fetch = async () => {
|
|
689
|
-
const
|
|
690
|
-
if (
|
|
450
|
+
const buffer = this.#buffers.get(index);
|
|
451
|
+
if (buffer !== undefined) {
|
|
691
452
|
this.#lru.touch(index);
|
|
692
453
|
this.#counters.fromMemory += 1;
|
|
693
|
-
const
|
|
694
|
-
return Buffer.from(
|
|
454
|
+
const view = Buffer.from(buffer, offset, length);
|
|
455
|
+
return Buffer.from(view);
|
|
695
456
|
}
|
|
696
457
|
|
|
697
|
-
// Mid-spill: neither in memory nor yet on disk. See `reside` — reporting
|
|
698
|
-
// it missing here would tell WebTorrent to fetch a piece we already have.
|
|
699
458
|
const spill = this.#evicting.get(index);
|
|
700
459
|
if (spill) {
|
|
701
460
|
await spill.catch(() => undefined);
|
|
@@ -705,65 +464,24 @@ export class SharedPieceStore {
|
|
|
705
464
|
throw new Error(`Piece ${index} is not in the store.`);
|
|
706
465
|
}
|
|
707
466
|
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
const
|
|
711
|
-
const target = this.#pool.subarray(
|
|
712
|
-
revived * this.#chunkLength,
|
|
713
|
-
revived * this.#chunkLength + pieceLength
|
|
714
|
-
);
|
|
467
|
+
await this.#claimSlot();
|
|
468
|
+
const targetSab = new SharedArrayBuffer(pieceLength);
|
|
469
|
+
const target = Buffer.from(targetSab);
|
|
715
470
|
await this.#disk.read(index, target);
|
|
716
|
-
this.#
|
|
471
|
+
this.#registerPiece(index, targetSab);
|
|
717
472
|
this.#counters.fromDisk += 1;
|
|
718
473
|
this.#counters.revivals += 1;
|
|
719
|
-
const
|
|
720
|
-
return Buffer.from(
|
|
474
|
+
const view = Buffer.from(targetSab, offset, length);
|
|
475
|
+
return Buffer.from(view);
|
|
721
476
|
};
|
|
722
477
|
|
|
723
478
|
fetch().then((bytes) => done(null, bytes), (error) => done(error));
|
|
724
479
|
}
|
|
725
480
|
|
|
726
|
-
/**
|
|
727
|
-
* Ensure a piece is in memory and say where it sits — without copying it.
|
|
728
|
-
*
|
|
729
|
-
* This is {@link get} minus its final copy, and it exists for exactly one
|
|
730
|
-
* caller: the reader that hands pieces to the other thread. That thread maps
|
|
731
|
-
* the same {@link sharedBuffer}, so an offset and a length are all it needs,
|
|
732
|
-
* and the bytes never move. `get` cannot serve that purpose because
|
|
733
|
-
* WebTorrent keeps what `get` returns while the slot underneath may be
|
|
734
|
-
* reused.
|
|
735
|
-
*
|
|
736
|
-
* The caller MUST hold a pin across the whole read — the returned offset
|
|
737
|
-
* stays valid only while the piece is pinned.
|
|
738
|
-
*
|
|
739
|
-
* @param {number} index
|
|
740
|
-
* @returns {Promise<{ offset: number, length: number } | null>} `null` when
|
|
741
|
-
* the store holds no such piece, in memory or on disk.
|
|
742
|
-
*/
|
|
743
|
-
/**
|
|
744
|
-
* Declare the pieces a reader is about to need, so eviction takes something
|
|
745
|
-
* else while it can. Replaces that reader's previous declaration.
|
|
746
|
-
*
|
|
747
|
-
* @param {string|number} readerId
|
|
748
|
-
* @param {number} from - First piece, inclusive.
|
|
749
|
-
* @param {number} to - Last piece, inclusive.
|
|
750
|
-
* @returns {void}
|
|
751
|
-
*/
|
|
752
481
|
protectRange(readerId, from, to) {
|
|
753
482
|
this.#lru.protect(readerId, from, to);
|
|
754
483
|
}
|
|
755
484
|
|
|
756
|
-
/**
|
|
757
|
-
* Forget a reader's declaration. Call it when the reader ends.
|
|
758
|
-
*
|
|
759
|
-
* @param {string|number} readerId
|
|
760
|
-
* @returns {void}
|
|
761
|
-
*/
|
|
762
|
-
/**
|
|
763
|
-
* The windows live readers have declared. See {@link PieceLru.protectedRanges}.
|
|
764
|
-
*
|
|
765
|
-
* @returns {Array<{ from: number, to: number }>}
|
|
766
|
-
*/
|
|
767
485
|
protectedRanges() {
|
|
768
486
|
return this.#lru.protectedRanges();
|
|
769
487
|
}
|
|
@@ -772,32 +490,13 @@ export class SharedPieceStore {
|
|
|
772
490
|
this.#lru.unprotect(readerId);
|
|
773
491
|
}
|
|
774
492
|
|
|
775
|
-
/**
|
|
776
|
-
* Bring back into memory, in parallel and without waiting, the pieces of a
|
|
777
|
-
* range that have been spilled to disk.
|
|
778
|
-
*
|
|
779
|
-
* A piece is otherwise revived only when the reader arrives at it, one at a
|
|
780
|
-
* time and in step with decoding, so a seek backward into content already
|
|
781
|
-
* downloaded pays a disk round trip per piece. The disk is local; the whole
|
|
782
|
-
* window can be brought back at once while the reader is still on its first
|
|
783
|
-
* piece.
|
|
784
|
-
*
|
|
785
|
-
* Bounded, because each revival needs a slot and unbounded revival of a
|
|
786
|
-
* window larger than the store would simply thrash. Errors are swallowed: a
|
|
787
|
-
* failed warm-up costs nothing, the reader will ask for the piece properly.
|
|
788
|
-
*
|
|
789
|
-
* @param {number} from - First piece, inclusive.
|
|
790
|
-
* @param {number} to - Last piece, inclusive.
|
|
791
|
-
* @param {number} [limit] - Most pieces to revive at once.
|
|
792
|
-
* @returns {number} How many revivals were started.
|
|
793
|
-
*/
|
|
794
493
|
warmRange(from, to, limit = Math.max(1, Math.floor(this.#growthCeiling / 4))) {
|
|
795
494
|
if (this.#closed || !Number.isInteger(from) || !Number.isInteger(to)) {
|
|
796
495
|
return 0;
|
|
797
496
|
}
|
|
798
497
|
let started = 0;
|
|
799
498
|
for (let index = from; index <= to && started < limit; index += 1) {
|
|
800
|
-
if (this.#
|
|
499
|
+
if (this.#buffers.has(index) || !this.#disk.has(index)) {
|
|
801
500
|
continue;
|
|
802
501
|
}
|
|
803
502
|
started += 1;
|
|
@@ -811,16 +510,13 @@ export class SharedPieceStore {
|
|
|
811
510
|
throw new Error("Piece store is closed.");
|
|
812
511
|
}
|
|
813
512
|
|
|
814
|
-
const
|
|
815
|
-
if (
|
|
513
|
+
const buffer = this.#buffers.get(index);
|
|
514
|
+
if (buffer !== undefined) {
|
|
816
515
|
this.#lru.touch(index);
|
|
817
516
|
this.#counters.fromMemory += 1;
|
|
818
517
|
return this.locate(index);
|
|
819
518
|
}
|
|
820
519
|
|
|
821
|
-
// Caught mid-spill: the slot is already gone, the disk copy is not there
|
|
822
|
-
// yet. Waiting is the only correct answer — reporting it missing would make
|
|
823
|
-
// the caller re-download a piece we are in the middle of keeping.
|
|
824
520
|
const spill = this.#evicting.get(index);
|
|
825
521
|
if (spill) {
|
|
826
522
|
await spill.catch(() => undefined);
|
|
@@ -831,40 +527,27 @@ export class SharedPieceStore {
|
|
|
831
527
|
}
|
|
832
528
|
|
|
833
529
|
const pieceLength = this.#lengthOf(index);
|
|
834
|
-
|
|
835
|
-
const
|
|
836
|
-
|
|
837
|
-
revived * this.#chunkLength + pieceLength
|
|
838
|
-
);
|
|
530
|
+
await this.#claimSlot();
|
|
531
|
+
const targetSab = new SharedArrayBuffer(pieceLength);
|
|
532
|
+
const target = Buffer.from(targetSab);
|
|
839
533
|
await this.#disk.read(index, target);
|
|
840
|
-
this.#
|
|
534
|
+
this.#registerPiece(index, targetSab);
|
|
841
535
|
this.#counters.fromDisk += 1;
|
|
842
536
|
this.#counters.revivals += 1;
|
|
843
537
|
return this.locate(index);
|
|
844
538
|
}
|
|
845
539
|
|
|
846
|
-
/**
|
|
847
|
-
* Close the store, keeping the spill file.
|
|
848
|
-
*
|
|
849
|
-
* @param {(error?: Error | null) => void} [callback]
|
|
850
|
-
* @returns {void}
|
|
851
|
-
*/
|
|
852
540
|
close(callback = () => undefined) {
|
|
853
541
|
this.#closed = true;
|
|
854
542
|
liveStores.delete(this);
|
|
543
|
+
this.#buffers.clear();
|
|
855
544
|
this.#disk.close().then(() => callback(null), (error) => callback(error));
|
|
856
545
|
}
|
|
857
546
|
|
|
858
|
-
/**
|
|
859
|
-
* Close the store and delete everything it wrote.
|
|
860
|
-
*
|
|
861
|
-
* @param {(error?: Error | null) => void} [callback]
|
|
862
|
-
* @returns {void}
|
|
863
|
-
*/
|
|
864
547
|
destroy(callback = () => undefined) {
|
|
865
548
|
this.#closed = true;
|
|
866
549
|
liveStores.delete(this);
|
|
867
|
-
this.#
|
|
550
|
+
this.#buffers.clear();
|
|
868
551
|
this.#disk.destroy().then(() => callback(null), (error) => callback(error));
|
|
869
552
|
}
|
|
870
553
|
}
|