@torrent-tv/proxy 2.69.0 → 2.69.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/bin/cli.js +544 -533
- package/package.json +1 -1
- package/research/valgrind-replica-2026-08-27/logs.tar.gz +0 -0
- package/services/memory-report.js +77 -25
- package/services/piece-store/piece-lru.js +79 -3
- package/services/piece-store/shared-piece-store.js +103 -3
- package/services/torrent-worker/worker.js +61 -15
- package/test/memory-budget.test.js +58 -1
- package/test/piece-lru.test.js +63 -0
- package/test/piece-store-eviction.test.js +49 -0
package/package.json
CHANGED
|
Binary file
|
|
@@ -291,6 +291,12 @@ function megabytes(bytes) {
|
|
|
291
291
|
* @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
|
|
292
292
|
* @param {number | null} [reading.diskFreeBytes]
|
|
293
293
|
* @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
|
|
294
|
+
* @param {string} [reading.extra] - Figures the caller wants on the same line
|
|
295
|
+
* rather than on one of its own. The piece-buffer counters are read here so
|
|
296
|
+
* that the number of buffers alive and the off-heap mass they should account
|
|
297
|
+
* for are the SAME instant: printed on separate timers they were up to a
|
|
298
|
+
* minute apart, and 950 MB of `arrayBuffers` could not be checked against the
|
|
299
|
+
* 62 buffers a reading half a minute away said were alive (roadmap item 2).
|
|
294
300
|
* @returns {string}
|
|
295
301
|
*/
|
|
296
302
|
export function describeMemory({
|
|
@@ -302,7 +308,8 @@ export function describeMemory({
|
|
|
302
308
|
anonymousBytes = null,
|
|
303
309
|
mappings = null,
|
|
304
310
|
diskFreeBytes = null,
|
|
305
|
-
stores = []
|
|
311
|
+
stores = [],
|
|
312
|
+
extra = ""
|
|
306
313
|
}) {
|
|
307
314
|
const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
|
|
308
315
|
const storeResident = total("residentBytes");
|
|
@@ -321,8 +328,9 @@ export function describeMemory({
|
|
|
321
328
|
`heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)}` +
|
|
322
329
|
`${usage.heapLimit ? ` of ${megabytes(usage.heapLimit)} allowed` : ""} ` +
|
|
323
330
|
`external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
|
|
331
|
+
const tail = extra ? `; ${extra}` : "";
|
|
324
332
|
if (scope === "thread") {
|
|
325
|
-
return `memory (${label || "thread"}): ${isolate}; ${storesPart}`;
|
|
333
|
+
return `memory (${label || "thread"}): ${isolate}; ${storesPart}${tail}`;
|
|
326
334
|
}
|
|
327
335
|
const shape = mappings === null
|
|
328
336
|
? ""
|
|
@@ -337,10 +345,41 @@ export function describeMemory({
|
|
|
337
345
|
`${storesPart}; ` +
|
|
338
346
|
`machine has ${megabytes(availableBytes ?? 0)} available` +
|
|
339
347
|
`${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
|
|
340
|
-
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}`
|
|
348
|
+
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}` +
|
|
349
|
+
tail
|
|
341
350
|
);
|
|
342
351
|
}
|
|
343
352
|
|
|
353
|
+
/**
|
|
354
|
+
* The figures whose movement earns a line, each under its own name.
|
|
355
|
+
*
|
|
356
|
+
* Not the same question as what the scope WATCHES for a heap snapshot. A
|
|
357
|
+
* thread's heap is only one of the three ways its isolate holds memory, and on
|
|
358
|
+
* 2026-09-02 it was the one that did not move: `heapTotal` stayed at 31-173 MB
|
|
359
|
+
* through a session where `arrayBuffers` swung between 130 and 950 MB, so the
|
|
360
|
+
* change trigger was watching the one quantity that stood still and every
|
|
361
|
+
* reading of the one that grew came out on the quiet interval, a minute apart.
|
|
362
|
+
*
|
|
363
|
+
* Each figure is compared against its own last written value and any one of
|
|
364
|
+
* them moving is enough. Nothing is added together, because `arrayBuffers` is
|
|
365
|
+
* documented as part of `external` and reads larger than it on this runtime —
|
|
366
|
+
* a contradiction this code has no business resolving.
|
|
367
|
+
*
|
|
368
|
+
* @param {"process" | "thread"} scope
|
|
369
|
+
* @param {{ rss: number, heapTotal: number, external: number, arrayBuffers: number }} memory
|
|
370
|
+
* @returns {Record<string, number>}
|
|
371
|
+
*/
|
|
372
|
+
export function watchedFigures(scope, memory) {
|
|
373
|
+
if (scope === "thread") {
|
|
374
|
+
return {
|
|
375
|
+
heap: memory.heapTotal ?? 0,
|
|
376
|
+
external: memory.external ?? 0,
|
|
377
|
+
buffers: memory.arrayBuffers ?? 0
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
return { rss: memory.rss ?? 0 };
|
|
381
|
+
}
|
|
382
|
+
|
|
344
383
|
/**
|
|
345
384
|
* Whether this reading is worth writing down, and why.
|
|
346
385
|
*
|
|
@@ -397,11 +436,14 @@ export function readingIsWorthWriting({
|
|
|
397
436
|
* @param {number} [options.snapshotFloorBytes]
|
|
398
437
|
* @param {number} [options.snapshotGrowthBytes]
|
|
399
438
|
* @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
|
|
439
|
+
* @param {() => string} [options.readExtra] - Figures to append to the line,
|
|
440
|
+
* read at the same instant as the memory itself.
|
|
400
441
|
* @returns {{ stop: () => void }}
|
|
401
442
|
*/
|
|
402
443
|
export function startMemoryReport({
|
|
403
444
|
log,
|
|
404
445
|
readStores,
|
|
446
|
+
readExtra,
|
|
405
447
|
scope = "process",
|
|
406
448
|
label = "",
|
|
407
449
|
diskPath = "",
|
|
@@ -414,7 +456,8 @@ export function startMemoryReport({
|
|
|
414
456
|
keepSnapshots = 0
|
|
415
457
|
}) {
|
|
416
458
|
let highWater = 0;
|
|
417
|
-
|
|
459
|
+
/** @type {Record<string, number>} */
|
|
460
|
+
let lastWritten = {};
|
|
418
461
|
let lastWrittenAt = 0;
|
|
419
462
|
// The process watches what the kernel kills it for; a thread watches what the
|
|
420
463
|
// runtime kills IT for, which is its own heap and not the process's resident
|
|
@@ -472,25 +515,35 @@ export function startMemoryReport({
|
|
|
472
515
|
}
|
|
473
516
|
const processMemory = readProcessMemory();
|
|
474
517
|
const watched = watchedOf(processMemory);
|
|
518
|
+
const figures = watchedFigures(scope, processMemory);
|
|
475
519
|
const now = Date.now();
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
520
|
+
const sinceWrittenMs = lastWrittenAt === 0 ? Number.POSITIVE_INFINITY : now - lastWrittenAt;
|
|
521
|
+
const write = Object.entries(figures).some(([name, bytes]) => readingIsWorthWriting({
|
|
522
|
+
watchedBytes: bytes,
|
|
523
|
+
lastWrittenBytes: lastWritten[name] ?? 0,
|
|
524
|
+
sinceWrittenMs,
|
|
480
525
|
changeBytes,
|
|
481
526
|
quietMs
|
|
482
|
-
});
|
|
527
|
+
}));
|
|
483
528
|
|
|
484
529
|
let anonymousBytes = null;
|
|
485
|
-
if (
|
|
486
|
-
|
|
487
|
-
|
|
530
|
+
if (write) {
|
|
531
|
+
let extra = "";
|
|
532
|
+
try {
|
|
533
|
+
extra = typeof readExtra === "function" ? readExtra() ?? "" : "";
|
|
534
|
+
} catch {
|
|
535
|
+
// silent-ok: a caller's own figures are worth less than the line they
|
|
536
|
+
// would have taken down with them.
|
|
488
537
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
538
|
+
if (scope === "thread") {
|
|
539
|
+
log(describeMemory({ scope, label, process: processMemory, stores, extra }));
|
|
540
|
+
} else {
|
|
541
|
+
// Read only when the line is written. `smaps` is one entry per
|
|
542
|
+
// mapping and a busy process has thousands; the rollup and
|
|
543
|
+
// `/proc/meminfo` are single lines but still walk page tables, and
|
|
544
|
+
// the reading now happens once a second rather than once a minute.
|
|
545
|
+
const { bytes, measured } = await availableMemory();
|
|
546
|
+
anonymousBytes = await readAnonymousMemory();
|
|
494
547
|
log(describeMemory({
|
|
495
548
|
scope,
|
|
496
549
|
label,
|
|
@@ -498,17 +551,13 @@ export function startMemoryReport({
|
|
|
498
551
|
availableBytes: bytes,
|
|
499
552
|
availableMeasured: measured,
|
|
500
553
|
anonymousBytes,
|
|
501
|
-
// Read only when the line is written: `smaps` is one entry per
|
|
502
|
-
// mapping and a busy process has thousands, which is a different
|
|
503
|
-
// cost from the rollup's single line.
|
|
504
554
|
mappings: await readMappingSummary(),
|
|
505
|
-
diskFreeBytes,
|
|
506
|
-
stores
|
|
555
|
+
diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
|
|
556
|
+
stores,
|
|
557
|
+
extra
|
|
507
558
|
}));
|
|
508
559
|
}
|
|
509
|
-
|
|
510
|
-
if (write) {
|
|
511
|
-
lastWrittenBytes = watched;
|
|
560
|
+
lastWritten = figures;
|
|
512
561
|
lastWrittenAt = now;
|
|
513
562
|
}
|
|
514
563
|
|
|
@@ -524,6 +573,9 @@ export function startMemoryReport({
|
|
|
524
573
|
highWater = watched;
|
|
525
574
|
await takeSnapshot(watched, "a new high-water");
|
|
526
575
|
}
|
|
576
|
+
// Under `write` by construction: `anonymousBytes` is only read when the
|
|
577
|
+
// line is, and at one reading a second an unconditional warning would be
|
|
578
|
+
// a line a second for as long as the process stayed large.
|
|
527
579
|
if (scope !== "thread" && anonymousBytes !== null && processMemory.rss > 800 * 1024 * 1024) {
|
|
528
580
|
log(`memory: high rss=${megabytes(processMemory.rss)} anon=${megabytes(anonymousBytes)} heap=${megabytes(processMemory.heapUsed)} — watch for OOM`);
|
|
529
581
|
}
|
|
@@ -174,11 +174,31 @@ export class PieceLru {
|
|
|
174
174
|
* @returns {number | null}
|
|
175
175
|
*/
|
|
176
176
|
evictionCandidate() {
|
|
177
|
+
return this.evictionChoice().index;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The same choice, with the two facts that say whether the store is working
|
|
182
|
+
* or thrashing: whether protection had to yield, and how far the victim was
|
|
183
|
+
* from the nearest piece a reader declared it wants.
|
|
184
|
+
*
|
|
185
|
+
* Evicting a stale piece nobody asked for is the store doing its job.
|
|
186
|
+
* Evicting a piece inside a reader's own declared window is the store being
|
|
187
|
+
* asked to hold more than it has room for, and it comes back from disk
|
|
188
|
+
* moments later — 6565 spills and 7575 revivals in 44 minutes on
|
|
189
|
+
* 2026-09-02, with only 53.6% of reads served from memory. Nothing recorded
|
|
190
|
+
* which of the two was happening (roadmap item 9).
|
|
191
|
+
*
|
|
192
|
+
* @returns {{ index: number | null, protectionYielded: boolean, distance: number }}
|
|
193
|
+
* `distance` is in pieces from the nearest declared window, zero when the
|
|
194
|
+
* victim is inside one, and -1 when no reader has declared anything.
|
|
195
|
+
*/
|
|
196
|
+
evictionChoice() {
|
|
177
197
|
// First choice: the least recently used piece nobody is reading and nobody
|
|
178
198
|
// is about to read.
|
|
179
199
|
for (const index of this.#order) {
|
|
180
200
|
if (!this.#pins.has(index) && !this.#isProtected(index)) {
|
|
181
|
-
return index;
|
|
201
|
+
return { index, protectionYielded: false, distance: this.#distanceToWindow(index) };
|
|
182
202
|
}
|
|
183
203
|
}
|
|
184
204
|
// Nothing spare left. Protection yields — it is a preference, and refusing
|
|
@@ -186,10 +206,66 @@ export class PieceLru {
|
|
|
186
206
|
// yield: a piece being read now cannot have its memory taken away.
|
|
187
207
|
for (const index of this.#order) {
|
|
188
208
|
if (!this.#pins.has(index)) {
|
|
189
|
-
return index;
|
|
209
|
+
return { index, protectionYielded: true, distance: this.#distanceToWindow(index) };
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return { index: null, protectionYielded: false, distance: -1 };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* How many pieces the live readers between them are asking to keep, against
|
|
217
|
+
* how many this store may hold.
|
|
218
|
+
*
|
|
219
|
+
* The union, not the sum: two readers of one file overlap, and counting the
|
|
220
|
+
* overlap twice would say the store is short when it is not. This is the
|
|
221
|
+
* comparison that decides whether thrashing is a policy fault or arithmetic —
|
|
222
|
+
* a union wider than the capacity cannot be held however the eviction is
|
|
223
|
+
* ordered.
|
|
224
|
+
*
|
|
225
|
+
* @returns {{ readers: number, unionPieces: number, widestPieces: number, capacity: number }}
|
|
226
|
+
*/
|
|
227
|
+
demand() {
|
|
228
|
+
const ranges = [...this.#protected.values()]
|
|
229
|
+
.map((range) => ({ from: range.from, to: range.to }))
|
|
230
|
+
.sort((left, right) => left.from - right.from);
|
|
231
|
+
let unionPieces = 0;
|
|
232
|
+
let widestPieces = 0;
|
|
233
|
+
let coveredTo = -Infinity;
|
|
234
|
+
for (const range of ranges) {
|
|
235
|
+
const width = range.to - range.from + 1;
|
|
236
|
+
widestPieces = Math.max(widestPieces, width);
|
|
237
|
+
const from = Math.max(range.from, coveredTo + 1);
|
|
238
|
+
if (range.to >= from) {
|
|
239
|
+
unionPieces += range.to - from + 1;
|
|
240
|
+
coveredTo = range.to;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
readers: ranges.length,
|
|
245
|
+
unionPieces,
|
|
246
|
+
widestPieces,
|
|
247
|
+
capacity: this.#capacity
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Pieces from `index` to the nearest declared window, zero inside one and -1
|
|
253
|
+
* when nothing is declared.
|
|
254
|
+
*
|
|
255
|
+
* @param {number} index
|
|
256
|
+
* @returns {number}
|
|
257
|
+
*/
|
|
258
|
+
#distanceToWindow(index) {
|
|
259
|
+
let nearest = -1;
|
|
260
|
+
for (const range of this.#protected.values()) {
|
|
261
|
+
const gap = index < range.from
|
|
262
|
+
? range.from - index
|
|
263
|
+
: index > range.to ? index - range.to : 0;
|
|
264
|
+
if (nearest === -1 || gap < nearest) {
|
|
265
|
+
nearest = gap;
|
|
190
266
|
}
|
|
191
267
|
}
|
|
192
|
-
return
|
|
268
|
+
return nearest;
|
|
193
269
|
}
|
|
194
270
|
|
|
195
271
|
/**
|
|
@@ -151,6 +151,32 @@ const MIN_RESIDENT_PIECES = 2;
|
|
|
151
151
|
* never failing (field 2026-08-31).
|
|
152
152
|
*/
|
|
153
153
|
const PINNED_WAIT_MS = 5_000;
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* How many revival ages are kept for the median. A window, not a history: two
|
|
157
|
+
* hundred covers several minutes of the busiest session measured (7575
|
|
158
|
+
* revivals in 44 minutes) and costs two hundred numbers.
|
|
159
|
+
*/
|
|
160
|
+
const REVIVAL_AGE_SAMPLES = 200;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The middle value of a sample, or null when there is nothing to take a middle
|
|
164
|
+
* of. Null rather than zero: no revivals and instant revivals are different
|
|
165
|
+
* facts and must not print the same.
|
|
166
|
+
*
|
|
167
|
+
* @param {number[]} values
|
|
168
|
+
* @returns {number | null}
|
|
169
|
+
*/
|
|
170
|
+
function median(values) {
|
|
171
|
+
if (values.length === 0) {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
175
|
+
const middle = Math.floor(sorted.length / 2);
|
|
176
|
+
return sorted.length % 2 === 0
|
|
177
|
+
? Math.round((sorted[middle - 1] + sorted[middle]) / 2)
|
|
178
|
+
: sorted[middle];
|
|
179
|
+
}
|
|
154
180
|
const CLAIM_RETRY_MS = 50;
|
|
155
181
|
|
|
156
182
|
export class SharedPieceStore {
|
|
@@ -191,8 +217,25 @@ export class SharedPieceStore {
|
|
|
191
217
|
blockedByPins: 0,
|
|
192
218
|
waitedForPins: 0,
|
|
193
219
|
evictedOnRevise: 0,
|
|
194
|
-
spillFailures: 0
|
|
220
|
+
spillFailures: 0,
|
|
221
|
+
// Whether the store is doing its job or being asked to hold more than it
|
|
222
|
+
// has room for. An eviction that had to take a piece a reader declared it
|
|
223
|
+
// wants is the second, and it comes back from disk moments later
|
|
224
|
+
// (roadmap item 9).
|
|
225
|
+
evictedProtected: 0,
|
|
226
|
+
evictedDistanceSum: 0,
|
|
227
|
+
evictedWithDistance: 0
|
|
195
228
|
};
|
|
229
|
+
/** Piece index → when it was written out, for the age it comes back at. */
|
|
230
|
+
#spilledAt = new Map();
|
|
231
|
+
/**
|
|
232
|
+
* Ages, in milliseconds, of the last revivals — bounded, because the figure
|
|
233
|
+
* wanted is a median and not a history. A piece that comes back seconds
|
|
234
|
+
* after it left should not have left.
|
|
235
|
+
*
|
|
236
|
+
* @type {number[]}
|
|
237
|
+
*/
|
|
238
|
+
#revivalAges = [];
|
|
196
239
|
|
|
197
240
|
constructor(chunkLength, options = {}) {
|
|
198
241
|
if (!Number.isInteger(chunkLength) || chunkLength < 1) {
|
|
@@ -244,6 +287,14 @@ export class SharedPieceStore {
|
|
|
244
287
|
outstanding: this.#outstandingPieces,
|
|
245
288
|
spilled: this.#disk.size,
|
|
246
289
|
spilledBytes: this.#disk.size * this.#chunkLength,
|
|
290
|
+
// What the readers between them are asking this store to keep, against
|
|
291
|
+
// what it may hold. A union wider than the capacity cannot be held
|
|
292
|
+
// however the eviction is ordered, and that is the difference between a
|
|
293
|
+
// policy to fix and arithmetic to accept (roadmap item 9).
|
|
294
|
+
demand: this.#lru.demand(),
|
|
295
|
+
revivalAgeMedianMs: median(this.#revivalAges),
|
|
296
|
+
revivalAgeSamples: this.#revivalAges.length,
|
|
297
|
+
revivedWithinFiveSeconds: this.#revivalAges.filter((age) => age <= 5_000).length,
|
|
247
298
|
...this.#counters
|
|
248
299
|
};
|
|
249
300
|
}
|
|
@@ -263,7 +314,7 @@ export class SharedPieceStore {
|
|
|
263
314
|
// old growable pool. Eagerly evict excess to honour the new ceiling.
|
|
264
315
|
let evicted = 0;
|
|
265
316
|
while (this.#buffers.size > this.#growthCeiling) {
|
|
266
|
-
const victim = this.#lru.
|
|
317
|
+
const { index: victim, protectionYielded, distance } = this.#lru.evictionChoice();
|
|
267
318
|
if (victim === null) {
|
|
268
319
|
break;
|
|
269
320
|
}
|
|
@@ -276,6 +327,7 @@ export class SharedPieceStore {
|
|
|
276
327
|
this.#lru.remove(victim);
|
|
277
328
|
evicted += 1;
|
|
278
329
|
this.#counters.evictedOnRevise += 1;
|
|
330
|
+
this.#noteEviction(protectionYielded, distance);
|
|
279
331
|
// Nobody awaits this spill, so its failure has to end here. Rethrowing
|
|
280
332
|
// made it an unhandled rejection, and an unhandled rejection in the
|
|
281
333
|
// torrent worker ends the thread — a second way to lose the torrent
|
|
@@ -421,6 +473,7 @@ export class SharedPieceStore {
|
|
|
421
473
|
const spill = this.#disk.write(index, bytes).then(
|
|
422
474
|
() => {
|
|
423
475
|
this.#counters.spills += 1;
|
|
476
|
+
this.#spilledAt.set(index, Date.now());
|
|
424
477
|
this.#evicting.delete(index);
|
|
425
478
|
this.#noteProgress();
|
|
426
479
|
},
|
|
@@ -440,6 +493,48 @@ export class SharedPieceStore {
|
|
|
440
493
|
this.#wake();
|
|
441
494
|
}
|
|
442
495
|
|
|
496
|
+
/**
|
|
497
|
+
* Record how long a piece stayed on disk before it was wanted again.
|
|
498
|
+
*
|
|
499
|
+
* A piece that comes back seconds after it left was evicted from a working
|
|
500
|
+
* set that does not fit, and the write and the read were both waste. Kept as
|
|
501
|
+
* a bounded window of ages because the figure wanted is a median, not a
|
|
502
|
+
* history.
|
|
503
|
+
*
|
|
504
|
+
* @param {number} index
|
|
505
|
+
* @returns {void}
|
|
506
|
+
*/
|
|
507
|
+
#noteRevival(index) {
|
|
508
|
+
const spilledAt = this.#spilledAt.get(index);
|
|
509
|
+
if (spilledAt === undefined) {
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
this.#spilledAt.delete(index);
|
|
513
|
+
this.#revivalAges.push(Date.now() - spilledAt);
|
|
514
|
+
if (this.#revivalAges.length > REVIVAL_AGE_SAMPLES) {
|
|
515
|
+
this.#revivalAges.shift();
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Record what an eviction had to take.
|
|
521
|
+
*
|
|
522
|
+
* @param {boolean} protectionYielded - The victim was inside a window a
|
|
523
|
+
* reader had declared, and was taken anyway because nothing else was free.
|
|
524
|
+
* @param {number} distance - Pieces from the nearest declared window, -1 when
|
|
525
|
+
* no reader declared one.
|
|
526
|
+
* @returns {void}
|
|
527
|
+
*/
|
|
528
|
+
#noteEviction(protectionYielded, distance) {
|
|
529
|
+
if (protectionYielded) {
|
|
530
|
+
this.#counters.evictedProtected += 1;
|
|
531
|
+
}
|
|
532
|
+
if (distance >= 0) {
|
|
533
|
+
this.#counters.evictedDistanceSum += distance;
|
|
534
|
+
this.#counters.evictedWithDistance += 1;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
443
538
|
#wake() {
|
|
444
539
|
const waiting = this.#waiters;
|
|
445
540
|
this.#waiters = [];
|
|
@@ -456,7 +551,7 @@ export class SharedPieceStore {
|
|
|
456
551
|
return true;
|
|
457
552
|
}
|
|
458
553
|
|
|
459
|
-
const victim = this.#lru.
|
|
554
|
+
const { index: victim, protectionYielded, distance } = this.#lru.evictionChoice();
|
|
460
555
|
if (victim === null) {
|
|
461
556
|
// Nothing may leave. Wait while the store is still MOVING — a spill
|
|
462
557
|
// completing, a piece admitted, a pin released — and give up when it has
|
|
@@ -491,6 +586,7 @@ export class SharedPieceStore {
|
|
|
491
586
|
this.#buffers.delete(victim);
|
|
492
587
|
this.#lru.remove(victim);
|
|
493
588
|
this.#outstandingPieces += 1;
|
|
589
|
+
this.#noteEviction(protectionYielded, distance);
|
|
494
590
|
|
|
495
591
|
try {
|
|
496
592
|
await this.#spill(victim, victimBuffer);
|
|
@@ -538,6 +634,7 @@ export class SharedPieceStore {
|
|
|
538
634
|
this.#registerPiece(index, target);
|
|
539
635
|
this.#counters.fromDisk += 1;
|
|
540
636
|
this.#counters.revivals += 1;
|
|
637
|
+
this.#noteRevival(index);
|
|
541
638
|
return target;
|
|
542
639
|
} finally {
|
|
543
640
|
release();
|
|
@@ -593,6 +690,7 @@ export class SharedPieceStore {
|
|
|
593
690
|
await spill.catch(() => undefined);
|
|
594
691
|
}
|
|
595
692
|
this.#disk.forget(index);
|
|
693
|
+
this.#spilledAt.delete(index);
|
|
596
694
|
}
|
|
597
695
|
|
|
598
696
|
put(index, bytes, callback = () => undefined) {
|
|
@@ -707,6 +805,7 @@ export class SharedPieceStore {
|
|
|
707
805
|
this.#closed = true;
|
|
708
806
|
liveStores.delete(this);
|
|
709
807
|
this.#buffers.clear();
|
|
808
|
+
this.#spilledAt.clear();
|
|
710
809
|
// Whoever is waiting for a slot is woken and finds the store closed, which
|
|
711
810
|
// is an error they can report. Left asleep they simply never returned.
|
|
712
811
|
this.#wake();
|
|
@@ -717,6 +816,7 @@ export class SharedPieceStore {
|
|
|
717
816
|
this.#closed = true;
|
|
718
817
|
liveStores.delete(this);
|
|
719
818
|
this.#buffers.clear();
|
|
819
|
+
this.#spilledAt.clear();
|
|
720
820
|
this.#wake();
|
|
721
821
|
this.#disk.destroy().then(() => callback(null), (error) => callback(error));
|
|
722
822
|
}
|
|
@@ -522,6 +522,12 @@ parentPort.on("message", async (message) => {
|
|
|
522
522
|
startMemoryReport({
|
|
523
523
|
log,
|
|
524
524
|
readStores: collectStoreStats,
|
|
525
|
+
// Beside the isolate's own figures, and on the SAME line, because the
|
|
526
|
+
// question they answer together is whether the off-heap mass is buffers this
|
|
527
|
+
// thread still refers to or buffers the collector has not reached yet. On
|
|
528
|
+
// separate timers the two were up to a minute apart and could not be
|
|
529
|
+
// compared at all (roadmap item 2).
|
|
530
|
+
readExtra: describePieceBuffers,
|
|
525
531
|
scope: "thread",
|
|
526
532
|
label: "torrent worker",
|
|
527
533
|
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
@@ -559,7 +565,10 @@ setInterval(() => {
|
|
|
559
565
|
}
|
|
560
566
|
}
|
|
561
567
|
for (const stats of collectStoreStats()) {
|
|
562
|
-
const signature =
|
|
568
|
+
const signature =
|
|
569
|
+
`${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
|
|
570
|
+
`${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
|
|
571
|
+
`${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}`;
|
|
563
572
|
if (lastReported.get(stats.name) === signature) {
|
|
564
573
|
continue;
|
|
565
574
|
}
|
|
@@ -580,23 +589,60 @@ setInterval(() => {
|
|
|
580
589
|
(stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
|
|
581
590
|
(stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
|
|
582
591
|
);
|
|
592
|
+
// Why it spills, on its own line because it is a different question from
|
|
593
|
+
// how much it holds. Three facts, and between them they say whether the
|
|
594
|
+
// thrashing is a policy to fix or arithmetic to accept: what the readers
|
|
595
|
+
// together are asking to keep against what the store may hold; how many
|
|
596
|
+
// evictions had to take a piece a reader had declared it wants; and how
|
|
597
|
+
// long a piece stayed on disk before it was wanted back. On 2026-09-02 a
|
|
598
|
+
// session did 6565 spills and 7575 revivals with 53.6% of reads served
|
|
599
|
+
// from memory, and nothing recorded which of the three was the cause
|
|
600
|
+
// (roadmap item 9).
|
|
601
|
+
const demand = stats.demand;
|
|
602
|
+
if (demand && demand.readers > 0) {
|
|
603
|
+
const age = stats.revivalAgeMedianMs;
|
|
604
|
+
log(
|
|
605
|
+
`piece-store "${stats.name.slice(0, 40)}" demand: ${demand.readers} reader(s) want ` +
|
|
606
|
+
`${demand.unionPieces} piece(s) of ${demand.capacity} the store may hold ` +
|
|
607
|
+
`(widest window ${demand.widestPieces})` +
|
|
608
|
+
(stats.evictedProtected > 0
|
|
609
|
+
? `; ${stats.evictedProtected} of ${stats.spills} eviction(s) took a piece a reader had declared`
|
|
610
|
+
: "; no eviction has taken a declared piece") +
|
|
611
|
+
(stats.evictedWithDistance > 0
|
|
612
|
+
? `, a victim lay ${(stats.evictedDistanceSum / stats.evictedWithDistance).toFixed(1)} ` +
|
|
613
|
+
"piece(s) from the nearest window on average"
|
|
614
|
+
: "") +
|
|
615
|
+
(age === null
|
|
616
|
+
? "; nothing has come back from disk yet"
|
|
617
|
+
: `; a revived piece had been on disk ${(age / 1000).toFixed(1)}s (median of ` +
|
|
618
|
+
`${stats.revivalAgeSamples}, ${stats.revivedWithinFiveSeconds} of them within 5s)`)
|
|
619
|
+
);
|
|
620
|
+
}
|
|
583
621
|
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
622
|
+
}, STORE_REPORT_INTERVAL_MS).unref();
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* The piece buffers this thread has let go of against the ones it still holds.
|
|
626
|
+
*
|
|
627
|
+
* Not per store: the collector is per thread, and the question is about the
|
|
628
|
+
* thread. A gap that keeps widening means a reference of ours outlives the
|
|
629
|
+
* piece; a gap that does not means whatever grows is below us, in the
|
|
630
|
+
* allocator or in buffers the collector has not reached.
|
|
631
|
+
*
|
|
632
|
+
* @returns {string}
|
|
633
|
+
*/
|
|
634
|
+
function describePieceBuffers() {
|
|
588
635
|
const collection = pieceBufferCollection();
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
if (outstandingBuffers > heldNow) {
|
|
592
|
-
log(
|
|
593
|
-
`piece buffers: ${collection.released} let go, ${collection.collected} collected — ` +
|
|
594
|
-
`${outstandingBuffers} still alive against ${heldNow} the store holds. ` +
|
|
595
|
-
"A gap that keeps widening means a reference of ours outlives the piece; " +
|
|
596
|
-
"a gap that does not means whatever grows is below us, in the allocator"
|
|
597
|
-
);
|
|
636
|
+
if (collection.released === 0) {
|
|
637
|
+
return "";
|
|
598
638
|
}
|
|
599
|
-
|
|
639
|
+
const alive = collection.released - collection.collected;
|
|
640
|
+
const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
|
|
641
|
+
return (
|
|
642
|
+
`piece buffers ${collection.released} let go, ${collection.collected} collected, ` +
|
|
643
|
+
`${alive} still alive against ${held} the store holds`
|
|
644
|
+
);
|
|
645
|
+
}
|
|
600
646
|
|
|
601
647
|
/**
|
|
602
648
|
* Walk subtitle cues for every actively-read file of one torrent, and PUSH
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import test from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
describeMemory,
|
|
6
|
+
readingIsWorthWriting,
|
|
7
|
+
summariseMappings,
|
|
8
|
+
watchedFigures
|
|
9
|
+
} from "../services/memory-report.js";
|
|
5
10
|
import {
|
|
6
11
|
budgetForNewStore,
|
|
7
12
|
SharedPieceStore,
|
|
@@ -287,3 +292,55 @@ test("the mapping shape is said in the line, and left out when it is not known",
|
|
|
287
292
|
});
|
|
288
293
|
assert.ok(!withoutShape.includes("mappings="), "an unread breakdown is absent, not zeroed");
|
|
289
294
|
});
|
|
295
|
+
|
|
296
|
+
test("a thread's line is earned by any of its three figures, not by the heap alone", () => {
|
|
297
|
+
// 2026-09-02, the session that ended in two out-of-memory kills: the worker's
|
|
298
|
+
// heap stood at 33-45 MB for the whole of it while its buffers went from 130
|
|
299
|
+
// to 950 MB. Watching the heap alone, nothing ever earned a line and every
|
|
300
|
+
// reading of the quantity that grew came out on the quiet minute.
|
|
301
|
+
const before = watchedFigures("thread", {
|
|
302
|
+
rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 130 * MEGABYTE
|
|
303
|
+
});
|
|
304
|
+
const after = watchedFigures("thread", {
|
|
305
|
+
rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 515 * MEGABYTE
|
|
306
|
+
});
|
|
307
|
+
assert.equal(before.heap, after.heap, "the heap is what did NOT move");
|
|
308
|
+
|
|
309
|
+
const moved = Object.entries(after).some(([name, bytes]) => readingIsWorthWriting({
|
|
310
|
+
watchedBytes: bytes,
|
|
311
|
+
lastWrittenBytes: before[name],
|
|
312
|
+
sinceWrittenMs: 1_000,
|
|
313
|
+
changeBytes: 25 * MEGABYTE,
|
|
314
|
+
quietMs: 60_000
|
|
315
|
+
}));
|
|
316
|
+
assert.equal(moved, true, "buffers growing by 385 MB earns a line of its own");
|
|
317
|
+
|
|
318
|
+
assert.deepEqual(
|
|
319
|
+
Object.keys(watchedFigures("process", { rss: 5, heapTotal: 1, external: 2, arrayBuffers: 3 })),
|
|
320
|
+
["rss"],
|
|
321
|
+
"a process is killed for its resident memory and watches that"
|
|
322
|
+
);
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
test("figures a caller reads for itself land on the same line as the memory", () => {
|
|
326
|
+
const line = describeMemory({
|
|
327
|
+
scope: "thread",
|
|
328
|
+
label: "torrent worker",
|
|
329
|
+
process: {
|
|
330
|
+
rss: 0, heapUsed: 34 * MEGABYTE, heapTotal: 46 * MEGABYTE,
|
|
331
|
+
external: 54 * MEGABYTE, arrayBuffers: 393 * MEGABYTE, heapLimit: 0
|
|
332
|
+
},
|
|
333
|
+
stores: [],
|
|
334
|
+
extra: "piece buffers 21544 let go, 21482 collected, 62 still alive against 53 the store holds"
|
|
335
|
+
});
|
|
336
|
+
assert.match(line, /arrayBuffers=393MB/);
|
|
337
|
+
assert.match(line, /62 still alive against 53 the store holds$/);
|
|
338
|
+
assert.doesNotMatch(
|
|
339
|
+
describeMemory({
|
|
340
|
+
scope: "thread",
|
|
341
|
+
process: { rss: 0, heapUsed: 1, heapTotal: 2, external: 3, arrayBuffers: 4, heapLimit: 0 }
|
|
342
|
+
}),
|
|
343
|
+
/;\s*$/,
|
|
344
|
+
"nothing to add leaves no dangling separator"
|
|
345
|
+
);
|
|
346
|
+
});
|