@torrent-tv/proxy 2.69.1 → 2.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,7 +39,8 @@ import { startMemoryReport, WORKER_MEMORY_SAMPLE_MS } from "../memory-report.js"
39
39
  // the hook above had a chance to register. Verified the hard way: with a static
40
40
  // import the process still aborted, and the stack named the genuine polyfill.
41
41
  const { TorrentPool, resolveDhtBootstrap } = await import("../torrent-pool.js");
42
- const { collectStoreStats, pieceBufferCollection, reviseStoreBudgets } = await import("../piece-store/shared-piece-store.js");
42
+ const { collectStoreStats, machineReserveBytes, pieceBufferCollection, reviseStoreBudgets } =
43
+ await import("../piece-store/shared-piece-store.js");
43
44
 
44
45
  // Resolved before the client exists, because the client builds its DHT in its
45
46
  // own constructor and the addresses have to be in hand by then. Awaiting here
@@ -541,6 +542,9 @@ startMemoryReport({
541
542
 
542
543
  const STORE_REPORT_INTERVAL_MS = 60_000;
543
544
 
545
+ /** Last reported reserve, so an unchanged one stays silent. */
546
+ let lastReserveBytes = 0;
547
+
544
548
  /** Last reported figures per store, so unchanged ones stay silent. */
545
549
  const lastReported = new Map();
546
550
 
@@ -548,12 +552,24 @@ setInterval(() => {
548
552
  // What the machine can spare NOW, not what it could spare when each store was
549
553
  // created. With per-piece buffers a lowered ceiling is honoured immediately:
550
554
  // excess pieces are evicted to disk and their memory is reclaimable.
555
+ // What the machine has been seen to need for everything that is not us. It
556
+ // starts at nothing and grows only on evidence, so it is worth saying when it
557
+ // moves — it is the one term of the budget that comes from observation of
558
+ // other processes rather than from our own readers.
559
+ const reserveBefore = machineReserveBytes();
551
560
  for (const revised of reviseStoreBudgets()) {
552
561
  if (revised.evicted > 0) {
553
562
  log(
554
563
  `piece-store "${revised.name.slice(0, 40)}": allowance is now ` +
555
- `${Math.round(revised.ceilingBytes / 1048576)}MB, evicted ${revised.evicted} piece(s) to meet it` +
556
- `now ${Math.round(revised.committedBytes / 1048576)}MB committed`
564
+ `${Math.round(revised.ceilingBytes / 1048576)}MB, evicted ${revised.evicted} piece(s) to meet it` +
565
+ (revised.releasedBlocks > 0 ? `, gave back ${revised.releasedBlocks} block(s) of memory` : "") +
566
+ ` — now ${Math.round(revised.committedBytes / 1048576)}MB committed`
567
+ );
568
+ } else if (revised.belowAWindow) {
569
+ log(
570
+ `piece-store "${revised.name.slice(0, 40)}": the machine's share is smaller than one ` +
571
+ `reader's window, so the allowance is held at ${Math.round(revised.ceilingBytes / 1048576)}MB ` +
572
+ "anyway — a store that cannot hold the window of the read it is serving cannot finish that read"
557
573
  );
558
574
  } else if (revised.committedBytes > revised.ceilingBytes) {
559
575
  log(
@@ -564,8 +580,20 @@ setInterval(() => {
564
580
  );
565
581
  }
566
582
  }
583
+ const reserveNow = machineReserveBytes();
584
+ if (reserveNow !== reserveBefore || reserveNow !== lastReserveBytes) {
585
+ lastReserveBytes = reserveNow;
586
+ log(
587
+ `piece-store: leaving ${Math.round(reserveNow / 1048576)}MB for everything else on this ` +
588
+ "machine — the largest fall in available memory this process has seen and did not cause"
589
+ );
590
+ }
567
591
  for (const stats of collectStoreStats()) {
568
- const signature = `${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}`;
592
+ const signature =
593
+ `${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
594
+ `${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
595
+ `${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}/${stats.admittedToDisk}/` +
596
+ `${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}`;
569
597
  if (lastReported.get(stats.name) === signature) {
570
598
  continue;
571
599
  }
@@ -578,6 +606,7 @@ setInterval(() => {
578
606
  `(${Math.round((stats.residentBytes || 0) / 1048576)}MB of ` +
579
607
  `${Math.round((stats.budgetBytes || 0) / 1048576)}MB allowed) ` +
580
608
  `committed=${Math.round((stats.committedBytes || 0) / 1048576)}MB ` +
609
+ `blocks=${stats.blocksAllocated} (${stats.blocksFree} spare) ` +
581
610
  `on-disk=${Math.round((stats.spilledBytes || 0) / 1048576)}MB ` +
582
611
  `pinned=${stats.pinned} spilled=${stats.spilled} reads=${reads} (${fromMemoryShare}% from memory) ` +
583
612
  `spills=${stats.spills} revivals=${stats.revivals}` +
@@ -586,16 +615,60 @@ setInterval(() => {
586
615
  (stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
587
616
  (stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
588
617
  );
618
+ // Why it spills, on its own line because it is a different question from
619
+ // how much it holds. Three facts, and between them they say whether the
620
+ // thrashing is a policy to fix or arithmetic to accept: what the readers
621
+ // together are asking to keep against what the store may hold; how many
622
+ // evictions had to take a piece a reader had declared it wants; and how
623
+ // long a piece stayed on disk before it was wanted back. On 2026-09-02 a
624
+ // session did 6565 spills and 7575 revivals with 53.6% of reads served
625
+ // from memory, and nothing recorded which of the three was the cause
626
+ // (roadmap item 9).
627
+ const demand = stats.demand;
628
+ if (demand && demand.readers > 0) {
629
+ const age = stats.revivalAgeMedianMs;
630
+ log(
631
+ `piece-store "${stats.name.slice(0, 40)}" demand: ${demand.readers} reader(s) want ` +
632
+ `${demand.unionPieces} piece(s) of ${demand.capacity} the store may hold ` +
633
+ `(widest window ${demand.widestPieces})` +
634
+ (stats.evictedProtected > 0
635
+ ? `; ${stats.evictedProtected} of ${stats.spills} eviction(s) took a piece a reader had declared`
636
+ : "; no eviction has taken a declared piece") +
637
+ (stats.evictedWithDistance > 0
638
+ ? `, a victim lay ${(stats.evictedDistanceSum / stats.evictedWithDistance).toFixed(1)} ` +
639
+ "piece(s) from the nearest window on average"
640
+ : "") +
641
+ (age === null
642
+ ? "; nothing has come back from disk yet"
643
+ : `; a revived piece had been on disk ${(age / 1000).toFixed(1)}s (median of ` +
644
+ `${stats.revivalAgeSamples}, ${stats.revivedWithinFiveSeconds} of them within 5s)`) +
645
+ `; of ${stats.admittedInsideWindow + stats.admittedOutsideWindow} piece(s) admitted ` +
646
+ `${stats.admittedOutsideWindow} were in nobody's window, ${stats.admittedToDisk} of those ` +
647
+ "went straight to disk" +
648
+ `; ${stats.blocksAllocated} block(s) of memory exist, ${stats.blocksFree} of them spare` +
649
+ (stats.reuseGapMs === null
650
+ ? ", none re-used yet"
651
+ : `, a block waits up to ${(stats.reuseGapMs / 1000).toFixed(1)}s before it is wanted again`) +
652
+ `, ${stats.blocksReleased} given back` +
653
+ (stats.spillsSkipped > 0
654
+ ? `; ${stats.spillsSkipped} of ${stats.spills} eviction(s) needed no write, the disk already had them`
655
+ : "") +
656
+ (stats.returnedWhilePinned > 0
657
+ ? `; ${stats.returnedWhilePinned} BLOCK(S) WERE RECYCLED WHILE STILL BEING READ`
658
+ : "")
659
+ );
660
+ }
589
661
  }
590
662
  }, STORE_REPORT_INTERVAL_MS).unref();
591
663
 
592
664
  /**
593
- * The piece buffers this thread has let go of against the ones it still holds.
665
+ * The blocks of piece memory this thread has allocated against the pieces the
666
+ * stores hold in them.
594
667
  *
595
668
  * Not per store: the collector is per thread, and the question is about the
596
- * thread. A gap that keeps widening means a reference of ours outlives the
597
- * piece; a gap that does not means whatever grows is below us, in the
598
- * allocator or in buffers the collector has not reached.
669
+ * thread. With a pool one block serves many pieces, so a number of allocations
670
+ * that keeps climbing while the stores hold a steady number of pieces means
671
+ * blocks are being made and thrown away instead of re-used.
599
672
  *
600
673
  * @returns {string}
601
674
  */
@@ -607,8 +680,8 @@ function describePieceBuffers() {
607
680
  const alive = collection.released - collection.collected;
608
681
  const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
609
682
  return (
610
- `piece buffers ${collection.released} let go, ${collection.collected} collected, ` +
611
- `${alive} still alive against ${held} the store holds`
683
+ `memory blocks ${collection.released} allocated, ${collection.collected} collected, ` +
684
+ `${alive} still alive against ${held} piece(s) the stores hold`
612
685
  );
613
686
  }
614
687
 
@@ -8,9 +8,12 @@ import {
8
8
  watchedFigures
9
9
  } from "../services/memory-report.js";
10
10
  import {
11
- budgetForNewStore,
12
- SharedPieceStore,
13
- totalStoreBudgetBytes
11
+ divideAllowance,
12
+ forgetMachineMemory,
13
+ machineAllowanceBytes,
14
+ machineReserveBytes,
15
+ noteMachineMemory,
16
+ SharedPieceStore
14
17
  } from "../services/piece-store/shared-piece-store.js";
15
18
  import { mkdtemp, rm } from "node:fs/promises";
16
19
  import os from "node:os";
@@ -19,31 +22,69 @@ import path from "node:path";
19
22
  const MEGABYTE = 1024 * 1024;
20
23
  const GIGABYTE = 1024 * MEGABYTE;
21
24
 
22
- test("the whole of the torrent stores is bounded, not each one of them", () => {
23
- // The failure this replaces: the budget was per torrent, so two torrents took
24
- // two of it. Whatever the machine has, the total is one budget.
25
- const available = 8 * GIGABYTE;
26
- const alone = budgetForNewStore(available, 1);
27
- const withThree = budgetForNewStore(available, 3);
28
- assert.equal(alone, totalStoreBudgetBytes(available));
29
- assert.ok(withThree < alone, "a third store must not be given a first store's share");
30
- assert.ok(withThree * 3 <= totalStoreBudgetBytes(available) + 3);
25
+ test("the stores are allowed what the machine has, less what others were seen to need", () => {
26
+ // Not a share of what is free. A share is a number chosen out of nothing, and
27
+ // the three that were here until 2026-09-02 — a quarter, a 64 MB floor, a
28
+ // 512 MB ceiling all came from one observation of one host on 2026-08-03.
29
+ // `MemAvailable` is what can be taken ON TOP of what is held, so what the
30
+ // stores already hold is added back to get the ceiling they could reach.
31
+ assert.equal(
32
+ machineAllowanceBytes(2 * GIGABYTE, 300 * MEGABYTE, 0),
33
+ 2 * GIGABYTE + 300 * MEGABYTE
34
+ );
35
+ assert.equal(
36
+ machineAllowanceBytes(2 * GIGABYTE, 300 * MEGABYTE, 500 * MEGABYTE),
37
+ 2 * GIGABYTE - 200 * MEGABYTE
38
+ );
39
+ assert.equal(
40
+ machineAllowanceBytes(100 * MEGABYTE, 0, 4 * GIGABYTE),
41
+ 0,
42
+ "a reserve larger than everything leaves nothing, and says so rather than going negative"
43
+ );
31
44
  });
32
45
 
33
- test("the budget is a share of what the machine can give, capped", () => {
34
- // A quarter of two gigabytes is under the ceiling and is what is taken.
35
- assert.equal(totalStoreBudgetBytes(2 * GIGABYTE), 512 * MEGABYTE);
36
- // A quarter of one gigabyte is 256 MB below the ceiling, so not capped.
37
- assert.equal(totalStoreBudgetBytes(GIGABYTE), 256 * MEGABYTE);
38
- // And it never exceeds the ceiling however much is free.
39
- assert.equal(totalStoreBudgetBytes(64 * GIGABYTE), 512 * MEGABYTE);
46
+ test("what to leave for everyone else is measured, starts at nothing, and ages out", () => {
47
+ // Field 2026-09-02: while this proxy held 76-133 MB the machine's available
48
+ // memory fell from 2378 MB to 306 MB and came back. That fall was somebody
49
+ // else's, and it is the quantity to leave room for. On a quiet host the same
50
+ // reading stays near zero, which is the right answer there.
51
+ forgetMachineMemory();
52
+ assert.equal(noteMachineMemory(2378 * MEGABYTE, 100 * MEGABYTE), 0, "nothing is reserved on faith");
53
+
54
+ // A fall of 2072 MB while the stores grew by 20 MB: 2052 MB of it was theirs.
55
+ assert.equal(noteMachineMemory(306 * MEGABYTE, 120 * MEGABYTE), 2052 * MEGABYTE);
56
+
57
+ // Memory coming back does not lower what has been seen to be needed — the
58
+ // spike can happen again, and that is what there has to be room for.
59
+ assert.equal(noteMachineMemory(3567 * MEGABYTE, 120 * MEGABYTE), 2052 * MEGABYTE);
60
+
61
+ // But it does not stand for ever either. A window of observations, not a
62
+ // high-water: one spike hours ago must stop squeezing the stores, which is
63
+ // the same mistake the block re-use gap would make with an all-time maximum.
64
+ for (let quiet = 0; quiet < 60; quiet += 1) {
65
+ noteMachineMemory(3567 * MEGABYTE, 120 * MEGABYTE);
66
+ }
67
+ assert.equal(machineReserveBytes(), 0, "an hour of quiet leaves the spike behind");
40
68
  });
41
69
 
42
- test("a machine with almost nothing left still gets a workable floor", () => {
43
- // Refusing to serve is worse than exceeding the share, and the memory line
44
- // says plainly what is held either way.
45
- const tiny = budgetForNewStore(32 * MEGABYTE, 4);
46
- assert.equal(tiny, 64 * MEGABYTE);
70
+ test("everyone gets what they asked for while the asks fit", () => {
71
+ // The usual case by a wide margin: two readers of one film declare 32-192 MB
72
+ // between them against gigabytes of free memory, so the machine's limit never
73
+ // binds and the demand is what decides.
74
+ assert.deepEqual(
75
+ divideAllowance([100 * MEGABYTE, 50 * MEGABYTE], GIGABYTE),
76
+ [100 * MEGABYTE, 50 * MEGABYTE]
77
+ );
78
+
79
+ // When they do not fit, each is cut in proportion to what it asked, so a
80
+ // store wanting little is not cut to make room for one wanting much.
81
+ const cut = divideAllowance([300 * MEGABYTE, 100 * MEGABYTE], 200 * MEGABYTE);
82
+ assert.equal(cut[0], 150 * MEGABYTE);
83
+ assert.equal(cut[1], 50 * MEGABYTE);
84
+ assert.equal(cut[0] + cut[1], 200 * MEGABYTE);
85
+
86
+ assert.deepEqual(divideAllowance([], GIGABYTE), []);
87
+ assert.deepEqual(divideAllowance([0, 0], 0), [0, 0], "nobody asking takes nothing");
47
88
  });
48
89
 
49
90
  test("the memory line says bytes, and names what it could not measure", () => {
@@ -131,7 +172,7 @@ test("a thread's reading leaves out what belongs to the process", () => {
131
172
  assert.doesNotMatch(line, /machine has/);
132
173
  });
133
174
 
134
- test("a store's allowance follows the machine, and never passes its reservation", async () => {
175
+ test("a store's allowance follows the machine, in both directions", async () => {
135
176
  // The defect: a store created on an idle machine kept an idle machine's
136
177
  // allowance for life and went on growing into memory the host no longer had.
137
178
  const directory = await mkdtemp(path.join(os.tmpdir(), "budget-revision-"));
@@ -149,11 +190,14 @@ test("a store's allowance follows the machine, and never passes its reservation"
149
190
  assert.ok(lowered.ceilingBytes < born, "a busier machine buys fewer slots");
150
191
  assert.equal(store.stats().budgetBytes, lowered.ceilingBytes, "the line says what is allowed now");
151
192
 
152
- // The machine empties again: the ceiling may rise, but never above the
153
- // reservation, because `maxByteLength` was fixed from it and `grow()`
154
- // cannot pass it.
193
+ // The machine empties again: the ceiling rises with it, PAST where this
194
+ // store started. Until 2026-09-02 it could not the ceiling was capped at
195
+ // a figure computed once in the constructor, so a torrent opened while the
196
+ // machine was full kept a small allowance for its whole life however much
197
+ // memory was freed afterwards.
155
198
  const raised = store.reviseGrowthCeiling(1024 * 1024 * 1024);
156
- assert.equal(raised.ceilingBytes, born, "the reservation is the hard limit");
199
+ assert.ok(raised.ceilingBytes > born, "an emptier machine buys more slots than it started with");
200
+ assert.equal(store.stats().budgetBytes, raised.ceilingBytes);
157
201
  } finally {
158
202
  await new Promise((resolve) => store.destroy(resolve));
159
203
  await rm(directory, { recursive: true, force: true });
@@ -189,3 +189,66 @@ test("the capacity follows the store's live allowance", () => {
189
189
  lru.setCapacity(0);
190
190
  assert.equal(lru.capacity, 2, "a capacity below one is refused, not obeyed");
191
191
  });
192
+
193
+ test("the demand is the union of the readers' windows, not their sum", () => {
194
+ const lru = new PieceLru(88);
195
+ // Two readers of one file — picture and sound — overlap by construction.
196
+ // Summing them would say the store is short when it is not.
197
+ lru.protect("video", 100, 149);
198
+ lru.protect("audio", 130, 179);
199
+
200
+ const demand = lru.demand();
201
+ assert.equal(demand.readers, 2);
202
+ assert.equal(demand.unionPieces, 80, "100..179 is eighty pieces, not a hundred");
203
+ assert.equal(demand.widestPieces, 50);
204
+ assert.equal(demand.capacity, 88);
205
+
206
+ lru.protect("second-viewer", 900, 979);
207
+ assert.equal(lru.demand().unionPieces, 160, "windows that do not touch add up");
208
+
209
+ lru.unprotect("second-viewer");
210
+ lru.unprotect("audio");
211
+ lru.unprotect("video");
212
+ assert.deepEqual(
213
+ lru.demand(),
214
+ { readers: 0, unionPieces: 0, widestPieces: 0, capacity: 88 },
215
+ "no reader asking for anything is not the same as asking for one piece"
216
+ );
217
+ });
218
+
219
+ test("an eviction says whether it had to take a piece a reader declared", () => {
220
+ const lru = new PieceLru(3);
221
+ lru.touch(10);
222
+ lru.touch(11);
223
+ lru.touch(12);
224
+ lru.protect("video", 11, 12);
225
+
226
+ const spare = lru.evictionChoice();
227
+ assert.equal(spare.index, 10, "the piece outside every window goes first");
228
+ assert.equal(spare.protectionYielded, false);
229
+ assert.equal(spare.distance, 1, "one piece away from the window at 11");
230
+
231
+ // Nothing spare left: both survivors are inside the declared window.
232
+ lru.remove(10);
233
+ const forced = lru.evictionChoice();
234
+ assert.equal(forced.index, 11);
235
+ assert.equal(forced.protectionYielded, true, "the store is holding less than it is asked to");
236
+ assert.equal(forced.distance, 0, "inside a window");
237
+
238
+ assert.equal(lru.evictionCandidate(), 11, "the older answer is the same choice");
239
+ });
240
+
241
+ test("with nothing declared there is no distance to report", () => {
242
+ const lru = new PieceLru(2);
243
+ lru.touch(7);
244
+ const choice = lru.evictionChoice();
245
+ assert.equal(choice.index, 7);
246
+ assert.equal(choice.distance, -1, "-1 is absence, and 0 would read as inside a window");
247
+
248
+ lru.pin(7);
249
+ assert.deepEqual(
250
+ lru.evictionChoice(),
251
+ { index: null, protectionYielded: false, distance: -1 },
252
+ "a pinned piece is never a candidate, and says so without a distance"
253
+ );
254
+ });
@@ -132,3 +132,272 @@ test("pinned pieces are never evicted, and the pin count is reported", async ()
132
132
  await fs.rm(directory, { recursive: true, force: true });
133
133
  }
134
134
  });
135
+
136
+ test("the store says why it spills: what is asked of it, what it had to take, how soon it came back", async () => {
137
+ const capacity = 4;
138
+ const { store, directory } = await makeStore(capacity);
139
+ try {
140
+ // A reader declaring more than the store may hold. This is the shape the
141
+ // field session of 2026-09-02 is suspected of — 88 slots against an
142
+ // encoder running 120-380 s ahead of a viewer, half the reads missing —
143
+ // and nothing recorded it (roadmap item 9).
144
+ store.protectRange("video", 0, 9);
145
+ for (let index = 0; index < 10; index += 1) {
146
+ await put(store, index, pieceOf(index));
147
+ }
148
+
149
+ const asked = store.stats();
150
+ assert.equal(asked.demand.readers, 1);
151
+ assert.equal(asked.demand.unionPieces, 10);
152
+ assert.equal(asked.demand.capacity, capacity);
153
+ assert.ok(
154
+ asked.demand.unionPieces > asked.demand.capacity,
155
+ "a reader asking for more than the store holds is arithmetic, not a policy fault"
156
+ );
157
+ assert.ok(
158
+ asked.evictedProtected > 0,
159
+ "every eviction here had to take a piece the reader had declared"
160
+ );
161
+
162
+ // Read back the pieces that were spilled: each one comes home, and the
163
+ // store says how long it had been away.
164
+ for (let index = 0; index < 10; index += 1) {
165
+ const bytes = await get(store, index);
166
+ assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
167
+ }
168
+
169
+ const after = store.stats();
170
+ assert.ok(after.revivalAgeSamples > 0, "pieces came back and their age was recorded");
171
+ assert.equal(typeof after.revivalAgeMedianMs, "number");
172
+ assert.ok(
173
+ after.revivedWithinFiveSeconds > 0,
174
+ "a piece wanted again seconds after it left should not have left"
175
+ );
176
+
177
+ store.releaseProtection("video");
178
+ assert.equal(store.stats().demand.readers, 0, "a reader that ends stops being counted");
179
+ } finally {
180
+ store.destroy(() => undefined);
181
+ await fs.rm(directory, { recursive: true, force: true });
182
+ }
183
+ });
184
+
185
+ test("a piece nobody has declared does not push out one that is being read", async () => {
186
+ const capacity = 4;
187
+ const { store, directory } = await makeStore(capacity);
188
+ try {
189
+ // A reader declares the pieces it will need, and they are put in memory.
190
+ store.protectRange("video", 0, 3);
191
+ for (let index = 0; index < 4; index += 1) {
192
+ await put(store, index, pieceOf(index));
193
+ }
194
+ assert.equal(store.stats().resident, capacity, "the declared window fills the store");
195
+
196
+ // Now pieces arrive that the download fetched ahead of every reader. Before
197
+ // 2026-09-02 each of them claimed a slot and evicted one of the four above,
198
+ // which was then read back from disk moments later: 6565 spills and 7575
199
+ // revivals in 44 minutes, 53.6% of reads served from memory.
200
+ for (let index = 100; index < 110; index += 1) {
201
+ await put(store, index, pieceOf(index));
202
+ }
203
+
204
+ const stats = store.stats();
205
+ assert.equal(stats.admittedToDisk, 10, "every undeclared arrival went straight to disk");
206
+ assert.equal(stats.admittedOutsideWindow, 10);
207
+ assert.equal(stats.admittedInsideWindow, 4);
208
+ assert.equal(stats.spills, 0, "and nothing had to be pushed out to make room");
209
+
210
+ // The declared pieces are still in memory, and every piece reads back as
211
+ // itself from whichever tier holds it.
212
+ assert.equal(stats.resident, capacity);
213
+ for (const index of [0, 1, 2, 3, 100, 105, 109]) {
214
+ const bytes = await get(store, index);
215
+ assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
216
+ }
217
+ } finally {
218
+ store.destroy(() => undefined);
219
+ await fs.rm(directory, { recursive: true, force: true });
220
+ }
221
+ });
222
+
223
+ test("before any reader has declared anything, an arriving piece still goes to memory", async () => {
224
+ const { store, directory } = await makeStore(4);
225
+ try {
226
+ // No window declared: there is no basis for calling a piece unwanted, so
227
+ // the store behaves as it always did. This is the initial download, and the
228
+ // warm-up fetches of the header and the tail.
229
+ for (let index = 0; index < 8; index += 1) {
230
+ await put(store, index, pieceOf(index));
231
+ }
232
+ const stats = store.stats();
233
+ assert.equal(stats.admittedToDisk, 0, "nothing was refused memory on a guess");
234
+ assert.equal(stats.admittedOutsideWindow, 8);
235
+ assert.ok(stats.spills > 0, "the store filled and evicted, as it did before");
236
+ } finally {
237
+ store.destroy(() => undefined);
238
+ await fs.rm(directory, { recursive: true, force: true });
239
+ }
240
+ });
241
+
242
+ test("the store asks for what its readers declared, and for a whole window at least", async () => {
243
+ const { store, directory } = await makeStore(64);
244
+ try {
245
+ // With nobody reading there is no demand to speak of, so the store asks for
246
+ // what it is already allowed and the first revision after a read begins
247
+ // brings it down.
248
+ const idle = store.wantedBytes;
249
+ assert.equal(idle, store.stats().budgetBytes);
250
+
251
+ // Two readers of one file — picture and sound — overlapping by
252
+ // construction. The ask is their union, not their sum.
253
+ store.protectRange("video", 10, 29);
254
+ store.protectRange("audio", 25, 44);
255
+ assert.equal(store.wantedBytes, 35 * PIECE, "10..44 is thirty-five pieces, not forty");
256
+
257
+ // One reader with a window wider than the union of nothing else: the ask
258
+ // never falls below a single window, or that reader's read cannot complete
259
+ // at all — every resident piece ends up pinned and the read returns zero
260
+ // bytes.
261
+ store.releaseProtection("audio");
262
+ store.releaseProtection("video");
263
+ store.protectRange("video", 0, 49);
264
+ assert.equal(store.wantedBytes, 50 * PIECE);
265
+ } finally {
266
+ store.destroy(() => undefined);
267
+ await fs.rm(directory, { recursive: true, force: true });
268
+ }
269
+ });
270
+
271
+ test("a block is re-used instead of a new one being allocated for every piece", async () => {
272
+ const capacity = 4;
273
+ const { store, directory } = await makeStore(capacity);
274
+ try {
275
+ // Twenty pieces through a store that may hold four. Before 2026-09-02 that
276
+ // was twenty allocations of a piece each, every one of them released only
277
+ // when the collector got to it — 7575 of them in 44 minutes in the field,
278
+ // and 1.86 GB held while the store's own accounting said 352 MB.
279
+ for (let index = 0; index < 20; index += 1) {
280
+ await put(store, index, pieceOf(index));
281
+ }
282
+
283
+ const stats = store.stats();
284
+ assert.ok(
285
+ stats.blocksAllocated <= capacity,
286
+ `the pool never exceeds the allowance: ${stats.blocksAllocated} blocks for ${capacity} slots`
287
+ );
288
+ assert.equal(stats.committedBytes, stats.blocksAllocated * PIECE);
289
+ assert.equal(stats.returnedWhilePinned, 0);
290
+ assert.ok(stats.reuseGapMs !== null, "blocks were taken from the free list, not freshly made");
291
+
292
+ // And every piece still reads back as itself: a re-used block must not
293
+ // carry the last piece's bytes into the next one.
294
+ for (let index = 0; index < 20; index += 1) {
295
+ const bytes = await get(store, index);
296
+ assert.ok(bytes.equals(pieceOf(index)), `piece ${index} came back changed`);
297
+ }
298
+ } finally {
299
+ store.destroy(() => undefined);
300
+ await fs.rm(directory, { recursive: true, force: true });
301
+ }
302
+ });
303
+
304
+ test("a spare block is given up once it has sat longer than the store's own working rhythm", async () => {
305
+ const { store, directory } = await makeStore(8);
306
+ try {
307
+ for (let index = 0; index < 12; index += 1) {
308
+ await put(store, index, pieceOf(index));
309
+ }
310
+ // The allowance falls: pieces are written out and their blocks fall spare.
311
+ store.reviseGrowthCeiling(2 * PIECE);
312
+ const spare = store.stats().blocksFree;
313
+
314
+ // Nothing is given up while the blocks are younger than the longest wait
315
+ // this store has actually seen between a block falling free and being
316
+ // wanted again.
317
+ assert.equal(store.sweepFreeBlocks(Date.now()), 0, "a block in use moments ago is not spare");
318
+ assert.equal(store.stats().blocksFree, spare);
319
+
320
+ // An hour later they plainly are.
321
+ const released = store.sweepFreeBlocks(Date.now() + 3_600_000);
322
+ assert.equal(released, spare);
323
+ assert.equal(store.stats().blocksFree, 0);
324
+ assert.equal(store.stats().blocksReleased, released);
325
+ } finally {
326
+ store.destroy(() => undefined);
327
+ await fs.rm(directory, { recursive: true, force: true });
328
+ }
329
+ });
330
+
331
+ test("evicting a piece the disk already holds costs no second write", async () => {
332
+ const { store, directory } = await makeStore(2);
333
+ try {
334
+ // Three pieces through two slots: piece 0 is written out.
335
+ for (let index = 0; index < 3; index += 1) {
336
+ await put(store, index, pieceOf(index));
337
+ }
338
+ const written = store.stats().spills;
339
+ assert.ok(written > 0);
340
+ assert.equal(store.stats().spillsSkipped, 0, "the first write of a piece is a real one");
341
+
342
+ // Read it back — it returns to memory and the copy stays on disk. Evicting
343
+ // it again writes bytes that are already there, byte for byte, because only
344
+ // `put` removes the disk copy and no `put` has happened.
345
+ assert.ok((await get(store, 0)).equals(pieceOf(0)));
346
+ for (let index = 10; index < 13; index += 1) {
347
+ await put(store, index, pieceOf(index));
348
+ }
349
+ assert.ok(store.stats().spillsSkipped > 0, "the second write of the same bytes is skipped");
350
+
351
+ // And the piece still comes back correctly from the disk copy.
352
+ assert.ok((await get(store, 0)).equals(pieceOf(0)));
353
+ } finally {
354
+ store.destroy(() => undefined);
355
+ await fs.rm(directory, { recursive: true, force: true });
356
+ }
357
+ });
358
+
359
+ test("a store whose readers have gone asks for nothing, one that never had them keeps its opening", async () => {
360
+ const { store, directory } = await makeStore(16);
361
+ try {
362
+ // Never had a reader: this is the initial download and the warm-up fetches
363
+ // of the header and the tail, with a read on its way.
364
+ const opening = store.wantedBytes;
365
+ assert.equal(opening, store.stats().budgetBytes);
366
+
367
+ store.protectRange("video", 0, 9);
368
+ assert.equal(store.wantedBytes, 10 * PIECE);
369
+
370
+ // The read ends. Its torrent sits until the pool's idle timer removes it,
371
+ // and that timer needs a refcount of zero and can be a quarter of an hour
372
+ // away. Holding the pieces for a reader that has gone is memory taken from
373
+ // the machine for nothing.
374
+ store.releaseProtection("video");
375
+ assert.ok(store.wantedBytes < opening, "a store with no readers left asks for nothing");
376
+ } finally {
377
+ store.destroy(() => undefined);
378
+ await fs.rm(directory, { recursive: true, force: true });
379
+ }
380
+ });
381
+
382
+ test("the allowance is never cut below one reader's whole window", async () => {
383
+ const { store, directory } = await makeStore(16);
384
+ try {
385
+ store.protectRange("video", 0, 9);
386
+ // The machine says this store may have two pieces. Obeying that would leave
387
+ // it unable to finish the read it is serving: every resident piece pinned,
388
+ // zero bytes returned, and ffmpeg taking that for the end of the file —
389
+ // which killed every encoder on that file in the field on 2026-08-15.
390
+ const revised = store.reviseGrowthCeiling(2 * PIECE);
391
+ assert.equal(revised.ceilingBytes, 10 * PIECE, "one whole window is the floor");
392
+ assert.equal(revised.belowAWindow, true, "and the store says the share was smaller than that");
393
+
394
+ // With no reader there is no window to protect and the share is obeyed.
395
+ store.releaseProtection("video");
396
+ const obeyed = store.reviseGrowthCeiling(2 * PIECE);
397
+ assert.equal(obeyed.ceilingBytes, 2 * PIECE);
398
+ assert.equal(obeyed.belowAWindow, false);
399
+ } finally {
400
+ store.destroy(() => undefined);
401
+ await fs.rm(directory, { recursive: true, force: true });
402
+ }
403
+ });