@torrent-tv/proxy 2.82.0 → 2.83.1

package/services/piece-store/piece-disk-store.js

@@ -31,6 +31,7 @@
  */
 
 import fs from "node:fs/promises";
+import { readdirSync, statSync } from "node:fs";
 import path from "node:path";
 
 /**
@@ -72,10 +73,16 @@ export class PieceDiskStore {
 
   #evictions = 0;
 
+  /** How many were thrown away for being behind every reader. */
+  #behind = 0;
+
   #bytes = 0;
 
   #now;
 
+  /** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
+  #readHeads;
+
   /**
    * @param {object} params
    * @param {string} params.directory - Where this torrent's pieces live.
@@ -88,11 +95,65 @@ export class PieceDiskStore {
    *   means nobody has said yet, and nothing is evicted until somebody does.
    * @param {() => number} [params.now]
    */
-  constructor({ directory, name, chunkLength, allowanceBytes = null, now = Date.now }) {
+  constructor({ directory, name, chunkLength, allowanceBytes = null, now = Date.now, readHeads = () => [] }) {
     this.#directory = path.join(directory, name);
     this.#chunkLength = chunkLength;
     this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
     this.#now = now;
+    this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
+    this.#adoptWhatIsAlreadyHere();
+  }
+
+  /**
+   * Take up the pieces a previous life of this torrent left in this directory.
+   *
+   * The directory is the torrent's own, so what is in it belongs to it — and
+   * without this, nothing ever reads those files again: a torrent that is torn
+   * down and added back gets a store whose index starts empty, answers "not on
+   * disk" for every piece it in fact has, and downloads the film a second time
+   * while the first copy sits beside it. That is what a torrent destroyed by an
+   * error left behind until 2026-09-11, and what the pool's own restart leaves
+   * behind every time.
+   *
+   * Read once, synchronously, because `has()` is answered synchronously and the
+   * torrent asks it immediately — a piece reported missing while a scan is
+   * still running is a piece fetched again. One directory listing per torrent.
+   *
+   * Correctness is not taken on trust: the torrent hashes every piece it means
+   * to use, so a file here that does not match is refused by the layer above
+   * and downloaded again.
+   *
+   * @returns {void}
+   */
+  #adoptWhatIsAlreadyHere() {
+    let entries = [];
+    try {
+      entries = readdirSync(this.#directory, { withFileTypes: true });
+    } catch {
+      // No directory yet: this torrent is new here, which is the ordinary case.
+      return;
+    }
+    const born = this.#now();
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith(".piece")) {
+        continue;
+      }
+      const index = Number.parseInt(entry.name.slice(0, -".piece".length), 10);
+      if (!Number.isInteger(index) || index < 0) {
+        continue;
+      }
+      try {
+        const { size } = statSync(path.join(this.#directory, entry.name));
+        if (size <= 0) {
+          continue;
+        }
+        this.#stored.set(index, size);
+        this.#touched.set(index, born);
+        this.#bytes += size;
+      } catch {
+        // Gone between the listing and the reading: not ours to worry about.
+      }
+    }
   }
 
   /** Where this store's pieces live, for logging and cleanup. */
@@ -100,6 +161,15 @@ export class PieceDiskStore {
     return this.#directory;
   }
 
+  /**
+   * Every piece this tier holds, as numbers.
+   *
+   * @returns {number[]}
+   */
+  indexes() {
+    return [...this.#stored.keys()];
+  }
+
   /** How many pieces are on disk. */
   get size() {
     return this.#stored.size;
@@ -139,6 +209,45 @@ export class PieceDiskStore {
     return this.#stored.has(index);
   }
 
+  /**
+   * Throw away what no reader will ask for again, without waiting for the disk
+   * to be short.
+   *
+   * THE SECOND RULE, and it answers a different question from the ceiling.
+   * Material nobody needs should not sit on somebody's disk merely because
+   * there is room for it — the ceiling here is a share of what is free, and on
+   * a roomy host that is tens of gigabytes against a measured growth of 14 400
+   * MB in one viewing, so the ceiling alone never binds and nothing is ever
+   * removed until the torrent itself goes.
+   *
+   * A piece BEHIND every read head has been read and will not be read again
+   * unless somebody seeks back — and a seek back re-downloads it, which is the
+   * bargain this tier already makes when it drops a piece for room. Nothing is
+   * thrown away while any reader might still reach it.
+   *
+   * With no reader at all nothing is removed: a store between reads is not a
+   * store nobody wants, and the torrent going idle is what empties it whole.
+   *
+   * @param {number[]} readHeads - The first piece each live reader still wants.
+   * @returns {number} How many pieces were thrown away.
+   */
+  forgetBehind(readHeads) {
+    const heads = (readHeads ?? []).filter((at) => Number.isInteger(at));
+    if (heads.length === 0) {
+      return 0;
+    }
+    const earliest = Math.min(...heads);
+    let removed = 0;
+    for (const index of [...this.#stored.keys()]) {
+      if (index < earliest && !this.#reading.has(index)) {
+        this.forget(index);
+        this.#behind += 1;
+        removed += 1;
+      }
+    }
+    return removed;
+  }
+
   /**
    * Write a piece out, making room for it first.
    *
@@ -167,11 +276,18 @@ export class PieceDiskStore {
   /**
    * Read a piece back into a buffer the caller already owns.
    *
+   * PART of a piece, when the caller asks for one. A peer asks for 16 KB at a
+   * time and a piece here is megabytes, so reading the whole of it to answer
+   * one request is the difference between 16 KB and 4 MB off the disk — field
+   * 2026-09-11: 63 416 reads of which 21.6 % came from memory, 49 696 pieces
+   * revived whole, to serve an upload capped at 512 KB/s.
+   *
    * @param {number} index
    * @param {Uint8Array} target - Destination; its length is what gets read.
+   * @param {number} [at] - Offset within the piece to start at.
    * @returns {Promise<number>} Bytes read.
    */
-  async read(index, target) {
+  async read(index, target, at = 0) {
     if (!this.#stored.has(index)) {
       throw new Error(`Piece ${index} is not on disk.`);
     }
@@ -183,7 +299,7 @@ export class PieceDiskStore {
     let handle = null;
     try {
       handle = await fs.open(this.#pathOf(index), "r");
-      const { bytesRead } = await handle.read(target, 0, target.length, 0);
+      const { bytesRead } = await handle.read(target, 0, target.length, Math.max(0, at));
       this.#touched.set(index, this.#now());
       return bytesRead;
     } finally {
@@ -234,14 +350,15 @@ export class PieceDiskStore {
   /**
    * What it holds, what it may hold, and what it has had to throw away.
    *
-   * @returns {{ pieces: number, bytes: number, allowanceBytes: number | null, evictions: number }}
+   * @returns {{ pieces: number, bytes: number, allowanceBytes: number | null, evictions: number, behind: number }}
    */
   stats() {
     return {
       pieces: this.#stored.size,
       bytes: this.#bytes,
       allowanceBytes: this.#allowanceBytes,
-      evictions: this.#evictions
+      evictions: this.#evictions,
+      behind: this.#behind
     };
   }
 
@@ -349,14 +466,32 @@ export class PieceDiskStore {
    * @returns {number | null}
    */
   #leastRecentlyUsed(except) {
+    // WHERE THE READERS STAND DECIDES, and last use only settles ties.
+    //
+    // What lies behind every read head has been read and will not be read again
+    // unless somebody seeks back, so it goes before anything ahead of them,
+    // furthest behind first. It is the order the segments are given one layer
+    // up, and the order the priority map states, read from the other end.
+    // Without the heads there is nothing to order by and last use is all that
+    // is left — which is what this was, and what said nothing about what
+    // anybody is about to read.
+    const heads = this.#readHeads().filter((at) => Number.isInteger(at));
+    const earliest = heads.length > 0 ? Math.min(...heads) : null;
     let victim = null;
-    let oldest = Number.POSITIVE_INFINITY;
+    let worst = null;
     for (const [index, at] of this.#touched) {
       if (index === except || this.#reading.has(index)) {
         continue;
       }
-      if (at < oldest) {
-        oldest = at;
+      const behind = earliest !== null && index < earliest;
+      const score = { behind, distance: behind ? earliest - index : 0, at };
+      if (
+        worst === null
+        || (score.behind && !worst.behind)
+        || (score.behind === worst.behind && score.distance > worst.distance)
+        || (score.behind === worst.behind && score.distance === worst.distance && score.at < worst.at)
+      ) {
+        worst = score;
         victim = index;
       }
     }

package/services/piece-store/piece-lru.js

@@ -258,6 +258,23 @@ export class PieceLru {
     return want;
   }
 
+  /**
+   * Where the live readers stand, as the first piece each of them still wants.
+   *
+   * A declared range begins at the piece its reader is about to read, so the
+   * earliest of those beginnings is the point behind which nothing will be
+   * asked for again unless somebody seeks back. That is what makes a spilled
+   * piece disposable without waiting for the disk to be short: a piece behind
+   * every reader has been read and will not be read again.
+   *
+   * @returns {number[]} One number per reader, unsorted. Empty when nobody has
+   *   declared anything, which is a stronger statement than any position — no
+   *   piece of this file is spoken for at all.
+   */
+  readHeads() {
+    return [...this.#protected.values()].map((range) => range.from);
+  }
+
   /**
    * How many pieces the live readers between them are asking to keep, against
    * how many this store may hold.