@torrent-tv/proxy 2.87.19 → 2.87.21

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 2.87.21
+
+- **Fix**: Give waiting source reads ordered memory reservations ahead of new piece arrivals, share concurrent revival work for one piece, and reserve memory before allocating assembled-file bytes.
+
+## 2.87.20
+
+- **Fix**: Expose completed files through the worker pool adapter so source HTTP reads use the assembled file; open it before sending headers and return to torrent reads if it has already been removed.
+
 ## 2.87.19
 
 - **Fix**: Return shared source bytes after whole-file assembly and retain the buffer held by concurrent readers instead of replacing it; fail malformed fragments explicitly rather than skipping source bytes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.87.19",
+  "version": "2.87.21",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/routes/stream/get.js

@@ -6,7 +6,7 @@
  * and HTTP 200 for full-file requests.
  */
 
-import { createReadStream } from "node:fs";
+import { open } from "node:fs/promises";
 import { parseRange } from "../../utils/parse-range.js";
 import { logger } from "../../utils/logger.js";
 
@@ -107,9 +107,10 @@ function wholeFileFor(torrentPool, sourceKey, fileIndex) {
  * @param {import("fastify").FastifyRequest} req
  * @param {import("fastify").FastifyReply} reply
  * @param {{ path: string, length: number, name: string }} file
+ * @param {import("node:fs/promises").FileHandle} handle - Already open; closed for HEAD.
  * @returns {Promise<void> | void}
  */
-function serveWholeFile(req, reply, file) {
+function serveWholeFile(req, reply, file, handle) {
   const disposition = `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`;
   if (req.method === "HEAD") {
     reply.hijack();
@@ -128,13 +129,13 @@ function serveWholeFile(req, reply, file) {
   const range = parseRange(req.headers.range, file.length);
   if (!range) {
     reply.header("Content-Length", String(file.length));
-    return reply.send(createReadStream(file.path));
+    return reply.send(handle.createReadStream());
   }
   const { start, end } = range;
   reply.code(206);
   reply.header("Content-Length", String(end - start + 1));
   reply.header("Content-Range", `bytes ${start}-${end}/${file.length}`);
-  return reply.send(createReadStream(file.path, { start, end }));
+  return reply.send(handle.createReadStream({ start, end }));
 }
 
 export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null }) {
@@ -157,7 +158,25 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
   // nothing the torrent would have told us.
   const whole = wholeFileFor(torrentPool, sourceKey, fileIndex);
   if (whole) {
-    return serveWholeFile(req, reply, whole);
+    let handle;
+    try {
+      // Open before committing headers: the disk owner may have removed an
+      // announced file. An open descriptor also preserves a live POSIX read.
+      handle = await open(whole.path, "r");
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+      const key = `${sourceKey.slice("torrent:".length).toLowerCase()}/${fileIndex}`;
+      if (torrentPool.wholeFiles.get(key) === whole) torrentPool.wholeFiles.delete(key);
+    }
+    if (handle) {
+      if (req.method === "HEAD") await handle.close();
+      try {
+        return serveWholeFile(req, reply, whole, handle);
+      } catch (error) {
+        await handle.close().catch(() => undefined);
+        throw error;
+      }
+    }
   }
 
   let torrent;

package/services/storage/piece-store/shared-piece-store.js

@@ -405,6 +405,12 @@ export class SharedPieceStore {
    */
   #readElsewhere = null;
 
+  /** Reads waiting for a reservation, in arrival order. */
+  #readerClaims = [];
+
+  /** One revival per piece, shared by concurrent readers. */
+  #revivals = new Map();
+
   /**
    * Whether a piece can be had elsewhere, asked without reading it — so a
    * spilled copy that has become a duplicate can be dropped.
@@ -858,7 +864,7 @@ export class SharedPieceStore {
     }
   }
 
-  async #claimSlot() {
+  async #claimSlot(forReader = false) {
     // How long THIS claim has been trying, kept here and not on the store.
     // It was a field, `#pinnedWaitStartedAt`, shared by the two waits inside
     // `#claimSlotOnce` — the one for the disk and the one for a piece that may
@@ -870,23 +876,36 @@ export class SharedPieceStore {
     // claimants a shared field is wrong anyway, since one caller giving up
     // would reset the wait of every other.
     const waitingSince = Date.now();
-    for (;;) {
-      if (this.#closed) {
-        throw new Error("Piece store is closed.");
+    const claimant = forReader ? {} : null;
+    if (claimant) this.#readerClaims.push(claimant);
+    try {
+      for (;;) {
+        if (this.#closed) {
+          throw new Error("Piece store is closed.");
+        }
+        // New arrivals must not take every released block ahead of an encoder
+        // already waiting to read. Readers share one ordered reservation queue.
+        const hasTurn = this.#readerClaims.length === 0 || this.#readerClaims[0] === claimant;
+        const isReserved = hasTurn && await this.#claimSlotOnce(waitingSince);
+        if (isReserved) {
+          let released = false;
+          return () => {
+            if (released) {
+              return;
+            }
+            released = true;
+            this.#outstandingPieces -= 1;
+            this.#noteProgress();
+          };
+        }
+        await this.#waitForSlot();
       }
-      const isReserved = await this.#claimSlotOnce(waitingSince);
-      if (isReserved) {
-        let released = false;
-        return () => {
-          if (released) {
-            return;
-          }
-          released = true;
-          this.#outstandingPieces -= 1;
-          this.#noteProgress();
-        };
+    } finally {
+      if (claimant) {
+        const at = this.#readerClaims.indexOf(claimant);
+        if (at >= 0) this.#readerClaims.splice(at, 1);
+        this.#wake();
       }
-      await this.#waitForSlot();
     }
   }
 
@@ -1395,15 +1414,14 @@ export class SharedPieceStore {
       // Not spilled, but the file it belongs to may be here whole — which is
       // the ordinary case once a film has been assembled and its spilled copy
       // dropped as the duplicate it had become.
-      const whole = await this.#fromWholeFiles(index);
-      if (!whole) {
-        return null;
-      }
-      this.#counters.fromWholeFile += 1;
+      if (this.#isElsewhere && !this.isInWholeFiles(index)) return null;
       // Fragment readers require shared memory, including when the bytes now
-      // live in an assembled file. Wait for the same reservation as disk reads.
-      const release = await this.#claimSlot();
+      // live in an assembled file. Reserve before allocating ordinary bytes.
+      const release = await this.#claimSlot(true);
       try {
+        const whole = await this.#fromWholeFiles(index);
+        if (!whole) return null;
+        this.#counters.fromWholeFile += 1;
         return this.#registerPiece(index, this.#copyIntoNewBuffer(index, whole));
       } finally {
         release();
@@ -1415,7 +1433,7 @@ export class SharedPieceStore {
     const releaseDisk = this.#disk.hold?.(index) ?? (() => {});
     let release = null;
     try {
-      release = await this.#claimSlot();
+      release = await this.#claimSlot(true);
       // Another caller may have brought it back while this one waited for a
       // slot. Registering a second buffer for the same piece would leave
       // whoever holds the first reading memory nothing evicts.
@@ -1860,7 +1878,12 @@ export class SharedPieceStore {
       return this.locate(index);
     }
 
-    const revived = await this.#revive(index);
+    let revival = this.#revivals.get(index);
+    if (!revival) {
+      revival = this.#revive(index).finally(() => this.#revivals.delete(index));
+      this.#revivals.set(index, revival);
+    }
+    const revived = await revival;
     if (revived === null) {
       return null;
     }

package/services/torrent/worker/pool-adapter.js

@@ -36,9 +36,10 @@ export class WorkerTorrentPool {
 
   /**
    * @param {{ memoryBytes?: number, stateDir?: string }} [options]
+   * @param {object | null} [client] - The worker interface; tests supply a fake.
    */
-  constructor(options = {}) {
-    this.#client = new TorrentWorkerClient(options);
+  constructor(options = {}, client = null) {
+    this.#client = client ?? new TorrentWorkerClient(options);
   }
 
   /**
@@ -95,6 +96,11 @@ export class WorkerTorrentPool {
     return this.#client.wholeFileBytes ?? 0;
   }
 
+  /** Completed files announced by the worker, shared with the HTTP reader. */
+  get wholeFiles() {
+    return this.#client.wholeFiles;
+  }
+
   allowWholeFileBytes(bytes) {
     return this.#client.allowWholeFileBytes(bytes);
   }

package/test/piece-store-reservations.test.js

@@ -34,6 +34,10 @@ function makeDisk({ failWrite = false, failRead = false, holdWrites = false } =
         resume();
       }
     },
+    finishWrites() {
+      holdWrites = false;
+      this.releaseWrites();
+    },
     get heldCount() {
       return held.length;
     },
@@ -150,15 +154,20 @@ test("a revival holds disk bytes across the asynchronous memory reservation", as
 
 test("an assembled-file revival returns shared bytes and preserves a concurrent pinned reader", async () => {
   const disk = makeDisk();
+  let reads = 0;
   const store = new SharedPieceStore(CHUNK, {
     length: CHUNK * 16, memoryBytes: CHUNK * 4, disk,
-    readPieceElsewhere: async ({ index }) => piece(index), name: "whole-file-read"
+    readPieceElsewhere: async ({ index }) => {
+      reads += 1;
+      return piece(index);
+    }, name: "whole-file-read"
   });
   try {
     store.pin(0);
     const [first, second] = await Promise.all([store.reside(0), store.reside(0)]);
     assert.ok(first.buffer instanceof SharedArrayBuffer);
     assert.equal(first.buffer, second.buffer);
+    assert.equal(reads, 1, "concurrent readers repeated the whole-file allocation and read");
     await put(store, 1);
     await put(store, 2);
     assert.deepEqual(Buffer.from(first.buffer, first.offset, first.length), piece(0));
@@ -170,6 +179,35 @@ test("an assembled-file revival returns shared bytes and preserves a concurrent
   }
 });
 
+test("a waiting source reader receives a released slot before queued arrivals", async () => {
+  const disk = makeDisk({ holdWrites: true });
+  const { store } = makeStore({ pieces: 2, disk });
+  let arrived = false;
+  try {
+    await put(store, 0);
+    await put(store, 1);
+    store.pin(1);
+    disk.stored.set(4, piece(4));
+    const spilling = put(store, 2);
+    await until(() => disk.heldCount > 0, "an arriving piece occupies the spill slot");
+    const arrival = put(store, 3).then(() => { arrived = true; });
+    store.pin(4);
+    const reading = store.reside(4);
+    await until(() => store.stats().waitedForDisk >= 2, "both claims wait behind the spill");
+    disk.finishWrites();
+    const located = await reading;
+    assert.deepEqual(Buffer.from(located.buffer, located.offset, located.length), piece(4));
+    assert.equal(arrived, false, "an arriving piece overtook an already waiting reader");
+    store.unpin(1);
+    store.unpin(4);
+    await Promise.all([spilling, arrival]);
+    assert.equal(store.stats().outstanding, 0);
+  } finally {
+    disk.finishWrites();
+    await new Promise((resolve) => store.destroy(resolve));
+  }
+});
+
 /**
  * Wait until a condition holds, rather than for a chosen interval — a test that
  * sleeps samples, it does not check (roadmap item 54).

package/test/stream-route.test.js

@@ -14,6 +14,7 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { handleStreamGet } from "../routes/stream/get.js";
+import { WorkerTorrentPool } from "../services/torrent/worker/pool-adapter.js";
 
 /**
  * Minimal stand-ins for the parts of Fastify and the pool this route touches.
@@ -151,13 +152,18 @@ test("a file downloaded whole is served from disk without touching the torrent",
   try {
     const { req, reply, sent, deps } = harness({ method: "GET", range: "bytes=2-6" });
     let asked = false;
-    deps.torrentPool.getTorrent = async () => {
-      asked = true;
-      throw new Error("the torrent was asked for, which is what this avoids");
-    };
-    deps.torrentPool.wholeFiles = new Map([
+    const wholeFiles = new Map([
       ["abc/0", { path: where, length: bytes.length, name: "film.mkv" }]
     ]);
+    // Supply the worker boundary explicitly: no Worker or torrent is started.
+    deps.torrentPool = new WorkerTorrentPool({}, {
+      wholeFiles,
+      async getTorrent() {
+        asked = true;
+        throw new Error("the worker must not be asked for completed bytes");
+      }
+    });
+    assert.equal(deps.torrentPool.wholeFiles, wholeFiles);
     // The source key IS the identity — `torrent:<infohash>` — and it is all the
     // route needs to find the file.
     deps.sourceRegistry = { get: () => ({ sourceType: "torrent", source: "magnet:?xt=urn:btih:abc" }) };
@@ -169,11 +175,32 @@ test("a file downloaded whole is served from disk without touching the torrent",
     assert.equal(sent.code, 206);
     assert.equal(sent.headers["content-range"], `bytes 2-6/${bytes.length}`);
     assert.equal(sent.headers["content-length"], "5");
+    const chunks = [];
+    for await (const chunk of sent.body) chunks.push(chunk);
+    assert.deepEqual(Buffer.concat(chunks), bytes.subarray(2, 7));
   } finally {
     await fs.rm(root, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
+test("an announced file removed before opening falls back to the torrent before headers", async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "stream-removed-"));
+  try {
+    const { req, reply, sent, opened, deps } = harness({ method: "GET", range: "bytes=2-6" });
+    deps.torrentPool.wholeFiles = new Map([
+      ["abc/0", { path: path.join(root, "missing"), length: 10, name: "gone.mkv" }]
+    ]);
+    deps.sourceRegistry = { get: () => ({ sourceType: "magnet", source: "magnet:?xt=urn:btih:abc" }) };
+    req.query = { sourceKey: "torrent:abc", fileIndex: "0" };
+    await handleStreamGet(req, reply, deps);
+    assert.deepEqual(opened, ["2-6"]);
+    assert.equal(deps.torrentPool.wholeFiles.size, 0);
+    assert.equal(sent.headers["content-range"], "bytes 2-6/5869669065");
+  } finally {
+    await fs.rm(root, { recursive: true, force: true });
+  }
+});
+
 test("a file this proxy does not have whole still goes to the torrent", async () => {
   const { req, reply, state, deps } = harness({ method: "GET", range: "bytes=0-99" });
   deps.torrentPool.wholeFiles = new Map([["other/7", { path: "/nowhere", length: 1, name: "x" }]]);