@torrent-tv/proxy 2.9.73 → 2.9.74

package/services/torrent-pool.js

@@ -12,6 +12,7 @@ import path from "node:path";
 import { rmSync, statfsSync } from "node:fs";
 import WebTorrent from "webtorrent";
 import { logger } from "../utils/logger.js";
+import { SharedPieceStore } from "./piece-store/shared-piece-store.js";
 
 // WebTorrent's default download root (see webtorrent lib/torrent.js: TMP =
 // path.join(os.tmpdir(), 'webtorrent')). We use the default store, so all
@@ -308,6 +309,9 @@ export class TorrentPool {
   /** Global disk cap in bytes (0 = disabled). */
   #maxDiskBytes = 0;
 
+  /** Memory budget per torrent for resident pieces; undefined = store default. */
+  #memoryBytes;
+
   /** Periodic disk-cap enforcement timer. */
   #diskSweepTimer = null;
 
@@ -323,7 +327,9 @@ export class TorrentPool {
    *   default is computed from free disk (min(10 GB, half free)). Pass 0 to
    *   disable the cap.
    */
-  constructor({ maxDiskBytes } = {}) {
+  constructor({ maxDiskBytes, memoryBytes } = {}) {
+    this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
+
     // Sweep orphaned torrent data left by a previous hard kill (no graceful
     // shutdown ran, so destroyAll never cleaned the store). Safe here: no
     // torrents are loaded yet at construction. Best-effort, synchronous so it
@@ -558,7 +564,17 @@ export class TorrentPool {
         reject(error);
       };
       this.client.once("error", onError);
-      this.client.add(torrentId, (readyTorrent) => {
+      // Our own store, and WebTorrent's piece cache switched off in front of it
+      // (`storeCacheSlots: 0`). That cache is what made the thread split fail:
+      // it hands out the buffer it keeps and re-slices it later, so moving a
+      // piece across threads detached memory still in use. Ours owns what it
+      // hands out, holds pieces in shared memory the main thread can read
+      // directly, and spills to disk instead of losing them.
+      this.client.add(torrentId, {
+        store: SharedPieceStore,
+        storeCacheSlots: 0,
+        storeOpts: { memoryBytes: this.#memoryBytes }
+      }, (readyTorrent) => {
         this.client.off("error", onError);
         this.torrents.set(key, readyTorrent);
         this.#lastAccess.set(readyTorrent, Date.now());

package/services/torrent-worker/channel.js

@@ -183,20 +183,24 @@ export function createSendStream({ port, requestId }) {
         return;
       }
       inFlight += 1;
-      // Transfer the underlying memory rather than copying it — the whole point
-      // of the design, and the difference between 4.8 ms and 37 ms per 10 MB.
+      // Copy into memory this transport allocated, then transfer THAT.
       //
-      // But only memory this chunk owns OUTRIGHT may be transferred. Node hands
-      // out small buffers from a shared 8 KB pool: several unrelated buffers sit
-      // in one region, each viewing its own slice (verified: a 1 KB buffer
-      // reports an 8192-byte region at offset 8). Transferring that region
-      // detaches it from every other buffer living there — which is what broke
-      // reads in the field 2026-08-02: headers were produced in 325 ms and the
-      // body never arrived, because chunks from the network are small enough to
-      // be pooled while local disk reads, which is all the local test exercised,
-      // are not.
-      const ownsRegion = bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength;
-      const payload = ownsRegion ? bytes : new Uint8Array(bytes);
+      // Transferring the caller's buffer is faster and was what shipped, but it
+      // is only correct if the caller owns the memory outright — and no test at
+      // this boundary can establish that. 2.9.73 tried to decide it by
+      // inspection (`byteOffset === 0 && byteLength === buffer.byteLength`),
+      // which answers "does this view cover its region", not "did we allocate
+      // it". WebTorrent's piece cache returns a buffer covering its whole
+      // region and keeps using it, so the check passed and the transfer
+      // detached the cache: every later read failed with a detached
+      // ArrayBuffer, and because the error never reached the reader it looked
+      // like an empty file (`Stream ends prematurely at 0`).
+      //
+      // The copy costs 3.64 ms per 8 MB on the field host, against 37 ms for a
+      // structured clone. It disappears entirely for pieces read through the
+      // shared piece store, which the main thread reads by offset without any
+      // hand-over at all.
+      const payload = new Uint8Array(bytes);
       port.postMessage(
         { type: Event.CHUNK, id: requestId, bytes: payload },
         [payload.buffer]

package/services/torrent-worker/install-webrtc-shim.js

@@ -0,0 +1,32 @@
+/**
+ * @file Point `webrtc-polyfill` at the JavaScript WebRTC stack, in this thread only.
+ *
+ * Imported for its side effect, and imported FIRST by `worker.js` — ES module
+ * bodies run in import order, so registering the hook here happens before
+ * `torrent-pool.js` pulls in WebTorrent, which is what reaches
+ * `@thaunknown/simple-peer` and, through it, `webrtc-polyfill`.
+ *
+ * Scope is deliberately narrow. The hook lives in the worker's isolate, so the
+ * main thread keeps using node-datachannel directly for the browser's video
+ * channel — see `webrtc-shim.js` for why the two cannot share one process
+ * isolate at all.
+ */
+
+import { registerHooks } from "node:module";
+
+const SHIM_URL = new URL("./webrtc-shim.js", import.meta.url).href;
+
+registerHooks({
+  /**
+   * @param {string} specifier
+   * @param {object} context
+   * @param {(specifier: string, context: object) => { url: string }} nextResolve
+   * @returns {{ url: string, shortCircuit?: boolean }}
+   */
+  resolve(specifier, context, nextResolve) {
+    if (specifier === "webrtc-polyfill") {
+      return { url: SHIM_URL, shortCircuit: true };
+    }
+    return nextResolve(specifier, context);
+  }
+});

package/services/torrent-worker/webrtc-shim.js

@@ -0,0 +1,219 @@
+/**
+ * @file `webrtc-polyfill`'s interface, served by a pure-JavaScript WebRTC stack.
+ *
+ * **Why this exists.** node-datachannel is native, and its native side is not
+ * safe to use from two V8 isolates at once: the process aborts with
+ * `HandleScope: Entering the V8 API without proper locking in place`. Measured
+ * on win32/x64 and linux/arm64 alike — one isolate is fine (either the main
+ * thread or a worker), two at the same time is fatal, and preloading in both
+ * does not help. The upstream issue about workers (#129) was closed in 0.4.0
+ * but only covers use from a worker ALONE, which does work.
+ *
+ * We need it twice: `webrtc-manager.js` runs the video channel to the browser on
+ * the main thread, and the torrent client — which is now on its own thread —
+ * creates peer connections of its own to announce on `wss://` trackers. Before
+ * the thread split both lived in one isolate and nothing was wrong; afterwards
+ * any torrent carrying a wss tracker took the whole proxy down.
+ *
+ * So the native stack stays where it earns its keep — the main thread, carrying
+ * video — and the torrent's trackers get a JavaScript implementation, where the
+ * traffic is a handful of signalling messages. `services/torrent-worker/worker.js`
+ * installs a module resolution hook that points `webrtc-polyfill` here; nothing
+ * outside the worker thread is affected, and no dependency is patched (the addon
+ * installs with `--ignore-scripts`, so a postinstall patch would never run).
+ *
+ * **What the shim has to fix.** werift's peer connection matches the interface
+ * `simple-peer` expects, but its data channel differs in two ways that matter:
+ *
+ *  - it has no `binaryType`, and hands `onmessage` a `Buffer`. `simple-peer`
+ *    only recognises `ArrayBuffer`; anything else goes through `text2arr`,
+ *    which would corrupt every byte of torrent payload.
+ *  - it has no `onbufferedamountlow`. `simple-peer` builds its backpressure on
+ *    that event, so without it a send that hits the high-water mark never
+ *    resumes.
+ */
+
+import {
+  RTCPeerConnection as WeriftPeerConnection,
+  RTCIceCandidate
+} from "werift";
+
+/**
+ * A session description built the way browsers build it — from one object.
+ *
+ * werift's own class takes two positional arguments, `(sdp, type)`. Callers
+ * written against the browser pass `{ type, sdp }`, so with werift's class the
+ * type lands in the sdp slot and the description is rejected: field 2026-08-03,
+ * wss announces succeeded but every peer connection died with "Connection
+ * error: invalid sessionDescription".
+ *
+ * `RTCIceCandidate` needs no such treatment — werift already takes an object.
+ */
+class ShimSessionDescription {
+  /**
+   * @param {{ type?: string, sdp?: string } | string} init
+   * @param {string} [type] - Tolerated for callers using werift's own order.
+   */
+  constructor(init, type) {
+    if (typeof init === "string") {
+      this.sdp = init;
+      this.type = type;
+      return;
+    }
+    this.type = init?.type;
+    this.sdp = init?.sdp;
+  }
+
+  toJSON() {
+    return { type: this.type, sdp: this.sdp };
+  }
+}
+
+/**
+ * A werift data channel wearing the browser's interface.
+ *
+ * Only what `simple-peer` touches is implemented — promising more would be
+ * pretending, since anything else has no caller and would never be exercised.
+ */
+class ShimDataChannel {
+  #channel;
+  /** @type {"blob" | "arraybuffer"} */
+  binaryType = "blob";
+  /** @type {((event: { data: ArrayBuffer | Buffer }) => void) | null} */
+  onmessage = null;
+  /** @type {(() => void) | null} */
+  onopen = null;
+  /** @type {(() => void) | null} */
+  onclose = null;
+  /** @type {((event: { error?: Error }) => void) | null} */
+  onerror = null;
+  /** @type {(() => void) | null} */
+  onbufferedamountlow = null;
+
+  /**
+   * @param {object} channel - werift's `RTCDataChannel`.
+   */
+  constructor(channel) {
+    this.#channel = channel;
+
+    channel.onmessage = (event) => {
+      const data = event?.data ?? event;
+      this.onmessage?.({ data: this.#toWireFormat(data) });
+    };
+    channel.onopen = () => this.onopen?.();
+    channel.onclose = () => this.onclose?.();
+    channel.onerror = (event) => this.onerror?.(event ?? {});
+
+    // werift reports this as an observable rather than a handler property.
+    channel.bufferedAmountLow?.subscribe?.(() => this.onbufferedamountlow?.());
+  }
+
+  /**
+   * Match what a browser would deliver for the requested `binaryType`.
+   *
+   * `simple-peer` checks `instanceof ArrayBuffer` and sends everything else
+   * through a text decoder, so a `Buffer` handed over unchanged arrives
+   * mangled.
+   *
+   * @param {unknown} data
+   * @returns {ArrayBuffer | unknown}
+   */
+  #toWireFormat(data) {
+    if (this.binaryType !== "arraybuffer" || typeof data === "string") {
+      return data;
+    }
+    if (ArrayBuffer.isView(data)) {
+      return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+    }
+    return data;
+  }
+
+  get label() {
+    return this.#channel.label;
+  }
+
+  get readyState() {
+    return this.#channel.readyState;
+  }
+
+  get bufferedAmount() {
+    return this.#channel.bufferedAmount;
+  }
+
+  get bufferedAmountLowThreshold() {
+    return this.#channel.bufferedAmountLowThreshold;
+  }
+
+  set bufferedAmountLowThreshold(value) {
+    this.#channel.bufferedAmountLowThreshold = value;
+  }
+
+  /**
+   * @param {string | ArrayBuffer | ArrayBufferView} data
+   * @returns {void}
+   */
+  send(data) {
+    if (typeof data === "string") {
+      this.#channel.send(data);
+      return;
+    }
+    this.#channel.send(ArrayBuffer.isView(data) ? Buffer.from(data.buffer, data.byteOffset, data.byteLength) : Buffer.from(data));
+  }
+
+  close() {
+    this.#channel.close();
+  }
+}
+
+/**
+ * werift's peer connection, handing out data channels that carry the browser's
+ * interface. Everything else is inherited unchanged — the peer connection side
+ * already matches.
+ */
+class ShimPeerConnection extends WeriftPeerConnection {
+  /**
+   * @param {...unknown} args - Passed through to werift.
+   */
+  constructor(...args) {
+    super(...args);
+
+    // `ondatachannel` has to be redefined on the instance, not declared as an
+    // accessor on this class: werift assigns it as an own field in its own
+    // constructor, and an own property shadows a prototype accessor — so a
+    // subclass setter is simply never called, and the caller receives werift's
+    // bare channel instead of the wrapped one.
+    let handler = null;
+    const deliver = (event) => {
+      handler?.({ ...event, channel: new ShimDataChannel(event?.channel ?? event) });
+    };
+    Object.defineProperty(this, "ondatachannel", {
+      configurable: true,
+      // werift reads this property to dispatch, so it must hand back the
+      // wrapper rather than what the caller assigned.
+      get: () => (handler ? deliver : null),
+      set: (value) => {
+        handler = value;
+      }
+    });
+  }
+
+  /**
+   * @param {string} label
+   * @param {object} [options]
+   * @returns {ShimDataChannel}
+   */
+  createDataChannel(label, options) {
+    return new ShimDataChannel(super.createDataChannel(label, options));
+  }
+}
+
+export {
+  ShimPeerConnection as RTCPeerConnection,
+  ShimSessionDescription as RTCSessionDescription,
+  RTCIceCandidate
+};
+export default {
+  RTCPeerConnection: ShimPeerConnection,
+  RTCSessionDescription: ShimSessionDescription,
+  RTCIceCandidate
+};

package/services/torrent-worker/worker.js

@@ -17,11 +17,22 @@
  * threads.
  */
 
+// MUST stay first: it redirects `webrtc-polyfill` to a JavaScript WebRTC stack
+// before WebTorrent can reach the native one. Two isolates using
+// node-datachannel at once abort the process, and the torrent's wss trackers
+// create peer connections of their own.
+import "./install-webrtc-shim.js";
 import { parentPort, workerData } from "node:worker_threads";
-import { TorrentPool } from "../torrent-pool.js";
 import { createSendStream } from "./channel.js";
 import { Command, Event, STREAM_CHUNK_BYTES } from "./protocol.js";
 
+// Imported dynamically, and that is load-bearing: static imports are RESOLVED
+// during linking, before any module body runs, so a statically imported pool
+// would drag in WebTorrent — and with it the real `webrtc-polyfill` — before
+// the hook above had a chance to register. Verified the hard way: with a static
+// import the process still aborted, and the stack named the genuine polyfill.
+const { TorrentPool } = await import("../torrent-pool.js");
+
 const pool = new TorrentPool({ maxDiskBytes: workerData?.maxDiskBytes });
 
 /** Torrents by sourceKey — the main thread names them, this thread owns them. */

package/test/piece-lru.test.js

@@ -0,0 +1,109 @@
+/**
+ * @file Recency and pinning for resident pieces.
+ *
+ * The pinning cases matter more than the ordering ones: proxy 2.9.71 removed a
+ * torrent's data while a reader was mid-read, and every later read hung. Here
+ * that is meant to be impossible by construction, so it is worth stating in
+ * tests rather than trusting to eviction order.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { PieceLru } from "../services/piece-store/piece-lru.js";
+
+test("evicts the least recently used piece", () => {
+  const lru = new PieceLru(3);
+  lru.touch(1);
+  lru.touch(2);
+  lru.touch(3);
+
+  assert.equal(lru.evictionCandidate(), 1);
+});
+
+test("using a piece again moves it out of the firing line", () => {
+  const lru = new PieceLru(3);
+  lru.touch(1);
+  lru.touch(2);
+  lru.touch(3);
+  lru.touch(1);
+
+  assert.equal(lru.evictionCandidate(), 2);
+});
+
+test("a pinned piece is never offered for eviction", () => {
+  const lru = new PieceLru(3);
+  lru.touch(1);
+  lru.touch(2);
+  lru.touch(3);
+  lru.pin(1);
+
+  assert.equal(lru.evictionCandidate(), 2, "the pinned piece was offered up");
+});
+
+test("pins nest, so one reader leaving does not expose the piece", () => {
+  const lru = new PieceLru(2);
+  lru.touch(1);
+  lru.touch(2);
+
+  // Two sessions reading the same piece — the union-window case.
+  lru.pin(1);
+  lru.pin(1);
+  lru.unpin(1);
+
+  assert.ok(lru.isPinned(1), "the piece stopped being held while a reader still had it");
+  assert.equal(lru.evictionCandidate(), 2);
+
+  lru.unpin(1);
+  assert.equal(lru.isPinned(1), false);
+});
+
+test("reports no candidate rather than evicting a piece in use", () => {
+  const lru = new PieceLru(2);
+  lru.touch(1);
+  lru.touch(2);
+  lru.pin(1);
+  lru.pin(2);
+
+  assert.equal(
+    lru.evictionCandidate(),
+    null,
+    "with every piece held, the caller must wait — not have memory taken from under it"
+  );
+});
+
+test("unpinning something that was never pinned is harmless", () => {
+  const lru = new PieceLru(2);
+  lru.touch(1);
+  lru.unpin(1);
+  lru.unpin(1);
+
+  assert.equal(lru.isPinned(1), false);
+  assert.equal(lru.evictionCandidate(), 1);
+});
+
+test("removal takes a piece out of the ordering", () => {
+  const lru = new PieceLru(3);
+  lru.touch(1);
+  lru.touch(2);
+  lru.remove(1);
+
+  assert.equal(lru.has(1), false);
+  assert.equal(lru.evictionCandidate(), 2);
+  assert.equal(lru.size, 1);
+});
+
+test("fullness follows capacity", () => {
+  const lru = new PieceLru(2);
+  assert.equal(lru.isFull(), false);
+  lru.touch(1);
+  lru.touch(2);
+  assert.equal(lru.isFull(), true);
+  lru.remove(1);
+  assert.equal(lru.isFull(), false);
+});
+
+test("a nonsensical capacity is refused at construction", () => {
+  assert.throws(() => new PieceLru(0), /positive integer/);
+  assert.throws(() => new PieceLru(-1), /positive integer/);
+  assert.throws(() => new PieceLru(1.5), /positive integer/);
+});