@torrent-tv/proxy 2.88.6 → 2.89.0

package/services/torrent/worker/client.js

@@ -375,7 +375,10 @@ export class TorrentWorkerClient {
    */
   async readHeld({ sourceKey, fileIndex, start, end }) {
     const answer = await this.#caller.call(Command.READ_HELD, { sourceKey, fileIndex, start, end });
-    return answer?.bytes ? Buffer.from(answer.bytes) : null;
+    const bytes = answer?.bytes;
+    // The memory was handed over by the worker, so it is this thread's alone:
+    // viewed, not copied a second time.
+    return bytes ? Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) : null;
   }
 
   async warmResumePosition({ sourceKey, fileIndex, positionSeconds, durationSeconds }) {
@@ -601,7 +604,7 @@ export class TorrentWorkerClient {
    * not reachable from here.
    *
    * @param {{ sourceKey: string, sourceType: "magnet" | "torrent", source: string }} params
-   * @returns {Promise<{ infoHash: string, name: string, sourceKey: string, files: object[] }>}
+   * @returns {Promise<{ infoHash: string, name: string, pieceLength: number, sourceKey: string, files: object[] }>}
    */
   async getTorrent({ sourceKey, sourceType, source }) {
     const info = await this.addSource({ sourceKey, sourceType, source });
@@ -609,6 +612,7 @@ export class TorrentWorkerClient {
     return {
       infoHash: info.infoHash,
       name: info.name,
+      pieceLength: Number(info.pieceLength) || 0,
       // Carried so helpers that receive only the torrent can still name it to
       // the worker.
       sourceKey,

package/services/torrent/worker/held-bytes.js

@@ -88,13 +88,42 @@ function rangeWithin(firstPiece, lastPiece, pieceLength, offset, length) {
 }
 
 /**
- * How long a read of bytes the torrent already holds may take before it is
- * given up.
+ * Whether every piece under an inclusive range of a file has arrived.
  *
- * Not a measurement and nothing is derived from it: such a read either answers
- * or it does not, and this is the point past which it is presumed lost — so
- * that one stream which never ends cannot hold a file's walk, and with it the
- * browser's own request for its subtitles, for the rest of the session.
+ * @param {object} torrent
+ * @param {number} fileIndex
+ * @param {number} start
+ * @param {number} end - Inclusive.
+ * @returns {boolean}
+ */
+export function isRangeHeld(torrent, fileIndex, start, end) {
+  const file = torrent?.files?.[fileIndex];
+  const pieceLength = Number(torrent?.pieceLength);
+  if (!file || !Number.isFinite(pieceLength) || pieceLength <= 0 || !torrent?.bitfield) {
+    return false;
+  }
+  const offset = Number(file.offset) || 0;
+  const first = Math.floor((offset + start) / pieceLength);
+  const last = Math.floor((offset + end) / pieceLength);
+  for (let index = first; index <= last; index += 1) {
+    if (!torrent.bitfield.get(index)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * How long a read of bytes the torrent HOLDS may take before it is taken to be
+ * a fault.
+ *
+ * Not a measurement and nothing is derived from it: the bytes are checked to be
+ * held before the read starts, so a read that does not finish is a store that
+ * failed to serve what it has, and the line it writes says so. Until 2026-10-01
+ * this read was started on bytes that had NOT arrived, the stream waited for
+ * them, and this was the bound that gave it up — 36 such reads gave up at once
+ * at a torrent's open, and one of them was the Cues table of the file being
+ * watched.
  */
 const READ_ABANDON_MS = 30_000;
 
@@ -102,6 +131,14 @@ const READ_ABANDON_MS = 30_000;
  * Read a byte range of a file straight from the store, without asking the swarm
  * for anything.
  *
+ * Answered at once with null where any piece under the range has not arrived:
+ * "not here" is the whole answer, and nothing is waited for or requested.
+ *
+ * The bytes come back in a buffer that owns its whole memory — not a slice of
+ * Node's shared pool and not a view of the store's own blocks — so the reply can
+ * hand that memory to the other thread instead of copying it, and nothing the
+ * store or another read still uses goes with it.
+ *
  * @param {object} torrent
  * @param {number} fileIndex
  * @param {number} start
@@ -115,6 +152,9 @@ export function readHeldBytes(torrent, fileIndex, start, end, logger = null) {
     return Promise.resolve(null);
   }
   const last = Math.min(end, Number(file.length) - 1);
+  if (!isRangeHeld(torrent, fileIndex, start, last)) {
+    return Promise.resolve(null);
+  }
   return new Promise((resolve) => {
     const chunks = [];
     let stream;
@@ -143,13 +183,49 @@ export function readHeldBytes(torrent, fileIndex, start, end, logger = null) {
     abandon = setTimeout(() => {
       logger?.info(
         `subtitles: a read of ${start}-${last} in "${String(file.name).slice(0, 40)}" ` +
-        `did not finish in ${READ_ABANDON_MS / 1000}s and was given up`
+        `was held but not served in ${READ_ABANDON_MS / 1000}s and was given up — the store failed to serve bytes it has`
       );
       settle(null);
     }, READ_ABANDON_MS);
     abandon.unref?.();
     stream.on("data", (chunk) => chunks.push(chunk));
-    stream.on("end", () => settle(Buffer.concat(chunks)));
+    stream.on("end", () => settle(ownedCopyOf(chunks)));
     stream.on("error", () => settle(null));
   });
 }
+
+/**
+ * The chunks as one buffer over memory of its own.
+ *
+ * @param {Uint8Array[]} chunks
+ * @returns {Buffer}
+ */
+function ownedCopyOf(chunks) {
+  const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  // `allocUnsafeSlow` never takes from the shared pool, so the buffer is the
+  // whole of its ArrayBuffer and may be transferred.
+  const owned = Buffer.allocUnsafeSlow(total);
+  let at = 0;
+  for (const chunk of chunks) {
+    owned.set(chunk, at);
+    at += chunk.length;
+  }
+  return owned;
+}
+
+/**
+ * Whether a buffer may be TRANSFERRED to another thread: it must be the whole of
+ * a plain ArrayBuffer. A slice of the shared pool would take every other
+ * buffer in the pool with it, and shared memory cannot be transferred at all.
+ *
+ * @param {unknown} bytes
+ * @returns {boolean}
+ */
+export function ownsItsMemory(bytes) {
+  return (
+    bytes instanceof Uint8Array &&
+    bytes.buffer instanceof ArrayBuffer &&
+    bytes.byteOffset === 0 &&
+    bytes.byteLength === bytes.buffer.byteLength
+  );
+}

package/services/torrent/worker/worker.js

@@ -30,7 +30,7 @@ import {
 } from "./resume-warm.js";
 import { fillFileInBackground } from "./background-fill.js";
 import { demandFor } from "../download/registry.js";
-import { heldRangesOf, readHeldBytes } from "./held-bytes.js";
+import { heldRangesOf, ownsItsMemory, readHeldBytes } from "./held-bytes.js";
 import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
 import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
 import { Command, Event } from "./protocol.js";
@@ -343,6 +343,10 @@ async function runCommand(command, params, id) {
       return {
         infoHash: torrent.infoHash,
         name: torrent.name,
+        // The unit the swarm delivers and the store keeps. Said to the main
+        // thread as a number so a reading there can size its portions to it
+        // without knowing what a piece is.
+        pieceLength: Number(torrent.pieceLength) || 0,
         // Files cross as plain data; the objects stay here.
         files: (torrent.files ?? []).map((file, index) => ({
           index,
@@ -567,7 +571,11 @@ parentPort.on("message", async (message) => {
   const { command, id, params } = message ?? {};
   try {
     const result = await runCommand(command, params ?? {}, id);
-    parentPort.postMessage({ type: Event.RESULT, id, result });
+    // A held read's bytes are handed over rather than copied, and only when
+    // the buffer is the whole of its own memory (`ownsItsMemory`): anything
+    // else would take memory the store or another read still uses with it.
+    const transfer = command === Command.READ_HELD && ownsItsMemory(result?.bytes) ? [result.bytes.buffer] : [];
+    parentPort.postMessage({ type: Event.RESULT, id, result }, transfer);
   } catch (error) {
     parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error) });
   }

package/services/transport/data-channel-handler.js

@@ -772,12 +772,16 @@ export function createDataChannelHandler({
    * are tiny (kilobytes at most for a whole track), so this is one message,
    * not a stream.
    *
+   * `withdrawn` names, by their found-order numbers, cues sent earlier that the
+   * proxy has taken back — read from a stretch that turned out not to be a
+   * cluster. Each cue carries its own number (`seq`) for that purpose.
+   *
    * @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[],
-   *   language: string, detectedLanguage: { code: string, name: string } | null,
+   *   withdrawn?: number[], language: string, detectedLanguage: { code: string, name: string } | null,
    *   cursor: number }} event
    * @returns {void}
    */
-  function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, detectedLanguage, cursor }) {
+  function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, withdrawn = [], language, detectedLanguage, cursor }) {
     const names = viewersWantingCues(sourceKey, fileIndex);
     if (names.length === 0) {
       log(
@@ -787,7 +791,7 @@ export function createDataChannelHandler({
       return;
     }
     const message = JSON.stringify({
-      type: "subtitle-cues", fileIndex, trackIndex, cues, language, detectedLanguage, cursor
+      type: "subtitle-cues", fileIndex, trackIndex, cues, withdrawn, language, detectedLanguage, cursor
     });
     let sent = 0;
     for (const consumerId of names) {

package/services/viewer/PriorityMap.js

@@ -257,6 +257,37 @@ export function mapForViewer({ atSeconds, durationSeconds, allowanceSeconds, pla
   return map;
 }
 
+/**
+ * Where viewers stand, read off a map: every second whose priority rises above
+ * the second before it.
+ *
+ * A viewer's own map is highest where they stand and falls away in front of
+ * them, so in a merged map each viewer's position is a step UP — the one place
+ * a value is higher than the value just behind it. A viewer standing inside the
+ * stretch somebody else is walking towards adds nothing that map does not
+ * already ask for, and is not a step; neither is somebody behind everybody.
+ *
+ * Read by whoever orders work by where people are and must not know what a
+ * viewer is — the subtitle walk reads the clusters at these times first.
+ *
+ * @param {PriorityMap | null | undefined} map
+ * @returns {number[]} Seconds, ascending.
+ */
+export function viewerStartsOn(map) {
+  const starts = [];
+  if (!map || !(map.durationSeconds > 0)) {
+    return starts;
+  }
+  for (let second = 0; second < map.durationSeconds; second += 1) {
+    const value = map.priority[second];
+    const before = second === 0 ? NOBODY_IS_COMING : map.priority[second - 1];
+    if (value > before && value > NOBODY_IS_COMING) {
+      starts.push(second);
+    }
+  }
+  return starts;
+}
+
 /**
  * Every viewer's map as one.
  *

package/test/audio-calibration.test.js

@@ -1,23 +1,23 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-import { audioReadingFor } from "../services/encode/audio-calibration.js";
-
-const readings = [
-  { codec: "ac3", operation: "aac", channels: 2, samplingFrequency: 48000, speed: 120 },
-  { codec: "aac", operation: "copy", channels: 2, samplingFrequency: 48000, speed: 200,
-    bytes: 64000, durationSeconds: 4 }
-];
-
-test("audio processing uses codec and sample work instead of a video-copy rate", () => {
-  assert.equal(audioReadingFor(readings, { codec: "ac3", transcode: true, channels: 6,
-    samplingFrequency: 48000 }).speed, 40);
-  assert.equal(audioReadingFor(readings, { codec: "ac3", transcode: true, channels: 2,
-    samplingFrequency: 24000 }).speed, 240);
-  assert.equal(audioReadingFor(readings, { codec: "dts", transcode: true }), null);
-});
-
-test("copying uses encoded byte work when the source declares its bitrate", () => {
-  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: false, bitrateKbps: 256 }).speed, 100);
-  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: false }).speed, 200);
-  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: true }), null);
-});
+import assert from "node:assert/strict";
+import test from "node:test";
+import { audioReadingFor } from "../services/encode/audio-calibration.js";
+
+const readings = [
+  { codec: "ac3", operation: "aac", channels: 2, samplingFrequency: 48000, speed: 120 },
+  { codec: "aac", operation: "copy", channels: 2, samplingFrequency: 48000, speed: 200,
+    bytes: 64000, durationSeconds: 4 }
+];
+
+test("audio processing uses codec and sample work instead of a video-copy rate", () => {
+  assert.equal(audioReadingFor(readings, { codec: "ac3", transcode: true, channels: 6,
+    samplingFrequency: 48000 }).speed, 40);
+  assert.equal(audioReadingFor(readings, { codec: "ac3", transcode: true, channels: 2,
+    samplingFrequency: 24000 }).speed, 240);
+  assert.equal(audioReadingFor(readings, { codec: "dts", transcode: true }), null);
+});
+
+test("copying uses encoded byte work when the source declares its bitrate", () => {
+  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: false, bitrateKbps: 256 }).speed, 100);
+  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: false }).speed, 200);
+  assert.equal(audioReadingFor(readings, { codec: "aac", transcode: true }), null);
+});

package/test/coalesce.test.js

@@ -0,0 +1,39 @@
+/**
+ * @file A request that arrives during a run gives one more run, never a lost
+ * one and never a queue.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { coalescing } from "../utils/coalesce.js";
+
+test("asked during a run: one more run after it, however often it was asked", async () => {
+  const runs = [];
+  let release;
+  const run = coalescing(async (label) => {
+    runs.push(label);
+    if (runs.length === 1) {
+      await new Promise((resolve) => {
+        release = resolve;
+      });
+    }
+  });
+  const first = run("file", "first");
+  await run("file", "second");
+  await run("file", "third");
+  assert.deepEqual(runs, ["first"], "nothing ran beside the run in progress");
+  release();
+  await first;
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.deepEqual(runs, ["first", "third"], "exactly one more run, with the latest arguments");
+});
+
+test("different keys run side by side", async () => {
+  const runs = [];
+  const run = coalescing(async (label) => {
+    runs.push(label);
+  });
+  await Promise.all([run("a", "a"), run("b", "b")]);
+  assert.deepEqual(runs.sort(), ["a", "b"]);
+});

package/test/held-read-ownership.test.js

@@ -0,0 +1,77 @@
+/**
+ * @file A read of what is downloaded asks for nothing, and hands the other
+ * thread memory that is its own.
+ *
+ * Until 2026-10-01 the read opened the torrent's own file stream on bytes that
+ * had not arrived; that stream selects pieces itself, outside the demand
+ * register, and waited up to thirty seconds — 36 such reads at a torrent's open
+ * competed with the file being watched. And the bytes it returns are now handed
+ * to the main thread instead of copied, which is safe only for a buffer that is
+ * the whole of its own memory. Fakes only.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { Readable } from "node:stream";
+import { MessageChannel } from "node:worker_threads";
+import { createHash } from "node:crypto";
+
+import { ownsItsMemory, readHeldBytes } from "../services/torrent/worker/held-bytes.js";
+
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+/** A torrent of one file whose stream serves views of one shared store buffer. */
+function torrentOver(store, held) {
+  const streams = [];
+  const file = {
+    name: "film.mkv",
+    length: store.length,
+    offset: 0,
+    createReadStream({ start, end }) {
+      streams.push({ start, end });
+      return Readable.from([store.subarray(start, end + 1)]);
+    }
+  };
+  return {
+    streams,
+    torrent: { pieceLength: 16, bitfield: { get: (index) => held(index) }, files: [file] }
+  };
+}
+
+test("a range not wholly downloaded is answered at once with nothing, and no stream is opened", async () => {
+  const store = Buffer.alloc(64, 1);
+  const { torrent, streams } = torrentOver(store, (index) => index !== 2);
+  assert.equal(await readHeldBytes(torrent, 0, 0, 63), null);
+  assert.equal(streams.length, 0, "nothing was asked of the torrent");
+  assert.notEqual(await readHeldBytes(torrent, 0, 0, 31), null, "a range that is here is read");
+});
+
+test("what a held read returns owns its memory; a pool slice and shared memory do not", async () => {
+  const store = Buffer.from(Array.from({ length: 64 }, (_, index) => index));
+  const { torrent } = torrentOver(store, () => true);
+  const bytes = await readHeldBytes(torrent, 0, 3, 40);
+  assert.equal(ownsItsMemory(bytes), true);
+  assert.equal(ownsItsMemory(Buffer.from("small")), false, "a slice of Node's pool");
+  assert.equal(ownsItsMemory(new Uint8Array(new SharedArrayBuffer(8))), false, "shared memory");
+  assert.equal(ownsItsMemory(store.subarray(4, 8)), false, "a view into another buffer");
+});
+
+test("handing a read's bytes to another thread leaves the store and an overlapping read as they were", async () => {
+  const store = Buffer.from(Array.from({ length: 256 }, (_, index) => index % 251));
+  const before = digest(store);
+  const { torrent } = torrentOver(store, () => true);
+  const [first, second] = await Promise.all([readHeldBytes(torrent, 0, 10, 200), readHeldBytes(torrent, 0, 50, 120)]);
+  const secondBefore = digest(second);
+
+  const { port1, port2 } = new MessageChannel();
+  const received = new Promise((resolve) => port2.once("message", resolve));
+  port1.postMessage({ bytes: first }, [first.buffer]);
+  const message = await received;
+  port1.close();
+  port2.close();
+
+  assert.equal(first.byteLength, 0, "the sender no longer holds the memory it handed over");
+  assert.deepEqual(Buffer.from(message.bytes), store.subarray(10, 201), "the receiver has every byte");
+  assert.equal(digest(store), before, "the store's own bytes are untouched");
+  assert.equal(digest(second), secondBefore, "the overlapping read is untouched");
+});

package/test/helpers/held-file.js

@@ -5,25 +5,73 @@
  * (`torrent/worker/held-bytes.js`), so a check that walks cues exercises both
  * sides of the seam rather than a stand-in for one of them: the bitfield
  * becoming a list of ranges, and a range read that never fetches.
+ *
+ * The file's container is the one a `ContainerOrchestrator` builds, over a
+ * reader that FETCHES — the fake file's own stream, which serves any range —
+ * because the head and the Cues table are read that way in the product.
  */
 
 import { heldRangesOf, readHeldBytes } from "../../services/torrent/worker/held-bytes.js";
+import { ContainerOrchestrator } from "../../services/media/ContainerOrchestrator.js";
+
+/**
+ * Every byte of a range from the fake file's stream.
+ *
+ * @param {object} file
+ * @param {number} start
+ * @param {number} end
+ * @returns {Promise<Buffer>}
+ */
+async function readThroughStream(file, start, end) {
+  const chunks = [];
+  for await (const chunk of file.createReadStream({ start, end })) {
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks);
+}
+
+/** One orchestrator for the checks that do not bring their own. */
+const sharedContainers = new ContainerOrchestrator();
 
 /**
  * @param {object} torrent - The fake a check builds: `pieceLength`, `bitfield`,
  *   `files[]` with `offset`, `length`, `name`, `createReadStream`.
  * @param {number} fileIndex
  * @param {string} sourceKey
+ * @param {{ containers?: ContainerOrchestrator, readRange?: (start: number, end: number) => Promise<Buffer | null>, wantedSeconds?: () => number[], portionBytes?: number }} [options]
  * @returns {import("../../services/media/SubtitleCues.js").HeldFile}
  */
-export function heldFileOver(torrent, fileIndex, sourceKey) {
+export function heldFileOver(torrent, fileIndex, sourceKey, options = {}) {
   const file = torrent?.files?.[fileIndex] ?? {};
+  const containers = options.containers ?? sharedContainers;
+  const readRange = options.readRange ?? ((start, end) => readThroughStream(file, start, end));
   return {
     sourceKey,
     fileIndex,
     name: String(file.name ?? ""),
     length: Number(file.length) || 0,
+    portionBytes: options.portionBytes,
+    container: () =>
+      containers.containerFor({
+        sourceKey,
+        fileIndex,
+        readRange,
+        fileSize: Number(file.length) || 0,
+        label: String(file.name ?? ""),
+        portionBytes: options.portionBytes
+      }),
     heldRanges: async () => heldRangesOf(torrent, fileIndex),
-    readHeld: (start, end) => readHeldBytes(torrent, fileIndex, start, end)
+    readHeld: (start, end) => readHeldBytes(torrent, fileIndex, start, end),
+    wantedSeconds: options.wantedSeconds
   };
 }
+
+/**
+ * Forget what the shared orchestrator built for a source.
+ *
+ * @param {string} sourceKey
+ * @returns {void}
+ */
+export function forgetContainers(sourceKey) {
+  sharedContainers.forget(sourceKey);
+}