@torrent-tv/proxy 2.83.3 → 2.83.4

package/services/torrent-worker/protocol.js

@@ -47,10 +47,6 @@
 export const Command = {
   /** Add (or join) a torrent; resolves when metadata is ready. */
   ADD_SOURCE: "add-source",
-  /** Claim a file for reading, so it is not evicted while in use. */
-  ACQUIRE_FILE: "acquire-file",
-  /** Drop a claim; the worker applies its own idle-removal policy. */
-  RELEASE_FILE: "release-file",
   /** File list and metadata for a source. */
   LIST_FILES: "list-files",
   /** Live download figures for the progress display. */

package/services/torrent-worker/worker.js

@@ -11,10 +11,9 @@
  * with the torrent for this thread, which is exactly the problem being solved.
  *
  * The existing `TorrentPool` is reused wholesale rather than reimplemented. It
- * already carries the parts that took field failures to get right — refcounted
- * file claims, idle removal, the global disk cap with LRU eviction, seek-aware
- * piece prioritisation, adaptive upload — and none of that changes by moving
- * threads.
+ * already carries the parts that took field failures to get right — idle
+ * removal, the global disk cap with LRU eviction, seek-aware piece
+ * prioritisation, adaptive upload — and none of that changes by moving threads.
  */
 
 // MUST stay first: it redirects `webrtc-polyfill` to a JavaScript WebRTC stack
@@ -25,7 +24,6 @@ import { isUsableTorrentHandle } from "./handle-state.js";
 import "./install-webrtc-shim.js";
 import { parentPort, workerData } from "node:worker_threads";
 import { createSendStream } from "./channel.js";
-import { createFileClaims } from "./file-claims.js";
 import { readFragments, supplyFiguresFor } from "./piece-reader.js";
 import { cuesHeldFor, declaredSubtitleTracksOf, subtitleTracksOf, warmSubtitleCues } from "./subtitle-cues.js";
 import {
@@ -36,6 +34,7 @@ import {
   warmResumePosition
 } from "./container-tracks.js";
 import { fillFileInBackground } from "./background-fill.js";
+import { demandFor } from "../download/registry.js";
 import { CompletedFiles, completedFilesRoot } from "../files/CompletedFiles.js";
 import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../files/piece-from-whole-file.js";
 import { Command, Event } from "./protocol.js";
@@ -93,8 +92,6 @@ const torrentsByKey = new Map();
  * @type {Map<string, { sourceType: string, source: string }>}
  */
 const sourceRecipes = new Map();
-/** File claims, each with its own identity — see `file-claims.js`. */
-const fileClaims = createFileClaims();
 /** In-flight reads, so a cancel can stop one mid-body. */
 const readsById = new Map();
 
@@ -127,6 +124,26 @@ function log(message) {
  * @param {string} sourceKey
  * @returns {Promise<import("webtorrent").Torrent>}
  */
+/**
+ * The torrent for a sourceKey IF there is one, and never one that has to be
+ * built to answer.
+ *
+ * The other half of the pair above, for the questions that are about something
+ * going away: a departure is not a reason to add anything, and the answer
+ * "there is nothing here" is a complete answer to them.
+ *
+ * @param {string} sourceKey
+ * @returns {Promise<import("webtorrent").Torrent | null>}
+ */
+async function knownTorrent(sourceKey) {
+  const pending = torrentsByKey.get(sourceKey);
+  if (!pending) {
+    return null;
+  }
+  const torrent = await pending.catch(() => null);
+  return isUsableTorrentHandle(torrent) ? torrent : null;
+}
+
 async function requireTorrent(sourceKey) {
   const pending = torrentsByKey.get(sourceKey);
   if (!pending) {
@@ -241,15 +258,6 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
   const sender = createSendStream({ port: parentPort, requestId: id });
   readsById.set(id, sender);
 
-  // Hold the file for as long as this read runs. The caller also acquires it,
-  // but that acquire and its release are separate messages from another thread
-  // and can be reordered; this one cannot, because it lives entirely inside the
-  // read. Without it the idle sweep saw a zero reader count and removed the
-  // torrent AND its store mid-read — field 2026-08-02: "removed idle torrent
-  // ... and its store", after which every subsequent read hung and ffmpeg got
-  // an empty input.
-  const releaseRead = pool.acquireFile(torrent, fileIndex);
-
   const rangeStart = start ?? 0;
   const rangeEnd = end ?? file.length - 1;
 
@@ -290,7 +298,6 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
     // Any fragment still awaiting confirmation will never get one now; settling
     // it here releases its pin rather than leaking a held slot.
     settleFragment(id);
-    releaseRead();
     if (!failed) {
       sender.end();
     }
@@ -356,27 +363,6 @@ async function runCommand(command, params, id) {
       }));
     }
 
-    case Command.ACQUIRE_FILE: {
-      const torrent = await requireTorrent(params.sourceKey);
-      // Every acquire is its own claim. Sharing one per file meant the first
-      // reader to finish released the hold while others were still reading.
-      return fileClaims.open(
-        params.sourceKey,
-        params.fileIndex,
-        pool.acquireFile(torrent, params.fileIndex)
-      );
-    }
-
-    case Command.RELEASE_FILE: {
-      const released = fileClaims.close(params.claimId);
-      if (!released) {
-        // Not fatal — but it means a release arrived twice or after teardown,
-        // and silence here is what let the previous scheme look healthy.
-        log(`release for unknown file claim ${params.claimId}`);
-      }
-      return released;
-    }
-
     case Command.HELD_TORRENTS: {
       // Which films this proxy has right now, and how much of each. Answered
       // from the live client rather than from the main thread's map of
@@ -537,13 +523,18 @@ async function runCommand(command, params, id) {
     }
 
     case Command.PRIORITY_MAP: {
-      const torrent = await requireTorrent(params.sourceKey);
-      pool.applyPriorityMap(
-        torrent,
-        params.fileIndex,
-        params.zones,
-        params.durationSeconds
-      );
+      // A MAP WITH NOTHING IN IT MUST NOT BRING A TORRENT BACK. It is what is
+      // said when the last viewer of a file leaves, which is also when the
+      // torrent may be on its way out — and `requireTorrent` rebuilds a dead
+      // handle from its recipe, so asking that way would re-add a torrent in
+      // order to be told that nothing is wanted of it.
+      const zones = Array.isArray(params.zones) ? params.zones : [];
+      const torrent = zones.length === 0
+        ? await knownTorrent(params.sourceKey)
+        : await requireTorrent(params.sourceKey);
+      if (torrent) {
+        pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
+      }
       return true;
     }
 
@@ -590,7 +581,6 @@ async function runCommand(command, params, id) {
     }
 
     case Command.DESTROY_ALL: {
-      fileClaims.closeAll();
       torrentsByKey.clear();
       sourceRecipes.clear();
       await pool.destroyAll();
@@ -827,11 +817,11 @@ function describePieceBuffers() {
  * @returns {void}
  */
 function warmActiveFiles(sourceKey, torrent) {
-  const usage = pool.fileUsageByTorrent.get(torrent);
-  if (!usage) {
-    return;
-  }
-  for (const fileIndex of usage.keys()) {
+  // The files anything is stated for — a viewer's own picture and soundtrack
+  // through the priority map, and the ends of a file that is open. It replaces
+  // a count of readers, which said the same thing by keeping a second copy of
+  // it.
+  for (const fileIndex of demandFor(torrent).register.files()) {
     const key = `${sourceKey}:${fileIndex}`;
     // A trigger that arrives while the previous pass is still walking is
     // dropped, not queued. `verified` fires per piece, so on a fast download
@@ -1009,18 +999,21 @@ async function keepWholeFiles() {
     if (!infoHash || !Array.isArray(torrent.files)) {
       continue;
     }
-    const usage = pool.fileUsageByTorrent?.get?.(torrent);
+    // What anybody wants of this torrent. A file something is stated for is a
+    // file somebody may be reading, and this is the same list the reader counts
+    // used to give.
+    const wanted = new Set(demandFor(torrent).register.files());
     for (const [fileIndex, file] of torrent.files.entries()) {
       const key = `${infoHash}/${fileIndex}`;
       if (file?.done !== true || completedFiles.find(infoHash, fileIndex) || beingKept.has(key)) {
         continue;
       }
-      // NOT WHILE SOMEBODY IS READING IT. Writing a film out is a read of the
-      // whole of it and a write of the whole of it — a gigabyte and a half on
-      // the file this was measured against — and doing that beside a viewer
-      // takes the disk and the piece store from them for nothing they asked
-      // for. The file is complete; it will still be complete when they leave.
-      if (usage?.has?.(fileIndex)) {
+      // NOT WHILE ANYBODY WANTS IT. Writing a film out is a read of the whole
+      // of it and a write of the whole of it — a gigabyte and a half on the
+      // file this was measured against — and doing that beside a viewer takes
+      // the disk and the piece store from them for nothing they asked for. The
+      // file is complete; it will still be complete when they leave.
+      if (wanted.has(fileIndex)) {
         continue;
       }
       beingKept.add(key);
@@ -1074,7 +1067,7 @@ async function keepWholeFiles() {
     const isWhole =
       torrent.done === true &&
       torrent.files.every((unused, fileIndex) => completedFiles.find(infoHash, fileIndex) !== null);
-    if (isWhole && !(usage?.size > 0)) {
+    if (isWhole && wanted.size === 0) {
       logger.info(
         `whole files: "${torrent.name}" is downloaded whole and saved — removing the torrent, keeping the files`
       );

package/test/file-edges.test.js

@@ -0,0 +1,191 @@
+/**
+ * @file THE TWO ENDS OF A FILE ARE THE FILE'S, NOT A READ'S.
+ *
+ * A container keeps its directory at one end or the other: `ftyp` and an EBML
+ * header at the front, and for an MP4 that was not written for streaming the
+ * `moov` at the very back. Nothing can be read of such a file until they have
+ * arrived, and they go on being wanted for as long as it is open — a player
+ * asks for the file's shape again at every seek.
+ *
+ * Until 2.83.4 they were wanted by nobody. The prefetch that fetches them is an
+ * ordinary read, and a read withdraws what it states the moment it finishes, so
+ * the ends of an open film were held by nothing at all once the codec probe was
+ * done.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { TorrentPool, stateFileEdges, withdrawFileEdges } from "../services/torrent-pool.js";
+import { demandFor, forgetTorrent } from "../services/download/registry.js";
+import { Urgency } from "../services/demand/index.js";
+
+/** A torrent of two files, of which the second is the film. */
+function torrentOf({ length = 1_000_000 } = {}) {
+  return {
+    infoHash: `hash-${Math.random().toString(36).slice(2)}`,
+    pieceLength: 1024,
+    files: [
+      { offset: 0, length: 4096, name: "notes.nfo" },
+      { offset: 4096, length, name: "film.mkv" }
+    ],
+    _selections: { _items: [] },
+    _select() {},
+    _deselect() {},
+    critical() {}
+  };
+}
+
+/**
+ * @param {object} torrent
+ * @returns {object[]}
+ */
+function edges(torrent) {
+  return demandFor(torrent)
+    .register.windows()
+    .filter((one) => String(one.claimant).startsWith("file-edges:"));
+}
+
+test("one byte is claimed at each end, which is one piece at each end", () => {
+  const torrent = torrentOf({ length: 1_000_000 });
+  try {
+    assert.equal(stateFileEdges(torrent, 1, Urgency.TAIL), true);
+
+    const stated = edges(torrent);
+    assert.equal(stated.length, 2);
+    const head = stated.find((one) => String(one.claimant).endsWith(":head"));
+    const tail = stated.find((one) => String(one.claimant).endsWith(":tail"));
+    // File-relative, like everything else stated about a file: the claim says
+    // "the first byte of this file" and "its last", and what a byte costs to
+    // fetch — a whole piece — is the swarm's business and nobody else's.
+    assert.deepEqual([head.byteStart, head.byteEnd], [0, 0]);
+    assert.deepEqual([tail.byteStart, tail.byteEnd], [999_999, 999_999]);
+    assert.equal(head.fileIndex, 1);
+    assert.equal(tail.fileIndex, 1);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("while somebody is waiting for them they are urgent, and afterwards they are kept", () => {
+  const torrent = torrentOf();
+  try {
+    // The playback plan cannot answer until the file has said what is in it,
+    // and a person is watching a loading screen for as long as that takes.
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.NEAR, Urgency.NEAR]
+    );
+
+    // Once it has answered, nobody is waiting — but a seek will want them
+    // again, so they stay stated at the level of something nobody is waiting
+    // for. Re-stating replaces; it does not pile a second claim on the first.
+    stateFileEdges(torrent, 1, Urgency.TAIL, { lower: true });
+    assert.equal(edges(torrent).length, 2);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.TAIL, Urgency.TAIL]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("each file of a torrent has its own ends", () => {
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 0, Urgency.TAIL);
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    assert.equal(edges(torrent).length, 4);
+
+    withdrawFileEdges(torrent, 0);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.fileIndex),
+      [1, 1]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a file of unknown length has no ends to claim", () => {
+  const torrent = torrentOf();
+  try {
+    assert.equal(stateFileEdges(torrent, 7, Urgency.TAIL), false);
+    assert.equal(stateFileEdges({ files: [{ length: 0 }] }, 0, Urgency.TAIL), false);
+    assert.equal(edges(torrent).length, 0);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a map with nothing in it takes the ends back too", () => {
+  const torrent = torrentOf();
+  const applyPriorityMap = TorrentPool.prototype.applyPriorityMap;
+  try {
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+    applyPriorityMap.call(null, torrent, 1, [{ from: 0, to: 600, priority: 100 }], 600);
+    assert.equal(edges(torrent).length, 2);
+
+    // Nobody wants this file any more, and its ends are kept only for as long
+    // as it is open.
+    applyPriorityMap.call(null, torrent, 1, [], 600);
+    assert.equal(edges(torrent).length, 0);
+    assert.deepEqual(demandFor(torrent).register.windows(), []);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("another file's departure leaves these ends alone", () => {
+  const torrent = torrentOf();
+  const applyPriorityMap = TorrentPool.prototype.applyPriorityMap;
+  try {
+    stateFileEdges(torrent, 0, Urgency.TAIL);
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+
+    applyPriorityMap.call(null, torrent, 0, [], 600);
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.fileIndex),
+      [1, 1]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a warm-up cannot take the urgency away from a plan that is waiting", () => {
+  // The two callers of the same ends: a playback plan, which somebody is
+  // watching a loading screen for, and a warm-up, which by its whole purpose
+  // nobody is waiting for. The warm-up fires off the files beside the picture
+  // without awaiting them, so it can arrive second — and lowering the plan's
+  // level there would put a person behind a film somebody else is watching.
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.NEAR, Urgency.NEAR]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("the read that is over is what puts them back down", () => {
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    stateFileEdges(torrent, 1, Urgency.TAIL, { lower: true });
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.TAIL, Urgency.TAIL]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});

package/test/priority-map-emptied.test.js

@@ -0,0 +1,207 @@
+/**
+ * @file WHEN NOBODY WANTS A FILE ANY MORE, THE MAP SAYS SO.
+ *
+ * The map is the only statement of what is wanted, and until 2.83.4 it could
+ * make only one of the two statements it exists for. A file whose viewers had
+ * gone was deleted from the orchestrator's own memory and published nowhere, so
+ * the bands it had stated on the swarm's behalf stood until the torrent itself
+ * was removed — read from the demand register, a film nobody had watched for an
+ * hour was indistinguishable from one being watched now.
+ *
+ * Two halves, and both are here: the side that builds the map has to say it,
+ * and the side that receives it has to act on it without needing the file's
+ * length, the torrent's list, or the torrent to exist at all.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { PriorityOrchestrator } from "../services/priority/PriorityOrchestrator.js";
+import { Viewers } from "../services/viewer/Viewers.js";
+import { viewersOf } from "../services/viewer/Viewer.js";
+import { TorrentPool } from "../services/torrent-pool.js";
+import { demandFor, forgetTorrent } from "../services/download/registry.js";
+import { Urgency } from "../services/demand/index.js";
+
+const STALE_AFTER_MS = 60_000;
+
+/**
+ * A session, as much of one as the orchestrator reads.
+ *
+ * @param {{ id: string, fileIndex?: number }} params
+ * @returns {object}
+ */
+function outputOf({ id, fileIndex = 0 }) {
+  return {
+    id,
+    outputKey: `out:${id}`,
+    sourceKey: "source-1",
+    fileIndex,
+    file: { key: `film-${fileIndex}`, durationSeconds: 600 }
+  };
+}
+
+/**
+ * The real orchestrator, with everything it publishes kept.
+ *
+ * @param {object[]} sessions
+ * @returns {{ priority: PriorityOrchestrator, viewers: Viewers, published: object[], publish: (sessions?: object[]) => void }}
+ */
+function over(sessions) {
+  const published = [];
+  const viewers = new Viewers();
+  const priority = new PriorityOrchestrator({
+    publish: (one) => published.push(one),
+    viewersOf: (session) => viewersOf(session),
+    allowanceFor: () => 10
+  });
+  return {
+    priority,
+    viewers,
+    published,
+    publish: (live = sessions) =>
+      priority.publishFor({ sessionGroups: [live], staleAfterMs: STALE_AFTER_MS })
+  };
+}
+
+test("a file whose viewers have all gone is published as wanting nothing", () => {
+  const picture = outputOf({ id: "pic" });
+  const { viewers, published, publish } = over([picture]);
+  const person = viewers.of(picture, "p");
+  person.moveTo(300);
+  publish();
+  assert.ok(published.at(-1).zones.length > 0, "somebody is watching, so something is wanted");
+
+  // They stop answering. The session is still there — it outlives them by half
+  // an hour — so this is the case that used to say nothing at all.
+  person.seen(Date.now() - STALE_AFTER_MS * 2);
+  publish();
+
+  const last = published.at(-1);
+  assert.deepEqual(last.zones, []);
+  assert.equal(last.sourceKey, "source-1");
+  assert.equal(last.fileIndex, 0);
+});
+
+test("a file whose session has gone is published as wanting nothing, once", () => {
+  const picture = outputOf({ id: "pic" });
+  const { viewers, published, publish } = over([picture]);
+  viewers.of(picture, "p").moveTo(300);
+  publish();
+  const said = published.length;
+
+  // The session is disposed: it is not among the live ones any more.
+  publish([]);
+  assert.deepEqual(published.at(-1).zones, []);
+  assert.equal(published.length, said + 1);
+
+  // And it is not repeated on every pass afterwards — the file is gone from
+  // this class's memory, and a departure is said once.
+  publish([]);
+  publish([]);
+  assert.equal(published.length, said + 1);
+});
+
+test("an unchanged map is still not republished", () => {
+  const picture = outputOf({ id: "pic" });
+  const { viewers, published, publish } = over([picture]);
+  viewers.of(picture, "p").moveTo(300);
+  publish();
+  const said = published.length;
+  publish();
+  publish();
+  assert.equal(published.length, said, "the downloading rebuilds its requests on every one");
+});
+
+/** A torrent that is nothing but one file of a known length. */
+function torrentOf({ length = 1_000_000 } = {}) {
+  return {
+    infoHash: `hash-${Math.random().toString(36).slice(2)}`,
+    pieceLength: 1024,
+    files: [{ offset: 0, length, name: "film.mkv" }],
+    _selections: { _items: [] },
+    _select() {},
+    _deselect() {},
+    critical() {}
+  };
+}
+
+const applyPriorityMap = TorrentPool.prototype.applyPriorityMap;
+
+/**
+ * @param {object} torrent
+ * @returns {object[]}
+ */
+function stated(torrent) {
+  return demandFor(torrent)
+    .register.windows()
+    .filter((one) => String(one.claimant).startsWith("priority-map:"));
+}
+
+test("a map with nothing in it withdraws everything that file had stated", () => {
+  const torrent = torrentOf();
+  try {
+    applyPriorityMap.call(null, torrent, 0, [
+      { from: 0, to: 100, priority: 100 },
+      { from: 100, to: 600, priority: 50 }
+    ], 600);
+    assert.equal(stated(torrent).length, 2);
+
+    applyPriorityMap.call(null, torrent, 0, [], 600);
+    assert.equal(stated(torrent).length, 0);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("it withdraws that file's bands and nobody else's", () => {
+  const torrent = torrentOf();
+  try {
+    const { register } = demandFor(torrent);
+    applyPriorityMap.call(null, torrent, 0, [{ from: 0, to: 600, priority: 100 }], 600);
+    // A read stopped on a piece right now, which only a read can say, and the
+    // background fill, which the pool states for itself. Neither is the map's
+    // to withdraw.
+    register.state({
+      claimant: "read-7:blocked",
+      fileIndex: 0,
+      byteStart: 0,
+      byteEnd: 999,
+      urgency: Urgency.BLOCKED
+    });
+    register.state({
+      claimant: "background-fill:0",
+      fileIndex: 0,
+      byteStart: 1000,
+      byteEnd: 9999,
+      urgency: Urgency.TAIL
+    });
+
+    applyPriorityMap.call(null, torrent, 0, [], 600);
+
+    assert.equal(stated(torrent).length, 0);
+    assert.deepEqual(
+      register.windows().map((one) => one.claimant).sort(),
+      ["background-fill:0", "read-7:blocked"]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a departure is answered even when nothing else about the file is known", () => {
+  // The length, the duration and the torrent's own list are what the ordinary
+  // path needs to turn seconds into bytes. A departure needs none of them, and
+  // it arrives exactly when they are going away: the map that says it is
+  // published as the last viewer leaves.
+  const torrent = torrentOf();
+  try {
+    applyPriorityMap.call(null, torrent, 0, [{ from: 0, to: 600, priority: 100 }], 600);
+    assert.equal(stated(torrent).length, 1);
+
+    torrent.files = [];
+    applyPriorityMap.call(null, torrent, 0, [], 0);
+    assert.equal(stated(torrent).length, 0);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});