@torrent-tv/proxy 2.43.2 → 2.44.0

package/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.44.0
+
+- **Fix**: The cues a browser is missing are found by the order they were READ, not by where they sit in the film. Cues come out of whichever clusters are downloaded, and a torrent does not arrive in film order, so the set of known cues grows in the MIDDLE as well as at the end — and the cursor shipped in 2.43.1 was a time. Measured 2026-08-20: a viewer at 272 s was answered with cues out to 1176 s, and from that moment every cue between the two was filtered away for the rest of the session, with 59 of 276 clusters read. The subtitles the viewer was about to need had become unreachable, while cues fifteen minutes ahead kept arriving. Each cue now carries the order it was found in, `?since=<n>` selects by that, and the answer states the next cursor in `X-Subtitle-Cursor`. `?after=<seconds>` still works, for a browser that has not been reloaded.
+- **Fix**: One walk over the container fills EVERY subtitle track, instead of one walk per track. A Matroska cluster carries the blocks of every track that has anything to say over its span, so the bytes that answer one track answer them all — but the set of clusters already read was kept per track, so the same bytes were fetched and parsed as many times as the film has subtitle tracks. On the field film that was five reads of everything, each costing 0.2-5.2 s, for cues that together weigh a few kilobytes. The union of the tracks' cluster lists is walked once and every track is filled from it, which is also why offering all of them costs no more than offering one.
+
 ## 2.43.2
 
 - **Fix**: The proxy no longer dies without a word in the middle of a film. It was a segmentation fault in the uTP native library — `on_uv_read` parsed a sender address that a FAILED read never produced, and libuv passes null there — so a read error dereferenced a null pointer on the thread that owns the torrent client. Three core dumps in two days, each about three and a half hours into an otherwise healthy run, each with the same top frame; the last one on a swarm of 63-75 peers delivering 13 MB/s, one segment after a successful 158 Mbit/s send. Fixed in our build (`@torrent-tv/utp-native@2.5.3-ttv.3`, which this now depends on) and absent from upstream master. Detail: `research/utp-native-null-addr-2026-08-20.md`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.43.2",
+  "version": "2.44.0",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/routes/api/subtitles/get.js

@@ -100,18 +100,30 @@ export async function handleApiSubtitlesGet(req, reply, { sourceRegistry, torren
   // read that way. Costs no network at all and answers with the part of the
   // film they are watching; the rest arrives as they watch it. Only when the
   // container cannot be read this way does the old extraction run.
-  // Where the browser's copy of this track ends. It already holds every cue
-  // before this second, so sending them again is bytes for nothing: measured
+  // How many cues this browser has already been sent, counted in the order
+  // they were FOUND. Sending them again is bytes for nothing: measured
   // 2026-08-19, one track is 76 KB and the browser asked for it every few
   // seconds while the film downloaded. Absent or unparsable means "send
   // everything", which is what a browser asking for the first time wants.
+  //
+  // Found-order, not time. A cue's time cannot serve as a cursor here: the
+  // cues are read out of whichever clusters are downloaded, and those are not
+  // contiguous, so the set grows in the middle as well as at the end. The old
+  // `?after=<seconds>` therefore threw away every cue that turned up BEHIND
+  // the furthest one already sent — which is the stretch the viewer is about
+  // to watch. Measured 2026-08-20: a viewer at 272 s was sent cues out to
+  // 1176 s, and from that moment nothing between the two could ever reach
+  // them, with 59 of 276 clusters read. `after` is still honoured so an older
+  // browser keeps working.
+  const since = Number.parseInt(String(req.query?.since ?? ""), 10);
   const after = Number.parseFloat(String(req.query?.after ?? ""));
   const held = await cuesFromDownloadedClusters(
     torrentPool,
     torrent,
     fileIndex,
     trackIndex,
-    Number.isFinite(after) ? after : null
+    Number.isFinite(after) ? after : null,
+    Number.isInteger(since) ? since : null
   );
   if (held !== null) {
     setLanguageHeaders(reply, held.language);
@@ -122,9 +134,11 @@ export async function handleApiSubtitlesGet(req, reply, { sourceRegistry, torren
     // as playback moves into clusters that were not downloaded yet.
     reply.header("X-Subtitle-Covered-Clusters", String(held.coveredClusters));
     reply.header("X-Subtitle-Indexed-Clusters", String(held.indexedClusters));
+    // What to send back as `?since=` next time.
+    reply.header("X-Subtitle-Cursor", String(held.cursor));
     reply.raw.setHeader(
       "Access-Control-Expose-Headers",
-      "X-Subtitle-Language, X-Subtitle-Language-Name, X-Subtitle-Covered-Clusters, X-Subtitle-Indexed-Clusters"
+      "X-Subtitle-Language, X-Subtitle-Language-Name, X-Subtitle-Covered-Clusters, X-Subtitle-Indexed-Clusters, X-Subtitle-Cursor"
     );
     return reply.send(held.vtt);
   }
@@ -264,7 +278,7 @@ function readFileFully(file, maxBytes) {
  * @returns {Promise<{ vtt: string, language: object | null, coveredClusters: number, indexedClusters: number } | null>}
  *   Null when this file cannot be read this way, and then the caller falls back.
  */
-async function cuesFromDownloadedClusters(torrentPool, torrent, fileIndex, trackIndex, after = null) {
+async function cuesFromDownloadedClusters(torrentPool, torrent, fileIndex, trackIndex, after = null, since = null) {
   if (typeof torrentPool?.getSubtitleTracks !== "function") {
     return null;
   }
@@ -290,9 +304,12 @@ async function cuesFromDownloadedClusters(torrentPool, torrent, fileIndex, track
   // Only what the browser does not have. The language is still detected from
   // EVERYTHING held, because three new lines say much less about a language
   // than the whole track does.
-  const fresh = Number.isFinite(after)
-    ? held.cues.filter((cue) => cue.startSeconds > after)
-    : held.cues;
+  const cursor = held.cues.reduce((highest, cue) => Math.max(highest, Number(cue.seq) || 0), 0);
+  const fresh = Number.isInteger(since)
+    ? held.cues.filter((cue) => (Number(cue.seq) || 0) > since)
+    : Number.isFinite(after)
+      ? held.cues.filter((cue) => cue.startSeconds > after)
+      : held.cues;
   const vtt = cuesToVtt(fresh, held.codecId);
   const language = held.cues.length > 0
     ? detectLanguage(held.cues.map((cue) => cue.text).join("\n"))
@@ -300,6 +317,7 @@ async function cuesFromDownloadedClusters(torrentPool, torrent, fileIndex, track
   return {
     vtt,
     language,
+    cursor,
     coveredClusters: held.coveredClusters ?? 0,
     indexedClusters: held.indexedClusters ?? 0
   };

package/services/torrent-worker/subtitle-cues.js

@@ -99,7 +99,7 @@ function readHeld(file, start, end) {
 async function planFor(torrent, fileIndex, key) {
   let state = byFile.get(key);
   if (!state) {
-    state = { plan: null, harvested: new Map(), cues: new Map() };
+    state = { plan: null, harvested: new Map(), cues: new Map(), seq: new Map(), walked: new Set() };
     byFile.set(key, state);
   }
   if (state.plan !== null) {
@@ -154,6 +154,31 @@ async function planFor(torrent, fileIndex, key) {
   return state.plan;
 }
 
+/**
+ * The order a cue was FOUND in, which is the only cursor a browser can follow.
+ *
+ * A cue's TIME cannot serve as one. Cues are harvested out of whichever
+ * clusters happen to be downloaded, and those are not contiguous, so the set
+ * grows in the middle as well as at the end. A browser that remembered "the
+ * latest time I hold" and asked for everything past it would never be sent the
+ * cues that turn up BEHIND that mark afterwards — which is exactly the stretch
+ * it is about to play. Measured 2026-08-20 on a viewer at 272 s: one answer
+ * carried cues out to 1176 s, and from then on every cue between the two was
+ * filtered away for the rest of the session, with 59 of 276 clusters read.
+ *
+ * Found-order is monotonic by construction, so `?since=<n>` is exact however
+ * the file arrives.
+ *
+ * @param {{ seq: Map<number, number> }} state
+ * @param {number} trackNumber
+ * @returns {number}
+ */
+function nextSeq(state, trackNumber) {
+  const next = (state.seq.get(trackNumber) ?? 0) + 1;
+  state.seq.set(trackNumber, next);
+  return next;
+}
+
 /**
  * Every cue of one track that can be read from what is already downloaded.
  *
@@ -201,7 +226,12 @@ export async function cuesHeldFor(torrent, fileIndex, sourceKey, trackNumber) {
       harvested.add(sample.offset);
       const text = decodeSubtitleSample(bytes, track.codecId);
       if (text) {
-        cues.push({ startSeconds: sample.startSeconds, endSeconds: sample.endSeconds, text });
+        cues.push({
+          startSeconds: sample.startSeconds,
+          endSeconds: sample.endSeconds,
+          text,
+          seq: nextSeq(state, trackNumber)
+        });
       }
     }
     cues.sort((left, right) => left.startSeconds - right.startSeconds);
@@ -213,8 +243,24 @@ export async function cuesHeldFor(torrent, fileIndex, sourceKey, trackNumber) {
     };
   }
 
-  for (const position of track.clusterPositions) {
-    if (harvested.has(position)) {
+  // ONE walk for the whole file, not one per track. A Matroska cluster carries
+  // the blocks of every track that has anything to say over its span, so the
+  // bytes that answer one track answer them all — and reading them once per
+  // track meant the same cluster was fetched and parsed as many times as the
+  // film has subtitle tracks. Measured 2026-08-20 on a film with five: five
+  // requests every fifteen seconds, each costing 0.2-5.2 s of container
+  // reading, for cues that together weigh a few kilobytes.
+  //
+  // The union of the tracks' cluster lists is what gets walked: each track's
+  // list comes from its own Cues entries, so they overlap but do not coincide.
+  const positions = new Set();
+  for (const candidate of plan.tracks) {
+    for (const position of candidate.clusterPositions ?? []) {
+      positions.add(position);
+    }
+  }
+  for (const position of [...positions].sort((left, right) => left - right)) {
+    if (state.walked.has(position)) {
       continue;
     }
     // The header first: it says how long the cluster is, and a cluster whose
@@ -225,7 +271,7 @@ export async function cuesHeldFor(torrent, fileIndex, sourceKey, trackNumber) {
     const probe = await readHeld(file, position, Math.min(file.length - 1, position + CLUSTER_HEADER_PROBE - 1));
     const header = probe && [...iterateElements(probe, 0, probe.length)][0];
     if (!header || header.size <= 0 || header.size > MAX_CLUSTER_BYTES) {
-      harvested.add(position); // not a cluster we can read; do not look again
+      state.walked.add(position); // not a cluster we can read; do not look again
       continue;
     }
     const last = Math.min(file.length - 1, position + header.dataOffset + header.size - 1);
@@ -236,18 +282,33 @@ export async function cuesHeldFor(torrent, fileIndex, sourceKey, trackNumber) {
     if (!bytes) {
       continue;
     }
-    harvested.add(position);
-    for (const cue of harvestCluster(bytes, trackNumber, plan.secondsPerTick)) {
-      cues.push(cue);
+    state.walked.add(position);
+    for (const candidate of plan.tracks) {
+      let into = state.cues.get(candidate.trackNumber);
+      if (!into) {
+        into = [];
+        state.cues.set(candidate.trackNumber, into);
+      }
+      let found = false;
+      for (const cue of harvestCluster(bytes, candidate.trackNumber, plan.secondsPerTick)) {
+        cue.seq = nextSeq(state, candidate.trackNumber);
+        into.push(cue);
+        found = true;
+      }
+      if (found) {
+        into.sort((left, right) => left.startSeconds - right.startSeconds);
+      }
     }
   }
-  cues.sort((left, right) => left.startSeconds - right.startSeconds);
   return {
-    cues,
-    coveredClusters: harvested.size,
+    cues: state.cues.get(trackNumber) ?? [],
+    // Every track is filled by the same walk, so this is a fact about the FILE
+    // and reads the same whichever track asked.
+    coveredClusters: state.walked.size,
     indexedClusters: track.clusterPositions.length,
     track
   };
+
 }
 
 /**

package/test/subtitle-cursor.test.js

@@ -0,0 +1,63 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+/**
+ * The rule this pins: a subtitle cursor counts the order cues were FOUND, not
+ * where they sit in the film.
+ *
+ * Cues are read out of whichever clusters happen to be downloaded, and a
+ * torrent does not arrive in film order — a seek pulls a later stretch first,
+ * and the earlier one fills in afterwards. So the set of known cues grows in
+ * the MIDDLE as well as at the end.
+ *
+ * A cursor in film time cannot survive that: measured 2026-08-20 on a viewer at
+ * 272 s, one answer carried cues out to 1176 s, and from that moment every cue
+ * between the two was filtered away for the rest of the session — the stretch
+ * they were about to watch. 59 of 276 clusters had been read.
+ *
+ * The filter below is the route's, written out so the property can be checked
+ * without a torrent: `?since=<n>` selects by found-order, `?after=<seconds>` is
+ * the old behaviour kept for an older browser.
+ */
+const bySince = (cues, since) => cues.filter((cue) => (Number(cue.seq) || 0) > since);
+const byAfter = (cues, after) => cues.filter((cue) => cue.startSeconds > after);
+
+/** A late-arriving cluster from EARLIER in the film than what is already held. */
+const held = [
+  { startSeconds: 40, text: "found first, early in the film", seq: 1 },
+  { startSeconds: 1176, text: "found second, far ahead", seq: 2 },
+  { startSeconds: 300, text: "found third, behind the furthest held", seq: 3 }
+];
+
+test("a cue found after a further-ahead one is still delivered", () => {
+  // The browser holds seq 1 and 2 and asks for what came after.
+  const fresh = bySince(held, 2);
+  assert.deepEqual(fresh.map((cue) => cue.startSeconds), [300]);
+});
+
+test("the same case in film time loses that cue for ever", () => {
+  // This is what shipped in 2.43.1 and what the field session showed: the
+  // browser's furthest cue is 1176 s, so the 300 s cue can never reach it.
+  const fresh = byAfter(held, 1176);
+  assert.deepEqual(fresh.map((cue) => cue.startSeconds), []);
+});
+
+test("a browser asking for the first time is sent everything", () => {
+  assert.equal(bySince(held, 0).length, 3);
+});
+
+test("nothing new is answered with nothing", () => {
+  assert.deepEqual(bySince(held, 3), []);
+});
+
+test("the cursor to send back is the highest found-order held", () => {
+  const cursor = held.reduce((highest, cue) => Math.max(highest, Number(cue.seq) || 0), 0);
+  assert.equal(cursor, 3);
+});
+
+test("a cursor is unaffected by the order cues are sorted into", () => {
+  // The list is kept in film order for the WebVTT it becomes; the cursor must
+  // not depend on that.
+  const sorted = [...held].sort((left, right) => left.startSeconds - right.startSeconds);
+  assert.deepEqual(bySince(sorted, 2).map((cue) => cue.startSeconds), [300]);
+});