@torrent-tv/proxy 2.67.0 → 2.69.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.67.0",
+  "version": "2.69.0",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/routes/api/subtitles/get.js

@@ -25,7 +25,7 @@
  */
 
 import { spawn } from "node:child_process";
-import { detectLanguage } from "../../../services/language-detect.js";
+import { detectLanguageFromVtt } from "../../../services/language-detect.js";
 import { SubtitleController } from "../../../services/controllers/SubtitleController.js";
 import { logger } from "../../../utils/logger.js";
 
@@ -137,7 +137,7 @@ export async function handleApiSubtitlesGet(req, reply, { sourceRegistry, torren
  * Extractions by `sourceKey:fileIndex:trackIndex`, so the scan happens once per
  * track however many times it is asked for.
  *
- * @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: string, error?: string }>}
+ * @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: { code: string, name: string } | null, error?: string }>}
  */
 const extractions = new Map();
 
@@ -190,7 +190,12 @@ function startExtraction({ key, ffmpegBin, localBaseUrl, sourceKey, fileIndex, t
       logger.warn(`subtitles ${key}: nothing produced after ${seconds}s`);
       return;
     }
-    extractions.set(key, { state: "done", body, language: detectLanguage(String(body.subarray(0, 4096))) });
+    // The whole document, decoded as one string, and only its cue text.
+    // Detecting on the first 4096 BYTES was wrong twice over: a byte cut lands
+    // mid-character on any non-Latin track, and most of those bytes are
+    // timestamps rather than words. This runs once per track in the background,
+    // so reading all of it costs nothing anybody waits for.
+    extractions.set(key, { state: "done", body, language: detectLanguageFromVtt(body.toString("utf8")) });
     logger.info(`subtitles ${key}: ${body.length} bytes in ${seconds}s`);
   };
   ffmpeg.once("close", settle);

package/services/controllers/SubtitleController.js

@@ -9,7 +9,7 @@
 
 import { subtitleOrchestrator } from "../orchestrators/SubtitleOrchestrator.js";
 import { convertSubtitleToVtt, decodeSubtitleBytes } from "../subtitle-convert.js";
-import { detectLanguage } from "../language-detect.js";
+import { detectLanguage, detectLanguageFromVtt } from "../language-detect.js";
 import { finalizeCues } from "../torrent-worker/subtitle-cues.js";
 
 const EXTERNAL_MAX_BYTES = 8 * 1024 * 1024;
@@ -75,7 +75,12 @@ export class SubtitleController {
         const text = decodeSubtitleBytes(bytes);
         const vtt = convertSubtitleToVtt(text, ext);
         if (!vtt) return { error: `Unsupported subtitle format: ${ext}`, status: 422 };
-        return { vtt, language: detectLanguage(text), headers: {} };
+        // The language is read from the CONVERTED document, not from the file.
+        // The conversion has already dropped everything that is not the words —
+        // and on an ASS file that is half of it, in Latin letters, which is what
+        // made a Russian track answer `en` (field 2026-09-01, and the whole of
+        // `research/subtitle-language-ass-markup-2026-09-01.md`).
+        return { vtt, language: detectLanguageFromVtt(vtt), headers: {} };
       } catch (e) {
         return { error: `Could not read subtitle file: ${e?.message ?? e}`, status: 502 };
       } finally {
@@ -105,8 +110,23 @@ export class SubtitleController {
       const cursor = held.cues.reduce((h, c) => Math.max(h, Number(c.seq) || 0), 0);
       const fresh = Number.isInteger(since) ? held.cues.filter((c) => (Number(c.seq) || 0) > since)
         : Number.isFinite(after) ? held.cues.filter((c) => c.startSeconds > after) : held.cues;
-      const vtt = cuesToVtt(fresh, held.track?.codecId ?? track?.codecId ?? "");
-      const language = held.cues.length > 0 ? detectLanguage(held.cues.map((c) => c.text).join("\n")) : null;
+      const codecId = held.track?.codecId ?? track?.codecId ?? "";
+      const vtt = cuesToVtt(fresh, codecId);
+      // Two things this reads, and each of them was wrong before 2.68.1.
+      //
+      // It reads the cues through `finalizeCues`, which is what turns an ASS
+      // dialogue row into the words: a raw cue carries the nine
+      // comma-separated fields of the row and its `{\…}` override groups, and
+      // those are Latin on a Russian track. Detecting on the raw text is the
+      // same fault as detecting on a whole `.ass` file, one layer down.
+      //
+      // And it reads EVERY cue held so far, not the `fresh` subset that is
+      // being sent. A re-subscription after a reconnect asks only for what this
+      // page missed, which can be three lines, and three lines are not a sample
+      // of a language.
+      const language = detectLanguage(
+        finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
+      );
       return {
         vtt,
         language,

package/services/data-channel-handler.js

@@ -590,10 +590,12 @@ export function createDataChannelHandler({
    * are tiny (kilobytes at most for a whole track), so this is one message,
    * not a stream.
    *
-   * @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string, cursor: number }} event
+   * @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[],
+   *   language: string, detectedLanguage: { code: string, name: string } | null,
+   *   cursor: number }} event
    * @returns {void}
    */
-  function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, cursor }) {
+  function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, detectedLanguage, cursor }) {
     const set = subtitleSubscribers.get(`${sourceKey}:${fileIndex}`);
     if (!set || set.size === 0) {
       log(
@@ -602,7 +604,7 @@ export function createDataChannelHandler({
       );
       return;
     }
-    const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, cursor };
+    const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, detectedLanguage, cursor };
     const total = set.size;
     let sent = 0;
     for (const channel of set) {

package/services/language-detect.js

@@ -54,20 +54,175 @@ const LANG_3_TO_1 = {
 
 const ONLY = Object.keys(LANG_3_TO_1);
 
+/**
+ * The least text, in characters, that supports an answer of each language.
+ *
+ * MEASURED, not chosen — `research/franc-boundary-2026-09-02.md`. Method:
+ * Wikipedia extracts per language (deliberately NOT the UDHR, which is what
+ * franc's own profiles are built from and would read optimistically), 120
+ * windows cut at random from them at each length of a ladder from 40 to 1300
+ * characters, and the figure recorded is the shortest length from which franc
+ * answered correctly in at least 95 % of trials AND kept doing so at every
+ * longer length measured.
+ *
+ * Why it is per language rather than one number: the answer is not equally hard
+ * to reach, and the spread is fivefold. Greek and Korean settle at 40
+ * characters because their script settles it; English needs 130; Russian and
+ * Czech need 650, because each competes with neighbours in this very list for
+ * the same trigrams — Russian with Bulgarian, Serbian and Ukrainian, Czech with
+ * Slovak.
+ *
+ * Two languages are deliberately ABSENT. Swedish and Chinese did not settle
+ * anywhere in the ladder on the corpora collected, so no figure for them is
+ * measured and none is invented; they take the fallback below.
+ *
+ * A language with no entry gets the WORST measured figure. That is the
+ * conservative reading and it is still a measurement rather than a guess: it
+ * says "no better than the hardest language we have measured".
+ *
+ * @type {Record<string, number>}
+ */
+const LEAST_TEXT = {
+  bel: 100,
+  bul: 80,
+  ces: 650,
+  deu: 200,
+  ell: 40,
+  eng: 130,
+  fra: 80,
+  heb: 60,
+  ita: 160,
+  kor: 40,
+  nld: 130,
+  pol: 200,
+  por: 200,
+  rus: 650,
+  spa: 100,
+  srp: 100,
+  tur: 160,
+  ukr: 250
+};
+
+/** The worst measured figure, used for any language not in the table. */
+const LEAST_TEXT_WORST = Math.max(...Object.values(LEAST_TEXT), 0);
+
+/** franc's own floor: below this it is not asked at all. */
+const FRANC_FLOOR = 15;
+
+/** One space between words, nothing else — the form every figure above is in. */
+function normalise(text) {
+  return typeof text === "string" ? text.replace(/\s+/g, " ").trim() : "";
+}
+
+/**
+ * franc's answer for this text, or null when it has none.
+ *
+ * @param {string} text
+ * @returns {string | null} ISO 639-3.
+ */
+function ask(text) {
+  if (text.length < FRANC_FLOOR) {
+    return null;
+  }
+  const iso3 = franc(text, { only: ONLY, minLength: FRANC_FLOOR });
+  return iso3 === "und" ? null : iso3;
+}
+
 /**
  * Best-effort detect the language of subtitle text.
  *
- * @param {string} text - Decoded subtitle text (VTT/SRT/ASS — franc ignores markup well enough).
+ * **Give this the words a viewer reads and nothing else.** franc scores letter
+ * trigrams over the whole string it is handed, so anything around the words
+ * competes with them. An ASS file is markup by half: measured 2026-09-01 on
+ * `[HorribleSubs] Drifters - 03 [1080p].ass`, 5040 Latin characters of Aegisub
+ * headers, style and font names, `Format:`/`Dialogue:` field prefixes and
+ * `{\…}` override groups against 5983 Cyrillic characters of dialogue —
+ * `franc(the file) = eng`, `franc(the dialogue) = rus`. The proxy had already
+ * built the markup-free text and detected on the file anyway, so a Russian
+ * track was offered to the viewer as English.
+ *
+ * `detectLanguageFromVtt` below is the safe entry point for a whole document;
+ * this one is for text that is already only text.
+ *
+ * @param {string} text - Subtitle text with no markup left in it.
  * @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
  */
 export function detectLanguage(text) {
-  if (typeof text !== "string" || text.trim().length < 15) {
+  const words = normalise(text);
+  if (words.length < FRANC_FLOOR) {
+    return null;
+  }
+  const candidate = ask(words);
+  if (candidate === null) {
+    return null;
+  }
+  // Enough text to support THIS answer. The figure is the language's own,
+  // because the languages are not alike: Russian shares its trigrams with
+  // Bulgarian, Serbian and Ukrainian and needs several times what English does.
+  if (words.length < (LEAST_TEXT[candidate] ?? LEAST_TEXT_WORST)) {
     return null;
   }
-  // Restrict to plausible subtitle languages; require a little text.
-  const iso3 = franc(text, { only: ONLY, minLength: 15 });
-  if (iso3 === "und") {
+  // And an answer that does not survive losing half the text was an accident of
+  // where the text happened to stop, not a reading of it. Free — franc costs
+  // about 2 ms whatever the size, measured — and it needs no figure of its own,
+  // because the test is taken on the text in hand.
+  const middle = Math.floor(words.length / 2);
+  if (ask(words.slice(0, middle)) !== candidate || ask(words.slice(middle)) !== candidate) {
     return null;
   }
-  return LANG_3_TO_1[iso3] ?? null;
+  return LANG_3_TO_1[candidate] ?? null;
+}
+
+/**
+ * The words of a WebVTT document — what a viewer reads, with everything the
+ * format puts around them removed.
+ *
+ * A WebVTT document is a series of blocks separated by blank lines. A block
+ * that holds a timing line (`00:00:12.060 --> 00:00:13.270`) is a cue, and the
+ * lines after that timing line are its text; the lines before it are the cue's
+ * optional identifier. A block with NO timing line is the `WEBVTT` header or a
+ * `NOTE` / `STYLE` / `REGION` block, and none of those is anybody's language.
+ * That one rule removes the identifiers, the timings and the headers together.
+ *
+ * What is left can still carry WebVTT's own inline markup — `<v Speaker>`,
+ * `<i>`, `<c.yellow>` — and character references. Both are dropped: a speaker
+ * name and a class name are written in whatever language the releaser's tooling
+ * used, which is not the language of the film.
+ *
+ * @param {string} vtt - A WebVTT document.
+ * @returns {string} The cue text, blocks joined by newlines.
+ */
+export function cueTextOfVtt(vtt) {
+  if (typeof vtt !== "string") {
+    return "";
+  }
+  const blocks = vtt.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split(/\n{2,}/);
+  const spoken = [];
+  for (const block of blocks) {
+    const lines = block.split("\n");
+    const timingAt = lines.findIndex((line) => line.includes("-->"));
+    if (timingAt < 0) {
+      continue;
+    }
+    for (const line of lines.slice(timingAt + 1)) {
+      spoken.push(line);
+    }
+  }
+  return spoken
+    .join("\n")
+    .replace(/<[^>]*>/g, "")
+    // A character reference stands for one character and never for a word, so a
+    // space in its place keeps the neighbouring words apart and adds nothing.
+    .replace(/&[a-z]+;|&#\d+;|&#x[0-9a-f]+;/gi, " ")
+    .trim();
+}
+
+/**
+ * Detect the language of a WebVTT document, reading only its cue text.
+ *
+ * @param {string} vtt - A WebVTT document.
+ * @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
+ */
+export function detectLanguageFromVtt(vtt) {
+  return detectLanguage(cueTextOfVtt(vtt));
 }

package/services/torrent-worker/background-fill.js

@@ -28,7 +28,7 @@
  */
 
 import { logger } from "../../utils/logger.js";
-import { readersAreBlockedOn } from "./piece-reader.js";
+import { readersAreBlockedOn, stallsSeenOn } from "./piece-reader.js";
 
 /**
  * How long to stand aside after finding the viewer's own reading blocked.
@@ -101,7 +101,7 @@ function readRange(file, start, end) {
  * @param {object} torrent
  * @param {number} fileIndex
  * @param {string} sourceKey
- * @param {{ chunkBytes?: number, isBlocked?: (infoHash: string) => boolean }} [options]
+ * @param {{ chunkBytes?: number, isBlocked?: (infoHash: string) => boolean, stallsSeen?: (infoHash: string) => number }} [options]
  *   `isBlocked` answers "is the viewer's own reading starving right now"; it is
  *   the tier boundary, and it is a parameter so it can be exercised without a
  *   swarm.
@@ -123,7 +123,8 @@ export function fillFileInBackground(torrent, fileIndex, sourceKey, options = {}
     ? options.chunkBytes
     : (Number(torrent?.pieceLength) || 4 * 1024 * 1024);
   const isBlocked = typeof options.isBlocked === "function" ? options.isBlocked : readersAreBlockedOn;
-  const work = fill(torrent, file, fileIndex, chunkBytes, isBlocked).finally(() => {
+  const stallsSeen = typeof options.stallsSeen === "function" ? options.stallsSeen : stallsSeenOn;
+  const work = fill(torrent, file, fileIndex, chunkBytes, isBlocked, stallsSeen).finally(() => {
     running.delete(key);
   });
   running.set(key, work);
@@ -136,25 +137,41 @@ export function fillFileInBackground(torrent, fileIndex, sourceKey, options = {}
  * @param {number} fileIndex
  * @param {number} chunkBytes
  * @param {(infoHash: string) => boolean} isBlocked
+ * @param {(infoHash: string) => number} stallsSeen
  * @returns {Promise<void>}
  */
-async function fill(torrent, file, fileIndex, chunkBytes, isBlocked) {
+async function fill(torrent, file, fileIndex, chunkBytes, isBlocked, stallsSeen) {
   const startedAt = Date.now();
   const infoHash = String(torrent?.infoHash ?? "");
   let read = 0;
   let stoodAsideMs = 0;
+  // The stall count this fill last saw. A chunk is fetched only when it has not
+  // moved since the previous one.
+  let quietSince = stallsSeen(infoHash);
   logger.info(
     `background-fill: "${String(file.name).slice(0, 40)}" (${(file.length / 1e6).toFixed(1)}MB) will be ` +
       "fetched whole while the viewer's own reading leaves room"
   );
   for (let start = 0; start < file.length; start += chunkBytes) {
-    // Stand aside for as long as anything the viewer is watching is waiting on
-    // the swarm. This is the tier boundary, and it is checked before every
-    // chunk rather than once at the beginning: a torrent that was healthy a
-    // moment ago is not evidence about the next second.
-    while (isBlocked(infoHash)) {
+    // Stand aside while anything the viewer is watching is waiting on the
+    // swarm — AND for as long after it as it takes for a quiet stretch to
+    // pass. Pausing only DURING a stall is not enough: on a swarm delivering
+    // exactly what the film needs, this still takes bandwidth between stalls,
+    // and the stalls themselves are the proof there was none to spare. Field
+    // 2026-08-31, the case that forced this: 200-600 KB/s delivered against
+    // the 399 KB/s the film eats, one piece waited 101 s, and the picture
+    // stood still 145.6 s.
+    //
+    // "A quiet stretch" is measured, not chosen: the stall counter must not
+    // have moved while the previous chunk was being fetched. On a starving
+    // swarm it moves constantly and this stops altogether, which is the right
+    // answer — there is no spare room to use.
+    while (isBlocked(infoHash) || stallsSeen(infoHash) !== quietSince) {
       stoodAsideMs += STAND_ASIDE_MS;
       await pause(STAND_ASIDE_MS);
+      // Re-baselined after the pause, so a stretch that passes without a new
+      // stall lets the fill go on. Without this it could never resume.
+      quietSince = stallsSeen(infoHash);
     }
     // The torrent may have been destroyed under us — a viewer who left, the
     // disk sweep, a restart. Reading a destroyed file throws, and there is
@@ -162,6 +179,11 @@ async function fill(torrent, file, fileIndex, chunkBytes, isBlocked) {
     if (!torrent?.files?.[fileIndex]) {
       return;
     }
+    // Taken BEFORE the read, and deliberately not refreshed after it: a stall
+    // that happens while this chunk is in flight must still be visible to the
+    // next iteration. Refreshing afterwards erased exactly that evidence, which
+    // is the defect a test caught here.
+    quietSince = stallsSeen(infoHash);
     const bytes = await readRange(file, start, Math.min(start + chunkBytes, file.length) - 1);
     if (bytes === 0) {
       logger.info(

package/services/torrent-worker/piece-reader.js

@@ -617,6 +617,14 @@ function describeSteering(key) {
  */
 const blockedReaders = new Map();
 
+/**
+ * How many stalls a torrent's readers have had, ever. Only differences between
+ * two readings of it mean anything.
+ *
+ * @type {Map<string, number>}
+ */
+const stallsSeen = new Map();
+
 /**
  * Whether any reader on this torrent is waiting for a piece right now.
  *
@@ -627,6 +635,25 @@ export function readersAreBlockedOn(infoHash) {
   return (blockedReaders.get(infoHash) ?? 0) > 0;
 }
 
+/**
+ * How many times a reader on this torrent has been blocked since the process
+ * started.
+ *
+ * Exists so that work of lower importance can ask "did the viewer stall while I
+ * was busy?" — which is a different and stricter question than "is the viewer
+ * stalled right now". On a swarm delivering exactly what the film needs, a
+ * background fetch that only pauses DURING a stall still takes bandwidth
+ * between them, and the stalls are the proof it had none to spare. Field
+ * 2026-08-31: the swarm delivered 200-600 KB/s against the 399 KB/s the film
+ * needs, and the picture stood still 145.6 s.
+ *
+ * @param {string} infoHash
+ * @returns {number}
+ */
+export function stallsSeenOn(infoHash) {
+  return stallsSeen.get(infoHash) ?? 0;
+}
+
 /**
  * @param {string} infoHash
  * @param {number} delta
@@ -636,6 +663,9 @@ function countBlockedReader(infoHash, delta) {
   if (!infoHash) {
     return;
   }
+  if (delta > 0) {
+    stallsSeen.set(infoHash, (stallsSeen.get(infoHash) ?? 0) + 1);
+  }
   const next = (blockedReaders.get(infoHash) ?? 0) + delta;
   if (next > 0) {
     blockedReaders.set(infoHash, next);

package/services/torrent-worker/subtitle-cues.js

@@ -20,6 +20,7 @@
 import { readSubtitlePlan, harvestCluster } from "../container-index/matroska-subtitles.js";
 import { decodeSubtitleSample, readMp4SubtitlePlan } from "../container-index/mp4-subtitles.js";
 import { iterateElements } from "../container-index/ebml-reader.js";
+import { detectLanguage } from "../language-detect.js";
 import { logger } from "../../utils/logger.js";
 
 /** Enough to read any cluster's own element header. */
@@ -483,12 +484,23 @@ export async function warmSubtitleCues(torrent, fileIndex, sourceKey) {
     }
     const highest = newCues.reduce((max, cue) => Math.max(max, Number(cue.seq) || 0), since);
     state.pushed.set(track.trackNumber, highest);
-    const cues = finalizeCues(newCues, held.track?.codecId ?? track.codecId);
+    const codecId = held.track?.codecId ?? track.codecId;
+    const cues = finalizeCues(newCues, codecId);
     fresh.push({
       // ffmpeg's own numbering, which is the only one the browser knows.
       trackIndex: Number.isInteger(track.declaredIndex) ? track.declaredIndex : order,
       cues,
       language: held.track?.language ?? "",
+      // What the CUES say the language is, re-read on every push over every cue
+      // held so far rather than over this batch. A track whose container states
+      // no language is unreadable at the start of a session — a handful of cues
+      // is not a sample of a language, and the detector refuses to answer on one
+      // — so the answer has to be re-taken as the film downloads, and the label
+      // moved when it arrives. Costs about 6 ms per push, measured; pushes
+      // arrive about once a second per file being read.
+      detectedLanguage: detectLanguage(
+        finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
+      ),
       // Where the browser should resume from if it has to ask again — after a
       // reconnect, which loses the subscription these pushes ride on.
       cursor: highest,

package/test/background-fill.test.js

@@ -154,3 +154,37 @@ test("the gate is re-asked before every chunk, not once at the start", async ()
   await waitFor(() => !fillIsRunning("source-f", 0));
   assert.equal(file.reads.length, 3);
 });
+
+test("a stall during a chunk makes the next one wait, not just a stall in progress", async () => {
+  // The case this exists for: a swarm delivering exactly what the film needs.
+  // Nothing is blocked at the moment the gate is asked, but the viewer stalled
+  // while the last chunk was in flight — which is the proof there was no room.
+  const file = fakeFile({ length: 12 });
+  const torrent = { infoHash: "ggg", pieceLength: 4, files: [file] };
+  let stalls = 0;
+  let starving = true;
+  // A stall happens DURING every chunk, exactly as it does on a swarm with no
+  // surplus — and nothing is blocked at the moment the gate is asked.
+  const originalRead = file.createReadStream.bind(file);
+  file.createReadStream = (range) => {
+    if (starving) {
+      stalls += 1;
+    }
+    return originalRead(range);
+  };
+
+  fillFileInBackground(torrent, 0, "source-g", {
+    chunkBytes: 4,
+    isBlocked: () => false,
+    stallsSeen: () => stalls
+  });
+
+  await waitFor(() => file.reads.length === 1);
+  await new Promise((resolve) => setTimeout(resolve, 300));
+  assert.equal(file.reads.length, 1, "a stall during the chunk holds the next one back");
+
+  // The swarm settles: no further stalls, so it may go on.
+  starving = false;
+  await waitFor(() => !fillIsRunning("source-g", 0), 8000);
+  assert.equal(file.reads.length, 3);
+});