@torrent-tv/proxy 2.68.0 → 2.69.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@torrent-tv/proxy",
3
- "version": "2.68.0",
3
+ "version": "2.69.1",
4
4
  "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
5
5
  "license": "GPL-3.0-or-later",
6
6
  "publishConfig": {
@@ -25,7 +25,7 @@
25
25
  */
26
26
 
27
27
  import { spawn } from "node:child_process";
28
- import { detectLanguage } from "../../../services/language-detect.js";
28
+ import { detectLanguageFromVtt } from "../../../services/language-detect.js";
29
29
  import { SubtitleController } from "../../../services/controllers/SubtitleController.js";
30
30
  import { logger } from "../../../utils/logger.js";
31
31
 
@@ -137,7 +137,7 @@ export async function handleApiSubtitlesGet(req, reply, { sourceRegistry, torren
137
137
  * Extractions by `sourceKey:fileIndex:trackIndex`, so the scan happens once per
138
138
  * track however many times it is asked for.
139
139
  *
140
- * @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: string, error?: string }>}
140
+ * @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: { code: string, name: string } | null, error?: string }>}
141
141
  */
142
142
  const extractions = new Map();
143
143
 
@@ -190,7 +190,12 @@ function startExtraction({ key, ffmpegBin, localBaseUrl, sourceKey, fileIndex, t
190
190
  logger.warn(`subtitles ${key}: nothing produced after ${seconds}s`);
191
191
  return;
192
192
  }
193
- extractions.set(key, { state: "done", body, language: detectLanguage(String(body.subarray(0, 4096))) });
193
+ // The whole document, decoded as one string, and only its cue text.
194
+ // Detecting on the first 4096 BYTES was wrong twice over: a byte cut lands
195
+ // mid-character on any non-Latin track, and most of those bytes are
196
+ // timestamps rather than words. This runs once per track in the background,
197
+ // so reading all of it costs nothing anybody waits for.
198
+ extractions.set(key, { state: "done", body, language: detectLanguageFromVtt(body.toString("utf8")) });
194
199
  logger.info(`subtitles ${key}: ${body.length} bytes in ${seconds}s`);
195
200
  };
196
201
  ffmpeg.once("close", settle);
@@ -9,7 +9,7 @@
9
9
 
10
10
  import { subtitleOrchestrator } from "../orchestrators/SubtitleOrchestrator.js";
11
11
  import { convertSubtitleToVtt, decodeSubtitleBytes } from "../subtitle-convert.js";
12
- import { detectLanguage } from "../language-detect.js";
12
+ import { detectLanguage, detectLanguageFromVtt } from "../language-detect.js";
13
13
  import { finalizeCues } from "../torrent-worker/subtitle-cues.js";
14
14
 
15
15
  const EXTERNAL_MAX_BYTES = 8 * 1024 * 1024;
@@ -75,7 +75,12 @@ export class SubtitleController {
75
75
  const text = decodeSubtitleBytes(bytes);
76
76
  const vtt = convertSubtitleToVtt(text, ext);
77
77
  if (!vtt) return { error: `Unsupported subtitle format: ${ext}`, status: 422 };
78
- return { vtt, language: detectLanguage(text), headers: {} };
78
+ // The language is read from the CONVERTED document, not from the file.
79
+ // The conversion has already dropped everything that is not the words —
80
+ // and on an ASS file that is half of it, in Latin letters, which is what
81
+ // made a Russian track answer `en` (field 2026-09-01, and the whole of
82
+ // `research/subtitle-language-ass-markup-2026-09-01.md`).
83
+ return { vtt, language: detectLanguageFromVtt(vtt), headers: {} };
79
84
  } catch (e) {
80
85
  return { error: `Could not read subtitle file: ${e?.message ?? e}`, status: 502 };
81
86
  } finally {
@@ -105,8 +110,23 @@ export class SubtitleController {
105
110
  const cursor = held.cues.reduce((h, c) => Math.max(h, Number(c.seq) || 0), 0);
106
111
  const fresh = Number.isInteger(since) ? held.cues.filter((c) => (Number(c.seq) || 0) > since)
107
112
  : Number.isFinite(after) ? held.cues.filter((c) => c.startSeconds > after) : held.cues;
108
- const vtt = cuesToVtt(fresh, held.track?.codecId ?? track?.codecId ?? "");
109
- const language = held.cues.length > 0 ? detectLanguage(held.cues.map((c) => c.text).join("\n")) : null;
113
+ const codecId = held.track?.codecId ?? track?.codecId ?? "";
114
+ const vtt = cuesToVtt(fresh, codecId);
115
+ // Two things this reads, and each of them was wrong before 2.68.1.
116
+ //
117
+ // It reads the cues through `finalizeCues`, which is what turns an ASS
118
+ // dialogue row into the words: a raw cue carries the nine
119
+ // comma-separated fields of the row and its `{\…}` override groups, and
120
+ // those are Latin on a Russian track. Detecting on the raw text is the
121
+ // same fault as detecting on a whole `.ass` file, one layer down.
122
+ //
123
+ // And it reads EVERY cue held so far, not the `fresh` subset that is
124
+ // being sent. A re-subscription after a reconnect asks only for what this
125
+ // page missed, which can be three lines, and three lines are not a sample
126
+ // of a language.
127
+ const language = detectLanguage(
128
+ finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
129
+ );
110
130
  return {
111
131
  vtt,
112
132
  language,
@@ -590,10 +590,12 @@ export function createDataChannelHandler({
590
590
  * are tiny (kilobytes at most for a whole track), so this is one message,
591
591
  * not a stream.
592
592
  *
593
- * @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string, cursor: number }} event
593
+ * @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[],
594
+ * language: string, detectedLanguage: { code: string, name: string } | null,
595
+ * cursor: number }} event
594
596
  * @returns {void}
595
597
  */
596
- function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, cursor }) {
598
+ function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, detectedLanguage, cursor }) {
597
599
  const set = subtitleSubscribers.get(`${sourceKey}:${fileIndex}`);
598
600
  if (!set || set.size === 0) {
599
601
  log(
@@ -602,7 +604,7 @@ export function createDataChannelHandler({
602
604
  );
603
605
  return;
604
606
  }
605
- const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, cursor };
607
+ const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, detectedLanguage, cursor };
606
608
  const total = set.size;
607
609
  let sent = 0;
608
610
  for (const channel of set) {
@@ -54,20 +54,175 @@ const LANG_3_TO_1 = {
54
54
 
55
55
  const ONLY = Object.keys(LANG_3_TO_1);
56
56
 
57
+ /**
58
+ * The least text, in characters, that supports an answer of each language.
59
+ *
60
+ * MEASURED, not chosen — `research/franc-boundary-2026-09-02.md`. Method:
61
+ * Wikipedia extracts per language (deliberately NOT the UDHR, which is what
62
+ * franc's own profiles are built from and would read optimistically), 120
63
+ * windows cut at random from them at each length of a ladder from 40 to 1300
64
+ * characters, and the figure recorded is the shortest length from which franc
65
+ * answered correctly in at least 95 % of trials AND kept doing so at every
66
+ * longer length measured.
67
+ *
68
+ * Why it is per language rather than one number: the answer is not equally hard
69
+ * to reach, and the spread is fivefold. Greek and Korean settle at 40
70
+ * characters because their script settles it; English needs 130; Russian and
71
+ * Czech need 650, because each competes with neighbours in this very list for
72
+ * the same trigrams — Russian with Bulgarian, Serbian and Ukrainian, Czech with
73
+ * Slovak.
74
+ *
75
+ * Two languages are deliberately ABSENT. Swedish and Chinese did not settle
76
+ * anywhere in the ladder on the corpora collected, so no figure for them is
77
+ * measured and none is invented; they take the fallback below.
78
+ *
79
+ * A language with no entry gets the WORST measured figure. That is the
80
+ * conservative reading and it is still a measurement rather than a guess: it
81
+ * says "no better than the hardest language we have measured".
82
+ *
83
+ * @type {Record<string, number>}
84
+ */
85
+ const LEAST_TEXT = {
86
+ bel: 100,
87
+ bul: 80,
88
+ ces: 650,
89
+ deu: 200,
90
+ ell: 40,
91
+ eng: 130,
92
+ fra: 80,
93
+ heb: 60,
94
+ ita: 160,
95
+ kor: 40,
96
+ nld: 130,
97
+ pol: 200,
98
+ por: 200,
99
+ rus: 650,
100
+ spa: 100,
101
+ srp: 100,
102
+ tur: 160,
103
+ ukr: 250
104
+ };
105
+
106
+ /** The worst measured figure, used for any language not in the table. */
107
+ const LEAST_TEXT_WORST = Math.max(...Object.values(LEAST_TEXT), 0);
108
+
109
+ /** franc's own floor: below this it is not asked at all. */
110
+ const FRANC_FLOOR = 15;
111
+
112
+ /** One space between words, nothing else — the form every figure above is in. */
113
+ function normalise(text) {
114
+ return typeof text === "string" ? text.replace(/\s+/g, " ").trim() : "";
115
+ }
116
+
117
+ /**
118
+ * franc's answer for this text, or null when it has none.
119
+ *
120
+ * @param {string} text
121
+ * @returns {string | null} ISO 639-3.
122
+ */
123
+ function ask(text) {
124
+ if (text.length < FRANC_FLOOR) {
125
+ return null;
126
+ }
127
+ const iso3 = franc(text, { only: ONLY, minLength: FRANC_FLOOR });
128
+ return iso3 === "und" ? null : iso3;
129
+ }
130
+
57
131
  /**
58
132
  * Best-effort detect the language of subtitle text.
59
133
  *
60
- * @param {string} text - Decoded subtitle text (VTT/SRT/ASS franc ignores markup well enough).
134
+ * **Give this the words a viewer reads and nothing else.** franc scores letter
135
+ * trigrams over the whole string it is handed, so anything around the words
136
+ * competes with them. An ASS file is markup by half: measured 2026-09-01 on
137
+ * `[HorribleSubs] Drifters - 03 [1080p].ass`, 5040 Latin characters of Aegisub
138
+ * headers, style and font names, `Format:`/`Dialogue:` field prefixes and
139
+ * `{\…}` override groups against 5983 Cyrillic characters of dialogue —
140
+ * `franc(the file) = eng`, `franc(the dialogue) = rus`. The proxy had already
141
+ * built the markup-free text and detected on the file anyway, so a Russian
142
+ * track was offered to the viewer as English.
143
+ *
144
+ * `detectLanguageFromVtt` below is the safe entry point for a whole document;
145
+ * this one is for text that is already only text.
146
+ *
147
+ * @param {string} text - Subtitle text with no markup left in it.
61
148
  * @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
62
149
  */
63
150
  export function detectLanguage(text) {
64
- if (typeof text !== "string" || text.trim().length < 15) {
151
+ const words = normalise(text);
152
+ if (words.length < FRANC_FLOOR) {
153
+ return null;
154
+ }
155
+ const candidate = ask(words);
156
+ if (candidate === null) {
157
+ return null;
158
+ }
159
+ // Enough text to support THIS answer. The figure is the language's own,
160
+ // because the languages are not alike: Russian shares its trigrams with
161
+ // Bulgarian, Serbian and Ukrainian and needs several times what English does.
162
+ if (words.length < (LEAST_TEXT[candidate] ?? LEAST_TEXT_WORST)) {
65
163
  return null;
66
164
  }
67
- // Restrict to plausible subtitle languages; require a little text.
68
- const iso3 = franc(text, { only: ONLY, minLength: 15 });
69
- if (iso3 === "und") {
165
+ // And an answer that does not survive losing half the text was an accident of
166
+ // where the text happened to stop, not a reading of it. Free — franc costs
167
+ // about 2 ms whatever the size, measured — and it needs no figure of its own,
168
+ // because the test is taken on the text in hand.
169
+ const middle = Math.floor(words.length / 2);
170
+ if (ask(words.slice(0, middle)) !== candidate || ask(words.slice(middle)) !== candidate) {
70
171
  return null;
71
172
  }
72
- return LANG_3_TO_1[iso3] ?? null;
173
+ return LANG_3_TO_1[candidate] ?? null;
174
+ }
175
+
176
+ /**
177
+ * The words of a WebVTT document — what a viewer reads, with everything the
178
+ * format puts around them removed.
179
+ *
180
+ * A WebVTT document is a series of blocks separated by blank lines. A block
181
+ * that holds a timing line (`00:00:12.060 --> 00:00:13.270`) is a cue, and the
182
+ * lines after that timing line are its text; the lines before it are the cue's
183
+ * optional identifier. A block with NO timing line is the `WEBVTT` header or a
184
+ * `NOTE` / `STYLE` / `REGION` block, and none of those is anybody's language.
185
+ * That one rule removes the identifiers, the timings and the headers together.
186
+ *
187
+ * What is left can still carry WebVTT's own inline markup — `<v Speaker>`,
188
+ * `<i>`, `<c.yellow>` — and character references. Both are dropped: a speaker
189
+ * name and a class name are written in whatever language the releaser's tooling
190
+ * used, which is not the language of the film.
191
+ *
192
+ * @param {string} vtt - A WebVTT document.
193
+ * @returns {string} The cue text, blocks joined by newlines.
194
+ */
195
+ export function cueTextOfVtt(vtt) {
196
+ if (typeof vtt !== "string") {
197
+ return "";
198
+ }
199
+ const blocks = vtt.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split(/\n{2,}/);
200
+ const spoken = [];
201
+ for (const block of blocks) {
202
+ const lines = block.split("\n");
203
+ const timingAt = lines.findIndex((line) => line.includes("-->"));
204
+ if (timingAt < 0) {
205
+ continue;
206
+ }
207
+ for (const line of lines.slice(timingAt + 1)) {
208
+ spoken.push(line);
209
+ }
210
+ }
211
+ return spoken
212
+ .join("\n")
213
+ .replace(/<[^>]*>/g, "")
214
+ // A character reference stands for one character and never for a word, so a
215
+ // space in its place keeps the neighbouring words apart and adds nothing.
216
+ .replace(/&[a-z]+;|&#\d+;|&#x[0-9a-f]+;/gi, " ")
217
+ .trim();
218
+ }
219
+
220
+ /**
221
+ * Detect the language of a WebVTT document, reading only its cue text.
222
+ *
223
+ * @param {string} vtt - A WebVTT document.
224
+ * @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
225
+ */
226
+ export function detectLanguageFromVtt(vtt) {
227
+ return detectLanguage(cueTextOfVtt(vtt));
73
228
  }
@@ -291,6 +291,12 @@ function megabytes(bytes) {
291
291
  * @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
292
292
  * @param {number | null} [reading.diskFreeBytes]
293
293
  * @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
294
+ * @param {string} [reading.extra] - Figures the caller wants on the same line
295
+ * rather than on one of its own. The piece-buffer counters are read here so
296
+ * that the number of buffers alive and the off-heap mass they should account
297
+ * for are the SAME instant: printed on separate timers they were up to a
298
+ * minute apart, and 950 MB of `arrayBuffers` could not be checked against the
299
+ * 62 buffers a reading half a minute away said were alive (roadmap item 2).
294
300
  * @returns {string}
295
301
  */
296
302
  export function describeMemory({
@@ -302,7 +308,8 @@ export function describeMemory({
302
308
  anonymousBytes = null,
303
309
  mappings = null,
304
310
  diskFreeBytes = null,
305
- stores = []
311
+ stores = [],
312
+ extra = ""
306
313
  }) {
307
314
  const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
308
315
  const storeResident = total("residentBytes");
@@ -321,8 +328,9 @@ export function describeMemory({
321
328
  `heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)}` +
322
329
  `${usage.heapLimit ? ` of ${megabytes(usage.heapLimit)} allowed` : ""} ` +
323
330
  `external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
331
+ const tail = extra ? `; ${extra}` : "";
324
332
  if (scope === "thread") {
325
- return `memory (${label || "thread"}): ${isolate}; ${storesPart}`;
333
+ return `memory (${label || "thread"}): ${isolate}; ${storesPart}${tail}`;
326
334
  }
327
335
  const shape = mappings === null
328
336
  ? ""
@@ -337,10 +345,41 @@ export function describeMemory({
337
345
  `${storesPart}; ` +
338
346
  `machine has ${megabytes(availableBytes ?? 0)} available` +
339
347
  `${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
340
- `${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}`
348
+ `${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}` +
349
+ tail
341
350
  );
342
351
  }
343
352
 
353
+ /**
354
+ * The figures whose movement earns a line, each under its own name.
355
+ *
356
+ * Not the same question as what the scope WATCHES for a heap snapshot. A
357
+ * thread's heap is only one of the three ways its isolate holds memory, and on
358
+ * 2026-09-02 it was the one that did not move: `heapTotal` stayed at 31-173 MB
359
+ * through a session where `arrayBuffers` swung between 130 and 950 MB, so the
360
+ * change trigger was watching the one quantity that stood still and every
361
+ * reading of the one that grew came out on the quiet interval, a minute apart.
362
+ *
363
+ * Each figure is compared against its own last written value and any one of
364
+ * them moving is enough. Nothing is added together, because `arrayBuffers` is
365
+ * documented as part of `external` and reads larger than it on this runtime —
366
+ * a contradiction this code has no business resolving.
367
+ *
368
+ * @param {"process" | "thread"} scope
369
+ * @param {{ rss: number, heapTotal: number, external: number, arrayBuffers: number }} memory
370
+ * @returns {Record<string, number>}
371
+ */
372
+ export function watchedFigures(scope, memory) {
373
+ if (scope === "thread") {
374
+ return {
375
+ heap: memory.heapTotal ?? 0,
376
+ external: memory.external ?? 0,
377
+ buffers: memory.arrayBuffers ?? 0
378
+ };
379
+ }
380
+ return { rss: memory.rss ?? 0 };
381
+ }
382
+
344
383
  /**
345
384
  * Whether this reading is worth writing down, and why.
346
385
  *
@@ -397,11 +436,14 @@ export function readingIsWorthWriting({
397
436
  * @param {number} [options.snapshotFloorBytes]
398
437
  * @param {number} [options.snapshotGrowthBytes]
399
438
  * @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
439
+ * @param {() => string} [options.readExtra] - Figures to append to the line,
440
+ * read at the same instant as the memory itself.
400
441
  * @returns {{ stop: () => void }}
401
442
  */
402
443
  export function startMemoryReport({
403
444
  log,
404
445
  readStores,
446
+ readExtra,
405
447
  scope = "process",
406
448
  label = "",
407
449
  diskPath = "",
@@ -414,7 +456,8 @@ export function startMemoryReport({
414
456
  keepSnapshots = 0
415
457
  }) {
416
458
  let highWater = 0;
417
- let lastWrittenBytes = 0;
459
+ /** @type {Record<string, number>} */
460
+ let lastWritten = {};
418
461
  let lastWrittenAt = 0;
419
462
  // The process watches what the kernel kills it for; a thread watches what the
420
463
  // runtime kills IT for, which is its own heap and not the process's resident
@@ -472,25 +515,35 @@ export function startMemoryReport({
472
515
  }
473
516
  const processMemory = readProcessMemory();
474
517
  const watched = watchedOf(processMemory);
518
+ const figures = watchedFigures(scope, processMemory);
475
519
  const now = Date.now();
476
- const write = readingIsWorthWriting({
477
- watchedBytes: watched,
478
- lastWrittenBytes,
479
- sinceWrittenMs: lastWrittenAt === 0 ? Number.POSITIVE_INFINITY : now - lastWrittenAt,
520
+ const sinceWrittenMs = lastWrittenAt === 0 ? Number.POSITIVE_INFINITY : now - lastWrittenAt;
521
+ const write = Object.entries(figures).some(([name, bytes]) => readingIsWorthWriting({
522
+ watchedBytes: bytes,
523
+ lastWrittenBytes: lastWritten[name] ?? 0,
524
+ sinceWrittenMs,
480
525
  changeBytes,
481
526
  quietMs
482
- });
527
+ }));
483
528
 
484
529
  let anonymousBytes = null;
485
- if (scope === "thread") {
486
- if (write) {
487
- log(describeMemory({ scope, label, process: processMemory, stores }));
530
+ if (write) {
531
+ let extra = "";
532
+ try {
533
+ extra = typeof readExtra === "function" ? readExtra() ?? "" : "";
534
+ } catch {
535
+ // silent-ok: a caller's own figures are worth less than the line they
536
+ // would have taken down with them.
488
537
  }
489
- } else {
490
- const { bytes, measured } = await availableMemory();
491
- anonymousBytes = await readAnonymousMemory();
492
- const diskFreeBytes = diskPath ? await readDiskFree(diskPath) : null;
493
- if (write) {
538
+ if (scope === "thread") {
539
+ log(describeMemory({ scope, label, process: processMemory, stores, extra }));
540
+ } else {
541
+ // Read only when the line is written. `smaps` is one entry per
542
+ // mapping and a busy process has thousands; the rollup and
543
+ // `/proc/meminfo` are single lines but still walk page tables, and
544
+ // the reading now happens once a second rather than once a minute.
545
+ const { bytes, measured } = await availableMemory();
546
+ anonymousBytes = await readAnonymousMemory();
494
547
  log(describeMemory({
495
548
  scope,
496
549
  label,
@@ -498,17 +551,13 @@ export function startMemoryReport({
498
551
  availableBytes: bytes,
499
552
  availableMeasured: measured,
500
553
  anonymousBytes,
501
- // Read only when the line is written: `smaps` is one entry per
502
- // mapping and a busy process has thousands, which is a different
503
- // cost from the rollup's single line.
504
554
  mappings: await readMappingSummary(),
505
- diskFreeBytes,
506
- stores
555
+ diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
556
+ stores,
557
+ extra
507
558
  }));
508
559
  }
509
- }
510
- if (write) {
511
- lastWrittenBytes = watched;
560
+ lastWritten = figures;
512
561
  lastWrittenAt = now;
513
562
  }
514
563
 
@@ -524,6 +573,9 @@ export function startMemoryReport({
524
573
  highWater = watched;
525
574
  await takeSnapshot(watched, "a new high-water");
526
575
  }
576
+ // Under `write` by construction: `anonymousBytes` is only read when the
577
+ // line is, and at one reading a second an unconditional warning would be
578
+ // a line a second for as long as the process stayed large.
527
579
  if (scope !== "thread" && anonymousBytes !== null && processMemory.rss > 800 * 1024 * 1024) {
528
580
  log(`memory: high rss=${megabytes(processMemory.rss)} anon=${megabytes(anonymousBytes)} heap=${megabytes(processMemory.heapUsed)} — watch for OOM`);
529
581
  }
@@ -20,6 +20,7 @@
20
20
  import { readSubtitlePlan, harvestCluster } from "../container-index/matroska-subtitles.js";
21
21
  import { decodeSubtitleSample, readMp4SubtitlePlan } from "../container-index/mp4-subtitles.js";
22
22
  import { iterateElements } from "../container-index/ebml-reader.js";
23
+ import { detectLanguage } from "../language-detect.js";
23
24
  import { logger } from "../../utils/logger.js";
24
25
 
25
26
  /** Enough to read any cluster's own element header. */
@@ -483,12 +484,23 @@ export async function warmSubtitleCues(torrent, fileIndex, sourceKey) {
483
484
  }
484
485
  const highest = newCues.reduce((max, cue) => Math.max(max, Number(cue.seq) || 0), since);
485
486
  state.pushed.set(track.trackNumber, highest);
486
- const cues = finalizeCues(newCues, held.track?.codecId ?? track.codecId);
487
+ const codecId = held.track?.codecId ?? track.codecId;
488
+ const cues = finalizeCues(newCues, codecId);
487
489
  fresh.push({
488
490
  // ffmpeg's own numbering, which is the only one the browser knows.
489
491
  trackIndex: Number.isInteger(track.declaredIndex) ? track.declaredIndex : order,
490
492
  cues,
491
493
  language: held.track?.language ?? "",
494
+ // What the CUES say the language is, re-read on every push over every cue
495
+ // held so far rather than over this batch. A track whose container states
496
+ // no language is unreadable at the start of a session — a handful of cues
497
+ // is not a sample of a language, and the detector refuses to answer on one
498
+ // — so the answer has to be re-taken as the film downloads, and the label
499
+ // moved when it arrives. Costs about 6 ms per push, measured; pushes
500
+ // arrive about once a second per file being read.
501
+ detectedLanguage: detectLanguage(
502
+ finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
503
+ ),
492
504
  // Where the browser should resume from if it has to ask again — after a
493
505
  // reconnect, which loses the subscription these pushes ride on.
494
506
  cursor: highest,
@@ -522,6 +522,12 @@ parentPort.on("message", async (message) => {
522
522
  startMemoryReport({
523
523
  log,
524
524
  readStores: collectStoreStats,
525
+ // Beside the isolate's own figures, and on the SAME line, because the
526
+ // question they answer together is whether the off-heap mass is buffers this
527
+ // thread still refers to or buffers the collector has not reached yet. On
528
+ // separate timers the two were up to a minute apart and could not be
529
+ // compared at all (roadmap item 2).
530
+ readExtra: describePieceBuffers,
525
531
  scope: "thread",
526
532
  label: "torrent worker",
527
533
  intervalMs: WORKER_MEMORY_SAMPLE_MS,
@@ -581,22 +587,30 @@ setInterval(() => {
581
587
  (stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
582
588
  );
583
589
  }
584
- // Not per store: the collector is per thread, and the question it answers —
585
- // is anything of ours outliving a piece — is about the thread. Printed
586
- // whenever the two disagree by more than the pieces actually held, which is
587
- // the only shape worth looking at (roadmap item 2).
590
+ }, STORE_REPORT_INTERVAL_MS).unref();
591
+
592
+ /**
593
+ * The piece buffers this thread has let go of against the ones it still holds.
594
+ *
595
+ * Not per store: the collector is per thread, and the question is about the
596
+ * thread. A gap that keeps widening means a reference of ours outlives the
597
+ * piece; a gap that does not means whatever grows is below us, in the
598
+ * allocator or in buffers the collector has not reached.
599
+ *
600
+ * @returns {string}
601
+ */
602
+ function describePieceBuffers() {
588
603
  const collection = pieceBufferCollection();
589
- const outstandingBuffers = collection.released - collection.collected;
590
- const heldNow = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
591
- if (outstandingBuffers > heldNow) {
592
- log(
593
- `piece buffers: ${collection.released} let go, ${collection.collected} collected — ` +
594
- `${outstandingBuffers} still alive against ${heldNow} the store holds. ` +
595
- "A gap that keeps widening means a reference of ours outlives the piece; " +
596
- "a gap that does not means whatever grows is below us, in the allocator"
597
- );
604
+ if (collection.released === 0) {
605
+ return "";
598
606
  }
599
- }, STORE_REPORT_INTERVAL_MS).unref();
607
+ const alive = collection.released - collection.collected;
608
+ const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
609
+ return (
610
+ `piece buffers ${collection.released} let go, ${collection.collected} collected, ` +
611
+ `${alive} still alive against ${held} the store holds`
612
+ );
613
+ }
600
614
 
601
615
  /**
602
616
  * Walk subtitle cues for every actively-read file of one torrent, and PUSH
@@ -1,7 +1,12 @@
1
1
  import test from "node:test";
2
2
  import assert from "node:assert/strict";
3
3
 
4
- import { describeMemory, readingIsWorthWriting, summariseMappings } from "../services/memory-report.js";
4
+ import {
5
+ describeMemory,
6
+ readingIsWorthWriting,
7
+ summariseMappings,
8
+ watchedFigures
9
+ } from "../services/memory-report.js";
5
10
  import {
6
11
  budgetForNewStore,
7
12
  SharedPieceStore,
@@ -287,3 +292,55 @@ test("the mapping shape is said in the line, and left out when it is not known",
287
292
  });
288
293
  assert.ok(!withoutShape.includes("mappings="), "an unread breakdown is absent, not zeroed");
289
294
  });
295
+
296
+ test("a thread's line is earned by any of its three figures, not by the heap alone", () => {
297
+ // 2026-09-02, the session that ended in two out-of-memory kills: the worker's
298
+ // heap stood at 33-45 MB for the whole of it while its buffers went from 130
299
+ // to 950 MB. Watching the heap alone, nothing ever earned a line and every
300
+ // reading of the quantity that grew came out on the quiet minute.
301
+ const before = watchedFigures("thread", {
302
+ rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 130 * MEGABYTE
303
+ });
304
+ const after = watchedFigures("thread", {
305
+ rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 515 * MEGABYTE
306
+ });
307
+ assert.equal(before.heap, after.heap, "the heap is what did NOT move");
308
+
309
+ const moved = Object.entries(after).some(([name, bytes]) => readingIsWorthWriting({
310
+ watchedBytes: bytes,
311
+ lastWrittenBytes: before[name],
312
+ sinceWrittenMs: 1_000,
313
+ changeBytes: 25 * MEGABYTE,
314
+ quietMs: 60_000
315
+ }));
316
+ assert.equal(moved, true, "buffers growing by 385 MB earns a line of its own");
317
+
318
+ assert.deepEqual(
319
+ Object.keys(watchedFigures("process", { rss: 5, heapTotal: 1, external: 2, arrayBuffers: 3 })),
320
+ ["rss"],
321
+ "a process is killed for its resident memory and watches that"
322
+ );
323
+ });
324
+
325
+ test("figures a caller reads for itself land on the same line as the memory", () => {
326
+ const line = describeMemory({
327
+ scope: "thread",
328
+ label: "torrent worker",
329
+ process: {
330
+ rss: 0, heapUsed: 34 * MEGABYTE, heapTotal: 46 * MEGABYTE,
331
+ external: 54 * MEGABYTE, arrayBuffers: 393 * MEGABYTE, heapLimit: 0
332
+ },
333
+ stores: [],
334
+ extra: "piece buffers 21544 let go, 21482 collected, 62 still alive against 53 the store holds"
335
+ });
336
+ assert.match(line, /arrayBuffers=393MB/);
337
+ assert.match(line, /62 still alive against 53 the store holds$/);
338
+ assert.doesNotMatch(
339
+ describeMemory({
340
+ scope: "thread",
341
+ process: { rss: 0, heapUsed: 1, heapTotal: 2, external: 3, arrayBuffers: 4, heapLimit: 0 }
342
+ }),
343
+ /;\s*$/,
344
+ "nothing to add leaves no dangling separator"
345
+ );
346
+ });