@torrent-tv/proxy 2.68.0 → 2.69.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/bin/cli.js +544 -533
- package/corpus.tmp.json +1 -0
- package/package.json +1 -1
- package/research/valgrind-replica-2026-08-27/logs.tar.gz +0 -0
- package/routes/api/subtitles/get.js +8 -3
- package/services/controllers/SubtitleController.js +24 -4
- package/services/data-channel-handler.js +5 -3
- package/services/language-detect.js +161 -6
- package/services/memory-report.js +77 -25
- package/services/torrent-worker/subtitle-cues.js +13 -1
- package/services/torrent-worker/worker.js +28 -14
- package/test/memory-budget.test.js +58 -1
- package/test/subtitle-language.test.js +241 -0
- package/test/worker-channel.test.js +9 -1
package/package.json
CHANGED
|
Binary file
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
import { spawn } from "node:child_process";
|
|
28
|
-
import {
|
|
28
|
+
import { detectLanguageFromVtt } from "../../../services/language-detect.js";
|
|
29
29
|
import { SubtitleController } from "../../../services/controllers/SubtitleController.js";
|
|
30
30
|
import { logger } from "../../../utils/logger.js";
|
|
31
31
|
|
|
@@ -137,7 +137,7 @@ export async function handleApiSubtitlesGet(req, reply, { sourceRegistry, torren
|
|
|
137
137
|
* Extractions by `sourceKey:fileIndex:trackIndex`, so the scan happens once per
|
|
138
138
|
* track however many times it is asked for.
|
|
139
139
|
*
|
|
140
|
-
* @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: string, error?: string }>}
|
|
140
|
+
* @type {Map<string, { state: "running" | "done" | "failed", body?: Buffer, language?: { code: string, name: string } | null, error?: string }>}
|
|
141
141
|
*/
|
|
142
142
|
const extractions = new Map();
|
|
143
143
|
|
|
@@ -190,7 +190,12 @@ function startExtraction({ key, ffmpegBin, localBaseUrl, sourceKey, fileIndex, t
|
|
|
190
190
|
logger.warn(`subtitles ${key}: nothing produced after ${seconds}s`);
|
|
191
191
|
return;
|
|
192
192
|
}
|
|
193
|
-
|
|
193
|
+
// The whole document, decoded as one string, and only its cue text.
|
|
194
|
+
// Detecting on the first 4096 BYTES was wrong twice over: a byte cut lands
|
|
195
|
+
// mid-character on any non-Latin track, and most of those bytes are
|
|
196
|
+
// timestamps rather than words. This runs once per track in the background,
|
|
197
|
+
// so reading all of it costs nothing anybody waits for.
|
|
198
|
+
extractions.set(key, { state: "done", body, language: detectLanguageFromVtt(body.toString("utf8")) });
|
|
194
199
|
logger.info(`subtitles ${key}: ${body.length} bytes in ${seconds}s`);
|
|
195
200
|
};
|
|
196
201
|
ffmpeg.once("close", settle);
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { subtitleOrchestrator } from "../orchestrators/SubtitleOrchestrator.js";
|
|
11
11
|
import { convertSubtitleToVtt, decodeSubtitleBytes } from "../subtitle-convert.js";
|
|
12
|
-
import { detectLanguage } from "../language-detect.js";
|
|
12
|
+
import { detectLanguage, detectLanguageFromVtt } from "../language-detect.js";
|
|
13
13
|
import { finalizeCues } from "../torrent-worker/subtitle-cues.js";
|
|
14
14
|
|
|
15
15
|
const EXTERNAL_MAX_BYTES = 8 * 1024 * 1024;
|
|
@@ -75,7 +75,12 @@ export class SubtitleController {
|
|
|
75
75
|
const text = decodeSubtitleBytes(bytes);
|
|
76
76
|
const vtt = convertSubtitleToVtt(text, ext);
|
|
77
77
|
if (!vtt) return { error: `Unsupported subtitle format: ${ext}`, status: 422 };
|
|
78
|
-
|
|
78
|
+
// The language is read from the CONVERTED document, not from the file.
|
|
79
|
+
// The conversion has already dropped everything that is not the words —
|
|
80
|
+
// and on an ASS file that is half of it, in Latin letters, which is what
|
|
81
|
+
// made a Russian track answer `en` (field 2026-09-01, and the whole of
|
|
82
|
+
// `research/subtitle-language-ass-markup-2026-09-01.md`).
|
|
83
|
+
return { vtt, language: detectLanguageFromVtt(vtt), headers: {} };
|
|
79
84
|
} catch (e) {
|
|
80
85
|
return { error: `Could not read subtitle file: ${e?.message ?? e}`, status: 502 };
|
|
81
86
|
} finally {
|
|
@@ -105,8 +110,23 @@ export class SubtitleController {
|
|
|
105
110
|
const cursor = held.cues.reduce((h, c) => Math.max(h, Number(c.seq) || 0), 0);
|
|
106
111
|
const fresh = Number.isInteger(since) ? held.cues.filter((c) => (Number(c.seq) || 0) > since)
|
|
107
112
|
: Number.isFinite(after) ? held.cues.filter((c) => c.startSeconds > after) : held.cues;
|
|
108
|
-
const
|
|
109
|
-
const
|
|
113
|
+
const codecId = held.track?.codecId ?? track?.codecId ?? "";
|
|
114
|
+
const vtt = cuesToVtt(fresh, codecId);
|
|
115
|
+
// Two things this reads, and each of them was wrong before 2.68.1.
|
|
116
|
+
//
|
|
117
|
+
// It reads the cues through `finalizeCues`, which is what turns an ASS
|
|
118
|
+
// dialogue row into the words: a raw cue carries the nine
|
|
119
|
+
// comma-separated fields of the row and its `{\…}` override groups, and
|
|
120
|
+
// those are Latin on a Russian track. Detecting on the raw text is the
|
|
121
|
+
// same fault as detecting on a whole `.ass` file, one layer down.
|
|
122
|
+
//
|
|
123
|
+
// And it reads EVERY cue held so far, not the `fresh` subset that is
|
|
124
|
+
// being sent. A re-subscription after a reconnect asks only for what this
|
|
125
|
+
// page missed, which can be three lines, and three lines are not a sample
|
|
126
|
+
// of a language.
|
|
127
|
+
const language = detectLanguage(
|
|
128
|
+
finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
|
|
129
|
+
);
|
|
110
130
|
return {
|
|
111
131
|
vtt,
|
|
112
132
|
language,
|
|
@@ -590,10 +590,12 @@ export function createDataChannelHandler({
|
|
|
590
590
|
* are tiny (kilobytes at most for a whole track), so this is one message,
|
|
591
591
|
* not a stream.
|
|
592
592
|
*
|
|
593
|
-
* @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[],
|
|
593
|
+
* @param {{ sourceKey: string, fileIndex: number, trackIndex: number, cues: object[],
|
|
594
|
+
* language: string, detectedLanguage: { code: string, name: string } | null,
|
|
595
|
+
* cursor: number }} event
|
|
594
596
|
* @returns {void}
|
|
595
597
|
*/
|
|
596
|
-
function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, cursor }) {
|
|
598
|
+
function publishSubtitleCues({ sourceKey, fileIndex, trackIndex, cues, language, detectedLanguage, cursor }) {
|
|
597
599
|
const set = subtitleSubscribers.get(`${sourceKey}:${fileIndex}`);
|
|
598
600
|
if (!set || set.size === 0) {
|
|
599
601
|
log(
|
|
@@ -602,7 +604,7 @@ export function createDataChannelHandler({
|
|
|
602
604
|
);
|
|
603
605
|
return;
|
|
604
606
|
}
|
|
605
|
-
const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, cursor };
|
|
607
|
+
const message = { type: "subtitle-cues", fileIndex, trackIndex, cues, language, detectedLanguage, cursor };
|
|
606
608
|
const total = set.size;
|
|
607
609
|
let sent = 0;
|
|
608
610
|
for (const channel of set) {
|
|
@@ -54,20 +54,175 @@ const LANG_3_TO_1 = {
|
|
|
54
54
|
|
|
55
55
|
const ONLY = Object.keys(LANG_3_TO_1);
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The least text, in characters, that supports an answer of each language.
|
|
59
|
+
*
|
|
60
|
+
* MEASURED, not chosen — `research/franc-boundary-2026-09-02.md`. Method:
|
|
61
|
+
* Wikipedia extracts per language (deliberately NOT the UDHR, which is what
|
|
62
|
+
* franc's own profiles are built from and would read optimistically), 120
|
|
63
|
+
* windows cut at random from them at each length of a ladder from 40 to 1300
|
|
64
|
+
* characters, and the figure recorded is the shortest length from which franc
|
|
65
|
+
* answered correctly in at least 95 % of trials AND kept doing so at every
|
|
66
|
+
* longer length measured.
|
|
67
|
+
*
|
|
68
|
+
* Why it is per language rather than one number: the answer is not equally hard
|
|
69
|
+
* to reach, and the spread is fivefold. Greek and Korean settle at 40
|
|
70
|
+
* characters because their script settles it; English needs 130; Russian and
|
|
71
|
+
* Czech need 650, because each competes with neighbours in this very list for
|
|
72
|
+
* the same trigrams — Russian with Bulgarian, Serbian and Ukrainian, Czech with
|
|
73
|
+
* Slovak.
|
|
74
|
+
*
|
|
75
|
+
* Two languages are deliberately ABSENT. Swedish and Chinese did not settle
|
|
76
|
+
* anywhere in the ladder on the corpora collected, so no figure for them is
|
|
77
|
+
* measured and none is invented; they take the fallback below.
|
|
78
|
+
*
|
|
79
|
+
* A language with no entry gets the WORST measured figure. That is the
|
|
80
|
+
* conservative reading and it is still a measurement rather than a guess: it
|
|
81
|
+
* says "no better than the hardest language we have measured".
|
|
82
|
+
*
|
|
83
|
+
* @type {Record<string, number>}
|
|
84
|
+
*/
|
|
85
|
+
const LEAST_TEXT = {
|
|
86
|
+
bel: 100,
|
|
87
|
+
bul: 80,
|
|
88
|
+
ces: 650,
|
|
89
|
+
deu: 200,
|
|
90
|
+
ell: 40,
|
|
91
|
+
eng: 130,
|
|
92
|
+
fra: 80,
|
|
93
|
+
heb: 60,
|
|
94
|
+
ita: 160,
|
|
95
|
+
kor: 40,
|
|
96
|
+
nld: 130,
|
|
97
|
+
pol: 200,
|
|
98
|
+
por: 200,
|
|
99
|
+
rus: 650,
|
|
100
|
+
spa: 100,
|
|
101
|
+
srp: 100,
|
|
102
|
+
tur: 160,
|
|
103
|
+
ukr: 250
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** The worst measured figure, used for any language not in the table. */
|
|
107
|
+
const LEAST_TEXT_WORST = Math.max(...Object.values(LEAST_TEXT), 0);
|
|
108
|
+
|
|
109
|
+
/** franc's own floor: below this it is not asked at all. */
|
|
110
|
+
const FRANC_FLOOR = 15;
|
|
111
|
+
|
|
112
|
+
/** One space between words, nothing else — the form every figure above is in. */
|
|
113
|
+
function normalise(text) {
|
|
114
|
+
return typeof text === "string" ? text.replace(/\s+/g, " ").trim() : "";
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* franc's answer for this text, or null when it has none.
|
|
119
|
+
*
|
|
120
|
+
* @param {string} text
|
|
121
|
+
* @returns {string | null} ISO 639-3.
|
|
122
|
+
*/
|
|
123
|
+
function ask(text) {
|
|
124
|
+
if (text.length < FRANC_FLOOR) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
const iso3 = franc(text, { only: ONLY, minLength: FRANC_FLOOR });
|
|
128
|
+
return iso3 === "und" ? null : iso3;
|
|
129
|
+
}
|
|
130
|
+
|
|
57
131
|
/**
|
|
58
132
|
* Best-effort detect the language of subtitle text.
|
|
59
133
|
*
|
|
60
|
-
*
|
|
134
|
+
* **Give this the words a viewer reads and nothing else.** franc scores letter
|
|
135
|
+
* trigrams over the whole string it is handed, so anything around the words
|
|
136
|
+
* competes with them. An ASS file is markup by half: measured 2026-09-01 on
|
|
137
|
+
* `[HorribleSubs] Drifters - 03 [1080p].ass`, 5040 Latin characters of Aegisub
|
|
138
|
+
* headers, style and font names, `Format:`/`Dialogue:` field prefixes and
|
|
139
|
+
* `{\…}` override groups against 5983 Cyrillic characters of dialogue —
|
|
140
|
+
* `franc(the file) = eng`, `franc(the dialogue) = rus`. The proxy had already
|
|
141
|
+
* built the markup-free text and detected on the file anyway, so a Russian
|
|
142
|
+
* track was offered to the viewer as English.
|
|
143
|
+
*
|
|
144
|
+
* `detectLanguageFromVtt` below is the safe entry point for a whole document;
|
|
145
|
+
* this one is for text that is already only text.
|
|
146
|
+
*
|
|
147
|
+
* @param {string} text - Subtitle text with no markup left in it.
|
|
61
148
|
* @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
|
|
62
149
|
*/
|
|
63
150
|
export function detectLanguage(text) {
|
|
64
|
-
|
|
151
|
+
const words = normalise(text);
|
|
152
|
+
if (words.length < FRANC_FLOOR) {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
const candidate = ask(words);
|
|
156
|
+
if (candidate === null) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
// Enough text to support THIS answer. The figure is the language's own,
|
|
160
|
+
// because the languages are not alike: Russian shares its trigrams with
|
|
161
|
+
// Bulgarian, Serbian and Ukrainian and needs several times what English does.
|
|
162
|
+
if (words.length < (LEAST_TEXT[candidate] ?? LEAST_TEXT_WORST)) {
|
|
65
163
|
return null;
|
|
66
164
|
}
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
165
|
+
// And an answer that does not survive losing half the text was an accident of
|
|
166
|
+
// where the text happened to stop, not a reading of it. Free — franc costs
|
|
167
|
+
// about 2 ms whatever the size, measured — and it needs no figure of its own,
|
|
168
|
+
// because the test is taken on the text in hand.
|
|
169
|
+
const middle = Math.floor(words.length / 2);
|
|
170
|
+
if (ask(words.slice(0, middle)) !== candidate || ask(words.slice(middle)) !== candidate) {
|
|
70
171
|
return null;
|
|
71
172
|
}
|
|
72
|
-
return LANG_3_TO_1[
|
|
173
|
+
return LANG_3_TO_1[candidate] ?? null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The words of a WebVTT document — what a viewer reads, with everything the
|
|
178
|
+
* format puts around them removed.
|
|
179
|
+
*
|
|
180
|
+
* A WebVTT document is a series of blocks separated by blank lines. A block
|
|
181
|
+
* that holds a timing line (`00:00:12.060 --> 00:00:13.270`) is a cue, and the
|
|
182
|
+
* lines after that timing line are its text; the lines before it are the cue's
|
|
183
|
+
* optional identifier. A block with NO timing line is the `WEBVTT` header or a
|
|
184
|
+
* `NOTE` / `STYLE` / `REGION` block, and none of those is anybody's language.
|
|
185
|
+
* That one rule removes the identifiers, the timings and the headers together.
|
|
186
|
+
*
|
|
187
|
+
* What is left can still carry WebVTT's own inline markup — `<v Speaker>`,
|
|
188
|
+
* `<i>`, `<c.yellow>` — and character references. Both are dropped: a speaker
|
|
189
|
+
* name and a class name are written in whatever language the releaser's tooling
|
|
190
|
+
* used, which is not the language of the film.
|
|
191
|
+
*
|
|
192
|
+
* @param {string} vtt - A WebVTT document.
|
|
193
|
+
* @returns {string} The cue text, blocks joined by newlines.
|
|
194
|
+
*/
|
|
195
|
+
export function cueTextOfVtt(vtt) {
|
|
196
|
+
if (typeof vtt !== "string") {
|
|
197
|
+
return "";
|
|
198
|
+
}
|
|
199
|
+
const blocks = vtt.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split(/\n{2,}/);
|
|
200
|
+
const spoken = [];
|
|
201
|
+
for (const block of blocks) {
|
|
202
|
+
const lines = block.split("\n");
|
|
203
|
+
const timingAt = lines.findIndex((line) => line.includes("-->"));
|
|
204
|
+
if (timingAt < 0) {
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
for (const line of lines.slice(timingAt + 1)) {
|
|
208
|
+
spoken.push(line);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return spoken
|
|
212
|
+
.join("\n")
|
|
213
|
+
.replace(/<[^>]*>/g, "")
|
|
214
|
+
// A character reference stands for one character and never for a word, so a
|
|
215
|
+
// space in its place keeps the neighbouring words apart and adds nothing.
|
|
216
|
+
.replace(/&[a-z]+;|&#\d+;|&#x[0-9a-f]+;/gi, " ")
|
|
217
|
+
.trim();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Detect the language of a WebVTT document, reading only its cue text.
|
|
222
|
+
*
|
|
223
|
+
* @param {string} vtt - A WebVTT document.
|
|
224
|
+
* @returns {{ code: string, name: string } | null} Detected language, or null when uncertain.
|
|
225
|
+
*/
|
|
226
|
+
export function detectLanguageFromVtt(vtt) {
|
|
227
|
+
return detectLanguage(cueTextOfVtt(vtt));
|
|
73
228
|
}
|
|
@@ -291,6 +291,12 @@ function megabytes(bytes) {
|
|
|
291
291
|
* @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
|
|
292
292
|
* @param {number | null} [reading.diskFreeBytes]
|
|
293
293
|
* @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
|
|
294
|
+
* @param {string} [reading.extra] - Figures the caller wants on the same line
|
|
295
|
+
* rather than on one of its own. The piece-buffer counters are read here so
|
|
296
|
+
* that the number of buffers alive and the off-heap mass they should account
|
|
297
|
+
* for are the SAME instant: printed on separate timers they were up to a
|
|
298
|
+
* minute apart, and 950 MB of `arrayBuffers` could not be checked against the
|
|
299
|
+
* 62 buffers a reading half a minute away said were alive (roadmap item 2).
|
|
294
300
|
* @returns {string}
|
|
295
301
|
*/
|
|
296
302
|
export function describeMemory({
|
|
@@ -302,7 +308,8 @@ export function describeMemory({
|
|
|
302
308
|
anonymousBytes = null,
|
|
303
309
|
mappings = null,
|
|
304
310
|
diskFreeBytes = null,
|
|
305
|
-
stores = []
|
|
311
|
+
stores = [],
|
|
312
|
+
extra = ""
|
|
306
313
|
}) {
|
|
307
314
|
const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
|
|
308
315
|
const storeResident = total("residentBytes");
|
|
@@ -321,8 +328,9 @@ export function describeMemory({
|
|
|
321
328
|
`heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)}` +
|
|
322
329
|
`${usage.heapLimit ? ` of ${megabytes(usage.heapLimit)} allowed` : ""} ` +
|
|
323
330
|
`external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
|
|
331
|
+
const tail = extra ? `; ${extra}` : "";
|
|
324
332
|
if (scope === "thread") {
|
|
325
|
-
return `memory (${label || "thread"}): ${isolate}; ${storesPart}`;
|
|
333
|
+
return `memory (${label || "thread"}): ${isolate}; ${storesPart}${tail}`;
|
|
326
334
|
}
|
|
327
335
|
const shape = mappings === null
|
|
328
336
|
? ""
|
|
@@ -337,10 +345,41 @@ export function describeMemory({
|
|
|
337
345
|
`${storesPart}; ` +
|
|
338
346
|
`machine has ${megabytes(availableBytes ?? 0)} available` +
|
|
339
347
|
`${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
|
|
340
|
-
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}`
|
|
348
|
+
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}` +
|
|
349
|
+
tail
|
|
341
350
|
);
|
|
342
351
|
}
|
|
343
352
|
|
|
353
|
+
/**
|
|
354
|
+
* The figures whose movement earns a line, each under its own name.
|
|
355
|
+
*
|
|
356
|
+
* Not the same question as what the scope WATCHES for a heap snapshot. A
|
|
357
|
+
* thread's heap is only one of the three ways its isolate holds memory, and on
|
|
358
|
+
* 2026-09-02 it was the one that did not move: `heapTotal` stayed at 31-173 MB
|
|
359
|
+
* through a session where `arrayBuffers` swung between 130 and 950 MB, so the
|
|
360
|
+
* change trigger was watching the one quantity that stood still and every
|
|
361
|
+
* reading of the one that grew came out on the quiet interval, a minute apart.
|
|
362
|
+
*
|
|
363
|
+
* Each figure is compared against its own last written value and any one of
|
|
364
|
+
* them moving is enough. Nothing is added together, because `arrayBuffers` is
|
|
365
|
+
* documented as part of `external` and reads larger than it on this runtime —
|
|
366
|
+
* a contradiction this code has no business resolving.
|
|
367
|
+
*
|
|
368
|
+
* @param {"process" | "thread"} scope
|
|
369
|
+
* @param {{ rss: number, heapTotal: number, external: number, arrayBuffers: number }} memory
|
|
370
|
+
* @returns {Record<string, number>}
|
|
371
|
+
*/
|
|
372
|
+
export function watchedFigures(scope, memory) {
|
|
373
|
+
if (scope === "thread") {
|
|
374
|
+
return {
|
|
375
|
+
heap: memory.heapTotal ?? 0,
|
|
376
|
+
external: memory.external ?? 0,
|
|
377
|
+
buffers: memory.arrayBuffers ?? 0
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
return { rss: memory.rss ?? 0 };
|
|
381
|
+
}
|
|
382
|
+
|
|
344
383
|
/**
|
|
345
384
|
* Whether this reading is worth writing down, and why.
|
|
346
385
|
*
|
|
@@ -397,11 +436,14 @@ export function readingIsWorthWriting({
|
|
|
397
436
|
* @param {number} [options.snapshotFloorBytes]
|
|
398
437
|
* @param {number} [options.snapshotGrowthBytes]
|
|
399
438
|
* @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
|
|
439
|
+
* @param {() => string} [options.readExtra] - Figures to append to the line,
|
|
440
|
+
* read at the same instant as the memory itself.
|
|
400
441
|
* @returns {{ stop: () => void }}
|
|
401
442
|
*/
|
|
402
443
|
export function startMemoryReport({
|
|
403
444
|
log,
|
|
404
445
|
readStores,
|
|
446
|
+
readExtra,
|
|
405
447
|
scope = "process",
|
|
406
448
|
label = "",
|
|
407
449
|
diskPath = "",
|
|
@@ -414,7 +456,8 @@ export function startMemoryReport({
|
|
|
414
456
|
keepSnapshots = 0
|
|
415
457
|
}) {
|
|
416
458
|
let highWater = 0;
|
|
417
|
-
|
|
459
|
+
/** @type {Record<string, number>} */
|
|
460
|
+
let lastWritten = {};
|
|
418
461
|
let lastWrittenAt = 0;
|
|
419
462
|
// The process watches what the kernel kills it for; a thread watches what the
|
|
420
463
|
// runtime kills IT for, which is its own heap and not the process's resident
|
|
@@ -472,25 +515,35 @@ export function startMemoryReport({
|
|
|
472
515
|
}
|
|
473
516
|
const processMemory = readProcessMemory();
|
|
474
517
|
const watched = watchedOf(processMemory);
|
|
518
|
+
const figures = watchedFigures(scope, processMemory);
|
|
475
519
|
const now = Date.now();
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
520
|
+
const sinceWrittenMs = lastWrittenAt === 0 ? Number.POSITIVE_INFINITY : now - lastWrittenAt;
|
|
521
|
+
const write = Object.entries(figures).some(([name, bytes]) => readingIsWorthWriting({
|
|
522
|
+
watchedBytes: bytes,
|
|
523
|
+
lastWrittenBytes: lastWritten[name] ?? 0,
|
|
524
|
+
sinceWrittenMs,
|
|
480
525
|
changeBytes,
|
|
481
526
|
quietMs
|
|
482
|
-
});
|
|
527
|
+
}));
|
|
483
528
|
|
|
484
529
|
let anonymousBytes = null;
|
|
485
|
-
if (
|
|
486
|
-
|
|
487
|
-
|
|
530
|
+
if (write) {
|
|
531
|
+
let extra = "";
|
|
532
|
+
try {
|
|
533
|
+
extra = typeof readExtra === "function" ? readExtra() ?? "" : "";
|
|
534
|
+
} catch {
|
|
535
|
+
// silent-ok: a caller's own figures are worth less than the line they
|
|
536
|
+
// would have taken down with them.
|
|
488
537
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
538
|
+
if (scope === "thread") {
|
|
539
|
+
log(describeMemory({ scope, label, process: processMemory, stores, extra }));
|
|
540
|
+
} else {
|
|
541
|
+
// Read only when the line is written. `smaps` is one entry per
|
|
542
|
+
// mapping and a busy process has thousands; the rollup and
|
|
543
|
+
// `/proc/meminfo` are single lines but still walk page tables, and
|
|
544
|
+
// the reading now happens once a second rather than once a minute.
|
|
545
|
+
const { bytes, measured } = await availableMemory();
|
|
546
|
+
anonymousBytes = await readAnonymousMemory();
|
|
494
547
|
log(describeMemory({
|
|
495
548
|
scope,
|
|
496
549
|
label,
|
|
@@ -498,17 +551,13 @@ export function startMemoryReport({
|
|
|
498
551
|
availableBytes: bytes,
|
|
499
552
|
availableMeasured: measured,
|
|
500
553
|
anonymousBytes,
|
|
501
|
-
// Read only when the line is written: `smaps` is one entry per
|
|
502
|
-
// mapping and a busy process has thousands, which is a different
|
|
503
|
-
// cost from the rollup's single line.
|
|
504
554
|
mappings: await readMappingSummary(),
|
|
505
|
-
diskFreeBytes,
|
|
506
|
-
stores
|
|
555
|
+
diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
|
|
556
|
+
stores,
|
|
557
|
+
extra
|
|
507
558
|
}));
|
|
508
559
|
}
|
|
509
|
-
|
|
510
|
-
if (write) {
|
|
511
|
-
lastWrittenBytes = watched;
|
|
560
|
+
lastWritten = figures;
|
|
512
561
|
lastWrittenAt = now;
|
|
513
562
|
}
|
|
514
563
|
|
|
@@ -524,6 +573,9 @@ export function startMemoryReport({
|
|
|
524
573
|
highWater = watched;
|
|
525
574
|
await takeSnapshot(watched, "a new high-water");
|
|
526
575
|
}
|
|
576
|
+
// Under `write` by construction: `anonymousBytes` is only read when the
|
|
577
|
+
// line is, and at one reading a second an unconditional warning would be
|
|
578
|
+
// a line a second for as long as the process stayed large.
|
|
527
579
|
if (scope !== "thread" && anonymousBytes !== null && processMemory.rss > 800 * 1024 * 1024) {
|
|
528
580
|
log(`memory: high rss=${megabytes(processMemory.rss)} anon=${megabytes(anonymousBytes)} heap=${megabytes(processMemory.heapUsed)} — watch for OOM`);
|
|
529
581
|
}
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
import { readSubtitlePlan, harvestCluster } from "../container-index/matroska-subtitles.js";
|
|
21
21
|
import { decodeSubtitleSample, readMp4SubtitlePlan } from "../container-index/mp4-subtitles.js";
|
|
22
22
|
import { iterateElements } from "../container-index/ebml-reader.js";
|
|
23
|
+
import { detectLanguage } from "../language-detect.js";
|
|
23
24
|
import { logger } from "../../utils/logger.js";
|
|
24
25
|
|
|
25
26
|
/** Enough to read any cluster's own element header. */
|
|
@@ -483,12 +484,23 @@ export async function warmSubtitleCues(torrent, fileIndex, sourceKey) {
|
|
|
483
484
|
}
|
|
484
485
|
const highest = newCues.reduce((max, cue) => Math.max(max, Number(cue.seq) || 0), since);
|
|
485
486
|
state.pushed.set(track.trackNumber, highest);
|
|
486
|
-
const
|
|
487
|
+
const codecId = held.track?.codecId ?? track.codecId;
|
|
488
|
+
const cues = finalizeCues(newCues, codecId);
|
|
487
489
|
fresh.push({
|
|
488
490
|
// ffmpeg's own numbering, which is the only one the browser knows.
|
|
489
491
|
trackIndex: Number.isInteger(track.declaredIndex) ? track.declaredIndex : order,
|
|
490
492
|
cues,
|
|
491
493
|
language: held.track?.language ?? "",
|
|
494
|
+
// What the CUES say the language is, re-read on every push over every cue
|
|
495
|
+
// held so far rather than over this batch. A track whose container states
|
|
496
|
+
// no language is unreadable at the start of a session — a handful of cues
|
|
497
|
+
// is not a sample of a language, and the detector refuses to answer on one
|
|
498
|
+
// — so the answer has to be re-taken as the film downloads, and the label
|
|
499
|
+
// moved when it arrives. Costs about 6 ms per push, measured; pushes
|
|
500
|
+
// arrive about once a second per file being read.
|
|
501
|
+
detectedLanguage: detectLanguage(
|
|
502
|
+
finalizeCues(held.cues, codecId).map((cue) => cue.text).join("\n")
|
|
503
|
+
),
|
|
492
504
|
// Where the browser should resume from if it has to ask again — after a
|
|
493
505
|
// reconnect, which loses the subscription these pushes ride on.
|
|
494
506
|
cursor: highest,
|
|
@@ -522,6 +522,12 @@ parentPort.on("message", async (message) => {
|
|
|
522
522
|
startMemoryReport({
|
|
523
523
|
log,
|
|
524
524
|
readStores: collectStoreStats,
|
|
525
|
+
// Beside the isolate's own figures, and on the SAME line, because the
|
|
526
|
+
// question they answer together is whether the off-heap mass is buffers this
|
|
527
|
+
// thread still refers to or buffers the collector has not reached yet. On
|
|
528
|
+
// separate timers the two were up to a minute apart and could not be
|
|
529
|
+
// compared at all (roadmap item 2).
|
|
530
|
+
readExtra: describePieceBuffers,
|
|
525
531
|
scope: "thread",
|
|
526
532
|
label: "torrent worker",
|
|
527
533
|
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
@@ -581,22 +587,30 @@ setInterval(() => {
|
|
|
581
587
|
(stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
|
|
582
588
|
);
|
|
583
589
|
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
590
|
+
}, STORE_REPORT_INTERVAL_MS).unref();
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* The piece buffers this thread has let go of against the ones it still holds.
|
|
594
|
+
*
|
|
595
|
+
* Not per store: the collector is per thread, and the question is about the
|
|
596
|
+
* thread. A gap that keeps widening means a reference of ours outlives the
|
|
597
|
+
* piece; a gap that does not means whatever grows is below us, in the
|
|
598
|
+
* allocator or in buffers the collector has not reached.
|
|
599
|
+
*
|
|
600
|
+
* @returns {string}
|
|
601
|
+
*/
|
|
602
|
+
function describePieceBuffers() {
|
|
588
603
|
const collection = pieceBufferCollection();
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
if (outstandingBuffers > heldNow) {
|
|
592
|
-
log(
|
|
593
|
-
`piece buffers: ${collection.released} let go, ${collection.collected} collected — ` +
|
|
594
|
-
`${outstandingBuffers} still alive against ${heldNow} the store holds. ` +
|
|
595
|
-
"A gap that keeps widening means a reference of ours outlives the piece; " +
|
|
596
|
-
"a gap that does not means whatever grows is below us, in the allocator"
|
|
597
|
-
);
|
|
604
|
+
if (collection.released === 0) {
|
|
605
|
+
return "";
|
|
598
606
|
}
|
|
599
|
-
|
|
607
|
+
const alive = collection.released - collection.collected;
|
|
608
|
+
const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
|
|
609
|
+
return (
|
|
610
|
+
`piece buffers ${collection.released} let go, ${collection.collected} collected, ` +
|
|
611
|
+
`${alive} still alive against ${held} the store holds`
|
|
612
|
+
);
|
|
613
|
+
}
|
|
600
614
|
|
|
601
615
|
/**
|
|
602
616
|
* Walk subtitle cues for every actively-read file of one torrent, and PUSH
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import test from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
describeMemory,
|
|
6
|
+
readingIsWorthWriting,
|
|
7
|
+
summariseMappings,
|
|
8
|
+
watchedFigures
|
|
9
|
+
} from "../services/memory-report.js";
|
|
5
10
|
import {
|
|
6
11
|
budgetForNewStore,
|
|
7
12
|
SharedPieceStore,
|
|
@@ -287,3 +292,55 @@ test("the mapping shape is said in the line, and left out when it is not known",
|
|
|
287
292
|
});
|
|
288
293
|
assert.ok(!withoutShape.includes("mappings="), "an unread breakdown is absent, not zeroed");
|
|
289
294
|
});
|
|
295
|
+
|
|
296
|
+
test("a thread's line is earned by any of its three figures, not by the heap alone", () => {
|
|
297
|
+
// 2026-09-02, the session that ended in two out-of-memory kills: the worker's
|
|
298
|
+
// heap stood at 33-45 MB for the whole of it while its buffers went from 130
|
|
299
|
+
// to 950 MB. Watching the heap alone, nothing ever earned a line and every
|
|
300
|
+
// reading of the quantity that grew came out on the quiet minute.
|
|
301
|
+
const before = watchedFigures("thread", {
|
|
302
|
+
rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 130 * MEGABYTE
|
|
303
|
+
});
|
|
304
|
+
const after = watchedFigures("thread", {
|
|
305
|
+
rss: 0, heapTotal: 38 * MEGABYTE, external: 14 * MEGABYTE, arrayBuffers: 515 * MEGABYTE
|
|
306
|
+
});
|
|
307
|
+
assert.equal(before.heap, after.heap, "the heap is what did NOT move");
|
|
308
|
+
|
|
309
|
+
const moved = Object.entries(after).some(([name, bytes]) => readingIsWorthWriting({
|
|
310
|
+
watchedBytes: bytes,
|
|
311
|
+
lastWrittenBytes: before[name],
|
|
312
|
+
sinceWrittenMs: 1_000,
|
|
313
|
+
changeBytes: 25 * MEGABYTE,
|
|
314
|
+
quietMs: 60_000
|
|
315
|
+
}));
|
|
316
|
+
assert.equal(moved, true, "buffers growing by 385 MB earns a line of its own");
|
|
317
|
+
|
|
318
|
+
assert.deepEqual(
|
|
319
|
+
Object.keys(watchedFigures("process", { rss: 5, heapTotal: 1, external: 2, arrayBuffers: 3 })),
|
|
320
|
+
["rss"],
|
|
321
|
+
"a process is killed for its resident memory and watches that"
|
|
322
|
+
);
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
test("figures a caller reads for itself land on the same line as the memory", () => {
|
|
326
|
+
const line = describeMemory({
|
|
327
|
+
scope: "thread",
|
|
328
|
+
label: "torrent worker",
|
|
329
|
+
process: {
|
|
330
|
+
rss: 0, heapUsed: 34 * MEGABYTE, heapTotal: 46 * MEGABYTE,
|
|
331
|
+
external: 54 * MEGABYTE, arrayBuffers: 393 * MEGABYTE, heapLimit: 0
|
|
332
|
+
},
|
|
333
|
+
stores: [],
|
|
334
|
+
extra: "piece buffers 21544 let go, 21482 collected, 62 still alive against 53 the store holds"
|
|
335
|
+
});
|
|
336
|
+
assert.match(line, /arrayBuffers=393MB/);
|
|
337
|
+
assert.match(line, /62 still alive against 53 the store holds$/);
|
|
338
|
+
assert.doesNotMatch(
|
|
339
|
+
describeMemory({
|
|
340
|
+
scope: "thread",
|
|
341
|
+
process: { rss: 0, heapUsed: 1, heapTotal: 2, external: 3, arrayBuffers: 4, heapLimit: 0 }
|
|
342
|
+
}),
|
|
343
|
+
/;\s*$/,
|
|
344
|
+
"nothing to add leaves no dangling separator"
|
|
345
|
+
);
|
|
346
|
+
});
|