@torrent-tv/proxy 2.80.7 → 2.80.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1666 -1654
- package/bin/cli.js +606 -596
- package/package.json +1 -1
- package/services/encode/CoverageMap.js +428 -401
- package/services/encode/EncodePlan.js +1104 -1104
- package/services/encode/SegmentStore.js +718 -669
- package/services/hls-session-manager.js +7 -19
- package/services/memory-report.js +596 -592
- package/services/orchestrators/EncodeOrchestrator.js +726 -579
- package/test/coverage-follows-the-disk.test.js +187 -0
- package/test/coverage-map.test.js +195 -178
- package/test/encode-plan.test.js +539 -540
- package/test/run-intervals.test.js +100 -100
- package/test/segment-store.test.js +216 -187
|
@@ -1,592 +1,596 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file What this process is holding, said out loud on a regular cadence.
|
|
3
|
-
*
|
|
4
|
-
* Written 2026-08-28, after the kernel killed the proxy and the log could not
|
|
5
|
-
* say why. The supervisor recorded `exit code 137` — SIGKILL, so no core dump —
|
|
6
|
-
* and the kernel ring buffer held the whole of what was known:
|
|
7
|
-
*
|
|
8
|
-
* Out of memory: Killed process 3036113 (MainThread)
|
|
9
|
-
* anon-rss: 2422628kB total-vm: 20856720kB oom_score_adj: 200
|
|
10
|
-
*
|
|
11
|
-
* Two point four gigabytes, on a host with under two free, and the addon is the
|
|
12
|
-
* first thing the kernel picks because Home Assistant gives addons a positive
|
|
13
|
-
* `oom_score_adj`. What the proxy had been logging all along was its share of a
|
|
14
|
-
* CPU. Nothing anywhere said how much memory it held, so the growth that ended
|
|
15
|
-
* in that line has no shape: one final reading taken by the kernel, and no
|
|
16
|
-
* series leading to it.
|
|
17
|
-
*
|
|
18
|
-
* This is that series. It costs one line a minute and reads only counters the
|
|
19
|
-
* runtime already maintains.
|
|
20
|
-
*/
|
|
21
|
-
|
|
22
|
-
import { readdir, readFile, rm, statfs } from "node:fs/promises";
|
|
23
|
-
import os from "node:os";
|
|
24
|
-
import v8 from "node:v8";
|
|
25
|
-
import path from "node:path";
|
|
26
|
-
|
|
27
|
-
/** How often the reading is taken and written. */
|
|
28
|
-
export const MEMORY_REPORT_INTERVAL_MS = 60_000;
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* How often the torrent worker takes its own reading.
|
|
32
|
-
*
|
|
33
|
-
* A minute cannot see what killed it. Three times — 2026-08-30 14:00 and 23:19,
|
|
34
|
-
* 2026-08-31 13:27 — the worker's heap read 28-36 MB in one sample and the
|
|
35
|
-
* thread was dead by the sample after next, with the whole rise from 30 MB to
|
|
36
|
-
* the 2240 MB heap limit fitting inside a single gap. A second is short enough
|
|
37
|
-
* that the rise is a curve rather than a step, and the line is only WRITTEN when
|
|
38
|
-
* something moved, so a quiet session costs what it costs today.
|
|
39
|
-
*/
|
|
40
|
-
export const WORKER_MEMORY_SAMPLE_MS = 1_000;
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* What the process is holding, from the runtime's own counters.
|
|
44
|
-
*
|
|
45
|
-
* `rss` is what the kernel counts against us and therefore what the OOM killer
|
|
46
|
-
* reads. The rest says where it went: the JavaScript heap, and everything held
|
|
47
|
-
* outside it — which for this proxy is where the interesting growth lives,
|
|
48
|
-
* since torrent pieces sit in a `SharedArrayBuffer` and segment bodies pass
|
|
49
|
-
* through buffers.
|
|
50
|
-
*
|
|
51
|
-
* `heapLimit` is this isolate's own ceiling, and it belongs beside the heap
|
|
52
|
-
* figures because it is what the runtime kills the thread for reaching — a
|
|
53
|
-
* worker created without `resourceLimits` inherits the main isolate's, 2240 MB
|
|
54
|
-
* on the addon host. Without it the log said 30 MB and gave no idea how far
|
|
55
|
-
* that was from the end.
|
|
56
|
-
*
|
|
57
|
-
* @returns {{ rss: number, heapUsed: number, heapTotal: number, external: number, arrayBuffers: number, heapLimit: number }}
|
|
58
|
-
*/
|
|
59
|
-
export function readProcessMemory() {
|
|
60
|
-
const usage = process.memoryUsage();
|
|
61
|
-
let heapLimit = 0;
|
|
62
|
-
try {
|
|
63
|
-
heapLimit = v8.getHeapStatistics().heap_size_limit ?? 0;
|
|
64
|
-
} catch {
|
|
65
|
-
// silent-ok: a missing ceiling leaves the term out, it does not cost the
|
|
66
|
-
// reading beside it.
|
|
67
|
-
}
|
|
68
|
-
return {
|
|
69
|
-
rss: usage.rss,
|
|
70
|
-
heapUsed: usage.heapUsed,
|
|
71
|
-
heapTotal: usage.heapTotal,
|
|
72
|
-
external: usage.external,
|
|
73
|
-
arrayBuffers: usage.arrayBuffers ?? 0,
|
|
74
|
-
heapLimit
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* How much memory the machine could still give out, in bytes.
|
|
80
|
-
*
|
|
81
|
-
* `os.freemem()` is the wrong quantity on Linux and the difference is not
|
|
82
|
-
* academic: it counts only pages that are free RIGHT NOW, while the kernel
|
|
83
|
-
* deliberately keeps that number low by filling the rest with reclaimable page
|
|
84
|
-
* cache. `MemAvailable` is the kernel's own estimate of what a new allocation
|
|
85
|
-
* could actually obtain, cache included. Reading the estimate the kernel
|
|
86
|
-
* publishes beats recomputing a worse one.
|
|
87
|
-
*
|
|
88
|
-
* @returns {Promise<number | null>} Bytes, or null where /proc is not there.
|
|
89
|
-
*/
|
|
90
|
-
export async function readAvailableMemory() {
|
|
91
|
-
try {
|
|
92
|
-
const text = await readFile("/proc/meminfo", "utf8");
|
|
93
|
-
const match = /^MemAvailable:\s+(\d+)\s+kB$/m.exec(text);
|
|
94
|
-
if (match) {
|
|
95
|
-
return Number(match[1]) * 1024;
|
|
96
|
-
}
|
|
97
|
-
} catch {
|
|
98
|
-
// silent-ok: not Linux, or /proc is not mounted. The fallback below is a
|
|
99
|
-
// worse answer, and saying so is the point of returning it separately.
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Anonymous memory grouped by the SHAPE of the mappings holding it.
|
|
106
|
-
*
|
|
107
|
-
* The rollup says how much there is; this says what it looks like, and the
|
|
108
|
-
* three shapes it can take are three different diagnoses of the same number:
|
|
109
|
-
*
|
|
110
|
-
* - **one growing `[heap]`** — the allocator's break-managed arena. Freed
|
|
111
|
-
* blocks stay in it, and on musl there is no `malloc_trim` to ask for them
|
|
112
|
-
* back. Nothing above the allocator is holding anything.
|
|
113
|
-
* - **many large anonymous mappings** — one per big allocation, which is what
|
|
114
|
-
* a 4 MiB piece buffer is. If their count tracks the pieces the store says
|
|
115
|
-
* it holds, the memory is accounted for; if it keeps climbing while the
|
|
116
|
-
* store's count does not, the buffers are being kept alive by somebody.
|
|
117
|
-
* - **many medium ones** — the allocator's own per-thread arenas, taken and
|
|
118
|
-
* not returned.
|
|
119
|
-
*
|
|
120
|
-
* The field failure of 2026-08-31 is 700 MB that is none of the JavaScript
|
|
121
|
-
* heaps, none of the piece store, and none of ffmpeg. Which of the three
|
|
122
|
-
* shapes it has decides what to change, and no reading so far can tell them
|
|
123
|
-
* apart (roadmap item 2, step 4).
|
|
124
|
-
*
|
|
125
|
-
* @param {string} text - The contents of `/proc/self/smaps`.
|
|
126
|
-
* @returns {{ heapBytes: number, largeBytes: number, largeCount: number,
|
|
127
|
-
* largestBytes: number, smallBytes: number, smallCount: number,
|
|
128
|
-
* fileBytes: number }}
|
|
129
|
-
*/
|
|
130
|
-
export function summariseMappings(text) {
|
|
131
|
-
const summary = {
|
|
132
|
-
heapBytes: 0,
|
|
133
|
-
largeBytes: 0,
|
|
134
|
-
largeCount: 0,
|
|
135
|
-
largestBytes: 0,
|
|
136
|
-
smallBytes: 0,
|
|
137
|
-
smallCount: 0,
|
|
138
|
-
fileBytes: 0
|
|
139
|
-
};
|
|
140
|
-
// A mapping is a header line followed by its fields; only `Rss` is wanted,
|
|
141
|
-
// because a mapping that is reserved and untouched costs no memory.
|
|
142
|
-
let pathName = null;
|
|
143
|
-
for (const line of String(text ?? "").split("\n")) {
|
|
144
|
-
const header = /^[0-9a-f]+-[0-9a-f]+ \S{4} [0-9a-f]+ \S+ \d+\s*(.*)$/.exec(line);
|
|
145
|
-
if (header) {
|
|
146
|
-
pathName = header[1].trim();
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
const rss = /^Rss:\s+(\d+)\s+kB$/.exec(line);
|
|
150
|
-
if (!rss || pathName === null) {
|
|
151
|
-
continue;
|
|
152
|
-
}
|
|
153
|
-
const bytes = Number(rss[1]) * 1024;
|
|
154
|
-
if (bytes === 0) {
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (pathName === "[heap]") {
|
|
158
|
-
summary.heapBytes += bytes;
|
|
159
|
-
} else if (pathName !== "" && !pathName.startsWith("[")) {
|
|
160
|
-
// Backed by a file: the executable, the libraries, anything mapped in.
|
|
161
|
-
// Counted so the anonymous figures can be checked against `rss`.
|
|
162
|
-
summary.fileBytes += bytes;
|
|
163
|
-
} else if (bytes >= LARGE_MAPPING_BYTES) {
|
|
164
|
-
summary.largeBytes += bytes;
|
|
165
|
-
summary.largeCount += 1;
|
|
166
|
-
summary.largestBytes = Math.max(summary.largestBytes, bytes);
|
|
167
|
-
} else {
|
|
168
|
-
summary.smallBytes += bytes;
|
|
169
|
-
summary.smallCount += 1;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
return summary;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
* Where "large" begins. Two megabytes, so a 4 MiB piece buffer is always large
|
|
177
|
-
* and an allocator's ordinary arena is not.
|
|
178
|
-
*/
|
|
179
|
-
const LARGE_MAPPING_BYTES = 2 * 1024 * 1024;
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* The mapping summary for this process, or null where /proc is not there.
|
|
183
|
-
*
|
|
184
|
-
* @returns {Promise<ReturnType<typeof summariseMappings> | null>}
|
|
185
|
-
*/
|
|
186
|
-
export async function readMappingSummary() {
|
|
187
|
-
try {
|
|
188
|
-
return summariseMappings(await readFile("/proc/self/smaps", "utf8"));
|
|
189
|
-
} catch {
|
|
190
|
-
// silent-ok: not Linux, or the kernel does not publish it. The line leaves
|
|
191
|
-
// the term out rather than printing a worse one.
|
|
192
|
-
}
|
|
193
|
-
return null;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
/**
|
|
197
|
-
* Available memory, falling back to what the runtime can offer.
|
|
198
|
-
*
|
|
199
|
-
* @returns {Promise<{ bytes: number, measured: boolean }>}
|
|
200
|
-
*/
|
|
201
|
-
export async function availableMemory() {
|
|
202
|
-
const fromKernel = await readAvailableMemory();
|
|
203
|
-
if (fromKernel !== null) {
|
|
204
|
-
return { bytes: fromKernel, measured: true };
|
|
205
|
-
}
|
|
206
|
-
return { bytes: os.freemem(), measured: false };
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/**
|
|
210
|
-
* Anonymous memory this process holds, from the kernel's own rollup.
|
|
211
|
-
*
|
|
212
|
-
* `process.memoryUsage()` sees what V8 knows about. It cannot see memory the
|
|
213
|
-
* allocator has taken and not returned, and on musl — which is what the addon
|
|
214
|
-
* runs on — a heavy churn of 8 MiB pieces leaves exactly that. The rollup's
|
|
215
|
-
* `Anonymous` counts every mapping backed by nothing but memory, which is
|
|
216
|
-
* where both a live `SharedArrayBuffer` and a freed-but-retained span sit, so
|
|
217
|
-
* the difference between it and what the isolates admit to is the size of what
|
|
218
|
-
* neither can explain (roadmap item 2, step 4).
|
|
219
|
-
*
|
|
220
|
-
* @returns {Promise<number | null>} Bytes, or null off Linux.
|
|
221
|
-
*/
|
|
222
|
-
export async function readAnonymousMemory() {
|
|
223
|
-
try {
|
|
224
|
-
const text = await readFile("/proc/self/smaps_rollup", "utf8");
|
|
225
|
-
const match = /^Anonymous:\s+(\d+)\s+kB$/m.exec(text);
|
|
226
|
-
if (match) {
|
|
227
|
-
return Number(match[1]) * 1024;
|
|
228
|
-
}
|
|
229
|
-
} catch {
|
|
230
|
-
// silent-ok: not Linux, or the kernel is too old for the rollup. The line
|
|
231
|
-
// simply leaves the term out rather than printing a worse one.
|
|
232
|
-
}
|
|
233
|
-
return null;
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
/**
|
|
237
|
-
* How much room is left where this proxy spills pieces and writes segments.
|
|
238
|
-
*
|
|
239
|
-
* A budget for memory alone is half a budget: pieces evicted from memory go to
|
|
240
|
-
* disk, so a store that is well behaved about RAM can still fill the card an
|
|
241
|
-
* addon host boots from. Both limits are the machine's, and neither was
|
|
242
|
-
* measured before (roadmap item 2).
|
|
243
|
-
*
|
|
244
|
-
* @param {string} directory
|
|
245
|
-
* @returns {Promise<number | null>} Bytes free, or null where it cannot be read.
|
|
246
|
-
*/
|
|
247
|
-
export async function readDiskFree(directory) {
|
|
248
|
-
try {
|
|
249
|
-
const stats = await statfs(directory);
|
|
250
|
-
return Number(stats.bavail) * Number(stats.bsize);
|
|
251
|
-
} catch {
|
|
252
|
-
// silent-ok: `statfs` is not everywhere, and a missing disk figure must not
|
|
253
|
-
// cost the memory reading beside it.
|
|
254
|
-
}
|
|
255
|
-
return null;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/**
|
|
259
|
-
* Render a size the way a person reads one.
|
|
260
|
-
*
|
|
261
|
-
* @param {number} bytes
|
|
262
|
-
* @returns {string}
|
|
263
|
-
*/
|
|
264
|
-
function megabytes(bytes) {
|
|
265
|
-
return `${Math.round(bytes / (1024 * 1024))}MB`;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* One line saying what the process holds and what the machine has left.
|
|
270
|
-
*
|
|
271
|
-
* Pure, so the wording and the arithmetic can be pinned without a running
|
|
272
|
-
* process. Store figures are given in BYTES rather than in pieces: the piece
|
|
273
|
-
* count is meaningless without the piece size, and the piece size differs per
|
|
274
|
-
* torrent — on the film this proxy died under, 63 pieces meant 504 MB.
|
|
275
|
-
*
|
|
276
|
-
* Two scopes, because two things are being asked and only one of them is
|
|
277
|
-
* per-thread. `rss` and the kernel's rollup belong to the PROCESS, so they are
|
|
278
|
-
* read once, on the main thread. The heap, external and arrayBuffer figures
|
|
279
|
-
* belong to an ISOLATE, and the torrent worker's are the ones that matter —
|
|
280
|
-
* the piece pool is a `SharedArrayBuffer` allocated there, which the main
|
|
281
|
-
* isolate cannot see at all. Reporting only the main thread's was half the
|
|
282
|
-
* reason 650 MB of a 893 MB process had no explanation on 2026-08-28.
|
|
283
|
-
*
|
|
284
|
-
* @param {Object} reading
|
|
285
|
-
* @param {"process" | "thread"} [reading.scope]
|
|
286
|
-
* @param {string} [reading.label] - Which thread the isolate figures are of.
|
|
287
|
-
* @param {{ rss: number, heapUsed: number, heapTotal: number, external: number, arrayBuffers: number }} reading.process
|
|
288
|
-
* @param {number} [reading.availableBytes]
|
|
289
|
-
* @param {boolean} [reading.availableMeasured]
|
|
290
|
-
* @param {number | null} [reading.anonymousBytes]
|
|
291
|
-
* @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
|
|
292
|
-
* @param {number | null} [reading.diskFreeBytes]
|
|
293
|
-
* @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
|
|
294
|
-
* @param {string} [reading.extra] - Figures the caller wants on the same line
|
|
295
|
-
* rather than on one of its own. The piece-buffer counters are read here so
|
|
296
|
-
* that the number of buffers alive and the off-heap mass they should account
|
|
297
|
-
* for are the SAME instant: printed on separate timers they were up to a
|
|
298
|
-
* minute apart, and 950 MB of `arrayBuffers` could not be checked against the
|
|
299
|
-
* 62 buffers a reading half a minute away said were alive (roadmap item 2).
|
|
300
|
-
* @returns {string}
|
|
301
|
-
*/
|
|
302
|
-
export function describeMemory({
|
|
303
|
-
scope = "process",
|
|
304
|
-
label = "",
|
|
305
|
-
process: usage,
|
|
306
|
-
availableBytes,
|
|
307
|
-
availableMeasured,
|
|
308
|
-
anonymousBytes = null,
|
|
309
|
-
mappings = null,
|
|
310
|
-
diskFreeBytes = null,
|
|
311
|
-
stores = [],
|
|
312
|
-
extra = ""
|
|
313
|
-
}) {
|
|
314
|
-
const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
|
|
315
|
-
const storeResident = total("residentBytes");
|
|
316
|
-
const storeCommitted = total("committedBytes");
|
|
317
|
-
const storeSpilled = total("spilledBytes");
|
|
318
|
-
const storeBudget = total("budgetBytes");
|
|
319
|
-
// Holding and having taken are different quantities, and the gap between
|
|
320
|
-
// them is the whole of the growth this series exists to find: the pool only
|
|
321
|
-
// grows, so a spilled piece frees a slot and no memory.
|
|
322
|
-
const storesPart = stores.length === 0
|
|
323
|
-
? "no torrent stores"
|
|
324
|
-
: `${stores.length} torrent store(s) holding ${megabytes(storeResident)}, ` +
|
|
325
|
-
`committed ${megabytes(storeCommitted)} of ${megabytes(storeBudget)} allowed, ` +
|
|
326
|
-
`${megabytes(storeSpilled)} spilled to disk`;
|
|
327
|
-
const isolate =
|
|
328
|
-
`heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)}` +
|
|
329
|
-
`${usage.heapLimit ? ` of ${megabytes(usage.heapLimit)} allowed` : ""} ` +
|
|
330
|
-
`external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
|
|
331
|
-
const tail = extra ? `; ${extra}` : "";
|
|
332
|
-
if (scope === "thread") {
|
|
333
|
-
return `memory (${label || "thread"}): ${isolate}; ${storesPart}${tail}`;
|
|
334
|
-
}
|
|
335
|
-
const shape = mappings === null
|
|
336
|
-
? ""
|
|
337
|
-
: ` mappings=[heap ${megabytes(mappings.heapBytes)}, ` +
|
|
338
|
-
`${mappings.largeCount} anon ≥2MB = ${megabytes(mappings.largeBytes)} ` +
|
|
339
|
-
`(largest ${megabytes(mappings.largestBytes)}), ` +
|
|
340
|
-
`${mappings.smallCount} anon <2MB = ${megabytes(mappings.smallBytes)}, ` +
|
|
341
|
-
`files ${megabytes(mappings.fileBytes)}]`;
|
|
342
|
-
return (
|
|
343
|
-
`memory: rss=${megabytes(usage.rss)} ${isolate}` +
|
|
344
|
-
`${anonymousBytes === null ? "" : ` anon=${megabytes(anonymousBytes)}`}${shape}; ` +
|
|
345
|
-
`${storesPart}; ` +
|
|
346
|
-
`machine has ${megabytes(availableBytes ?? 0)} available` +
|
|
347
|
-
`${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
|
|
348
|
-
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}` +
|
|
349
|
-
tail
|
|
350
|
-
);
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
/**
|
|
354
|
-
* The figures whose movement earns a line, each under its own name.
|
|
355
|
-
*
|
|
356
|
-
* Not the same question as what the scope WATCHES for a heap snapshot. A
|
|
357
|
-
* thread's heap is only one of the three ways its isolate holds memory, and on
|
|
358
|
-
* 2026-09-02 it was the one that did not move: `heapTotal` stayed at 31-173 MB
|
|
359
|
-
* through a session where `arrayBuffers` swung between 130 and 950 MB, so the
|
|
360
|
-
* change trigger was watching the one quantity that stood still and every
|
|
361
|
-
* reading of the one that grew came out on the quiet interval, a minute apart.
|
|
362
|
-
*
|
|
363
|
-
* Each figure is compared against its own last written value and any one of
|
|
364
|
-
* them moving is enough. Nothing is added together, because `arrayBuffers` is
|
|
365
|
-
* documented as part of `external` and reads larger than it on this runtime —
|
|
366
|
-
* a contradiction this code has no business resolving.
|
|
367
|
-
*
|
|
368
|
-
* @param {"process" | "thread"} scope
|
|
369
|
-
* @param {{ rss: number, heapTotal: number, external: number, arrayBuffers: number }} memory
|
|
370
|
-
* @returns {Record<string, number>}
|
|
371
|
-
*/
|
|
372
|
-
export function watchedFigures(scope, memory) {
|
|
373
|
-
if (scope === "thread") {
|
|
374
|
-
return {
|
|
375
|
-
heap: memory.heapTotal ?? 0,
|
|
376
|
-
external: memory.external ?? 0,
|
|
377
|
-
buffers: memory.arrayBuffers ?? 0
|
|
378
|
-
};
|
|
379
|
-
}
|
|
380
|
-
return { rss: memory.rss ?? 0 };
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
/**
|
|
384
|
-
* Whether this reading is worth writing down, and why.
|
|
385
|
-
*
|
|
386
|
-
* A series taken every second and printed every second is unreadable, and one
|
|
387
|
-
* printed every minute cannot see a rise that takes forty seconds. So the
|
|
388
|
-
* cadence of the READING and the cadence of the LINE are separate: the figure
|
|
389
|
-
* is taken often, and written when it has moved or when the quiet interval has
|
|
390
|
-
* passed. Pure, so the rule can be pinned without a clock.
|
|
391
|
-
*
|
|
392
|
-
* @param {Object} state
|
|
393
|
-
* @param {number} state.watchedBytes - The figure this scope is watching.
|
|
394
|
-
* @param {number} state.lastWrittenBytes
|
|
395
|
-
* @param {number} state.sinceWrittenMs
|
|
396
|
-
* @param {number} state.changeBytes - Movement that earns a line of its own.
|
|
397
|
-
* @param {number} state.quietMs - How long silence may last regardless.
|
|
398
|
-
* @returns {boolean}
|
|
399
|
-
*/
|
|
400
|
-
export function readingIsWorthWriting({
|
|
401
|
-
watchedBytes,
|
|
402
|
-
lastWrittenBytes,
|
|
403
|
-
sinceWrittenMs,
|
|
404
|
-
changeBytes,
|
|
405
|
-
quietMs
|
|
406
|
-
}) {
|
|
407
|
-
if (quietMs <= 0 || sinceWrittenMs >= quietMs) {
|
|
408
|
-
return true;
|
|
409
|
-
}
|
|
410
|
-
if (changeBytes <= 0) {
|
|
411
|
-
return false;
|
|
412
|
-
}
|
|
413
|
-
return Math.abs(watchedBytes - lastWrittenBytes) >= changeBytes;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/**
|
|
417
|
-
* Report memory on a timer until stopped.
|
|
418
|
-
*
|
|
419
|
-
* @param {Object} options
|
|
420
|
-
* @param {(message: string) => void} options.log
|
|
421
|
-
* @param {() => { name: string, residentBytes: number, budgetBytes: number }[]} [options.readStores]
|
|
422
|
-
* @param {"process" | "thread"} [options.scope] - `thread` leaves out the
|
|
423
|
-
* process-wide figures, so the torrent worker can report its own isolate
|
|
424
|
-
* without reading /proc twice.
|
|
425
|
-
* @param {string} [options.label] - Which thread the isolate figures are of.
|
|
426
|
-
* @param {string} [options.diskPath] - Where pieces spill, for the free-space
|
|
427
|
-
* reading. Omitted, the disk term is left out rather than guessed.
|
|
428
|
-
* @param {number} [options.intervalMs] - How often the figure is READ.
|
|
429
|
-
* @param {number} [options.quietMs] - How long the line may stay silent while
|
|
430
|
-
* nothing moves. Zero writes every reading, which is what the process scope
|
|
431
|
-
* has always done.
|
|
432
|
-
* @param {number} [options.changeBytes] - Movement that earns a line before the
|
|
433
|
-
* quiet interval is up.
|
|
434
|
-
* @param {string} [options.snapshotDir] - Where heap snapshots are written.
|
|
435
|
-
* Defaults to the temporary directory, as the process scope always did.
|
|
436
|
-
* @param {number} [options.snapshotFloorBytes]
|
|
437
|
-
* @param {number} [options.snapshotGrowthBytes]
|
|
438
|
-
* @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
|
|
439
|
-
* @param {() => string} [options.readExtra] - Figures to append to the line,
|
|
440
|
-
* read at the same instant as the memory itself.
|
|
441
|
-
* @returns {{ stop: () => void }}
|
|
442
|
-
*/
|
|
443
|
-
export function startMemoryReport({
|
|
444
|
-
log,
|
|
445
|
-
readStores,
|
|
446
|
-
readExtra,
|
|
447
|
-
scope = "process",
|
|
448
|
-
label = "",
|
|
449
|
-
diskPath = "",
|
|
450
|
-
intervalMs = MEMORY_REPORT_INTERVAL_MS,
|
|
451
|
-
quietMs = 0,
|
|
452
|
-
changeBytes = 0,
|
|
453
|
-
snapshotDir = "",
|
|
454
|
-
snapshotFloorBytes = 500 * 1024 * 1024,
|
|
455
|
-
snapshotGrowthBytes = 100 * 1024 * 1024,
|
|
456
|
-
keepSnapshots = 0
|
|
457
|
-
}) {
|
|
458
|
-
let highWater = 0;
|
|
459
|
-
/** @type {Record<string, number>} */
|
|
460
|
-
let lastWritten = {};
|
|
461
|
-
let lastWrittenAt = 0;
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
//
|
|
482
|
-
//
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
for
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
535
|
-
//
|
|
536
|
-
//
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
//
|
|
565
|
-
//
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
//
|
|
570
|
-
//
|
|
571
|
-
//
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file What this process is holding, said out loud on a regular cadence.
|
|
3
|
+
*
|
|
4
|
+
* Written 2026-08-28, after the kernel killed the proxy and the log could not
|
|
5
|
+
* say why. The supervisor recorded `exit code 137` — SIGKILL, so no core dump —
|
|
6
|
+
* and the kernel ring buffer held the whole of what was known:
|
|
7
|
+
*
|
|
8
|
+
* Out of memory: Killed process 3036113 (MainThread)
|
|
9
|
+
* anon-rss: 2422628kB total-vm: 20856720kB oom_score_adj: 200
|
|
10
|
+
*
|
|
11
|
+
* Two point four gigabytes, on a host with under two free, and the addon is the
|
|
12
|
+
* first thing the kernel picks because Home Assistant gives addons a positive
|
|
13
|
+
* `oom_score_adj`. What the proxy had been logging all along was its share of a
|
|
14
|
+
* CPU. Nothing anywhere said how much memory it held, so the growth that ended
|
|
15
|
+
* in that line has no shape: one final reading taken by the kernel, and no
|
|
16
|
+
* series leading to it.
|
|
17
|
+
*
|
|
18
|
+
* This is that series. It costs one line a minute and reads only counters the
|
|
19
|
+
* runtime already maintains.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { readdir, readFile, rm, statfs } from "node:fs/promises";
|
|
23
|
+
import os from "node:os";
|
|
24
|
+
import v8 from "node:v8";
|
|
25
|
+
import path from "node:path";
|
|
26
|
+
|
|
27
|
+
/** How often the reading is taken and written. */
|
|
28
|
+
export const MEMORY_REPORT_INTERVAL_MS = 60_000;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* How often the torrent worker takes its own reading.
|
|
32
|
+
*
|
|
33
|
+
* A minute cannot see what killed it. Three times — 2026-08-30 14:00 and 23:19,
|
|
34
|
+
* 2026-08-31 13:27 — the worker's heap read 28-36 MB in one sample and the
|
|
35
|
+
* thread was dead by the sample after next, with the whole rise from 30 MB to
|
|
36
|
+
* the 2240 MB heap limit fitting inside a single gap. A second is short enough
|
|
37
|
+
* that the rise is a curve rather than a step, and the line is only WRITTEN when
|
|
38
|
+
* something moved, so a quiet session costs what it costs today.
|
|
39
|
+
*/
|
|
40
|
+
export const WORKER_MEMORY_SAMPLE_MS = 1_000;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* What the process is holding, from the runtime's own counters.
|
|
44
|
+
*
|
|
45
|
+
* `rss` is what the kernel counts against us and therefore what the OOM killer
|
|
46
|
+
* reads. The rest says where it went: the JavaScript heap, and everything held
|
|
47
|
+
* outside it — which for this proxy is where the interesting growth lives,
|
|
48
|
+
* since torrent pieces sit in a `SharedArrayBuffer` and segment bodies pass
|
|
49
|
+
* through buffers.
|
|
50
|
+
*
|
|
51
|
+
* `heapLimit` is this isolate's own ceiling, and it belongs beside the heap
|
|
52
|
+
* figures because it is what the runtime kills the thread for reaching — a
|
|
53
|
+
* worker created without `resourceLimits` inherits the main isolate's, 2240 MB
|
|
54
|
+
* on the addon host. Without it the log said 30 MB and gave no idea how far
|
|
55
|
+
* that was from the end.
|
|
56
|
+
*
|
|
57
|
+
* @returns {{ rss: number, heapUsed: number, heapTotal: number, external: number, arrayBuffers: number, heapLimit: number }}
|
|
58
|
+
*/
|
|
59
|
+
export function readProcessMemory() {
|
|
60
|
+
const usage = process.memoryUsage();
|
|
61
|
+
let heapLimit = 0;
|
|
62
|
+
try {
|
|
63
|
+
heapLimit = v8.getHeapStatistics().heap_size_limit ?? 0;
|
|
64
|
+
} catch {
|
|
65
|
+
// silent-ok: a missing ceiling leaves the term out, it does not cost the
|
|
66
|
+
// reading beside it.
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
rss: usage.rss,
|
|
70
|
+
heapUsed: usage.heapUsed,
|
|
71
|
+
heapTotal: usage.heapTotal,
|
|
72
|
+
external: usage.external,
|
|
73
|
+
arrayBuffers: usage.arrayBuffers ?? 0,
|
|
74
|
+
heapLimit
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* How much memory the machine could still give out, in bytes.
|
|
80
|
+
*
|
|
81
|
+
* `os.freemem()` is the wrong quantity on Linux and the difference is not
|
|
82
|
+
* academic: it counts only pages that are free RIGHT NOW, while the kernel
|
|
83
|
+
* deliberately keeps that number low by filling the rest with reclaimable page
|
|
84
|
+
* cache. `MemAvailable` is the kernel's own estimate of what a new allocation
|
|
85
|
+
* could actually obtain, cache included. Reading the estimate the kernel
|
|
86
|
+
* publishes beats recomputing a worse one.
|
|
87
|
+
*
|
|
88
|
+
* @returns {Promise<number | null>} Bytes, or null where /proc is not there.
|
|
89
|
+
*/
|
|
90
|
+
export async function readAvailableMemory() {
|
|
91
|
+
try {
|
|
92
|
+
const text = await readFile("/proc/meminfo", "utf8");
|
|
93
|
+
const match = /^MemAvailable:\s+(\d+)\s+kB$/m.exec(text);
|
|
94
|
+
if (match) {
|
|
95
|
+
return Number(match[1]) * 1024;
|
|
96
|
+
}
|
|
97
|
+
} catch {
|
|
98
|
+
// silent-ok: not Linux, or /proc is not mounted. The fallback below is a
|
|
99
|
+
// worse answer, and saying so is the point of returning it separately.
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Anonymous memory grouped by the SHAPE of the mappings holding it.
|
|
106
|
+
*
|
|
107
|
+
* The rollup says how much there is; this says what it looks like, and the
|
|
108
|
+
* three shapes it can take are three different diagnoses of the same number:
|
|
109
|
+
*
|
|
110
|
+
* - **one growing `[heap]`** — the allocator's break-managed arena. Freed
|
|
111
|
+
* blocks stay in it, and on musl there is no `malloc_trim` to ask for them
|
|
112
|
+
* back. Nothing above the allocator is holding anything.
|
|
113
|
+
* - **many large anonymous mappings** — one per big allocation, which is what
|
|
114
|
+
* a 4 MiB piece buffer is. If their count tracks the pieces the store says
|
|
115
|
+
* it holds, the memory is accounted for; if it keeps climbing while the
|
|
116
|
+
* store's count does not, the buffers are being kept alive by somebody.
|
|
117
|
+
* - **many medium ones** — the allocator's own per-thread arenas, taken and
|
|
118
|
+
* not returned.
|
|
119
|
+
*
|
|
120
|
+
* The field failure of 2026-08-31 is 700 MB that is none of the JavaScript
|
|
121
|
+
* heaps, none of the piece store, and none of ffmpeg. Which of the three
|
|
122
|
+
* shapes it has decides what to change, and no reading so far can tell them
|
|
123
|
+
* apart (roadmap item 2, step 4).
|
|
124
|
+
*
|
|
125
|
+
* @param {string} text - The contents of `/proc/self/smaps`.
|
|
126
|
+
* @returns {{ heapBytes: number, largeBytes: number, largeCount: number,
|
|
127
|
+
* largestBytes: number, smallBytes: number, smallCount: number,
|
|
128
|
+
* fileBytes: number }}
|
|
129
|
+
*/
|
|
130
|
+
export function summariseMappings(text) {
|
|
131
|
+
const summary = {
|
|
132
|
+
heapBytes: 0,
|
|
133
|
+
largeBytes: 0,
|
|
134
|
+
largeCount: 0,
|
|
135
|
+
largestBytes: 0,
|
|
136
|
+
smallBytes: 0,
|
|
137
|
+
smallCount: 0,
|
|
138
|
+
fileBytes: 0
|
|
139
|
+
};
|
|
140
|
+
// A mapping is a header line followed by its fields; only `Rss` is wanted,
|
|
141
|
+
// because a mapping that is reserved and untouched costs no memory.
|
|
142
|
+
let pathName = null;
|
|
143
|
+
for (const line of String(text ?? "").split("\n")) {
|
|
144
|
+
const header = /^[0-9a-f]+-[0-9a-f]+ \S{4} [0-9a-f]+ \S+ \d+\s*(.*)$/.exec(line);
|
|
145
|
+
if (header) {
|
|
146
|
+
pathName = header[1].trim();
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const rss = /^Rss:\s+(\d+)\s+kB$/.exec(line);
|
|
150
|
+
if (!rss || pathName === null) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
const bytes = Number(rss[1]) * 1024;
|
|
154
|
+
if (bytes === 0) {
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
if (pathName === "[heap]") {
|
|
158
|
+
summary.heapBytes += bytes;
|
|
159
|
+
} else if (pathName !== "" && !pathName.startsWith("[")) {
|
|
160
|
+
// Backed by a file: the executable, the libraries, anything mapped in.
|
|
161
|
+
// Counted so the anonymous figures can be checked against `rss`.
|
|
162
|
+
summary.fileBytes += bytes;
|
|
163
|
+
} else if (bytes >= LARGE_MAPPING_BYTES) {
|
|
164
|
+
summary.largeBytes += bytes;
|
|
165
|
+
summary.largeCount += 1;
|
|
166
|
+
summary.largestBytes = Math.max(summary.largestBytes, bytes);
|
|
167
|
+
} else {
|
|
168
|
+
summary.smallBytes += bytes;
|
|
169
|
+
summary.smallCount += 1;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return summary;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Where "large" begins. Two megabytes, so a 4 MiB piece buffer is always large
|
|
177
|
+
* and an allocator's ordinary arena is not.
|
|
178
|
+
*/
|
|
179
|
+
const LARGE_MAPPING_BYTES = 2 * 1024 * 1024;
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The mapping summary for this process, or null where /proc is not there.
|
|
183
|
+
*
|
|
184
|
+
* @returns {Promise<ReturnType<typeof summariseMappings> | null>}
|
|
185
|
+
*/
|
|
186
|
+
export async function readMappingSummary() {
|
|
187
|
+
try {
|
|
188
|
+
return summariseMappings(await readFile("/proc/self/smaps", "utf8"));
|
|
189
|
+
} catch {
|
|
190
|
+
// silent-ok: not Linux, or the kernel does not publish it. The line leaves
|
|
191
|
+
// the term out rather than printing a worse one.
|
|
192
|
+
}
|
|
193
|
+
return null;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Available memory, falling back to what the runtime can offer.
|
|
198
|
+
*
|
|
199
|
+
* @returns {Promise<{ bytes: number, measured: boolean }>}
|
|
200
|
+
*/
|
|
201
|
+
export async function availableMemory() {
|
|
202
|
+
const fromKernel = await readAvailableMemory();
|
|
203
|
+
if (fromKernel !== null) {
|
|
204
|
+
return { bytes: fromKernel, measured: true };
|
|
205
|
+
}
|
|
206
|
+
return { bytes: os.freemem(), measured: false };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Anonymous memory this process holds, from the kernel's own rollup.
|
|
211
|
+
*
|
|
212
|
+
* `process.memoryUsage()` sees what V8 knows about. It cannot see memory the
|
|
213
|
+
* allocator has taken and not returned, and on musl — which is what the addon
|
|
214
|
+
* runs on — a heavy churn of 8 MiB pieces leaves exactly that. The rollup's
|
|
215
|
+
* `Anonymous` counts every mapping backed by nothing but memory, which is
|
|
216
|
+
* where both a live `SharedArrayBuffer` and a freed-but-retained span sit, so
|
|
217
|
+
* the difference between it and what the isolates admit to is the size of what
|
|
218
|
+
* neither can explain (roadmap item 2, step 4).
|
|
219
|
+
*
|
|
220
|
+
* @returns {Promise<number | null>} Bytes, or null off Linux.
|
|
221
|
+
*/
|
|
222
|
+
export async function readAnonymousMemory() {
|
|
223
|
+
try {
|
|
224
|
+
const text = await readFile("/proc/self/smaps_rollup", "utf8");
|
|
225
|
+
const match = /^Anonymous:\s+(\d+)\s+kB$/m.exec(text);
|
|
226
|
+
if (match) {
|
|
227
|
+
return Number(match[1]) * 1024;
|
|
228
|
+
}
|
|
229
|
+
} catch {
|
|
230
|
+
// silent-ok: not Linux, or the kernel is too old for the rollup. The line
|
|
231
|
+
// simply leaves the term out rather than printing a worse one.
|
|
232
|
+
}
|
|
233
|
+
return null;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* How much room is left where this proxy spills pieces and writes segments.
|
|
238
|
+
*
|
|
239
|
+
* A budget for memory alone is half a budget: pieces evicted from memory go to
|
|
240
|
+
* disk, so a store that is well behaved about RAM can still fill the card an
|
|
241
|
+
* addon host boots from. Both limits are the machine's, and neither was
|
|
242
|
+
* measured before (roadmap item 2).
|
|
243
|
+
*
|
|
244
|
+
* @param {string} directory
|
|
245
|
+
* @returns {Promise<number | null>} Bytes free, or null where it cannot be read.
|
|
246
|
+
*/
|
|
247
|
+
export async function readDiskFree(directory) {
|
|
248
|
+
try {
|
|
249
|
+
const stats = await statfs(directory);
|
|
250
|
+
return Number(stats.bavail) * Number(stats.bsize);
|
|
251
|
+
} catch {
|
|
252
|
+
// silent-ok: `statfs` is not everywhere, and a missing disk figure must not
|
|
253
|
+
// cost the memory reading beside it.
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Render a size the way a person reads one.
|
|
260
|
+
*
|
|
261
|
+
* @param {number} bytes
|
|
262
|
+
* @returns {string}
|
|
263
|
+
*/
|
|
264
|
+
function megabytes(bytes) {
|
|
265
|
+
return `${Math.round(bytes / (1024 * 1024))}MB`;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* One line saying what the process holds and what the machine has left.
|
|
270
|
+
*
|
|
271
|
+
* Pure, so the wording and the arithmetic can be pinned without a running
|
|
272
|
+
* process. Store figures are given in BYTES rather than in pieces: the piece
|
|
273
|
+
* count is meaningless without the piece size, and the piece size differs per
|
|
274
|
+
* torrent — on the film this proxy died under, 63 pieces meant 504 MB.
|
|
275
|
+
*
|
|
276
|
+
* Two scopes, because two things are being asked and only one of them is
|
|
277
|
+
* per-thread. `rss` and the kernel's rollup belong to the PROCESS, so they are
|
|
278
|
+
* read once, on the main thread. The heap, external and arrayBuffer figures
|
|
279
|
+
* belong to an ISOLATE, and the torrent worker's are the ones that matter —
|
|
280
|
+
* the piece pool is a `SharedArrayBuffer` allocated there, which the main
|
|
281
|
+
* isolate cannot see at all. Reporting only the main thread's was half the
|
|
282
|
+
* reason 650 MB of a 893 MB process had no explanation on 2026-08-28.
|
|
283
|
+
*
|
|
284
|
+
* @param {Object} reading
|
|
285
|
+
* @param {"process" | "thread"} [reading.scope]
|
|
286
|
+
* @param {string} [reading.label] - Which thread the isolate figures are of.
|
|
287
|
+
* @param {{ rss: number, heapUsed: number, heapTotal: number, external: number, arrayBuffers: number }} reading.process
|
|
288
|
+
* @param {number} [reading.availableBytes]
|
|
289
|
+
* @param {boolean} [reading.availableMeasured]
|
|
290
|
+
* @param {number | null} [reading.anonymousBytes]
|
|
291
|
+
* @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
|
|
292
|
+
* @param {number | null} [reading.diskFreeBytes]
|
|
293
|
+
* @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
|
|
294
|
+
* @param {string} [reading.extra] - Figures the caller wants on the same line
|
|
295
|
+
* rather than on one of its own. The piece-buffer counters are read here so
|
|
296
|
+
* that the number of buffers alive and the off-heap mass they should account
|
|
297
|
+
* for are the SAME instant: printed on separate timers they were up to a
|
|
298
|
+
* minute apart, and 950 MB of `arrayBuffers` could not be checked against the
|
|
299
|
+
* 62 buffers a reading half a minute away said were alive (roadmap item 2).
|
|
300
|
+
* @returns {string}
|
|
301
|
+
*/
|
|
302
|
+
export function describeMemory({
|
|
303
|
+
scope = "process",
|
|
304
|
+
label = "",
|
|
305
|
+
process: usage,
|
|
306
|
+
availableBytes,
|
|
307
|
+
availableMeasured,
|
|
308
|
+
anonymousBytes = null,
|
|
309
|
+
mappings = null,
|
|
310
|
+
diskFreeBytes = null,
|
|
311
|
+
stores = [],
|
|
312
|
+
extra = ""
|
|
313
|
+
}) {
|
|
314
|
+
const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
|
|
315
|
+
const storeResident = total("residentBytes");
|
|
316
|
+
const storeCommitted = total("committedBytes");
|
|
317
|
+
const storeSpilled = total("spilledBytes");
|
|
318
|
+
const storeBudget = total("budgetBytes");
|
|
319
|
+
// Holding and having taken are different quantities, and the gap between
|
|
320
|
+
// them is the whole of the growth this series exists to find: the pool only
|
|
321
|
+
// grows, so a spilled piece frees a slot and no memory.
|
|
322
|
+
const storesPart = stores.length === 0
|
|
323
|
+
? "no torrent stores"
|
|
324
|
+
: `${stores.length} torrent store(s) holding ${megabytes(storeResident)}, ` +
|
|
325
|
+
`committed ${megabytes(storeCommitted)} of ${megabytes(storeBudget)} allowed, ` +
|
|
326
|
+
`${megabytes(storeSpilled)} spilled to disk`;
|
|
327
|
+
const isolate =
|
|
328
|
+
`heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)}` +
|
|
329
|
+
`${usage.heapLimit ? ` of ${megabytes(usage.heapLimit)} allowed` : ""} ` +
|
|
330
|
+
`external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
|
|
331
|
+
const tail = extra ? `; ${extra}` : "";
|
|
332
|
+
if (scope === "thread") {
|
|
333
|
+
return `memory (${label || "thread"}): ${isolate}; ${storesPart}${tail}`;
|
|
334
|
+
}
|
|
335
|
+
const shape = mappings === null
|
|
336
|
+
? ""
|
|
337
|
+
: ` mappings=[heap ${megabytes(mappings.heapBytes)}, ` +
|
|
338
|
+
`${mappings.largeCount} anon ≥2MB = ${megabytes(mappings.largeBytes)} ` +
|
|
339
|
+
`(largest ${megabytes(mappings.largestBytes)}), ` +
|
|
340
|
+
`${mappings.smallCount} anon <2MB = ${megabytes(mappings.smallBytes)}, ` +
|
|
341
|
+
`files ${megabytes(mappings.fileBytes)}]`;
|
|
342
|
+
return (
|
|
343
|
+
`memory: rss=${megabytes(usage.rss)} ${isolate}` +
|
|
344
|
+
`${anonymousBytes === null ? "" : ` anon=${megabytes(anonymousBytes)}`}${shape}; ` +
|
|
345
|
+
`${storesPart}; ` +
|
|
346
|
+
`machine has ${megabytes(availableBytes ?? 0)} available` +
|
|
347
|
+
`${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
|
|
348
|
+
`${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}` +
|
|
349
|
+
tail
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* The figures whose movement earns a line, each under its own name.
|
|
355
|
+
*
|
|
356
|
+
* Not the same question as what the scope WATCHES for a heap snapshot. A
|
|
357
|
+
* thread's heap is only one of the three ways its isolate holds memory, and on
|
|
358
|
+
* 2026-09-02 it was the one that did not move: `heapTotal` stayed at 31-173 MB
|
|
359
|
+
* through a session where `arrayBuffers` swung between 130 and 950 MB, so the
|
|
360
|
+
* change trigger was watching the one quantity that stood still and every
|
|
361
|
+
* reading of the one that grew came out on the quiet interval, a minute apart.
|
|
362
|
+
*
|
|
363
|
+
* Each figure is compared against its own last written value and any one of
|
|
364
|
+
* them moving is enough. Nothing is added together, because `arrayBuffers` is
|
|
365
|
+
* documented as part of `external` and reads larger than it on this runtime —
|
|
366
|
+
* a contradiction this code has no business resolving.
|
|
367
|
+
*
|
|
368
|
+
* @param {"process" | "thread"} scope
|
|
369
|
+
* @param {{ rss: number, heapTotal: number, external: number, arrayBuffers: number }} memory
|
|
370
|
+
* @returns {Record<string, number>}
|
|
371
|
+
*/
|
|
372
|
+
export function watchedFigures(scope, memory) {
|
|
373
|
+
if (scope === "thread") {
|
|
374
|
+
return {
|
|
375
|
+
heap: memory.heapTotal ?? 0,
|
|
376
|
+
external: memory.external ?? 0,
|
|
377
|
+
buffers: memory.arrayBuffers ?? 0
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
return { rss: memory.rss ?? 0 };
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Whether this reading is worth writing down, and why.
|
|
385
|
+
*
|
|
386
|
+
* A series taken every second and printed every second is unreadable, and one
|
|
387
|
+
* printed every minute cannot see a rise that takes forty seconds. So the
|
|
388
|
+
* cadence of the READING and the cadence of the LINE are separate: the figure
|
|
389
|
+
* is taken often, and written when it has moved or when the quiet interval has
|
|
390
|
+
* passed. Pure, so the rule can be pinned without a clock.
|
|
391
|
+
*
|
|
392
|
+
* @param {Object} state
|
|
393
|
+
* @param {number} state.watchedBytes - The figure this scope is watching.
|
|
394
|
+
* @param {number} state.lastWrittenBytes
|
|
395
|
+
* @param {number} state.sinceWrittenMs
|
|
396
|
+
* @param {number} state.changeBytes - Movement that earns a line of its own.
|
|
397
|
+
* @param {number} state.quietMs - How long silence may last regardless.
|
|
398
|
+
* @returns {boolean}
|
|
399
|
+
*/
|
|
400
|
+
export function readingIsWorthWriting({
|
|
401
|
+
watchedBytes,
|
|
402
|
+
lastWrittenBytes,
|
|
403
|
+
sinceWrittenMs,
|
|
404
|
+
changeBytes,
|
|
405
|
+
quietMs
|
|
406
|
+
}) {
|
|
407
|
+
if (quietMs <= 0 || sinceWrittenMs >= quietMs) {
|
|
408
|
+
return true;
|
|
409
|
+
}
|
|
410
|
+
if (changeBytes <= 0) {
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
return Math.abs(watchedBytes - lastWrittenBytes) >= changeBytes;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Report memory on a timer until stopped.
|
|
418
|
+
*
|
|
419
|
+
* @param {Object} options
|
|
420
|
+
* @param {(message: string) => void} options.log
|
|
421
|
+
* @param {() => { name: string, residentBytes: number, budgetBytes: number }[]} [options.readStores]
|
|
422
|
+
* @param {"process" | "thread"} [options.scope] - `thread` leaves out the
|
|
423
|
+
* process-wide figures, so the torrent worker can report its own isolate
|
|
424
|
+
* without reading /proc twice.
|
|
425
|
+
* @param {string} [options.label] - Which thread the isolate figures are of.
|
|
426
|
+
* @param {string} [options.diskPath] - Where pieces spill, for the free-space
|
|
427
|
+
* reading. Omitted, the disk term is left out rather than guessed.
|
|
428
|
+
* @param {number} [options.intervalMs] - How often the figure is READ.
|
|
429
|
+
* @param {number} [options.quietMs] - How long the line may stay silent while
|
|
430
|
+
* nothing moves. Zero writes every reading, which is what the process scope
|
|
431
|
+
* has always done.
|
|
432
|
+
* @param {number} [options.changeBytes] - Movement that earns a line before the
|
|
433
|
+
* quiet interval is up.
|
|
434
|
+
* @param {string} [options.snapshotDir] - Where heap snapshots are written.
|
|
435
|
+
* Defaults to the temporary directory, as the process scope always did.
|
|
436
|
+
* @param {number} [options.snapshotFloorBytes]
|
|
437
|
+
* @param {number} [options.snapshotGrowthBytes]
|
|
438
|
+
* @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
|
|
439
|
+
* @param {() => string} [options.readExtra] - Figures to append to the line,
|
|
440
|
+
* read at the same instant as the memory itself.
|
|
441
|
+
* @returns {{ stop: () => void }}
|
|
442
|
+
*/
|
|
443
|
+
export function startMemoryReport({
|
|
444
|
+
log,
|
|
445
|
+
readStores,
|
|
446
|
+
readExtra,
|
|
447
|
+
scope = "process",
|
|
448
|
+
label = "",
|
|
449
|
+
diskPath = "",
|
|
450
|
+
intervalMs = MEMORY_REPORT_INTERVAL_MS,
|
|
451
|
+
quietMs = 0,
|
|
452
|
+
changeBytes = 0,
|
|
453
|
+
snapshotDir = "",
|
|
454
|
+
snapshotFloorBytes = 500 * 1024 * 1024,
|
|
455
|
+
snapshotGrowthBytes = 100 * 1024 * 1024,
|
|
456
|
+
keepSnapshots = 0
|
|
457
|
+
}) {
|
|
458
|
+
let highWater = 0;
|
|
459
|
+
/** @type {Record<string, number>} */
|
|
460
|
+
let lastWritten = {};
|
|
461
|
+
let lastWrittenAt = 0;
|
|
462
|
+
const slug = (label || scope).replace(/[^a-z0-9]+/gi, "-").toLowerCase();
|
|
463
|
+
const directory = snapshotDir || os.tmpdir();
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* @param {number} watchedBytes
|
|
467
|
+
* @param {string} why
|
|
468
|
+
* @returns {Promise<void>}
|
|
469
|
+
*/
|
|
470
|
+
const takeSnapshot = async (watchedBytes, why) => {
|
|
471
|
+
let snapPath = "";
|
|
472
|
+
try {
|
|
473
|
+
snapPath = path.join(directory, `heap-${slug}-${Date.now()}-${watchedBytes}.heapsnapshot`);
|
|
474
|
+
// Synchronous and proportional to the heap, so it stops this thread for
|
|
475
|
+
// as long as it takes to write. That is the price of the only reading
|
|
476
|
+
// that names what is holding the memory, and it is why it is bounded by
|
|
477
|
+
// a floor, by a growth step and by how many are kept.
|
|
478
|
+
v8.writeHeapSnapshot(snapPath);
|
|
479
|
+
log(`memory: wrote heap snapshot of the ${label || scope} to ${snapPath} (${megabytes(watchedBytes)}, ${why})`);
|
|
480
|
+
} catch {
|
|
481
|
+
// silent-ok: no snapshot is worse than a snapshot, and much better than
|
|
482
|
+
// ending the series that leads to the next one.
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
if (keepSnapshots <= 0) {
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
try {
|
|
489
|
+
const prefix = `heap-${slug}-`;
|
|
490
|
+
const mine = (await readdir(directory))
|
|
491
|
+
.filter((name) => name.startsWith(prefix) && name.endsWith(".heapsnapshot"))
|
|
492
|
+
.sort();
|
|
493
|
+
for (const name of mine.slice(0, Math.max(0, mine.length - keepSnapshots))) {
|
|
494
|
+
await rm(path.join(directory, name), { force: true });
|
|
495
|
+
}
|
|
496
|
+
} catch {
|
|
497
|
+
// silent-ok: a snapshot that could not be pruned is a disk-space problem
|
|
498
|
+
// for later, not a reason to lose the one just written.
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
const tick = async () => {
|
|
503
|
+
try {
|
|
504
|
+
let stores = [];
|
|
505
|
+
try {
|
|
506
|
+
stores = typeof readStores === "function" ? readStores() ?? [] : [];
|
|
507
|
+
} catch {
|
|
508
|
+
// silent-ok: a store list that cannot be read must not stop the reading
|
|
509
|
+
// that matters, which is the process's own.
|
|
510
|
+
}
|
|
511
|
+
const processMemory = readProcessMemory();
|
|
512
|
+
const figures = watchedFigures(scope, processMemory);
|
|
513
|
+
const now = Date.now();
|
|
514
|
+
const sinceWrittenMs = lastWrittenAt === 0 ? Number.POSITIVE_INFINITY : now - lastWrittenAt;
|
|
515
|
+
const write = Object.entries(figures).some(([name, bytes]) => readingIsWorthWriting({
|
|
516
|
+
watchedBytes: bytes,
|
|
517
|
+
lastWrittenBytes: lastWritten[name] ?? 0,
|
|
518
|
+
sinceWrittenMs,
|
|
519
|
+
changeBytes,
|
|
520
|
+
quietMs
|
|
521
|
+
}));
|
|
522
|
+
|
|
523
|
+
let anonymousBytes = null;
|
|
524
|
+
if (write) {
|
|
525
|
+
let extra = "";
|
|
526
|
+
try {
|
|
527
|
+
extra = typeof readExtra === "function" ? readExtra() ?? "" : "";
|
|
528
|
+
} catch {
|
|
529
|
+
// silent-ok: a caller's own figures are worth less than the line they
|
|
530
|
+
// would have taken down with them.
|
|
531
|
+
}
|
|
532
|
+
if (scope === "thread") {
|
|
533
|
+
log(describeMemory({ scope, label, process: processMemory, stores, extra }));
|
|
534
|
+
} else {
|
|
535
|
+
// Read only when the line is written. `smaps` is one entry per
|
|
536
|
+
// mapping and a busy process has thousands; the rollup and
|
|
537
|
+
// `/proc/meminfo` are single lines but still walk page tables, and
|
|
538
|
+
// the reading now happens once a second rather than once a minute.
|
|
539
|
+
const { bytes, measured } = await availableMemory();
|
|
540
|
+
anonymousBytes = await readAnonymousMemory();
|
|
541
|
+
log(describeMemory({
|
|
542
|
+
scope,
|
|
543
|
+
label,
|
|
544
|
+
process: processMemory,
|
|
545
|
+
availableBytes: bytes,
|
|
546
|
+
availableMeasured: measured,
|
|
547
|
+
anonymousBytes,
|
|
548
|
+
mappings: await readMappingSummary(),
|
|
549
|
+
diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
|
|
550
|
+
stores,
|
|
551
|
+
extra
|
|
552
|
+
}));
|
|
553
|
+
}
|
|
554
|
+
lastWritten = figures;
|
|
555
|
+
lastWrittenAt = now;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// A snapshot per high-water, so there is a file to open when the growth
|
|
559
|
+
// has to be named rather than described.
|
|
560
|
+
//
|
|
561
|
+
// Deliberately NOT one taken as the ceiling is approached: a snapshot is
|
|
562
|
+
// written by the isolate itself and is about the size of its heap, so
|
|
563
|
+
// asking for one at 1.9 GB on a machine with 600 MB left is a good way to
|
|
564
|
+
// cause the kill being studied. Whatever holds 1.6 GB is the same thing
|
|
565
|
+
// that holds 2.2 GB, and at one reading a second no step is ever missed.
|
|
566
|
+
//
|
|
567
|
+
// AND IT FOLLOWS THE JS HEAP, whatever this scope's LINE is written about.
|
|
568
|
+
// A snapshot can only ever explain the heap, and the process is killed by
|
|
569
|
+
// two different things: the kernel reads resident memory, V8 reads the
|
|
570
|
+
// heap. Taken on resident memory, 54 snapshots were written in one
|
|
571
|
+
// afternoon on 2026-09-07 for growth that was mostly native — each one
|
|
572
|
+
// stopping the main thread to write a file about the wrong quantity —
|
|
573
|
+
// and when V8 did kill the process at 1.99 GB of old space, no snapshot
|
|
574
|
+
// had ever been taken of the growth that did it.
|
|
575
|
+
const heapWatched = processMemory.heapTotal;
|
|
576
|
+
if (heapWatched > highWater + snapshotGrowthBytes && heapWatched > snapshotFloorBytes) {
|
|
577
|
+
highWater = heapWatched;
|
|
578
|
+
await takeSnapshot(heapWatched, "a new high-water of the heap");
|
|
579
|
+
}
|
|
580
|
+
// Under `write` by construction: `anonymousBytes` is only read when the
|
|
581
|
+
// line is, and at one reading a second an unconditional warning would be
|
|
582
|
+
// a line a second for as long as the process stayed large.
|
|
583
|
+
if (scope !== "thread" && anonymousBytes !== null && processMemory.rss > 800 * 1024 * 1024) {
|
|
584
|
+
log(`memory: high rss=${megabytes(processMemory.rss)} anon=${megabytes(anonymousBytes)} heap=${megabytes(processMemory.heapUsed)} — watch for OOM`);
|
|
585
|
+
}
|
|
586
|
+
} catch {
|
|
587
|
+
// silent-ok: a reading that fails is not worth ending the series over.
|
|
588
|
+
}
|
|
589
|
+
};
|
|
590
|
+
void tick();
|
|
591
|
+
const timer = setInterval(() => { void tick(); }, intervalMs);
|
|
592
|
+
if (typeof timer.unref === "function") {
|
|
593
|
+
timer.unref();
|
|
594
|
+
}
|
|
595
|
+
return { stop: () => clearInterval(timer) };
|
|
596
|
+
}
|