@torrent-tv/proxy 2.89.6 → 2.89.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +32 -12
- package/CHANGELOG.md +2340 -2330
- package/assets/calibration/audio/NOTICE.md +19 -19
- package/assets/calibration/audio/manifest.json +98 -98
- package/biome.json +412 -412
- package/docs/browser-proxy-contract.md +155 -155
- package/docs/container-architecture.md +273 -273
- package/docs/encode-run-state.md +1 -1
- package/docs/playback-start-forecast.md +113 -98
- package/package.json +51 -50
- package/routes/api/sources/files/get.js +106 -106
- package/scripts/build-audio-samples.mjs +31 -31
- package/services/encode/EncodeRun.js +67 -15
- package/services/encode/EncodeRuns.js +19 -7
- package/services/encode/OutputOpening.js +784 -784
- package/services/encode/audio-calibration.js +67 -67
- package/services/encode/audio-work-rate.js +15 -15
- package/services/encode/encode-exit.js +2 -0
- package/services/encode/hwaccel.js +1894 -1894
- package/services/encode/output/index.js +24 -24
- package/services/encode/piece-completeness.js +60 -2
- package/services/encode/run-command.js +757 -755
- package/services/encode/segment-formats/index.js +1 -0
- package/services/encode/segment-formats/mp4-boxes.js +8 -1
- package/services/media/ContainerOrchestrator.js +202 -202
- package/services/media/KeyframeTables.js +356 -356
- package/services/media/SubtitleCues.js +592 -592
- package/services/media/audio-inventory.js +299 -299
- package/services/media/container/AviContainer.js +268 -268
- package/services/media/container/Container.js +632 -632
- package/services/media/container/ContainerFactory.js +38 -38
- package/services/media/container/MatroskaContainer.js +1031 -1031
- package/services/media/container/Mp4Container.js +1354 -1354
- package/services/media/container/SubtitleFileContainer.js +339 -339
- package/services/media/container/ebml-stream.js +174 -174
- package/services/media/container/matroska-clusters.js +1165 -1165
- package/services/media/container/unavailable.js +77 -77
- package/services/media/tracks/ContainerTrack.js +84 -84
- package/services/media/tracks/TextSubtitleTrack.js +295 -295
- package/services/storage/piece-store/shared-piece-store.js +1937 -1937
- package/services/storage/segment-store/SegmentStore.js +1294 -1294
- package/services/torrent/Contents.js +382 -382
- package/services/torrent/episode-naming.js +182 -182
- package/services/torrent/torrent-pool.js +2688 -2688
- package/services/torrent/worker/client.js +836 -836
- package/services/torrent/worker/held-bytes.js +231 -231
- package/services/torrent/worker/worker.js +1137 -1137
- package/services/transport/data-channel-handler.js +1373 -1373
- package/services/viewer/PriorityMap.js +421 -421
- package/test/audio-calibration.test.js +23 -23
- package/test/auto-quality-step.test.js +762 -762
- package/test/coalesce.test.js +39 -39
- package/test/coverage-follows-the-disk.test.js +5 -0
- package/test/decode-cost.test.js +475 -475
- package/test/encode-run.test.js +58 -0
- package/test/episode-naming.test.js +131 -131
- package/test/failed-encoder-still-serves-what-exists.test.js +1 -1
- package/test/fixtures/audio-frame-boundary.json +1 -1
- package/test/fixtures/refused-audio-20261002.json +10 -0
- package/test/fmp4-presentation-clock.test.js +4 -2
- package/test/format-keyed-outputs.test.js +78 -78
- package/test/health-metrics.test.js +11 -13
- package/test/held-read-ownership.test.js +77 -77
- package/test/helpers/encode-run.js +14 -6
- package/test/helpers/held-file.js +77 -77
- package/test/helpers/manager.js +62 -62
- package/test/helpers/matroska-file.js +296 -296
- package/test/keyframe-table-budget.test.js +123 -123
- package/test/matroska-cluster-discovery.test.js +323 -323
- package/test/one-output-one-session.test.js +273 -273
- package/test/output-speed.test.js +6 -1
- package/test/piece-completeness.test.js +37 -1
- package/test/quality-variants.test.js +1274 -1270
- package/test/rate-control-in-the-output.test.js +4 -3
- package/test/reads-not-yet-arrived.test.js +218 -218
- package/test/seek-frees-the-viewers-requests.test.js +1 -1
- package/test/segment-closed-on-both-branches.test.js +114 -114
- package/test/segment-judged-by-its-own-header.test.js +222 -222
- package/test/segment-serve-wiring.test.js +481 -481
- package/test/serving-existing-output.test.js +111 -111
- package/test/source-files-route.test.js +113 -113
- package/test/subtitle-cue-walk.test.js +385 -385
- package/test/subtitle-metadata.test.js +15 -15
- package/test/subtitle-plan-waits.test.js +125 -125
- package/test/subtitle-track-numbering.test.js +391 -391
- package/test/two-viewers-one-picture.test.js +508 -508
- package/test/viewer-requests-admission.test.js +92 -92
- package/test/viewer-starts-on-map.test.js +27 -27
- package/test/viewer-states.test.js +5 -2
- package/test/viewing-generations.test.js +390 -390
- package/test/whole-file-held-read.test.js +30 -30
- package/utils/coalesce.js +41 -41
|
@@ -1,1294 +1,1294 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file One store of produced segments for the whole proxy, addressed by what
|
|
3
|
-
* the segments ARE rather than by who made them.
|
|
4
|
-
*
|
|
5
|
-
* Until now a segment lived under the id of the session whose encoder wrote it
|
|
6
|
-
* — under that session's own id, in a directory of that run's own — and the index
|
|
7
|
-
* over it was built per session, so a segment was visible only inside the
|
|
8
|
-
* session that made it. Two viewers of one film got two sessions with
|
|
9
|
-
* byte-identical output and neither could see the other's work (measured
|
|
10
|
-
* 2026-09-03, `research/two-viewers-one-file-2026-09-03.md`).
|
|
11
|
-
*
|
|
12
|
-
* Here the address is the output's own parameters. Which viewer asked never
|
|
13
|
-
* enters it, and neither does which encoder produced the bytes: a segment is
|
|
14
|
-
* the same segment whoever made it.
|
|
15
|
-
*
|
|
16
|
-
* **The directory carries its own identity.** Its NAME is a digest, because the
|
|
17
|
-
* key contains characters a path may not; the key itself is written inside it,
|
|
18
|
-
* in `key.txt`. That is what lets a new process, started after this one was
|
|
19
|
-
* killed, work out what it is looking at — without it, everything on disk after
|
|
20
|
-
* a kill is unidentifiable and can only be thrown away.
|
|
21
|
-
*
|
|
22
|
-
* **What proves a segment is closed: its NAME.** A piece being written is called
|
|
23
|
-
* something else — `making-40-00057.mp4`, tagged with the run writing it — and
|
|
24
|
-
* takes its served name only when the
|
|
25
|
-
* encoder has said it is closed, which it does on a channel of its own
|
|
26
|
-
* (`-segment_list pipe:3`). The `hls` branch needs nothing extra: its muxer
|
|
27
|
-
* writes through a temporary name of its own, so its files appear under their
|
|
28
|
-
* final name whole. One rule for both, and true whether or not this process is
|
|
29
|
-
* alive: **a file under its served name is complete.**
|
|
30
|
-
*
|
|
31
|
-
* It was "a segment is closed when the NEXT number exists". That is true of one
|
|
32
|
-
* writer walking forward and false the moment two runs share an output, because
|
|
33
|
-
* the next file is then written by another process while this one is still open
|
|
34
|
-
* — and two runs on one output is not a rare state, it is what the plan gives an
|
|
35
|
-
* output whenever it places a second encoder. Field 2026-09-08:
|
|
36
|
-
* `segment-00057.mp4` served at 2 268 361 bytes and then at 4 510 940, exactly
|
|
37
|
-
* half; the browser appended the half and refused the whole for the rest of the
|
|
38
|
-
* session, `bufferAppendError` fourteen times with the picture frozen at
|
|
39
|
-
* 319.66 s. `segment-00055.mp4` the same, 211 957 against 2 620 617.
|
|
40
|
-
*/
|
|
41
|
-
|
|
42
|
-
import { createHash } from "node:crypto";
|
|
43
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, watch, writeFileSync } from "node:fs";
|
|
44
|
-
import { rmSync } from "node:fs";
|
|
45
|
-
import os from "node:os";
|
|
46
|
-
import path from "node:path";
|
|
47
|
-
|
|
48
|
-
/** Where every output's segments live. One root for the process. */
|
|
49
|
-
export const DEFAULT_STORE_ROOT = path.join(os.tmpdir(), "torrent-tv-hls");
|
|
50
|
-
|
|
51
|
-
/** The file inside each directory that says which output it holds. */
|
|
52
|
-
const KEY_FILE = "key.txt";
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* The directory name for an output key.
|
|
56
|
-
*
|
|
57
|
-
* A digest rather than the key itself: the key carries `:` and `/`, which a
|
|
58
|
-
* path may not, and sanitising them would make two different keys collide.
|
|
59
|
-
* Sixteen hex characters is enough that a collision is not a thing that
|
|
60
|
-
* happens, and short enough to read in a log line.
|
|
61
|
-
*
|
|
62
|
-
* @param {string} key
|
|
63
|
-
* @returns {string}
|
|
64
|
-
*/
|
|
65
|
-
export function directoryNameFor(key) {
|
|
66
|
-
return createHash("sha256").update(String(key)).digest("hex").slice(0, 16);
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* What one output's directory holds, as last read.
|
|
71
|
-
*
|
|
72
|
-
* @typedef {object} HeldContents
|
|
73
|
-
* @property {number} readAt - The directory's modification time when it was read.
|
|
74
|
-
* @property {Map<number, string>} byNumber - Segment number to full path.
|
|
75
|
-
* @property {number} bytes - What those files weigh.
|
|
76
|
-
* @property {number} unproven - The highest number, whose closure nothing
|
|
77
|
-
* proves, or -1 when the directory holds no segments.
|
|
78
|
-
*/
|
|
79
|
-
|
|
80
|
-
export class SegmentStore {
|
|
81
|
-
/** @type {string} */
|
|
82
|
-
#root;
|
|
83
|
-
|
|
84
|
-
/** Output key → what its directory holds. @type {Map<string, HeldContents>} */
|
|
85
|
-
#held = new Map();
|
|
86
|
-
#mediaRanges = new Map();
|
|
87
|
-
|
|
88
|
-
/** Output key → how to read its file names. @type {Map<string, object>} */
|
|
89
|
-
#formats = new Map();
|
|
90
|
-
|
|
91
|
-
/** Output key → when it was last asked for. @type {Map<string, number>} */
|
|
92
|
-
#touched = new Map();
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* Output key → the init segment served for it, once one has been read whole.
|
|
96
|
-
*
|
|
97
|
-
* A property of the output's produced material, so it lives with that
|
|
98
|
-
* material and goes when it goes. The player fetches it once and never again,
|
|
99
|
-
* so the first whole one is kept and every later request gets the same bytes,
|
|
100
|
-
* even though a re-encoding run rewrites the file on disk.
|
|
101
|
-
*
|
|
102
|
-
* @type {Map<string, Buffer>}
|
|
103
|
-
*/
|
|
104
|
-
#inits = new Map();
|
|
105
|
-
|
|
106
|
-
/** Told of every segment published, whatever it is. @type {Set<(key: string, index: number) => void>} */
|
|
107
|
-
#publishedListeners = new Set();
|
|
108
|
-
|
|
109
|
-
/** Segment publication waiters by output key and segment number. @type {Map<string, Map<number, Set<Function>>>} */
|
|
110
|
-
#waiters = new Map();
|
|
111
|
-
|
|
112
|
-
/** Output key → the watch on its directory. @type {Map<string, import("node:fs").FSWatcher>} */
|
|
113
|
-
#watchers = new Map();
|
|
114
|
-
|
|
115
|
-
/** @type {{ info: Function, warn: Function }} */
|
|
116
|
-
#logger;
|
|
117
|
-
|
|
118
|
-
/** @type {() => number} */
|
|
119
|
-
#now;
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* @param {object} [params]
|
|
123
|
-
* @param {string} [params.root] - Where the directories live.
|
|
124
|
-
* @param {{ info: Function, warn: Function }} [params.logger]
|
|
125
|
-
* @param {() => number} [params.now]
|
|
126
|
-
*/
|
|
127
|
-
constructor({ root = DEFAULT_STORE_ROOT, logger = null, now = Date.now } = {}) {
|
|
128
|
-
this.#root = root;
|
|
129
|
-
this.#logger = logger ?? { info: () => {}, warn: () => {} };
|
|
130
|
-
this.#now = now;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/** @returns {string} */
|
|
134
|
-
get root() {
|
|
135
|
-
return this.#root;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/**
|
|
139
|
-
* Where this output's segments live, without making anything.
|
|
140
|
-
*
|
|
141
|
-
* Separate from {@link directoryFor} because a session works out its path
|
|
142
|
-
* long before it is sure it will exist: a probe or a keyframe read between
|
|
143
|
-
* the two can still fail, and a directory made in advance of that is a
|
|
144
|
-
* leftover nothing tracks — proxy 2.9.101 failed on every request and its
|
|
145
|
-
* abandoned directories were the only trace on disk.
|
|
146
|
-
*
|
|
147
|
-
* @param {string} key - `OutputSpec.toKey()`.
|
|
148
|
-
* @returns {string}
|
|
149
|
-
*/
|
|
150
|
-
pathFor(key) {
|
|
151
|
-
return path.join(this.#root, directoryNameFor(key));
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* The directory this output's segments live in, made if it is not there.
|
|
156
|
-
*
|
|
157
|
-
* @param {string} key - `OutputSpec.toKey()`.
|
|
158
|
-
* @returns {string}
|
|
159
|
-
*/
|
|
160
|
-
directoryFor(key) {
|
|
161
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
162
|
-
if (!existsSync(dir)) {
|
|
163
|
-
// A watch held on this path watches the directory that was removed, not
|
|
164
|
-
// the one about to be made, and while it is held `#watch` makes no other.
|
|
165
|
-
this.#unwatch(key);
|
|
166
|
-
mkdirSync(dir, { recursive: true });
|
|
167
|
-
// What this directory is, for whoever finds it after this process has
|
|
168
|
-
// been killed. Without it the sweep can only throw everything away.
|
|
169
|
-
writeFileSync(path.join(dir, KEY_FILE), `${key}\n`, "utf8");
|
|
170
|
-
}
|
|
171
|
-
this.#touched.set(key, this.#now());
|
|
172
|
-
// Production is about to begin here, and a piece closing in this directory
|
|
173
|
-
// is what every held request is waiting for.
|
|
174
|
-
this.#watch(key);
|
|
175
|
-
return dir;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* Say how this output's files are named, so the store can read its directory.
|
|
180
|
-
*
|
|
181
|
-
* @param {string} key
|
|
182
|
-
* @param {{ isSegmentFileName: (name: string) => boolean, segmentIndexFromName: (name: string) => number }} segmentFormat
|
|
183
|
-
*/
|
|
184
|
-
useFormat(key, segmentFormat) {
|
|
185
|
-
if (segmentFormat && typeof segmentFormat.isSegmentFileName === "function") {
|
|
186
|
-
this.#formats.set(key, segmentFormat);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
/**
|
|
191
|
-
* Re-read this output's directory if it has moved since last time.
|
|
192
|
-
*
|
|
193
|
-
* A directory's modification time changes when an entry is added or removed,
|
|
194
|
-
* so a quiet request costs one `stat` rather than a listing.
|
|
195
|
-
*
|
|
196
|
-
* @param {string} key
|
|
197
|
-
* @returns {HeldContents}
|
|
198
|
-
*/
|
|
199
|
-
refresh(key) {
|
|
200
|
-
const format = this.#formats.get(key);
|
|
201
|
-
const empty = { readAt: 0, byNumber: new Map(), bytes: 0, unproven: -1, largest: { index: -1, size: 0 } };
|
|
202
|
-
if (!format) {
|
|
203
|
-
return this.#held.get(key) ?? empty;
|
|
204
|
-
}
|
|
205
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
206
|
-
let mtime = 0;
|
|
207
|
-
try {
|
|
208
|
-
mtime = statSync(dir).mtimeMs;
|
|
209
|
-
} catch {
|
|
210
|
-
this.#held.delete(key);
|
|
211
|
-
return empty;
|
|
212
|
-
}
|
|
213
|
-
const known = this.#held.get(key);
|
|
214
|
-
if (known && known.readAt === mtime) {
|
|
215
|
-
return known;
|
|
216
|
-
}
|
|
217
|
-
const byNumber = new Map();
|
|
218
|
-
const sizes = new Map();
|
|
219
|
-
let bytes = 0;
|
|
220
|
-
let highest = -1;
|
|
221
|
-
// The biggest piece and which number it is. What reads it is the figure the
|
|
222
|
-
// master playlist declares: `BANDWIDTH` is the PEAK a link must carry, and
|
|
223
|
-
// the peak of a variable-bitrate source is nothing like its average — the
|
|
224
|
-
// field file of 2026-09-08 ran at 17.1 Mbit/s with a piece at 73. Which
|
|
225
|
-
// number it is matters because pieces are not all the same length, and only
|
|
226
|
-
// whoever holds the cut table can turn bytes into bits per second.
|
|
227
|
-
let largest = { index: -1, size: 0 };
|
|
228
|
-
try {
|
|
229
|
-
for (const name of readdirSync(dir)) {
|
|
230
|
-
if (!format.isSegmentFileName(name)) {
|
|
231
|
-
continue;
|
|
232
|
-
}
|
|
233
|
-
const index = format.segmentIndexFromName(name);
|
|
234
|
-
if (!Number.isInteger(index) || index < 0) {
|
|
235
|
-
continue;
|
|
236
|
-
}
|
|
237
|
-
const full = path.join(dir, name);
|
|
238
|
-
let size = 0;
|
|
239
|
-
try {
|
|
240
|
-
size = statSync(full).size;
|
|
241
|
-
} catch {
|
|
242
|
-
continue;
|
|
243
|
-
}
|
|
244
|
-
// A file of no bytes is not a segment, whatever its name says. It is
|
|
245
|
-
// what a run killed the instant after opening its next piece leaves,
|
|
246
|
-
// and taking it for a segment once convinced the look-ahead that a
|
|
247
|
-
// number had been produced and kept the encoder stopped for it.
|
|
248
|
-
if (size <= 0) {
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
byNumber.set(index, full);
|
|
252
|
-
sizes.set(index, size);
|
|
253
|
-
bytes += size;
|
|
254
|
-
if (index > highest) {
|
|
255
|
-
highest = index;
|
|
256
|
-
}
|
|
257
|
-
if (size > largest.size) {
|
|
258
|
-
largest = { index, size };
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
} catch {
|
|
262
|
-
this.#held.delete(key);
|
|
263
|
-
return empty;
|
|
264
|
-
}
|
|
265
|
-
const contents = { readAt: mtime, byNumber, sizes, bytes, unproven: highest, largest };
|
|
266
|
-
this.#held.set(key, contents);
|
|
267
|
-
return contents;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
/**
|
|
271
|
-
* The segment numbers this output holds that are finished.
|
|
272
|
-
*
|
|
273
|
-
* ONE PROOF, AND IT IS THE PIECE'S OWN NAME. A piece being written is called
|
|
274
|
-
* `making-40-00042.mp4`; it is renamed to `segment-00042.mp4` when its writer
|
|
275
|
-
* says it has closed it, and on the `hls` branch — which has no such channel —
|
|
276
|
-
* the muxer's own `+temp_file` does the same rename for the same reason. So a
|
|
277
|
-
* file under the served name is complete, whoever made it and whenever.
|
|
278
|
-
*
|
|
279
|
-
* WHAT THIS REPLACED, because the difference is what a viewer felt. Closure
|
|
280
|
-
* used to be inferred from the NEXT number existing, which is sound for one
|
|
281
|
-
* writer walking forward and false the moment two runs share an output — and
|
|
282
|
-
* one-piece intervals guarantee that. Field 2026-09-08:
|
|
283
|
-
* `segment-00057.mp4` was served at 2 268 361 bytes and then at 4 510 940, the
|
|
284
|
-
* browser appended the truncated body, and `bufferAppendError` repeated to the
|
|
285
|
-
* end of the log with the picture frozen at 319.66 s. It also left the last
|
|
286
|
-
* piece of every run unprovable for ever, since nothing follows it.
|
|
287
|
-
*
|
|
288
|
-
* @param {string} key
|
|
289
|
-
* @returns {number[]}
|
|
290
|
-
*/
|
|
291
|
-
provenNumbers(key) {
|
|
292
|
-
return [...this.refresh(key).byNumber.keys()].sort((left, right) => left - right);
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
/**
|
|
296
|
-
* How many numbered files this output holds, closed or not.
|
|
297
|
-
*
|
|
298
|
-
* What the disk has, against what has been proven closed: the two are printed
|
|
299
|
-
* side by side, so "the files are there and nobody reported them" reads
|
|
300
|
-
* differently from "there is nothing there".
|
|
301
|
-
*
|
|
302
|
-
* @param {string} key
|
|
303
|
-
* @returns {number}
|
|
304
|
-
*/
|
|
305
|
-
filesHeld(key) {
|
|
306
|
-
return this.refresh(key).byNumber.size;
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
/**
|
|
310
|
-
* The size in bytes of every finished piece of this output, by number.
|
|
311
|
-
*
|
|
312
|
-
* Read by whoever keeps what an output's segments carried (roadmap item 97,
|
|
313
|
-
* step 14). A reading, not a request: it does not mark the output as read,
|
|
314
|
-
* so it does not keep an output nobody is watching.
|
|
315
|
-
*
|
|
316
|
-
* @param {string} key
|
|
317
|
-
* @returns {Map<number, number>}
|
|
318
|
-
*/
|
|
319
|
-
sizesOf(key) {
|
|
320
|
-
return new Map(this.refresh(key).sizes ?? []);
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
/**
|
|
324
|
-
* The biggest piece this output has made, and which number it is.
|
|
325
|
-
*
|
|
326
|
-
* Read by whoever declares the variant's peak rate. Bytes alone cannot say
|
|
327
|
-
* it — pieces are not all the same length — so the number comes with them and
|
|
328
|
-
* whoever holds the cut table does the division.
|
|
329
|
-
*
|
|
330
|
-
* @param {string} key
|
|
331
|
-
* @returns {{ index: number, size: number }} An index of `-1` while nothing
|
|
332
|
-
* has been made, which is a statement and not a zero.
|
|
333
|
-
*/
|
|
334
|
-
largestPiece(key) {
|
|
335
|
-
return this.refresh(key).largest ?? { index: -1, size: 0 };
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
/**
|
|
340
|
-
* Every output whose pieces this store knows how to read: those in use and
|
|
341
|
-
* those a previous life of the process left.
|
|
342
|
-
*
|
|
343
|
-
* @returns {string[]}
|
|
344
|
-
*/
|
|
345
|
-
addresses() {
|
|
346
|
-
return [...this.#formats.keys()];
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* Whether this piece is finished, and may therefore be served.
|
|
351
|
-
*
|
|
352
|
-
* Its NAME is the proof, and there is no second one: a piece being written is
|
|
353
|
-
* called something else until whoever writes it says it is closed. That holds
|
|
354
|
-
* for a piece a live run is rewriting — the file standing there was closed by
|
|
355
|
-
* somebody, and it is replaced whole or not at all — and for a piece left by an
|
|
356
|
-
* earlier life of this process, which the startup sweep answers the same way.
|
|
357
|
-
*
|
|
358
|
-
* @param {string} key
|
|
359
|
-
* @param {number} index
|
|
360
|
-
* @returns {boolean}
|
|
361
|
-
*/
|
|
362
|
-
isClosed(key, index) {
|
|
363
|
-
return this.refresh(key).byNumber.has(index);
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
/**
|
|
367
|
-
* Remove the pieces an output was in the middle of writing.
|
|
368
|
-
*
|
|
369
|
-
* They are under working names, so they were never servable and nothing has
|
|
370
|
-
* to be un-proven — this is disk, not correctness. A process killed by the
|
|
371
|
-
* kernel leaves one per live run, and the kernel takes this process often
|
|
372
|
-
* enough for that to matter.
|
|
373
|
-
*
|
|
374
|
-
* @param {string} key
|
|
375
|
-
* @param {string} dir
|
|
376
|
-
* @param {{ makingTagOf?: (name: string) => string | null }} format
|
|
377
|
-
* @param {string | null} [tag] - One run's own tag, or null for every run's.
|
|
378
|
-
* @returns {number} How many were removed.
|
|
379
|
-
*/
|
|
380
|
-
#sweepUnfinished(key, dir, format, tag = null) {
|
|
381
|
-
let removed = 0;
|
|
382
|
-
let names = [];
|
|
383
|
-
try {
|
|
384
|
-
names = readdirSync(dir);
|
|
385
|
-
} catch {
|
|
386
|
-
return 0;
|
|
387
|
-
}
|
|
388
|
-
for (const name of names) {
|
|
389
|
-
const wroteIt = format?.makingTagOf?.(name) ?? null;
|
|
390
|
-
if (wroteIt === null || (tag !== null && wroteIt !== tag)) {
|
|
391
|
-
continue;
|
|
392
|
-
}
|
|
393
|
-
try {
|
|
394
|
-
rmSync(path.join(dir, name), { force: true });
|
|
395
|
-
removed += 1;
|
|
396
|
-
} catch {
|
|
397
|
-
// Then it stays, costing disk and nothing else.
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
if (removed > 0) {
|
|
401
|
-
this.#held.delete(key);
|
|
402
|
-
}
|
|
403
|
-
return removed;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
/**
|
|
407
|
-
* The encoder has closed a piece: give it the name it is served under.
|
|
408
|
-
*
|
|
409
|
-
* One rename inside the output's own directory — one filesystem operation, and
|
|
410
|
-
* atomic there. Before it the file is not a segment and no request can reach
|
|
411
|
-
* it; after it, its existence IS the proof that it is whole, and that is one
|
|
412
|
-
* rule for every branch whether or not our own process is alive.
|
|
413
|
-
*
|
|
414
|
-
* It replaced a rule that served half a segment: a piece was taken as finished
|
|
415
|
-
* when the NEXT file existed. That is true of one writer walking forward and
|
|
416
|
-
* false the moment two runs share an output, because the next file is then
|
|
417
|
-
* written by another process while this one is still open — and two runs on one
|
|
418
|
-
* output is not a rare state, it is what the plan gives an output whenever it
|
|
419
|
-
* places a second encoder.
|
|
420
|
-
*
|
|
421
|
-
* Field 2026-09-08: `segment-00057.mp4` was served at 2 268 361 bytes and then
|
|
422
|
-
* at 4 510 940 — exactly half of it. The browser appended the half and refused
|
|
423
|
-
* the whole for the rest of the session, `bufferAppendError` fourteen times
|
|
424
|
-
* over with the picture frozen at 319.66 s. `segment-00055.mp4` went the same
|
|
425
|
-
* way, 211 957 against 2 620 617.
|
|
426
|
-
*
|
|
427
|
-
* Whether the piece is whole is decided before this, by the encoding
|
|
428
|
-
* (`encode/piece-completeness.js`): the store keeps bytes and what is read
|
|
429
|
-
* from them, and does not judge them.
|
|
430
|
-
*
|
|
431
|
-
* @param {string} key
|
|
432
|
-
* @param {string} makingName - What the encoder called it while writing.
|
|
433
|
-
* @param {{ servedNameOf?: (name: string) => string | null, segmentIndexFromName?: (name: string) => number }} format
|
|
434
|
-
* @param {{ mediaRanges?: object | null }} [read] - The piece's media
|
|
435
|
-
* intervals as its format read them from the closed file, kept with it.
|
|
436
|
-
* @returns {string | null} The served name, or null where nothing was renamed.
|
|
437
|
-
*/
|
|
438
|
-
publish(key, makingName, format, { mediaRanges = null } = {}) {
|
|
439
|
-
const served = format?.servedNameOf?.(makingName) ?? null;
|
|
440
|
-
if (!served) {
|
|
441
|
-
return null;
|
|
442
|
-
}
|
|
443
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
444
|
-
const index = format?.segmentIndexFromName?.(served) ?? -1;
|
|
445
|
-
try {
|
|
446
|
-
renameSync(path.join(dir, makingName), path.join(dir, served));
|
|
447
|
-
} catch (error) {
|
|
448
|
-
// The file may already be gone — a process killed between closing the
|
|
449
|
-
// piece and this line. Said rather than swallowed: a piece the encoder
|
|
450
|
-
// reported and the disk does not have is worth knowing about.
|
|
451
|
-
this.#logger?.warn?.(
|
|
452
|
-
`segment store: could not publish ${makingName} of ${key.slice(0, 60)}: ` +
|
|
453
|
-
`${error instanceof Error ? error.message : String(error)}`
|
|
454
|
-
);
|
|
455
|
-
return null;
|
|
456
|
-
}
|
|
457
|
-
// What the directory holds has changed, so the memory of it is stale.
|
|
458
|
-
this.#held.delete(key);
|
|
459
|
-
if (index >= 0) {
|
|
460
|
-
if (mediaRanges) {
|
|
461
|
-
const byIndex = this.#mediaRanges.get(key) ?? new Map();
|
|
462
|
-
byIndex.set(index, mediaRanges);
|
|
463
|
-
this.#mediaRanges.set(key, byIndex);
|
|
464
|
-
}
|
|
465
|
-
this.announce(key, index);
|
|
466
|
-
for (const listener of this.#publishedListeners) {
|
|
467
|
-
listener(key, index);
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
return served;
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
/**
|
|
474
|
-
* The bytes of a file the encoder has closed under its working name, for the
|
|
475
|
-
* encoding to judge before it is published.
|
|
476
|
-
*
|
|
477
|
-
* @param {string} key
|
|
478
|
-
* @param {string} makingName
|
|
479
|
-
* @returns {Buffer}
|
|
480
|
-
*/
|
|
481
|
-
closedBytesOf(key, makingName) {
|
|
482
|
-
return readFileSync(path.join(this.#root, directoryNameFor(key), makingName));
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
/**
|
|
486
|
-
* The media intervals a stored piece holds, read once from its bytes and kept
|
|
487
|
-
* with them, or undefined where its format reads none or it is not here.
|
|
488
|
-
*
|
|
489
|
-
* @param {string} key
|
|
490
|
-
* @param {number} index
|
|
491
|
-
* @param {{ startSeconds?: number }} [where] - Where its cut puts it.
|
|
492
|
-
* @returns {object | undefined}
|
|
493
|
-
*/
|
|
494
|
-
mediaRangesOf(key, index, { startSeconds = 0 } = {}) {
|
|
495
|
-
const known = this.#mediaRanges.get(key)?.get(index);
|
|
496
|
-
const format = this.#formats.get(key);
|
|
497
|
-
if (!format?.readMediaRanges) return undefined;
|
|
498
|
-
if (known) return known;
|
|
499
|
-
const filePath = this.pathOf(key, index);
|
|
500
|
-
if (!filePath) return undefined;
|
|
501
|
-
try {
|
|
502
|
-
let initBytes = this.initOf(key);
|
|
503
|
-
if (!initBytes && format.initFileName) {
|
|
504
|
-
try {
|
|
505
|
-
initBytes = readFileSync(path.join(this.pathFor(key), format.initFileName));
|
|
506
|
-
} catch {
|
|
507
|
-
// A header not yet produced is not an empty media interval.
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
const ranges = format.readMediaRanges(readFileSync(filePath), { initBytes, startSeconds });
|
|
511
|
-
if (!ranges) return undefined;
|
|
512
|
-
const byIndex = this.#mediaRanges.get(key) ?? new Map();
|
|
513
|
-
byIndex.set(index, ranges);
|
|
514
|
-
this.#mediaRanges.set(key, byIndex);
|
|
515
|
-
return ranges;
|
|
516
|
-
} catch {
|
|
517
|
-
return [];
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
/**
|
|
522
|
-
* The init segment kept for this output, or null until one was read whole.
|
|
523
|
-
*
|
|
524
|
-
* @param {string} key
|
|
525
|
-
* @returns {Buffer | null}
|
|
526
|
-
*/
|
|
527
|
-
initOf(key) {
|
|
528
|
-
return this.#inits.get(key) ?? null;
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
/**
|
|
532
|
-
* Keep the init segment served for this output. The first whole one stands;
|
|
533
|
-
* an empty read is not an init (ffmpeg creates the file before it writes the
|
|
534
|
-
* header into it) and is refused.
|
|
535
|
-
*
|
|
536
|
-
* @param {string} key
|
|
537
|
-
* @param {Buffer | null} bytes
|
|
538
|
-
* @returns {Buffer | null} The init that now stands for this output.
|
|
539
|
-
*/
|
|
540
|
-
keepInit(key, bytes) {
|
|
541
|
-
if (!this.#inits.has(key) && bytes && bytes.length > 0) {
|
|
542
|
-
this.#inits.set(key, bytes);
|
|
543
|
-
}
|
|
544
|
-
return this.initOf(key);
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
/**
|
|
548
|
-
* Be told of every segment published: what the store holds has grown, which
|
|
549
|
-
* is when whether it still fits its share is worth asking.
|
|
550
|
-
*
|
|
551
|
-
* @param {(key: string, index: number) => void} listener
|
|
552
|
-
* @returns {() => void} Stop being told.
|
|
553
|
-
*/
|
|
554
|
-
onPublished(listener) {
|
|
555
|
-
this.#publishedListeners.add(listener);
|
|
556
|
-
return () => this.#publishedListeners.delete(listener);
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
/**
|
|
560
|
-
* Watch this output's directory, so a piece closed by ANY writer is an event.
|
|
561
|
-
*
|
|
562
|
-
* WHY IT EXISTS, and it is one fault rather than a precaution. A piece is
|
|
563
|
-
* finished in one of two ways, and only one of them said so. The `segment`
|
|
564
|
-
* muxer writes under a working name and reports the closure on a channel of
|
|
565
|
-
* its own (`publish`); the `hls` muxer writes through a temporary name of its
|
|
566
|
-
* own (`+temp_file`) and reports nothing, so the file simply appears under the
|
|
567
|
-
* name it is served as. That branch is taken by every re-encoded output on the
|
|
568
|
-
* even grid — the ordinary quality step — and for those `waitFor` could only
|
|
569
|
-
* end on its deadline: the segment was on disk and the request that wanted it
|
|
570
|
-
* went on waiting, up to a full minute, and was then answered 503.
|
|
571
|
-
*
|
|
572
|
-
* So the store watches what it owns. The rename into place moves the
|
|
573
|
-
* directory's modification time, the kernel says so, and both branches reach
|
|
574
|
-
* one statement: this number is now present. Nothing polls.
|
|
575
|
-
*
|
|
576
|
-
* Idempotent, and started from the two places that need it — the directory
|
|
577
|
-
* being made for a run, and a wait beginning on an output this process has
|
|
578
|
-
* adopted rather than made.
|
|
579
|
-
*
|
|
580
|
-
* `drop` stops the watch before it removes the directory. A DIRECTORY
|
|
581
|
-
* REMOVED BY ANYBODY ELSE ENDS ITS WATCH TOO, and it has to be noticed here,
|
|
582
|
-
* because no `error` is emitted for it. Measured 2026-10-02 on Windows, Node
|
|
583
|
-
* 24.2: the removal succeeds, and the watch then reports `rename` for the
|
|
584
|
-
* directory itself without end — 232 618 events in eight seconds — and that
|
|
585
|
-
* stream keeps the process alive although the watch is not persistent. Two
|
|
586
|
-
* test files never exited for it. Left in place, a dead watch also stops
|
|
587
|
-
* this method from watching the directory made again under the same name.
|
|
588
|
-
*
|
|
589
|
-
* @param {string} key
|
|
590
|
-
* @returns {void}
|
|
591
|
-
*/
|
|
592
|
-
#watch(key) {
|
|
593
|
-
if (this.#watchers.has(key)) {
|
|
594
|
-
return;
|
|
595
|
-
}
|
|
596
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
597
|
-
// Nothing has been made for this output yet. A wait that begins first will
|
|
598
|
-
// ask again — `directoryFor` is what a run calls before it writes — and
|
|
599
|
-
// asking the kernel to watch what is not there throws, once for every wait
|
|
600
|
-
// that begins meanwhile.
|
|
601
|
-
if (!existsSync(dir)) {
|
|
602
|
-
return;
|
|
603
|
-
}
|
|
604
|
-
try {
|
|
605
|
-
// Closes this watch only: by the time it is told, the key may already
|
|
606
|
-
// name the watch on a directory made again.
|
|
607
|
-
const end = () => {
|
|
608
|
-
if (this.#watchers.get(key) === watcher) {
|
|
609
|
-
this.#unwatch(key);
|
|
610
|
-
} else {
|
|
611
|
-
watcher.close();
|
|
612
|
-
}
|
|
613
|
-
};
|
|
614
|
-
const watcher = watch(dir, { persistent: false }, (event, name) => {
|
|
615
|
-
// A removal is a `rename`, and the writes of a piece are `change`, so
|
|
616
|
-
// the directory is asked about only where it can have gone.
|
|
617
|
-
if (event === "rename" && !existsSync(dir)) {
|
|
618
|
-
this.#held.delete(key);
|
|
619
|
-
end();
|
|
620
|
-
return;
|
|
621
|
-
}
|
|
622
|
-
this.#noticed(key, typeof name === "string" ? name : null);
|
|
623
|
-
});
|
|
624
|
-
// A directory that goes away takes its watch with it rather than leaving
|
|
625
|
-
// an error nobody reads.
|
|
626
|
-
watcher.on("error", end);
|
|
627
|
-
this.#watchers.set(key, watcher);
|
|
628
|
-
} catch (error) {
|
|
629
|
-
// Watching is how a wait ends early, not how a segment is found: every
|
|
630
|
-
// reader still reads the disk. Said rather than swallowed, because a
|
|
631
|
-
// store that cannot watch holds each request for its whole deadline, and
|
|
632
|
-
// that is worth knowing before the field says it.
|
|
633
|
-
this.#logger?.warn?.(
|
|
634
|
-
`segment store: cannot watch ${key.slice(0, 60)}: ` +
|
|
635
|
-
`${error instanceof Error ? error.message : String(error)}`
|
|
636
|
-
);
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
/**
|
|
641
|
-
* Stop watching one output's directory.
|
|
642
|
-
*
|
|
643
|
-
* @param {string} key
|
|
644
|
-
* @returns {void}
|
|
645
|
-
*/
|
|
646
|
-
#unwatch(key) {
|
|
647
|
-
const watcher = this.#watchers.get(key);
|
|
648
|
-
if (!watcher) {
|
|
649
|
-
return;
|
|
650
|
-
}
|
|
651
|
-
this.#watchers.delete(key);
|
|
652
|
-
try {
|
|
653
|
-
watcher.close();
|
|
654
|
-
} catch {
|
|
655
|
-
// Already closed with its directory.
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
/**
|
|
660
|
-
* The directory moved: say what is now there.
|
|
661
|
-
*
|
|
662
|
-
* A REMOVAL MOVES IT TOO, so the name is checked against the disk before
|
|
663
|
-
* anything is announced — otherwise clearing up after a dead run would read
|
|
664
|
-
* as that run's pieces arriving.
|
|
665
|
-
*
|
|
666
|
-
* @param {string} key
|
|
667
|
-
* @param {string | null} name - What changed, where the platform says.
|
|
668
|
-
* @returns {void}
|
|
669
|
-
*/
|
|
670
|
-
#noticed(key, name) {
|
|
671
|
-
// WHAT WAS READ OF THIS DIRECTORY IS STALE. Dropped rather than re-read:
|
|
672
|
-
// re-reading HERE is quadratic in the pieces of a film — one listing per
|
|
673
|
-
// file written, of a directory that grows with every file, measured at 110 s
|
|
674
|
-
// against 3 s over 482 pieces — while dropping costs one listing to whoever
|
|
675
|
-
// asks next, which is bounded by the questions and not by the writes.
|
|
676
|
-
//
|
|
677
|
-
// IT IS NOT ENOUGH TO LET `refresh` NOTICE BY ITSELF. It re-reads when the
|
|
678
|
-
// directory's modification time has moved, and that time is stated in
|
|
679
|
-
// milliseconds: a file written in the same millisecond as the last reading
|
|
680
|
-
// leaves it unmoved, and the piece is then reported absent although it is
|
|
681
|
-
// there. That is the hazard `refresh` documents and `publish` has always
|
|
682
|
-
// guarded against by dropping the memory.
|
|
683
|
-
this.#held.delete(key);
|
|
684
|
-
const format = this.#formats.get(key);
|
|
685
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
686
|
-
const index = name && format?.isSegmentFileName?.(name)
|
|
687
|
-
? (format.segmentIndexFromName?.(name) ?? -1)
|
|
688
|
-
: -1;
|
|
689
|
-
if (index >= 0) {
|
|
690
|
-
// A REMOVAL MOVES THE DIRECTORY TOO, so the one file is asked about
|
|
691
|
-
// before anything is announced — otherwise clearing up after a dead run
|
|
692
|
-
// would read as that run's pieces arriving.
|
|
693
|
-
if (!existsSync(path.join(dir, name))) {
|
|
694
|
-
return;
|
|
695
|
-
}
|
|
696
|
-
this.#announceArrival(key, index);
|
|
697
|
-
return;
|
|
698
|
-
}
|
|
699
|
-
if (name) {
|
|
700
|
-
return;
|
|
701
|
-
}
|
|
702
|
-
// The platform named nothing. Then whoever is waiting asks about their own
|
|
703
|
-
// piece, which is the same question with the same answer and no guessing.
|
|
704
|
-
const byIndex = this.#waiters.get(key);
|
|
705
|
-
if (!byIndex || !format?.segmentFileName) {
|
|
706
|
-
return;
|
|
707
|
-
}
|
|
708
|
-
for (const waiting of [...byIndex.keys()]) {
|
|
709
|
-
if (existsSync(path.join(dir, format.segmentFileName(waiting)))) {
|
|
710
|
-
this.#announceArrival(key, waiting);
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
/**
|
|
716
|
-
* One statement that a number is now present here.
|
|
717
|
-
*
|
|
718
|
-
* @param {string} key
|
|
719
|
-
* @param {number} index
|
|
720
|
-
* @returns {void}
|
|
721
|
-
*/
|
|
722
|
-
#announceArrival(key, index) {
|
|
723
|
-
this.announce(key, index);
|
|
724
|
-
for (const listener of this.#publishedListeners) {
|
|
725
|
-
listener(key, index);
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
/**
|
|
730
|
-
* Announce that a segment is now present. This stores no readiness state;
|
|
731
|
-
* the file under its served name remains the only fact.
|
|
732
|
-
*
|
|
733
|
-
* @param {string} key
|
|
734
|
-
* @param {number} index
|
|
735
|
-
*/
|
|
736
|
-
announce(key, index) {
|
|
737
|
-
const byIndex = this.#waiters.get(key);
|
|
738
|
-
const waiting = byIndex?.get(index);
|
|
739
|
-
if (!waiting) return;
|
|
740
|
-
byIndex.delete(index);
|
|
741
|
-
if (byIndex.size === 0) this.#waiters.delete(key);
|
|
742
|
-
for (const resolve of waiting) resolve(true);
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
/**
|
|
746
|
-
* How many requests are waiting on this output right now.
|
|
747
|
-
*
|
|
748
|
-
* The store's own state, and the one fact that says a wait EXISTS rather than
|
|
749
|
-
* is about to: a check that ends a wait it has not yet registered proves
|
|
750
|
-
* nothing, and passes just as well without the thing it is checking.
|
|
751
|
-
*
|
|
752
|
-
* @param {string} key
|
|
753
|
-
* @returns {number}
|
|
754
|
-
*/
|
|
755
|
-
waitingFor(key) {
|
|
756
|
-
let waiting = 0;
|
|
757
|
-
for (const holders of this.#waiters.get(key)?.values() ?? []) {
|
|
758
|
-
waiting += holders.size;
|
|
759
|
-
}
|
|
760
|
-
return waiting;
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
/**
|
|
764
|
-
* Wait for a segment publication event or one deadline. This does not poll
|
|
765
|
-
* the filesystem.
|
|
766
|
-
*
|
|
767
|
-
* @param {string} key
|
|
768
|
-
* @param {number} index
|
|
769
|
-
* @param {number} timeoutMs - May be infinite: then only the publication or
|
|
770
|
-
* `cancelled` ends the wait.
|
|
771
|
-
* @param {Promise<unknown> | null} [cancelled] - Settles when whoever waits
|
|
772
|
-
* has gone, so the waiter does not outlive them.
|
|
773
|
-
* @returns {Promise<boolean>}
|
|
774
|
-
*/
|
|
775
|
-
waitFor(key, index, timeoutMs, cancelled = null) {
|
|
776
|
-
if (this.pathOf(key, index)) return Promise.resolve(true);
|
|
777
|
-
// An output adopted from a previous process has a directory nobody made in
|
|
778
|
-
// this one, so the wait itself is the first moment it is watched.
|
|
779
|
-
this.#watch(key);
|
|
780
|
-
return new Promise((resolve) => {
|
|
781
|
-
const byIndex = this.#waiters.get(key) ?? new Map();
|
|
782
|
-
const waiting = byIndex.get(index) ?? new Set();
|
|
783
|
-
this.#waiters.set(key, byIndex);
|
|
784
|
-
byIndex.set(index, waiting);
|
|
785
|
-
let settled = false;
|
|
786
|
-
const finish = (published) => {
|
|
787
|
-
if (settled) return;
|
|
788
|
-
settled = true;
|
|
789
|
-
if (timer) clearTimeout(timer);
|
|
790
|
-
waiting.delete(finish);
|
|
791
|
-
// Removed only if they are still the sets this waiter joined: an
|
|
792
|
-
// announcement takes the set out before calling its waiters, and a new
|
|
793
|
-
// waiter may have registered a fresh one under the same number since.
|
|
794
|
-
if (waiting.size === 0 && byIndex.get(index) === waiting) byIndex.delete(index);
|
|
795
|
-
if (byIndex.size === 0 && this.#waiters.get(key) === byIndex) this.#waiters.delete(key);
|
|
796
|
-
resolve(published);
|
|
797
|
-
};
|
|
798
|
-
waiting.add(finish);
|
|
799
|
-
// No deadline is a wait that ends only on the publication: a timer given
|
|
800
|
-
// an infinite delay fires at once, which would turn the wait into a poll.
|
|
801
|
-
const timer = Number.isFinite(timeoutMs) ? setTimeout(() => finish(false), Math.max(0, timeoutMs)) : null;
|
|
802
|
-
timer?.unref?.();
|
|
803
|
-
cancelled?.then(() => finish(false));
|
|
804
|
-
// Publication may have landed after the first path check and before this
|
|
805
|
-
// waiter was registered. Recheck once after registration so that window
|
|
806
|
-
// cannot turn a present segment into a full-deadline wait.
|
|
807
|
-
if (this.pathOf(key, index)) finish(true);
|
|
808
|
-
});
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
/**
|
|
812
|
-
* Clear up after a run that has ended: remove what it left unfinished.
|
|
813
|
-
*
|
|
814
|
-
* Every file it left open carries its own tag, so this is a name match and
|
|
815
|
-
* nothing else — no stretch to search, no bytes to judge, and no chance of
|
|
816
|
-
* removing a piece somebody else closed.
|
|
817
|
-
*
|
|
818
|
-
* WHAT IT REPLACED, because the difference is the whole of the rename design.
|
|
819
|
-
* It used to take the highest SERVED name inside the stretch the ended run was
|
|
820
|
-
* given and judge whether its bytes looked usable — a guess, needed only
|
|
821
|
-
* because an unfinished piece was indistinguishable from a finished one. Under
|
|
822
|
-
* the naming rule it would now remove a complete segment: the highest served
|
|
823
|
-
* name in a dead run's stretch is a piece it closed.
|
|
824
|
-
*
|
|
825
|
-
* @param {string} key
|
|
826
|
-
* @param {number} startedAt - The run's first segment number, which is its tag.
|
|
827
|
-
* @returns {number} How many unfinished pieces were removed.
|
|
828
|
-
*/
|
|
829
|
-
clearUpAfter(key, startedAt) {
|
|
830
|
-
const format = this.#formats.get(key);
|
|
831
|
-
if (!format) {
|
|
832
|
-
return 0;
|
|
833
|
-
}
|
|
834
|
-
const removed = this.#sweepUnfinished(
|
|
835
|
-
key,
|
|
836
|
-
this.directoryFor(key),
|
|
837
|
-
format,
|
|
838
|
-
String(Number.isInteger(startedAt) && startedAt > 0 ? startedAt : 0)
|
|
839
|
-
);
|
|
840
|
-
if (removed > 0) {
|
|
841
|
-
this.#logger?.info?.(
|
|
842
|
-
`segment store: cleared up ${removed} unfinished piece(s) of the run at ` +
|
|
843
|
-
`#${startedAt} on ${key.slice(0, 60)}`
|
|
844
|
-
);
|
|
845
|
-
}
|
|
846
|
-
return removed;
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
/**
|
|
851
|
-
* Where the file of this name is, or null when this output does not hold it.
|
|
852
|
-
*
|
|
853
|
-
* By NAME rather than by number, because that is what a request carries. The
|
|
854
|
-
* conversion is the store's own business: it is told how this output's files
|
|
855
|
-
* are named (`useFormat`) and therefore needs nobody else to read one.
|
|
856
|
-
*
|
|
857
|
-
* @param {string} key
|
|
858
|
-
* @param {string} fileName
|
|
859
|
-
* @returns {string | null}
|
|
860
|
-
*/
|
|
861
|
-
pathOfName(key, fileName) {
|
|
862
|
-
const format = this.#formats.get(key);
|
|
863
|
-
const index = format?.segmentIndexFromName?.(fileName) ?? -1;
|
|
864
|
-
if (!Number.isInteger(index) || index < 0) {
|
|
865
|
-
return null;
|
|
866
|
-
}
|
|
867
|
-
return this.pathOf(key, index);
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
/**
|
|
871
|
-
* Forget what this output's directory was last seen to hold.
|
|
872
|
-
*
|
|
873
|
-
* What is held is re-read when the directory's modification time moves, which
|
|
874
|
-
* is how a quiet request costs one `stat`. A caller that has just removed a
|
|
875
|
-
* file and asks again in the same tick would otherwise be told it is still
|
|
876
|
-
* there — the time has moved, but only by as much as the clock's own
|
|
877
|
-
* granularity, and that is not something to depend on.
|
|
878
|
-
*
|
|
879
|
-
* @param {string} key
|
|
880
|
-
* @returns {void}
|
|
881
|
-
*/
|
|
882
|
-
forget(key) {
|
|
883
|
-
this.#held.delete(key);
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
/**
|
|
887
|
-
* Where a segment is, or null when this output does not hold it.
|
|
888
|
-
*
|
|
889
|
-
* @param {string} key
|
|
890
|
-
* @param {number} index
|
|
891
|
-
* @returns {string | null}
|
|
892
|
-
*/
|
|
893
|
-
pathOf(key, index) {
|
|
894
|
-
this.#touched.set(key, this.#now());
|
|
895
|
-
return this.refresh(key).byNumber.get(index) ?? null;
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
/**
|
|
899
|
-
* What every output in the store weighs.
|
|
900
|
-
*
|
|
901
|
-
* @returns {{ outputs: number, bytes: number }}
|
|
902
|
-
*/
|
|
903
|
-
stats() {
|
|
904
|
-
let bytes = 0;
|
|
905
|
-
for (const key of this.#formats.keys()) {
|
|
906
|
-
bytes += this.refresh(key).bytes;
|
|
907
|
-
}
|
|
908
|
-
return { outputs: this.#formats.size, bytes };
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
/**
|
|
912
|
-
* Throw one output's segments away.
|
|
913
|
-
*
|
|
914
|
-
* @param {string} key
|
|
915
|
-
* @param {string} because
|
|
916
|
-
*/
|
|
917
|
-
drop(key, because) {
|
|
918
|
-
const dir = path.join(this.#root, directoryNameFor(key));
|
|
919
|
-
this.#unwatch(key);
|
|
920
|
-
try {
|
|
921
|
-
rmSync(dir, { recursive: true, force: true });
|
|
922
|
-
} catch {
|
|
923
|
-
// Already gone, or in use; the next sweep sees it either way.
|
|
924
|
-
}
|
|
925
|
-
this.#held.delete(key);
|
|
926
|
-
this.#formats.delete(key);
|
|
927
|
-
this.#mediaRanges.delete(key);
|
|
928
|
-
this.#touched.delete(key);
|
|
929
|
-
this.#inits.delete(key);
|
|
930
|
-
this.#logger.info(`segment-store dropped ${directoryNameFor(key)} (${because})`);
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
/**
|
|
934
|
-
* When this output was last asked for, or null where it has never been.
|
|
935
|
-
*
|
|
936
|
-
* The one reading that makes the keeping period measurable rather than
|
|
937
|
-
* guessed: a session opened on an output this answers for IS a return, and
|
|
938
|
-
* this is its age.
|
|
939
|
-
*
|
|
940
|
-
* @param {string} key
|
|
941
|
-
* @returns {number | null}
|
|
942
|
-
*/
|
|
943
|
-
lastReadAt(key) {
|
|
944
|
-
return this.#touched.get(key) ?? null;
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
/**
|
|
948
|
-
* Throw away everything this store owns.
|
|
949
|
-
*
|
|
950
|
-
* For a clean exit. What is left on disk afterwards is by definition from a
|
|
951
|
-
* kill, which is the case the startup sweep exists for — and without this the
|
|
952
|
-
* sweep adopts, the exit leaves, and the next start adopts again, for ever.
|
|
953
|
-
*
|
|
954
|
-
* WHAT IT OWNS IS ITS DIRECTORIES, NOT THE ROOT. The root is a fixed path —
|
|
955
|
-
* `os.tmpdir()/torrent-tv-hls` — so it belongs to the machine and not to any
|
|
956
|
-
* one process, and this used to remove it whole. The comment that stood here
|
|
957
|
-
* named the hazard exactly ("another process may share the root") and did it
|
|
958
|
-
* anyway.
|
|
959
|
-
*
|
|
960
|
-
* Measured 2026-09-14, where a second process is ordinary: `node --test` runs
|
|
961
|
-
* every test file in its own process in parallel, and four files call this
|
|
962
|
-
* forty times between them. `segment-serve-wiring.test.js` then reported a
|
|
963
|
-
* segment missing that it had written itself a moment earlier, 2 to 4 checks
|
|
964
|
-
* per run, never the same ones — because another process had deleted the
|
|
965
|
-
* directory under it. The same thing happens to two proxies on one machine:
|
|
966
|
-
* a bare npm run beside the addon, or the wedge stand beside the product, and
|
|
967
|
-
* one of them exiting takes the other's segments mid-serve.
|
|
968
|
-
*
|
|
969
|
-
* A store removes what it made. It does not remove the container it made it
|
|
970
|
-
* in, and an empty root costs nothing.
|
|
971
|
-
*
|
|
972
|
-
* @param {string} because
|
|
973
|
-
* @returns {number} How many outputs went.
|
|
974
|
-
*/
|
|
975
|
-
dropAll(because) {
|
|
976
|
-
let dropped = 0;
|
|
977
|
-
for (const key of [...this.#formats.keys()]) {
|
|
978
|
-
this.drop(key, because);
|
|
979
|
-
dropped += 1;
|
|
980
|
-
}
|
|
981
|
-
// EVERY WATCH GOES, not only the watches of outputs that named their file
|
|
982
|
-
// format. A directory is made before anything says how its files are named
|
|
983
|
-
// — `directoryFor` is what a run calls, `useFormat` is a separate statement
|
|
984
|
-
// — so a store can hold a watch on a key that is in no format list, and
|
|
985
|
-
// dropping by that list left it open: on Windows the process then does not
|
|
986
|
-
// exit, which is how this was found.
|
|
987
|
-
for (const key of [...this.#watchers.keys()]) {
|
|
988
|
-
this.#unwatch(key);
|
|
989
|
-
}
|
|
990
|
-
return dropped;
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
/**
|
|
994
|
-
* Keep only what is still being read, and only as much of it as there is room
|
|
995
|
-
* for.
|
|
996
|
-
*
|
|
997
|
-
* **Not tied to a session.** An output is worth keeping while somebody may
|
|
998
|
-
* still ask for it, and a session ending says nothing about that: the viewer
|
|
999
|
-
* who left may come back, and a viewer who never had a session here may open
|
|
1000
|
-
* the same film a minute later and find every segment already made. So the
|
|
1001
|
-
* only question asked is when this output was last READ, and the only bound
|
|
1002
|
-
* is the disk.
|
|
1003
|
-
*
|
|
1004
|
-
* The idle period is deliberately long. Its job is not to reclaim space —
|
|
1005
|
-
* that is the cap's — but to stop an output nobody has touched in hours from
|
|
1006
|
-
* sitting there for the life of the process.
|
|
1007
|
-
*
|
|
1008
|
-
* TWO RULES, ANSWERING TWO QUESTIONS. Kept apart because they were briefly
|
|
1009
|
-
* proposed as one and that was wrong: material nobody needs should not sit on
|
|
1010
|
-
* the owner's disk merely because there is room for it, and material everyone
|
|
1011
|
-
* needs must still go when there is no room. The first is time, the second is
|
|
1012
|
-
* space.
|
|
1013
|
-
*
|
|
1014
|
-
* WHAT GOES FIRST WHEN THERE IS NO ROOM is decided by where the viewers are,
|
|
1015
|
-
* not by when a directory was last read. Behind every viewer of an output is
|
|
1016
|
-
* material that has been played and will not be asked for again unless
|
|
1017
|
-
* somebody seeks back; ahead of the furthest viewer is material that will be
|
|
1018
|
-
* asked for, eventually. So the order is: outputs nobody is watching at all,
|
|
1019
|
-
* then what lies behind the earliest viewer, furthest behind first, then what
|
|
1020
|
-
* lies ahead of the furthest viewer, furthest ahead first. It is the priority
|
|
1021
|
-
* map's own order read from the other end.
|
|
1022
|
-
*
|
|
1023
|
-
* A segment a viewer is standing on is never a victim.
|
|
1024
|
-
*
|
|
1025
|
-
* @param {object} params
|
|
1026
|
-
* @param {number} params.idleMs - Untouched for longer than this, and it goes.
|
|
1027
|
-
* @param {number} params.maxBytes - The most the whole store may hold.
|
|
1028
|
-
* @param {(key: string) => number[]} [params.viewersAt] - Where the viewers of
|
|
1029
|
-
* an output stand, as segment numbers. An empty answer means nobody is
|
|
1030
|
-
* watching it, which is what makes its segments the first to go. Absent, the
|
|
1031
|
-
* store has nothing to order by and falls back to the oldest directory —
|
|
1032
|
-
* which is what it did before it could be told.
|
|
1033
|
-
* @returns {{ droppedIdle: number, droppedForRoom: number, segmentsRemoved: number, bytes: number }}
|
|
1034
|
-
*/
|
|
1035
|
-
enforce({ idleMs, maxBytes, viewersAt = null }) {
|
|
1036
|
-
const now = this.#now();
|
|
1037
|
-
let droppedIdle = 0;
|
|
1038
|
-
for (const [key, touchedAt] of [...this.#touched]) {
|
|
1039
|
-
if (now - touchedAt > idleMs) {
|
|
1040
|
-
this.drop(key, `nothing has read it for ${Math.round((now - touchedAt) / 60000)} minutes`);
|
|
1041
|
-
droppedIdle += 1;
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
let held = this.stats().bytes;
|
|
1045
|
-
if (!Number.isFinite(maxBytes) || maxBytes <= 0 || held <= maxBytes) {
|
|
1046
|
-
return { droppedIdle, droppedForRoom: 0, segmentsRemoved: 0, bytes: held };
|
|
1047
|
-
}
|
|
1048
|
-
if (typeof viewersAt !== "function") {
|
|
1049
|
-
let droppedForRoom = 0;
|
|
1050
|
-
const byAge = [...this.#touched.entries()].sort((left, right) => left[1] - right[1]);
|
|
1051
|
-
for (const [key] of byAge) {
|
|
1052
|
-
if (held <= maxBytes) {
|
|
1053
|
-
break;
|
|
1054
|
-
}
|
|
1055
|
-
const size = this.refresh(key).bytes;
|
|
1056
|
-
this.drop(key, `the store is over its ${(maxBytes / 1073741824).toFixed(1)}GB allowance`);
|
|
1057
|
-
held -= size;
|
|
1058
|
-
droppedForRoom += 1;
|
|
1059
|
-
}
|
|
1060
|
-
return { droppedIdle, droppedForRoom, segmentsRemoved: 0, bytes: held };
|
|
1061
|
-
}
|
|
1062
|
-
|
|
1063
|
-
let segmentsRemoved = 0;
|
|
1064
|
-
for (const victim of this.#leastWantedFirst(viewersAt)) {
|
|
1065
|
-
if (held <= maxBytes) {
|
|
1066
|
-
break;
|
|
1067
|
-
}
|
|
1068
|
-
held -= this.#removeSegment(victim.key, victim.index);
|
|
1069
|
-
segmentsRemoved += 1;
|
|
1070
|
-
}
|
|
1071
|
-
if (segmentsRemoved > 0) {
|
|
1072
|
-
this.#logger.info(
|
|
1073
|
-
`segment-store removed ${segmentsRemoved} segment(s) for room: ` +
|
|
1074
|
-
`${megabytes(held)} of ${megabytes(maxBytes)} allowed`
|
|
1075
|
-
);
|
|
1076
|
-
}
|
|
1077
|
-
return { droppedIdle, droppedForRoom: 0, segmentsRemoved, bytes: held };
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
/**
|
|
1081
|
-
* Every segment in the store, least wanted first.
|
|
1082
|
-
*
|
|
1083
|
-
* @param {(key: string) => number[]} viewersAt
|
|
1084
|
-
* @returns {{ key: string, index: number }[]}
|
|
1085
|
-
*/
|
|
1086
|
-
#leastWantedFirst(viewersAt) {
|
|
1087
|
-
const candidates = [];
|
|
1088
|
-
for (const key of this.#formats.keys()) {
|
|
1089
|
-
const positions = (viewersAt(key) ?? []).filter((at) => Number.isInteger(at));
|
|
1090
|
-
const earliest = positions.length > 0 ? Math.min(...positions) : null;
|
|
1091
|
-
const furthest = positions.length > 0 ? Math.max(...positions) : null;
|
|
1092
|
-
for (const index of this.refresh(key).byNumber.keys()) {
|
|
1093
|
-
if (earliest === null) {
|
|
1094
|
-
// Nobody is watching this output at all. Everything it holds is worth
|
|
1095
|
-
// less than anything somebody is on their way to.
|
|
1096
|
-
candidates.push({ key, index, rank: 0, distance: index });
|
|
1097
|
-
continue;
|
|
1098
|
-
}
|
|
1099
|
-
if (positions.includes(index)) {
|
|
1100
|
-
continue;
|
|
1101
|
-
}
|
|
1102
|
-
if (index < earliest) {
|
|
1103
|
-
candidates.push({ key, index, rank: 1, distance: earliest - index });
|
|
1104
|
-
} else {
|
|
1105
|
-
candidates.push({ key, index, rank: 2, distance: index - /** @type {number} */ (furthest) });
|
|
1106
|
-
}
|
|
1107
|
-
}
|
|
1108
|
-
}
|
|
1109
|
-
return candidates
|
|
1110
|
-
.sort((left, right) => (left.rank !== right.rank ? left.rank - right.rank : right.distance - left.distance))
|
|
1111
|
-
.map(({ key, index }) => ({ key, index }));
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
/**
|
|
1115
|
-
* Take one segment off the disk because its owner says it is not usable.
|
|
1116
|
-
*
|
|
1117
|
-
* @param {string} key
|
|
1118
|
-
* @param {number} index
|
|
1119
|
-
* @param {string} because
|
|
1120
|
-
* @returns {void}
|
|
1121
|
-
*/
|
|
1122
|
-
remove(key, index, because) {
|
|
1123
|
-
if (this.#removeSegment(key, index) > 0) {
|
|
1124
|
-
this.#logger?.warn?.(`segment store: removed segment ${index} of ${key.slice(0, 60)}: ${because}`);
|
|
1125
|
-
}
|
|
1126
|
-
}
|
|
1127
|
-
|
|
1128
|
-
/**
|
|
1129
|
-
* Take one segment off the disk.
|
|
1130
|
-
*
|
|
1131
|
-
* @param {string} key
|
|
1132
|
-
* @param {number} index
|
|
1133
|
-
* @returns {number} What it weighed.
|
|
1134
|
-
*/
|
|
1135
|
-
#removeSegment(key, index) {
|
|
1136
|
-
const full = this.refresh(key).byNumber.get(index);
|
|
1137
|
-
if (!full) {
|
|
1138
|
-
return 0;
|
|
1139
|
-
}
|
|
1140
|
-
let size = 0;
|
|
1141
|
-
try {
|
|
1142
|
-
size = statSync(full, { throwIfNoEntry: false })?.size ?? 0;
|
|
1143
|
-
rmSync(full, { force: true });
|
|
1144
|
-
} catch {
|
|
1145
|
-
// Gone already, or refused. The next refresh reports what is really there.
|
|
1146
|
-
}
|
|
1147
|
-
this.#held.delete(key);
|
|
1148
|
-
this.#mediaRanges.get(key)?.delete(index);
|
|
1149
|
-
return size;
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
/**
|
|
1153
|
-
* What a previous life of this process left on the disk.
|
|
1154
|
-
*
|
|
1155
|
-
* This is the only record there is of a death nobody saw. The kernel kills
|
|
1156
|
-
* this process often enough to matter — two kills in one viewing on
|
|
1157
|
-
* 2026-09-02 — and when it does, no exit handler runs, nothing is cleared up,
|
|
1158
|
-
* and memory is reclaimed while the disk is not: `/tmp` in the addon
|
|
1159
|
-
* container is on the overlay filesystem, measured 2026-09-04, so the files
|
|
1160
|
-
* survive the process and its restart.
|
|
1161
|
-
*
|
|
1162
|
-
* So the sweep reports rather than deletes quietly. What it finds is the
|
|
1163
|
-
* evidence, and every directory it names is one abnormal ending that went
|
|
1164
|
-
* unrecorded.
|
|
1165
|
-
*
|
|
1166
|
-
* @returns {{ directories: number, segments: number, bytes: number, unidentified: number, found: {key: string, dir: string, segments: number, bytes: number}[] }}
|
|
1167
|
-
*/
|
|
1168
|
-
sweep() {
|
|
1169
|
-
const found = [];
|
|
1170
|
-
let unidentified = 0;
|
|
1171
|
-
let names = [];
|
|
1172
|
-
try {
|
|
1173
|
-
names = readdirSync(this.#root);
|
|
1174
|
-
} catch {
|
|
1175
|
-
return { directories: 0, segments: 0, bytes: 0, unidentified: 0, found: [] };
|
|
1176
|
-
}
|
|
1177
|
-
for (const name of names) {
|
|
1178
|
-
const dir = path.join(this.#root, name);
|
|
1179
|
-
let key = "";
|
|
1180
|
-
try {
|
|
1181
|
-
if (!statSync(dir).isDirectory()) {
|
|
1182
|
-
continue;
|
|
1183
|
-
}
|
|
1184
|
-
key = readFileSync(path.join(dir, KEY_FILE), "utf8").trim();
|
|
1185
|
-
} catch {
|
|
1186
|
-
key = "";
|
|
1187
|
-
}
|
|
1188
|
-
let segments = 0;
|
|
1189
|
-
let bytes = 0;
|
|
1190
|
-
try {
|
|
1191
|
-
for (const entry of readdirSync(dir)) {
|
|
1192
|
-
if (entry === KEY_FILE) {
|
|
1193
|
-
continue;
|
|
1194
|
-
}
|
|
1195
|
-
try {
|
|
1196
|
-
bytes += statSync(path.join(dir, entry)).size;
|
|
1197
|
-
segments += 1;
|
|
1198
|
-
} catch {
|
|
1199
|
-
// Vanished between the listing and the question.
|
|
1200
|
-
}
|
|
1201
|
-
}
|
|
1202
|
-
} catch {
|
|
1203
|
-
continue;
|
|
1204
|
-
}
|
|
1205
|
-
if (!key) {
|
|
1206
|
-
// A directory that cannot say what it holds is from before this layer,
|
|
1207
|
-
// or its key file did not survive. Nothing can be served out of it,
|
|
1208
|
-
// because nothing can match it to a request.
|
|
1209
|
-
unidentified += 1;
|
|
1210
|
-
}
|
|
1211
|
-
found.push({ key, dir, segments, bytes });
|
|
1212
|
-
}
|
|
1213
|
-
const totals = found.reduce(
|
|
1214
|
-
(sum, entry) => ({ segments: sum.segments + entry.segments, bytes: sum.bytes + entry.bytes }),
|
|
1215
|
-
{ segments: 0, bytes: 0 }
|
|
1216
|
-
);
|
|
1217
|
-
if (found.length > 0) {
|
|
1218
|
-
this.#logger.info(
|
|
1219
|
-
`segment-store startup sweep: ${found.length} directories left by a previous run, ` +
|
|
1220
|
-
`${totals.segments} segments, ${(totals.bytes / 1048576).toFixed(1)}MB, ` +
|
|
1221
|
-
`${unidentified} of them unidentifiable — each one is an encoder that ended ` +
|
|
1222
|
-
"without anything recording why"
|
|
1223
|
-
);
|
|
1224
|
-
}
|
|
1225
|
-
return {
|
|
1226
|
-
directories: found.length,
|
|
1227
|
-
segments: totals.segments,
|
|
1228
|
-
bytes: totals.bytes,
|
|
1229
|
-
unidentified,
|
|
1230
|
-
found
|
|
1231
|
-
};
|
|
1232
|
-
}
|
|
1233
|
-
|
|
1234
|
-
/**
|
|
1235
|
-
* Take back what a previous life left: keep what is proven, remove the rest.
|
|
1236
|
-
*
|
|
1237
|
-
* Deliberately not "throw everything away". A killed process leaves material
|
|
1238
|
-
* that is valid by construction — a copied segment's bytes depend only on the
|
|
1239
|
-
* source — and re-encoding it costs the machine that is already known to be
|
|
1240
|
-
* short of processor. What cannot be kept is a directory that cannot name
|
|
1241
|
-
* itself, and the one file per directory whose closure nothing proves.
|
|
1242
|
-
*
|
|
1243
|
-
* @param {(key: string) => object | null} formatFor - How to read the file
|
|
1244
|
-
* names of an output, given its key. Null when this proxy cannot serve that
|
|
1245
|
-
* output at all, and then the directory goes.
|
|
1246
|
-
* @returns {{ adopted: number, dropped: number, unprovenRemoved: number }}
|
|
1247
|
-
*/
|
|
1248
|
-
adoptWhatSurvived(formatFor) {
|
|
1249
|
-
const swept = this.sweep();
|
|
1250
|
-
let adopted = 0;
|
|
1251
|
-
let dropped = 0;
|
|
1252
|
-
let unprovenRemoved = 0;
|
|
1253
|
-
for (const entry of swept.found) {
|
|
1254
|
-
const format = entry.key ? formatFor(entry.key) : null;
|
|
1255
|
-
if (!format) {
|
|
1256
|
-
try {
|
|
1257
|
-
rmSync(entry.dir, { recursive: true, force: true });
|
|
1258
|
-
} catch {
|
|
1259
|
-
// Leave it; the next sweep reports it again.
|
|
1260
|
-
}
|
|
1261
|
-
dropped += 1;
|
|
1262
|
-
this.#logger.info(
|
|
1263
|
-
`segment-store discarded ${path.basename(entry.dir)}: ` +
|
|
1264
|
-
(entry.key ? "this proxy cannot serve that output" : "it does not say what it holds")
|
|
1265
|
-
);
|
|
1266
|
-
continue;
|
|
1267
|
-
}
|
|
1268
|
-
this.#formats.set(entry.key, format);
|
|
1269
|
-
// EVERY SEGMENT FOUND IS COMPLETE, because a piece is given its served
|
|
1270
|
-
// name only once the encoder has said it is closed. So there is nothing to
|
|
1271
|
-
// prove here and nothing to un-prove: what the directory holds under
|
|
1272
|
-
// served names is what a killed process finished.
|
|
1273
|
-
//
|
|
1274
|
-
// What it may also hold is pieces it was in the middle of, under their
|
|
1275
|
-
// working names, and those are swept — the file a run was writing when the
|
|
1276
|
-
// kernel took the process is exactly this.
|
|
1277
|
-
unprovenRemoved += this.#sweepUnfinished(entry.key, entry.dir, format);
|
|
1278
|
-
adopted += 1;
|
|
1279
|
-
this.#logger.info(
|
|
1280
|
-
`segment-store adopted ${path.basename(entry.dir)}: ${this.provenNumbers(entry.key).length} ` +
|
|
1281
|
-
"segments a killed process had already finished"
|
|
1282
|
-
);
|
|
1283
|
-
}
|
|
1284
|
-
return { adopted, dropped, unprovenRemoved };
|
|
1285
|
-
}
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
/**
|
|
1289
|
-
* @param {number} bytes
|
|
1290
|
-
* @returns {string}
|
|
1291
|
-
*/
|
|
1292
|
-
function megabytes(bytes) {
|
|
1293
|
-
return `${Math.round(Math.max(0, bytes) / (1024 * 1024))}MB`;
|
|
1294
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @file One store of produced segments for the whole proxy, addressed by what
|
|
3
|
+
* the segments ARE rather than by who made them.
|
|
4
|
+
*
|
|
5
|
+
* Until now a segment lived under the id of the session whose encoder wrote it
|
|
6
|
+
* — under that session's own id, in a directory of that run's own — and the index
|
|
7
|
+
* over it was built per session, so a segment was visible only inside the
|
|
8
|
+
* session that made it. Two viewers of one film got two sessions with
|
|
9
|
+
* byte-identical output and neither could see the other's work (measured
|
|
10
|
+
* 2026-09-03, `research/two-viewers-one-file-2026-09-03.md`).
|
|
11
|
+
*
|
|
12
|
+
* Here the address is the output's own parameters. Which viewer asked never
|
|
13
|
+
* enters it, and neither does which encoder produced the bytes: a segment is
|
|
14
|
+
* the same segment whoever made it.
|
|
15
|
+
*
|
|
16
|
+
* **The directory carries its own identity.** Its NAME is a digest, because the
|
|
17
|
+
* key contains characters a path may not; the key itself is written inside it,
|
|
18
|
+
* in `key.txt`. That is what lets a new process, started after this one was
|
|
19
|
+
* killed, work out what it is looking at — without it, everything on disk after
|
|
20
|
+
* a kill is unidentifiable and can only be thrown away.
|
|
21
|
+
*
|
|
22
|
+
* **What proves a segment is closed: its NAME.** A piece being written is called
|
|
23
|
+
* something else — `making-40-00057.mp4`, tagged with the run writing it — and
|
|
24
|
+
* takes its served name only when the
|
|
25
|
+
* encoder has said it is closed, which it does on a channel of its own
|
|
26
|
+
* (`-segment_list pipe:3`). The `hls` branch needs nothing extra: its muxer
|
|
27
|
+
* writes through a temporary name of its own, so its files appear under their
|
|
28
|
+
* final name whole. One rule for both, and true whether or not this process is
|
|
29
|
+
* alive: **a file under its served name is complete.**
|
|
30
|
+
*
|
|
31
|
+
* It was "a segment is closed when the NEXT number exists". That is true of one
|
|
32
|
+
* writer walking forward and false the moment two runs share an output, because
|
|
33
|
+
* the next file is then written by another process while this one is still open
|
|
34
|
+
* — and two runs on one output is not a rare state, it is what the plan gives an
|
|
35
|
+
* output whenever it places a second encoder. Field 2026-09-08:
|
|
36
|
+
* `segment-00057.mp4` served at 2 268 361 bytes and then at 4 510 940, exactly
|
|
37
|
+
* half; the browser appended the half and refused the whole for the rest of the
|
|
38
|
+
* session, `bufferAppendError` fourteen times with the picture frozen at
|
|
39
|
+
* 319.66 s. `segment-00055.mp4` the same, 211 957 against 2 620 617.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import { createHash } from "node:crypto";
|
|
43
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, watch, writeFileSync } from "node:fs";
|
|
44
|
+
import { rmSync } from "node:fs";
|
|
45
|
+
import os from "node:os";
|
|
46
|
+
import path from "node:path";
|
|
47
|
+
|
|
48
|
+
/** Where every output's segments live. One root for the process. */
|
|
49
|
+
export const DEFAULT_STORE_ROOT = path.join(os.tmpdir(), "torrent-tv-hls");
|
|
50
|
+
|
|
51
|
+
/** The file inside each directory that says which output it holds. */
|
|
52
|
+
const KEY_FILE = "key.txt";
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The directory name for an output key.
|
|
56
|
+
*
|
|
57
|
+
* A digest rather than the key itself: the key carries `:` and `/`, which a
|
|
58
|
+
* path may not, and sanitising them would make two different keys collide.
|
|
59
|
+
* Sixteen hex characters is enough that a collision is not a thing that
|
|
60
|
+
* happens, and short enough to read in a log line.
|
|
61
|
+
*
|
|
62
|
+
* @param {string} key
|
|
63
|
+
* @returns {string}
|
|
64
|
+
*/
|
|
65
|
+
export function directoryNameFor(key) {
|
|
66
|
+
return createHash("sha256").update(String(key)).digest("hex").slice(0, 16);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* What one output's directory holds, as last read.
|
|
71
|
+
*
|
|
72
|
+
* @typedef {object} HeldContents
|
|
73
|
+
* @property {number} readAt - The directory's modification time when it was read.
|
|
74
|
+
* @property {Map<number, string>} byNumber - Segment number to full path.
|
|
75
|
+
* @property {number} bytes - What those files weigh.
|
|
76
|
+
* @property {number} unproven - The highest number, whose closure nothing
|
|
77
|
+
* proves, or -1 when the directory holds no segments.
|
|
78
|
+
*/
|
|
79
|
+
|
|
80
|
+
export class SegmentStore {
|
|
81
|
+
/** @type {string} */
|
|
82
|
+
#root;
|
|
83
|
+
|
|
84
|
+
/** Output key → what its directory holds. @type {Map<string, HeldContents>} */
|
|
85
|
+
#held = new Map();
|
|
86
|
+
#mediaRanges = new Map();
|
|
87
|
+
|
|
88
|
+
/** Output key → how to read its file names. @type {Map<string, object>} */
|
|
89
|
+
#formats = new Map();
|
|
90
|
+
|
|
91
|
+
/** Output key → when it was last asked for. @type {Map<string, number>} */
|
|
92
|
+
#touched = new Map();
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Output key → the init segment served for it, once one has been read whole.
|
|
96
|
+
*
|
|
97
|
+
* A property of the output's produced material, so it lives with that
|
|
98
|
+
* material and goes when it goes. The player fetches it once and never again,
|
|
99
|
+
* so the first whole one is kept and every later request gets the same bytes,
|
|
100
|
+
* even though a re-encoding run rewrites the file on disk.
|
|
101
|
+
*
|
|
102
|
+
* @type {Map<string, Buffer>}
|
|
103
|
+
*/
|
|
104
|
+
#inits = new Map();
|
|
105
|
+
|
|
106
|
+
/** Told of every segment published, whatever it is. @type {Set<(key: string, index: number) => void>} */
|
|
107
|
+
#publishedListeners = new Set();
|
|
108
|
+
|
|
109
|
+
/** Segment publication waiters by output key and segment number. @type {Map<string, Map<number, Set<Function>>>} */
|
|
110
|
+
#waiters = new Map();
|
|
111
|
+
|
|
112
|
+
/** Output key → the watch on its directory. @type {Map<string, import("node:fs").FSWatcher>} */
|
|
113
|
+
#watchers = new Map();
|
|
114
|
+
|
|
115
|
+
/** @type {{ info: Function, warn: Function }} */
|
|
116
|
+
#logger;
|
|
117
|
+
|
|
118
|
+
/** @type {() => number} */
|
|
119
|
+
#now;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* @param {object} [params]
|
|
123
|
+
* @param {string} [params.root] - Where the directories live.
|
|
124
|
+
* @param {{ info: Function, warn: Function }} [params.logger]
|
|
125
|
+
* @param {() => number} [params.now]
|
|
126
|
+
*/
|
|
127
|
+
constructor({ root = DEFAULT_STORE_ROOT, logger = null, now = Date.now } = {}) {
|
|
128
|
+
this.#root = root;
|
|
129
|
+
this.#logger = logger ?? { info: () => {}, warn: () => {} };
|
|
130
|
+
this.#now = now;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** @returns {string} */
|
|
134
|
+
get root() {
|
|
135
|
+
return this.#root;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Where this output's segments live, without making anything.
|
|
140
|
+
*
|
|
141
|
+
* Separate from {@link directoryFor} because a session works out its path
|
|
142
|
+
* long before it is sure it will exist: a probe or a keyframe read between
|
|
143
|
+
* the two can still fail, and a directory made in advance of that is a
|
|
144
|
+
* leftover nothing tracks — proxy 2.9.101 failed on every request and its
|
|
145
|
+
* abandoned directories were the only trace on disk.
|
|
146
|
+
*
|
|
147
|
+
* @param {string} key - `OutputSpec.toKey()`.
|
|
148
|
+
* @returns {string}
|
|
149
|
+
*/
|
|
150
|
+
pathFor(key) {
|
|
151
|
+
return path.join(this.#root, directoryNameFor(key));
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* The directory this output's segments live in, made if it is not there.
|
|
156
|
+
*
|
|
157
|
+
* @param {string} key - `OutputSpec.toKey()`.
|
|
158
|
+
* @returns {string}
|
|
159
|
+
*/
|
|
160
|
+
directoryFor(key) {
|
|
161
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
162
|
+
if (!existsSync(dir)) {
|
|
163
|
+
// A watch held on this path watches the directory that was removed, not
|
|
164
|
+
// the one about to be made, and while it is held `#watch` makes no other.
|
|
165
|
+
this.#unwatch(key);
|
|
166
|
+
mkdirSync(dir, { recursive: true });
|
|
167
|
+
// What this directory is, for whoever finds it after this process has
|
|
168
|
+
// been killed. Without it the sweep can only throw everything away.
|
|
169
|
+
writeFileSync(path.join(dir, KEY_FILE), `${key}\n`, "utf8");
|
|
170
|
+
}
|
|
171
|
+
this.#touched.set(key, this.#now());
|
|
172
|
+
// Production is about to begin here, and a piece closing in this directory
|
|
173
|
+
// is what every held request is waiting for.
|
|
174
|
+
this.#watch(key);
|
|
175
|
+
return dir;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Say how this output's files are named, so the store can read its directory.
|
|
180
|
+
*
|
|
181
|
+
* @param {string} key
|
|
182
|
+
* @param {{ isSegmentFileName: (name: string) => boolean, segmentIndexFromName: (name: string) => number }} segmentFormat
|
|
183
|
+
*/
|
|
184
|
+
useFormat(key, segmentFormat) {
|
|
185
|
+
if (segmentFormat && typeof segmentFormat.isSegmentFileName === "function") {
|
|
186
|
+
this.#formats.set(key, segmentFormat);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Re-read this output's directory if it has moved since last time.
|
|
192
|
+
*
|
|
193
|
+
* A directory's modification time changes when an entry is added or removed,
|
|
194
|
+
* so a quiet request costs one `stat` rather than a listing.
|
|
195
|
+
*
|
|
196
|
+
* @param {string} key
|
|
197
|
+
* @returns {HeldContents}
|
|
198
|
+
*/
|
|
199
|
+
refresh(key) {
|
|
200
|
+
const format = this.#formats.get(key);
|
|
201
|
+
const empty = { readAt: 0, byNumber: new Map(), bytes: 0, unproven: -1, largest: { index: -1, size: 0 } };
|
|
202
|
+
if (!format) {
|
|
203
|
+
return this.#held.get(key) ?? empty;
|
|
204
|
+
}
|
|
205
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
206
|
+
let mtime = 0;
|
|
207
|
+
try {
|
|
208
|
+
mtime = statSync(dir).mtimeMs;
|
|
209
|
+
} catch {
|
|
210
|
+
this.#held.delete(key);
|
|
211
|
+
return empty;
|
|
212
|
+
}
|
|
213
|
+
const known = this.#held.get(key);
|
|
214
|
+
if (known && known.readAt === mtime) {
|
|
215
|
+
return known;
|
|
216
|
+
}
|
|
217
|
+
const byNumber = new Map();
|
|
218
|
+
const sizes = new Map();
|
|
219
|
+
let bytes = 0;
|
|
220
|
+
let highest = -1;
|
|
221
|
+
// The biggest piece and which number it is. What reads it is the figure the
|
|
222
|
+
// master playlist declares: `BANDWIDTH` is the PEAK a link must carry, and
|
|
223
|
+
// the peak of a variable-bitrate source is nothing like its average — the
|
|
224
|
+
// field file of 2026-09-08 ran at 17.1 Mbit/s with a piece at 73. Which
|
|
225
|
+
// number it is matters because pieces are not all the same length, and only
|
|
226
|
+
// whoever holds the cut table can turn bytes into bits per second.
|
|
227
|
+
let largest = { index: -1, size: 0 };
|
|
228
|
+
try {
|
|
229
|
+
for (const name of readdirSync(dir)) {
|
|
230
|
+
if (!format.isSegmentFileName(name)) {
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
const index = format.segmentIndexFromName(name);
|
|
234
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
const full = path.join(dir, name);
|
|
238
|
+
let size = 0;
|
|
239
|
+
try {
|
|
240
|
+
size = statSync(full).size;
|
|
241
|
+
} catch {
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
// A file of no bytes is not a segment, whatever its name says. It is
|
|
245
|
+
// what a run killed the instant after opening its next piece leaves,
|
|
246
|
+
// and taking it for a segment once convinced the look-ahead that a
|
|
247
|
+
// number had been produced and kept the encoder stopped for it.
|
|
248
|
+
if (size <= 0) {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
byNumber.set(index, full);
|
|
252
|
+
sizes.set(index, size);
|
|
253
|
+
bytes += size;
|
|
254
|
+
if (index > highest) {
|
|
255
|
+
highest = index;
|
|
256
|
+
}
|
|
257
|
+
if (size > largest.size) {
|
|
258
|
+
largest = { index, size };
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
} catch {
|
|
262
|
+
this.#held.delete(key);
|
|
263
|
+
return empty;
|
|
264
|
+
}
|
|
265
|
+
const contents = { readAt: mtime, byNumber, sizes, bytes, unproven: highest, largest };
|
|
266
|
+
this.#held.set(key, contents);
|
|
267
|
+
return contents;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The segment numbers this output holds that are finished.
|
|
272
|
+
*
|
|
273
|
+
* ONE PROOF, AND IT IS THE PIECE'S OWN NAME. A piece being written is called
|
|
274
|
+
* `making-40-00042.mp4`; it is renamed to `segment-00042.mp4` when its writer
|
|
275
|
+
* says it has closed it, and on the `hls` branch — which has no such channel —
|
|
276
|
+
* the muxer's own `+temp_file` does the same rename for the same reason. So a
|
|
277
|
+
* file under the served name is complete, whoever made it and whenever.
|
|
278
|
+
*
|
|
279
|
+
* WHAT THIS REPLACED, because the difference is what a viewer felt. Closure
|
|
280
|
+
* used to be inferred from the NEXT number existing, which is sound for one
|
|
281
|
+
* writer walking forward and false the moment two runs share an output — and
|
|
282
|
+
* one-piece intervals guarantee that. Field 2026-09-08:
|
|
283
|
+
* `segment-00057.mp4` was served at 2 268 361 bytes and then at 4 510 940, the
|
|
284
|
+
* browser appended the truncated body, and `bufferAppendError` repeated to the
|
|
285
|
+
* end of the log with the picture frozen at 319.66 s. It also left the last
|
|
286
|
+
* piece of every run unprovable for ever, since nothing follows it.
|
|
287
|
+
*
|
|
288
|
+
* @param {string} key
|
|
289
|
+
* @returns {number[]}
|
|
290
|
+
*/
|
|
291
|
+
provenNumbers(key) {
|
|
292
|
+
return [...this.refresh(key).byNumber.keys()].sort((left, right) => left - right);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* How many numbered files this output holds, closed or not.
|
|
297
|
+
*
|
|
298
|
+
* What the disk has, against what has been proven closed: the two are printed
|
|
299
|
+
* side by side, so "the files are there and nobody reported them" reads
|
|
300
|
+
* differently from "there is nothing there".
|
|
301
|
+
*
|
|
302
|
+
* @param {string} key
|
|
303
|
+
* @returns {number}
|
|
304
|
+
*/
|
|
305
|
+
filesHeld(key) {
|
|
306
|
+
return this.refresh(key).byNumber.size;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* The size in bytes of every finished piece of this output, by number.
|
|
311
|
+
*
|
|
312
|
+
* Read by whoever keeps what an output's segments carried (roadmap item 97,
|
|
313
|
+
* step 14). A reading, not a request: it does not mark the output as read,
|
|
314
|
+
* so it does not keep an output nobody is watching.
|
|
315
|
+
*
|
|
316
|
+
* @param {string} key
|
|
317
|
+
* @returns {Map<number, number>}
|
|
318
|
+
*/
|
|
319
|
+
sizesOf(key) {
|
|
320
|
+
return new Map(this.refresh(key).sizes ?? []);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* The biggest piece this output has made, and which number it is.
|
|
325
|
+
*
|
|
326
|
+
* Read by whoever declares the variant's peak rate. Bytes alone cannot say
|
|
327
|
+
* it — pieces are not all the same length — so the number comes with them and
|
|
328
|
+
* whoever holds the cut table does the division.
|
|
329
|
+
*
|
|
330
|
+
* @param {string} key
|
|
331
|
+
* @returns {{ index: number, size: number }} An index of `-1` while nothing
|
|
332
|
+
* has been made, which is a statement and not a zero.
|
|
333
|
+
*/
|
|
334
|
+
largestPiece(key) {
|
|
335
|
+
return this.refresh(key).largest ?? { index: -1, size: 0 };
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Every output whose pieces this store knows how to read: those in use and
|
|
341
|
+
* those a previous life of the process left.
|
|
342
|
+
*
|
|
343
|
+
* @returns {string[]}
|
|
344
|
+
*/
|
|
345
|
+
addresses() {
|
|
346
|
+
return [...this.#formats.keys()];
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Whether this piece is finished, and may therefore be served.
|
|
351
|
+
*
|
|
352
|
+
* Its NAME is the proof, and there is no second one: a piece being written is
|
|
353
|
+
* called something else until whoever writes it says it is closed. That holds
|
|
354
|
+
* for a piece a live run is rewriting — the file standing there was closed by
|
|
355
|
+
* somebody, and it is replaced whole or not at all — and for a piece left by an
|
|
356
|
+
* earlier life of this process, which the startup sweep answers the same way.
|
|
357
|
+
*
|
|
358
|
+
* @param {string} key
|
|
359
|
+
* @param {number} index
|
|
360
|
+
* @returns {boolean}
|
|
361
|
+
*/
|
|
362
|
+
isClosed(key, index) {
|
|
363
|
+
return this.refresh(key).byNumber.has(index);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Remove the pieces an output was in the middle of writing.
|
|
368
|
+
*
|
|
369
|
+
* They are under working names, so they were never servable and nothing has
|
|
370
|
+
* to be un-proven — this is disk, not correctness. A process killed by the
|
|
371
|
+
* kernel leaves one per live run, and the kernel takes this process often
|
|
372
|
+
* enough for that to matter.
|
|
373
|
+
*
|
|
374
|
+
* @param {string} key
|
|
375
|
+
* @param {string} dir
|
|
376
|
+
* @param {{ makingTagOf?: (name: string) => string | null }} format
|
|
377
|
+
* @param {string | null} [tag] - One run's own tag, or null for every run's.
|
|
378
|
+
* @returns {number} How many were removed.
|
|
379
|
+
*/
|
|
380
|
+
#sweepUnfinished(key, dir, format, tag = null) {
|
|
381
|
+
let removed = 0;
|
|
382
|
+
let names = [];
|
|
383
|
+
try {
|
|
384
|
+
names = readdirSync(dir);
|
|
385
|
+
} catch {
|
|
386
|
+
return 0;
|
|
387
|
+
}
|
|
388
|
+
for (const name of names) {
|
|
389
|
+
const wroteIt = format?.makingTagOf?.(name) ?? null;
|
|
390
|
+
if (wroteIt === null || (tag !== null && wroteIt !== tag)) {
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
try {
|
|
394
|
+
rmSync(path.join(dir, name), { force: true });
|
|
395
|
+
removed += 1;
|
|
396
|
+
} catch {
|
|
397
|
+
// Then it stays, costing disk and nothing else.
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
if (removed > 0) {
|
|
401
|
+
this.#held.delete(key);
|
|
402
|
+
}
|
|
403
|
+
return removed;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* The encoder has closed a piece: give it the name it is served under.
|
|
408
|
+
*
|
|
409
|
+
* One rename inside the output's own directory — one filesystem operation, and
|
|
410
|
+
* atomic there. Before it the file is not a segment and no request can reach
|
|
411
|
+
* it; after it, its existence IS the proof that it is whole, and that is one
|
|
412
|
+
* rule for every branch whether or not our own process is alive.
|
|
413
|
+
*
|
|
414
|
+
* It replaced a rule that served half a segment: a piece was taken as finished
|
|
415
|
+
* when the NEXT file existed. That is true of one writer walking forward and
|
|
416
|
+
* false the moment two runs share an output, because the next file is then
|
|
417
|
+
* written by another process while this one is still open — and two runs on one
|
|
418
|
+
* output is not a rare state, it is what the plan gives an output whenever it
|
|
419
|
+
* places a second encoder.
|
|
420
|
+
*
|
|
421
|
+
* Field 2026-09-08: `segment-00057.mp4` was served at 2 268 361 bytes and then
|
|
422
|
+
* at 4 510 940 — exactly half of it. The browser appended the half and refused
|
|
423
|
+
* the whole for the rest of the session, `bufferAppendError` fourteen times
|
|
424
|
+
* over with the picture frozen at 319.66 s. `segment-00055.mp4` went the same
|
|
425
|
+
* way, 211 957 against 2 620 617.
|
|
426
|
+
*
|
|
427
|
+
* Whether the piece is whole is decided before this, by the encoding
|
|
428
|
+
* (`encode/piece-completeness.js`): the store keeps bytes and what is read
|
|
429
|
+
* from them, and does not judge them.
|
|
430
|
+
*
|
|
431
|
+
* @param {string} key
|
|
432
|
+
* @param {string} makingName - What the encoder called it while writing.
|
|
433
|
+
* @param {{ servedNameOf?: (name: string) => string | null, segmentIndexFromName?: (name: string) => number }} format
|
|
434
|
+
* @param {{ mediaRanges?: object | null }} [read] - The piece's media
|
|
435
|
+
* intervals as its format read them from the closed file, kept with it.
|
|
436
|
+
* @returns {string | null} The served name, or null where nothing was renamed.
|
|
437
|
+
*/
|
|
438
|
+
publish(key, makingName, format, { mediaRanges = null } = {}) {
|
|
439
|
+
const served = format?.servedNameOf?.(makingName) ?? null;
|
|
440
|
+
if (!served) {
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
444
|
+
const index = format?.segmentIndexFromName?.(served) ?? -1;
|
|
445
|
+
try {
|
|
446
|
+
renameSync(path.join(dir, makingName), path.join(dir, served));
|
|
447
|
+
} catch (error) {
|
|
448
|
+
// The file may already be gone — a process killed between closing the
|
|
449
|
+
// piece and this line. Said rather than swallowed: a piece the encoder
|
|
450
|
+
// reported and the disk does not have is worth knowing about.
|
|
451
|
+
this.#logger?.warn?.(
|
|
452
|
+
`segment store: could not publish ${makingName} of ${key.slice(0, 60)}: ` +
|
|
453
|
+
`${error instanceof Error ? error.message : String(error)}`
|
|
454
|
+
);
|
|
455
|
+
return null;
|
|
456
|
+
}
|
|
457
|
+
// What the directory holds has changed, so the memory of it is stale.
|
|
458
|
+
this.#held.delete(key);
|
|
459
|
+
if (index >= 0) {
|
|
460
|
+
if (mediaRanges) {
|
|
461
|
+
const byIndex = this.#mediaRanges.get(key) ?? new Map();
|
|
462
|
+
byIndex.set(index, mediaRanges);
|
|
463
|
+
this.#mediaRanges.set(key, byIndex);
|
|
464
|
+
}
|
|
465
|
+
this.announce(key, index);
|
|
466
|
+
for (const listener of this.#publishedListeners) {
|
|
467
|
+
listener(key, index);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return served;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* The bytes of a file the encoder has closed under its working name, for the
|
|
475
|
+
* encoding to judge before it is published.
|
|
476
|
+
*
|
|
477
|
+
* @param {string} key
|
|
478
|
+
* @param {string} makingName
|
|
479
|
+
* @returns {Buffer}
|
|
480
|
+
*/
|
|
481
|
+
closedBytesOf(key, makingName) {
|
|
482
|
+
return readFileSync(path.join(this.#root, directoryNameFor(key), makingName));
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* The media intervals a stored piece holds, read once from its bytes and kept
|
|
487
|
+
* with them, or undefined where its format reads none or it is not here.
|
|
488
|
+
*
|
|
489
|
+
* @param {string} key
|
|
490
|
+
* @param {number} index
|
|
491
|
+
* @param {{ startSeconds?: number }} [where] - Where its cut puts it.
|
|
492
|
+
* @returns {object | undefined}
|
|
493
|
+
*/
|
|
494
|
+
mediaRangesOf(key, index, { startSeconds = 0 } = {}) {
|
|
495
|
+
const known = this.#mediaRanges.get(key)?.get(index);
|
|
496
|
+
const format = this.#formats.get(key);
|
|
497
|
+
if (!format?.readMediaRanges) return undefined;
|
|
498
|
+
if (known) return known;
|
|
499
|
+
const filePath = this.pathOf(key, index);
|
|
500
|
+
if (!filePath) return undefined;
|
|
501
|
+
try {
|
|
502
|
+
let initBytes = this.initOf(key);
|
|
503
|
+
if (!initBytes && format.initFileName) {
|
|
504
|
+
try {
|
|
505
|
+
initBytes = readFileSync(path.join(this.pathFor(key), format.initFileName));
|
|
506
|
+
} catch {
|
|
507
|
+
// A header not yet produced is not an empty media interval.
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
const ranges = format.readMediaRanges(readFileSync(filePath), { initBytes, startSeconds });
|
|
511
|
+
if (!ranges) return undefined;
|
|
512
|
+
const byIndex = this.#mediaRanges.get(key) ?? new Map();
|
|
513
|
+
byIndex.set(index, ranges);
|
|
514
|
+
this.#mediaRanges.set(key, byIndex);
|
|
515
|
+
return ranges;
|
|
516
|
+
} catch {
|
|
517
|
+
return [];
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/**
|
|
522
|
+
* The init segment kept for this output, or null until one was read whole.
|
|
523
|
+
*
|
|
524
|
+
* @param {string} key
|
|
525
|
+
* @returns {Buffer | null}
|
|
526
|
+
*/
|
|
527
|
+
initOf(key) {
|
|
528
|
+
return this.#inits.get(key) ?? null;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Keep the init segment served for this output. The first whole one stands;
|
|
533
|
+
* an empty read is not an init (ffmpeg creates the file before it writes the
|
|
534
|
+
* header into it) and is refused.
|
|
535
|
+
*
|
|
536
|
+
* @param {string} key
|
|
537
|
+
* @param {Buffer | null} bytes
|
|
538
|
+
* @returns {Buffer | null} The init that now stands for this output.
|
|
539
|
+
*/
|
|
540
|
+
keepInit(key, bytes) {
|
|
541
|
+
if (!this.#inits.has(key) && bytes && bytes.length > 0) {
|
|
542
|
+
this.#inits.set(key, bytes);
|
|
543
|
+
}
|
|
544
|
+
return this.initOf(key);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Be told of every segment published: what the store holds has grown, which
|
|
549
|
+
* is when whether it still fits its share is worth asking.
|
|
550
|
+
*
|
|
551
|
+
* @param {(key: string, index: number) => void} listener
|
|
552
|
+
* @returns {() => void} Stop being told.
|
|
553
|
+
*/
|
|
554
|
+
onPublished(listener) {
|
|
555
|
+
this.#publishedListeners.add(listener);
|
|
556
|
+
return () => this.#publishedListeners.delete(listener);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Watch this output's directory, so a piece closed by ANY writer is an event.
|
|
561
|
+
*
|
|
562
|
+
* WHY IT EXISTS, and it is one fault rather than a precaution. A piece is
|
|
563
|
+
* finished in one of two ways, and only one of them said so. The `segment`
|
|
564
|
+
* muxer writes under a working name and reports the closure on a channel of
|
|
565
|
+
* its own (`publish`); the `hls` muxer writes through a temporary name of its
|
|
566
|
+
* own (`+temp_file`) and reports nothing, so the file simply appears under the
|
|
567
|
+
* name it is served as. That branch is taken by every re-encoded output on the
|
|
568
|
+
* even grid — the ordinary quality step — and for those `waitFor` could only
|
|
569
|
+
* end on its deadline: the segment was on disk and the request that wanted it
|
|
570
|
+
* went on waiting, up to a full minute, and was then answered 503.
|
|
571
|
+
*
|
|
572
|
+
* So the store watches what it owns. The rename into place moves the
|
|
573
|
+
* directory's modification time, the kernel says so, and both branches reach
|
|
574
|
+
* one statement: this number is now present. Nothing polls.
|
|
575
|
+
*
|
|
576
|
+
* Idempotent, and started from the two places that need it — the directory
|
|
577
|
+
* being made for a run, and a wait beginning on an output this process has
|
|
578
|
+
* adopted rather than made.
|
|
579
|
+
*
|
|
580
|
+
* `drop` stops the watch before it removes the directory. A DIRECTORY
|
|
581
|
+
* REMOVED BY ANYBODY ELSE ENDS ITS WATCH TOO, and it has to be noticed here,
|
|
582
|
+
* because no `error` is emitted for it. Measured 2026-10-02 on Windows, Node
|
|
583
|
+
* 24.2: the removal succeeds, and the watch then reports `rename` for the
|
|
584
|
+
* directory itself without end — 232 618 events in eight seconds — and that
|
|
585
|
+
* stream keeps the process alive although the watch is not persistent. Two
|
|
586
|
+
* test files never exited for it. Left in place, a dead watch also stops
|
|
587
|
+
* this method from watching the directory made again under the same name.
|
|
588
|
+
*
|
|
589
|
+
* @param {string} key
|
|
590
|
+
* @returns {void}
|
|
591
|
+
*/
|
|
592
|
+
#watch(key) {
|
|
593
|
+
if (this.#watchers.has(key)) {
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
597
|
+
// Nothing has been made for this output yet. A wait that begins first will
|
|
598
|
+
// ask again — `directoryFor` is what a run calls before it writes — and
|
|
599
|
+
// asking the kernel to watch what is not there throws, once for every wait
|
|
600
|
+
// that begins meanwhile.
|
|
601
|
+
if (!existsSync(dir)) {
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
try {
|
|
605
|
+
// Closes this watch only: by the time it is told, the key may already
|
|
606
|
+
// name the watch on a directory made again.
|
|
607
|
+
const end = () => {
|
|
608
|
+
if (this.#watchers.get(key) === watcher) {
|
|
609
|
+
this.#unwatch(key);
|
|
610
|
+
} else {
|
|
611
|
+
watcher.close();
|
|
612
|
+
}
|
|
613
|
+
};
|
|
614
|
+
const watcher = watch(dir, { persistent: false }, (event, name) => {
|
|
615
|
+
// A removal is a `rename`, and the writes of a piece are `change`, so
|
|
616
|
+
// the directory is asked about only where it can have gone.
|
|
617
|
+
if (event === "rename" && !existsSync(dir)) {
|
|
618
|
+
this.#held.delete(key);
|
|
619
|
+
end();
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
this.#noticed(key, typeof name === "string" ? name : null);
|
|
623
|
+
});
|
|
624
|
+
// A directory that goes away takes its watch with it rather than leaving
|
|
625
|
+
// an error nobody reads.
|
|
626
|
+
watcher.on("error", end);
|
|
627
|
+
this.#watchers.set(key, watcher);
|
|
628
|
+
} catch (error) {
|
|
629
|
+
// Watching is how a wait ends early, not how a segment is found: every
|
|
630
|
+
// reader still reads the disk. Said rather than swallowed, because a
|
|
631
|
+
// store that cannot watch holds each request for its whole deadline, and
|
|
632
|
+
// that is worth knowing before the field says it.
|
|
633
|
+
this.#logger?.warn?.(
|
|
634
|
+
`segment store: cannot watch ${key.slice(0, 60)}: ` +
|
|
635
|
+
`${error instanceof Error ? error.message : String(error)}`
|
|
636
|
+
);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Stop watching one output's directory.
|
|
642
|
+
*
|
|
643
|
+
* @param {string} key
|
|
644
|
+
* @returns {void}
|
|
645
|
+
*/
|
|
646
|
+
#unwatch(key) {
|
|
647
|
+
const watcher = this.#watchers.get(key);
|
|
648
|
+
if (!watcher) {
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
this.#watchers.delete(key);
|
|
652
|
+
try {
|
|
653
|
+
watcher.close();
|
|
654
|
+
} catch {
|
|
655
|
+
// Already closed with its directory.
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* The directory moved: say what is now there.
|
|
661
|
+
*
|
|
662
|
+
* A REMOVAL MOVES IT TOO, so the name is checked against the disk before
|
|
663
|
+
* anything is announced — otherwise clearing up after a dead run would read
|
|
664
|
+
* as that run's pieces arriving.
|
|
665
|
+
*
|
|
666
|
+
* @param {string} key
|
|
667
|
+
* @param {string | null} name - What changed, where the platform says.
|
|
668
|
+
* @returns {void}
|
|
669
|
+
*/
|
|
670
|
+
#noticed(key, name) {
|
|
671
|
+
// WHAT WAS READ OF THIS DIRECTORY IS STALE. Dropped rather than re-read:
|
|
672
|
+
// re-reading HERE is quadratic in the pieces of a film — one listing per
|
|
673
|
+
// file written, of a directory that grows with every file, measured at 110 s
|
|
674
|
+
// against 3 s over 482 pieces — while dropping costs one listing to whoever
|
|
675
|
+
// asks next, which is bounded by the questions and not by the writes.
|
|
676
|
+
//
|
|
677
|
+
// IT IS NOT ENOUGH TO LET `refresh` NOTICE BY ITSELF. It re-reads when the
|
|
678
|
+
// directory's modification time has moved, and that time is stated in
|
|
679
|
+
// milliseconds: a file written in the same millisecond as the last reading
|
|
680
|
+
// leaves it unmoved, and the piece is then reported absent although it is
|
|
681
|
+
// there. That is the hazard `refresh` documents and `publish` has always
|
|
682
|
+
// guarded against by dropping the memory.
|
|
683
|
+
this.#held.delete(key);
|
|
684
|
+
const format = this.#formats.get(key);
|
|
685
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
686
|
+
const index = name && format?.isSegmentFileName?.(name)
|
|
687
|
+
? (format.segmentIndexFromName?.(name) ?? -1)
|
|
688
|
+
: -1;
|
|
689
|
+
if (index >= 0) {
|
|
690
|
+
// A REMOVAL MOVES THE DIRECTORY TOO, so the one file is asked about
|
|
691
|
+
// before anything is announced — otherwise clearing up after a dead run
|
|
692
|
+
// would read as that run's pieces arriving.
|
|
693
|
+
if (!existsSync(path.join(dir, name))) {
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
this.#announceArrival(key, index);
|
|
697
|
+
return;
|
|
698
|
+
}
|
|
699
|
+
if (name) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
// The platform named nothing. Then whoever is waiting asks about their own
|
|
703
|
+
// piece, which is the same question with the same answer and no guessing.
|
|
704
|
+
const byIndex = this.#waiters.get(key);
|
|
705
|
+
if (!byIndex || !format?.segmentFileName) {
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
for (const waiting of [...byIndex.keys()]) {
|
|
709
|
+
if (existsSync(path.join(dir, format.segmentFileName(waiting)))) {
|
|
710
|
+
this.#announceArrival(key, waiting);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
/**
|
|
716
|
+
* One statement that a number is now present here.
|
|
717
|
+
*
|
|
718
|
+
* @param {string} key
|
|
719
|
+
* @param {number} index
|
|
720
|
+
* @returns {void}
|
|
721
|
+
*/
|
|
722
|
+
#announceArrival(key, index) {
|
|
723
|
+
this.announce(key, index);
|
|
724
|
+
for (const listener of this.#publishedListeners) {
|
|
725
|
+
listener(key, index);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/**
|
|
730
|
+
* Announce that a segment is now present. This stores no readiness state;
|
|
731
|
+
* the file under its served name remains the only fact.
|
|
732
|
+
*
|
|
733
|
+
* @param {string} key
|
|
734
|
+
* @param {number} index
|
|
735
|
+
*/
|
|
736
|
+
announce(key, index) {
|
|
737
|
+
const byIndex = this.#waiters.get(key);
|
|
738
|
+
const waiting = byIndex?.get(index);
|
|
739
|
+
if (!waiting) return;
|
|
740
|
+
byIndex.delete(index);
|
|
741
|
+
if (byIndex.size === 0) this.#waiters.delete(key);
|
|
742
|
+
for (const resolve of waiting) resolve(true);
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/**
|
|
746
|
+
* How many requests are waiting on this output right now.
|
|
747
|
+
*
|
|
748
|
+
* The store's own state, and the one fact that says a wait EXISTS rather than
|
|
749
|
+
* is about to: a check that ends a wait it has not yet registered proves
|
|
750
|
+
* nothing, and passes just as well without the thing it is checking.
|
|
751
|
+
*
|
|
752
|
+
* @param {string} key
|
|
753
|
+
* @returns {number}
|
|
754
|
+
*/
|
|
755
|
+
waitingFor(key) {
|
|
756
|
+
let waiting = 0;
|
|
757
|
+
for (const holders of this.#waiters.get(key)?.values() ?? []) {
|
|
758
|
+
waiting += holders.size;
|
|
759
|
+
}
|
|
760
|
+
return waiting;
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/**
|
|
764
|
+
* Wait for a segment publication event or one deadline. This does not poll
|
|
765
|
+
* the filesystem.
|
|
766
|
+
*
|
|
767
|
+
* @param {string} key
|
|
768
|
+
* @param {number} index
|
|
769
|
+
* @param {number} timeoutMs - May be infinite: then only the publication or
|
|
770
|
+
* `cancelled` ends the wait.
|
|
771
|
+
* @param {Promise<unknown> | null} [cancelled] - Settles when whoever waits
|
|
772
|
+
* has gone, so the waiter does not outlive them.
|
|
773
|
+
* @returns {Promise<boolean>}
|
|
774
|
+
*/
|
|
775
|
+
waitFor(key, index, timeoutMs, cancelled = null) {
|
|
776
|
+
if (this.pathOf(key, index)) return Promise.resolve(true);
|
|
777
|
+
// An output adopted from a previous process has a directory nobody made in
|
|
778
|
+
// this one, so the wait itself is the first moment it is watched.
|
|
779
|
+
this.#watch(key);
|
|
780
|
+
return new Promise((resolve) => {
|
|
781
|
+
const byIndex = this.#waiters.get(key) ?? new Map();
|
|
782
|
+
const waiting = byIndex.get(index) ?? new Set();
|
|
783
|
+
this.#waiters.set(key, byIndex);
|
|
784
|
+
byIndex.set(index, waiting);
|
|
785
|
+
let settled = false;
|
|
786
|
+
const finish = (published) => {
|
|
787
|
+
if (settled) return;
|
|
788
|
+
settled = true;
|
|
789
|
+
if (timer) clearTimeout(timer);
|
|
790
|
+
waiting.delete(finish);
|
|
791
|
+
// Removed only if they are still the sets this waiter joined: an
|
|
792
|
+
// announcement takes the set out before calling its waiters, and a new
|
|
793
|
+
// waiter may have registered a fresh one under the same number since.
|
|
794
|
+
if (waiting.size === 0 && byIndex.get(index) === waiting) byIndex.delete(index);
|
|
795
|
+
if (byIndex.size === 0 && this.#waiters.get(key) === byIndex) this.#waiters.delete(key);
|
|
796
|
+
resolve(published);
|
|
797
|
+
};
|
|
798
|
+
waiting.add(finish);
|
|
799
|
+
// No deadline is a wait that ends only on the publication: a timer given
|
|
800
|
+
// an infinite delay fires at once, which would turn the wait into a poll.
|
|
801
|
+
const timer = Number.isFinite(timeoutMs) ? setTimeout(() => finish(false), Math.max(0, timeoutMs)) : null;
|
|
802
|
+
timer?.unref?.();
|
|
803
|
+
cancelled?.then(() => finish(false));
|
|
804
|
+
// Publication may have landed after the first path check and before this
|
|
805
|
+
// waiter was registered. Recheck once after registration so that window
|
|
806
|
+
// cannot turn a present segment into a full-deadline wait.
|
|
807
|
+
if (this.pathOf(key, index)) finish(true);
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* Clear up after a run that has ended: remove what it left unfinished.
|
|
813
|
+
*
|
|
814
|
+
* Every file it left open carries its own tag, so this is a name match and
|
|
815
|
+
* nothing else — no stretch to search, no bytes to judge, and no chance of
|
|
816
|
+
* removing a piece somebody else closed.
|
|
817
|
+
*
|
|
818
|
+
* WHAT IT REPLACED, because the difference is the whole of the rename design.
|
|
819
|
+
* It used to take the highest SERVED name inside the stretch the ended run was
|
|
820
|
+
* given and judge whether its bytes looked usable — a guess, needed only
|
|
821
|
+
* because an unfinished piece was indistinguishable from a finished one. Under
|
|
822
|
+
* the naming rule it would now remove a complete segment: the highest served
|
|
823
|
+
* name in a dead run's stretch is a piece it closed.
|
|
824
|
+
*
|
|
825
|
+
* @param {string} key
|
|
826
|
+
* @param {number} startedAt - The run's first segment number, which is its tag.
|
|
827
|
+
* @returns {number} How many unfinished pieces were removed.
|
|
828
|
+
*/
|
|
829
|
+
clearUpAfter(key, startedAt) {
|
|
830
|
+
const format = this.#formats.get(key);
|
|
831
|
+
if (!format) {
|
|
832
|
+
return 0;
|
|
833
|
+
}
|
|
834
|
+
const removed = this.#sweepUnfinished(
|
|
835
|
+
key,
|
|
836
|
+
this.directoryFor(key),
|
|
837
|
+
format,
|
|
838
|
+
String(Number.isInteger(startedAt) && startedAt > 0 ? startedAt : 0)
|
|
839
|
+
);
|
|
840
|
+
if (removed > 0) {
|
|
841
|
+
this.#logger?.info?.(
|
|
842
|
+
`segment store: cleared up ${removed} unfinished piece(s) of the run at ` +
|
|
843
|
+
`#${startedAt} on ${key.slice(0, 60)}`
|
|
844
|
+
);
|
|
845
|
+
}
|
|
846
|
+
return removed;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Where the file of this name is, or null when this output does not hold it.
|
|
852
|
+
*
|
|
853
|
+
* By NAME rather than by number, because that is what a request carries. The
|
|
854
|
+
* conversion is the store's own business: it is told how this output's files
|
|
855
|
+
* are named (`useFormat`) and therefore needs nobody else to read one.
|
|
856
|
+
*
|
|
857
|
+
* @param {string} key
|
|
858
|
+
* @param {string} fileName
|
|
859
|
+
* @returns {string | null}
|
|
860
|
+
*/
|
|
861
|
+
pathOfName(key, fileName) {
|
|
862
|
+
const format = this.#formats.get(key);
|
|
863
|
+
const index = format?.segmentIndexFromName?.(fileName) ?? -1;
|
|
864
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
865
|
+
return null;
|
|
866
|
+
}
|
|
867
|
+
return this.pathOf(key, index);
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
/**
|
|
871
|
+
* Forget what this output's directory was last seen to hold.
|
|
872
|
+
*
|
|
873
|
+
* What is held is re-read when the directory's modification time moves, which
|
|
874
|
+
* is how a quiet request costs one `stat`. A caller that has just removed a
|
|
875
|
+
* file and asks again in the same tick would otherwise be told it is still
|
|
876
|
+
* there — the time has moved, but only by as much as the clock's own
|
|
877
|
+
* granularity, and that is not something to depend on.
|
|
878
|
+
*
|
|
879
|
+
* @param {string} key
|
|
880
|
+
* @returns {void}
|
|
881
|
+
*/
|
|
882
|
+
forget(key) {
|
|
883
|
+
this.#held.delete(key);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/**
|
|
887
|
+
* Where a segment is, or null when this output does not hold it.
|
|
888
|
+
*
|
|
889
|
+
* @param {string} key
|
|
890
|
+
* @param {number} index
|
|
891
|
+
* @returns {string | null}
|
|
892
|
+
*/
|
|
893
|
+
pathOf(key, index) {
|
|
894
|
+
this.#touched.set(key, this.#now());
|
|
895
|
+
return this.refresh(key).byNumber.get(index) ?? null;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* What every output in the store weighs.
|
|
900
|
+
*
|
|
901
|
+
* @returns {{ outputs: number, bytes: number }}
|
|
902
|
+
*/
|
|
903
|
+
stats() {
|
|
904
|
+
let bytes = 0;
|
|
905
|
+
for (const key of this.#formats.keys()) {
|
|
906
|
+
bytes += this.refresh(key).bytes;
|
|
907
|
+
}
|
|
908
|
+
return { outputs: this.#formats.size, bytes };
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
/**
|
|
912
|
+
* Throw one output's segments away.
|
|
913
|
+
*
|
|
914
|
+
* @param {string} key
|
|
915
|
+
* @param {string} because
|
|
916
|
+
*/
|
|
917
|
+
drop(key, because) {
|
|
918
|
+
const dir = path.join(this.#root, directoryNameFor(key));
|
|
919
|
+
this.#unwatch(key);
|
|
920
|
+
try {
|
|
921
|
+
rmSync(dir, { recursive: true, force: true });
|
|
922
|
+
} catch {
|
|
923
|
+
// Already gone, or in use; the next sweep sees it either way.
|
|
924
|
+
}
|
|
925
|
+
this.#held.delete(key);
|
|
926
|
+
this.#formats.delete(key);
|
|
927
|
+
this.#mediaRanges.delete(key);
|
|
928
|
+
this.#touched.delete(key);
|
|
929
|
+
this.#inits.delete(key);
|
|
930
|
+
this.#logger.info(`segment-store dropped ${directoryNameFor(key)} (${because})`);
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* When this output was last asked for, or null where it has never been.
|
|
935
|
+
*
|
|
936
|
+
* The one reading that makes the keeping period measurable rather than
|
|
937
|
+
* guessed: a session opened on an output this answers for IS a return, and
|
|
938
|
+
* this is its age.
|
|
939
|
+
*
|
|
940
|
+
* @param {string} key
|
|
941
|
+
* @returns {number | null}
|
|
942
|
+
*/
|
|
943
|
+
lastReadAt(key) {
|
|
944
|
+
return this.#touched.get(key) ?? null;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* Throw away everything this store owns.
|
|
949
|
+
*
|
|
950
|
+
* For a clean exit. What is left on disk afterwards is by definition from a
|
|
951
|
+
* kill, which is the case the startup sweep exists for — and without this the
|
|
952
|
+
* sweep adopts, the exit leaves, and the next start adopts again, for ever.
|
|
953
|
+
*
|
|
954
|
+
* WHAT IT OWNS IS ITS DIRECTORIES, NOT THE ROOT. The root is a fixed path —
|
|
955
|
+
* `os.tmpdir()/torrent-tv-hls` — so it belongs to the machine and not to any
|
|
956
|
+
* one process, and this used to remove it whole. The comment that stood here
|
|
957
|
+
* named the hazard exactly ("another process may share the root") and did it
|
|
958
|
+
* anyway.
|
|
959
|
+
*
|
|
960
|
+
* Measured 2026-09-14, where a second process is ordinary: `node --test` runs
|
|
961
|
+
* every test file in its own process in parallel, and four files call this
|
|
962
|
+
* forty times between them. `segment-serve-wiring.test.js` then reported a
|
|
963
|
+
* segment missing that it had written itself a moment earlier, 2 to 4 checks
|
|
964
|
+
* per run, never the same ones — because another process had deleted the
|
|
965
|
+
* directory under it. The same thing happens to two proxies on one machine:
|
|
966
|
+
* a bare npm run beside the addon, or the wedge stand beside the product, and
|
|
967
|
+
* one of them exiting takes the other's segments mid-serve.
|
|
968
|
+
*
|
|
969
|
+
* A store removes what it made. It does not remove the container it made it
|
|
970
|
+
* in, and an empty root costs nothing.
|
|
971
|
+
*
|
|
972
|
+
* @param {string} because
|
|
973
|
+
* @returns {number} How many outputs went.
|
|
974
|
+
*/
|
|
975
|
+
dropAll(because) {
|
|
976
|
+
let dropped = 0;
|
|
977
|
+
for (const key of [...this.#formats.keys()]) {
|
|
978
|
+
this.drop(key, because);
|
|
979
|
+
dropped += 1;
|
|
980
|
+
}
|
|
981
|
+
// EVERY WATCH GOES, not only the watches of outputs that named their file
|
|
982
|
+
// format. A directory is made before anything says how its files are named
|
|
983
|
+
// — `directoryFor` is what a run calls, `useFormat` is a separate statement
|
|
984
|
+
// — so a store can hold a watch on a key that is in no format list, and
|
|
985
|
+
// dropping by that list left it open: on Windows the process then does not
|
|
986
|
+
// exit, which is how this was found.
|
|
987
|
+
for (const key of [...this.#watchers.keys()]) {
|
|
988
|
+
this.#unwatch(key);
|
|
989
|
+
}
|
|
990
|
+
return dropped;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
/**
|
|
994
|
+
* Keep only what is still being read, and only as much of it as there is room
|
|
995
|
+
* for.
|
|
996
|
+
*
|
|
997
|
+
* **Not tied to a session.** An output is worth keeping while somebody may
|
|
998
|
+
* still ask for it, and a session ending says nothing about that: the viewer
|
|
999
|
+
* who left may come back, and a viewer who never had a session here may open
|
|
1000
|
+
* the same film a minute later and find every segment already made. So the
|
|
1001
|
+
* only question asked is when this output was last READ, and the only bound
|
|
1002
|
+
* is the disk.
|
|
1003
|
+
*
|
|
1004
|
+
* The idle period is deliberately long. Its job is not to reclaim space —
|
|
1005
|
+
* that is the cap's — but to stop an output nobody has touched in hours from
|
|
1006
|
+
* sitting there for the life of the process.
|
|
1007
|
+
*
|
|
1008
|
+
* TWO RULES, ANSWERING TWO QUESTIONS. Kept apart because they were briefly
|
|
1009
|
+
* proposed as one and that was wrong: material nobody needs should not sit on
|
|
1010
|
+
* the owner's disk merely because there is room for it, and material everyone
|
|
1011
|
+
* needs must still go when there is no room. The first is time, the second is
|
|
1012
|
+
* space.
|
|
1013
|
+
*
|
|
1014
|
+
* WHAT GOES FIRST WHEN THERE IS NO ROOM is decided by where the viewers are,
|
|
1015
|
+
* not by when a directory was last read. Behind every viewer of an output is
|
|
1016
|
+
* material that has been played and will not be asked for again unless
|
|
1017
|
+
* somebody seeks back; ahead of the furthest viewer is material that will be
|
|
1018
|
+
* asked for, eventually. So the order is: outputs nobody is watching at all,
|
|
1019
|
+
* then what lies behind the earliest viewer, furthest behind first, then what
|
|
1020
|
+
* lies ahead of the furthest viewer, furthest ahead first. It is the priority
|
|
1021
|
+
* map's own order read from the other end.
|
|
1022
|
+
*
|
|
1023
|
+
* A segment a viewer is standing on is never a victim.
|
|
1024
|
+
*
|
|
1025
|
+
* @param {object} params
|
|
1026
|
+
* @param {number} params.idleMs - Untouched for longer than this, and it goes.
|
|
1027
|
+
* @param {number} params.maxBytes - The most the whole store may hold.
|
|
1028
|
+
* @param {(key: string) => number[]} [params.viewersAt] - Where the viewers of
|
|
1029
|
+
* an output stand, as segment numbers. An empty answer means nobody is
|
|
1030
|
+
* watching it, which is what makes its segments the first to go. Absent, the
|
|
1031
|
+
* store has nothing to order by and falls back to the oldest directory —
|
|
1032
|
+
* which is what it did before it could be told.
|
|
1033
|
+
* @returns {{ droppedIdle: number, droppedForRoom: number, segmentsRemoved: number, bytes: number }}
|
|
1034
|
+
*/
|
|
1035
|
+
enforce({ idleMs, maxBytes, viewersAt = null }) {
|
|
1036
|
+
const now = this.#now();
|
|
1037
|
+
let droppedIdle = 0;
|
|
1038
|
+
for (const [key, touchedAt] of [...this.#touched]) {
|
|
1039
|
+
if (now - touchedAt > idleMs) {
|
|
1040
|
+
this.drop(key, `nothing has read it for ${Math.round((now - touchedAt) / 60000)} minutes`);
|
|
1041
|
+
droppedIdle += 1;
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
let held = this.stats().bytes;
|
|
1045
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0 || held <= maxBytes) {
|
|
1046
|
+
return { droppedIdle, droppedForRoom: 0, segmentsRemoved: 0, bytes: held };
|
|
1047
|
+
}
|
|
1048
|
+
if (typeof viewersAt !== "function") {
|
|
1049
|
+
let droppedForRoom = 0;
|
|
1050
|
+
const byAge = [...this.#touched.entries()].sort((left, right) => left[1] - right[1]);
|
|
1051
|
+
for (const [key] of byAge) {
|
|
1052
|
+
if (held <= maxBytes) {
|
|
1053
|
+
break;
|
|
1054
|
+
}
|
|
1055
|
+
const size = this.refresh(key).bytes;
|
|
1056
|
+
this.drop(key, `the store is over its ${(maxBytes / 1073741824).toFixed(1)}GB allowance`);
|
|
1057
|
+
held -= size;
|
|
1058
|
+
droppedForRoom += 1;
|
|
1059
|
+
}
|
|
1060
|
+
return { droppedIdle, droppedForRoom, segmentsRemoved: 0, bytes: held };
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
let segmentsRemoved = 0;
|
|
1064
|
+
for (const victim of this.#leastWantedFirst(viewersAt)) {
|
|
1065
|
+
if (held <= maxBytes) {
|
|
1066
|
+
break;
|
|
1067
|
+
}
|
|
1068
|
+
held -= this.#removeSegment(victim.key, victim.index);
|
|
1069
|
+
segmentsRemoved += 1;
|
|
1070
|
+
}
|
|
1071
|
+
if (segmentsRemoved > 0) {
|
|
1072
|
+
this.#logger.info(
|
|
1073
|
+
`segment-store removed ${segmentsRemoved} segment(s) for room: ` +
|
|
1074
|
+
`${megabytes(held)} of ${megabytes(maxBytes)} allowed`
|
|
1075
|
+
);
|
|
1076
|
+
}
|
|
1077
|
+
return { droppedIdle, droppedForRoom: 0, segmentsRemoved, bytes: held };
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Every segment in the store, least wanted first.
|
|
1082
|
+
*
|
|
1083
|
+
* @param {(key: string) => number[]} viewersAt
|
|
1084
|
+
* @returns {{ key: string, index: number }[]}
|
|
1085
|
+
*/
|
|
1086
|
+
#leastWantedFirst(viewersAt) {
|
|
1087
|
+
const candidates = [];
|
|
1088
|
+
for (const key of this.#formats.keys()) {
|
|
1089
|
+
const positions = (viewersAt(key) ?? []).filter((at) => Number.isInteger(at));
|
|
1090
|
+
const earliest = positions.length > 0 ? Math.min(...positions) : null;
|
|
1091
|
+
const furthest = positions.length > 0 ? Math.max(...positions) : null;
|
|
1092
|
+
for (const index of this.refresh(key).byNumber.keys()) {
|
|
1093
|
+
if (earliest === null) {
|
|
1094
|
+
// Nobody is watching this output at all. Everything it holds is worth
|
|
1095
|
+
// less than anything somebody is on their way to.
|
|
1096
|
+
candidates.push({ key, index, rank: 0, distance: index });
|
|
1097
|
+
continue;
|
|
1098
|
+
}
|
|
1099
|
+
if (positions.includes(index)) {
|
|
1100
|
+
continue;
|
|
1101
|
+
}
|
|
1102
|
+
if (index < earliest) {
|
|
1103
|
+
candidates.push({ key, index, rank: 1, distance: earliest - index });
|
|
1104
|
+
} else {
|
|
1105
|
+
candidates.push({ key, index, rank: 2, distance: index - /** @type {number} */ (furthest) });
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
return candidates
|
|
1110
|
+
.sort((left, right) => (left.rank !== right.rank ? left.rank - right.rank : right.distance - left.distance))
|
|
1111
|
+
.map(({ key, index }) => ({ key, index }));
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/**
|
|
1115
|
+
* Take one segment off the disk because its owner says it is not usable.
|
|
1116
|
+
*
|
|
1117
|
+
* @param {string} key
|
|
1118
|
+
* @param {number} index
|
|
1119
|
+
* @param {string} because
|
|
1120
|
+
* @returns {void}
|
|
1121
|
+
*/
|
|
1122
|
+
remove(key, index, because) {
|
|
1123
|
+
if (this.#removeSegment(key, index) > 0) {
|
|
1124
|
+
this.#logger?.warn?.(`segment store: removed segment ${index} of ${key.slice(0, 60)}: ${because}`);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/**
|
|
1129
|
+
* Take one segment off the disk.
|
|
1130
|
+
*
|
|
1131
|
+
* @param {string} key
|
|
1132
|
+
* @param {number} index
|
|
1133
|
+
* @returns {number} What it weighed.
|
|
1134
|
+
*/
|
|
1135
|
+
#removeSegment(key, index) {
|
|
1136
|
+
const full = this.refresh(key).byNumber.get(index);
|
|
1137
|
+
if (!full) {
|
|
1138
|
+
return 0;
|
|
1139
|
+
}
|
|
1140
|
+
let size = 0;
|
|
1141
|
+
try {
|
|
1142
|
+
size = statSync(full, { throwIfNoEntry: false })?.size ?? 0;
|
|
1143
|
+
rmSync(full, { force: true });
|
|
1144
|
+
} catch {
|
|
1145
|
+
// Gone already, or refused. The next refresh reports what is really there.
|
|
1146
|
+
}
|
|
1147
|
+
this.#held.delete(key);
|
|
1148
|
+
this.#mediaRanges.get(key)?.delete(index);
|
|
1149
|
+
return size;
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
/**
|
|
1153
|
+
* What a previous life of this process left on the disk.
|
|
1154
|
+
*
|
|
1155
|
+
* This is the only record there is of a death nobody saw. The kernel kills
|
|
1156
|
+
* this process often enough to matter — two kills in one viewing on
|
|
1157
|
+
* 2026-09-02 — and when it does, no exit handler runs, nothing is cleared up,
|
|
1158
|
+
* and memory is reclaimed while the disk is not: `/tmp` in the addon
|
|
1159
|
+
* container is on the overlay filesystem, measured 2026-09-04, so the files
|
|
1160
|
+
* survive the process and its restart.
|
|
1161
|
+
*
|
|
1162
|
+
* So the sweep reports rather than deletes quietly. What it finds is the
|
|
1163
|
+
* evidence, and every directory it names is one abnormal ending that went
|
|
1164
|
+
* unrecorded.
|
|
1165
|
+
*
|
|
1166
|
+
* @returns {{ directories: number, segments: number, bytes: number, unidentified: number, found: {key: string, dir: string, segments: number, bytes: number}[] }}
|
|
1167
|
+
*/
|
|
1168
|
+
sweep() {
|
|
1169
|
+
const found = [];
|
|
1170
|
+
let unidentified = 0;
|
|
1171
|
+
let names = [];
|
|
1172
|
+
try {
|
|
1173
|
+
names = readdirSync(this.#root);
|
|
1174
|
+
} catch {
|
|
1175
|
+
return { directories: 0, segments: 0, bytes: 0, unidentified: 0, found: [] };
|
|
1176
|
+
}
|
|
1177
|
+
for (const name of names) {
|
|
1178
|
+
const dir = path.join(this.#root, name);
|
|
1179
|
+
let key = "";
|
|
1180
|
+
try {
|
|
1181
|
+
if (!statSync(dir).isDirectory()) {
|
|
1182
|
+
continue;
|
|
1183
|
+
}
|
|
1184
|
+
key = readFileSync(path.join(dir, KEY_FILE), "utf8").trim();
|
|
1185
|
+
} catch {
|
|
1186
|
+
key = "";
|
|
1187
|
+
}
|
|
1188
|
+
let segments = 0;
|
|
1189
|
+
let bytes = 0;
|
|
1190
|
+
try {
|
|
1191
|
+
for (const entry of readdirSync(dir)) {
|
|
1192
|
+
if (entry === KEY_FILE) {
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
try {
|
|
1196
|
+
bytes += statSync(path.join(dir, entry)).size;
|
|
1197
|
+
segments += 1;
|
|
1198
|
+
} catch {
|
|
1199
|
+
// Vanished between the listing and the question.
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
} catch {
|
|
1203
|
+
continue;
|
|
1204
|
+
}
|
|
1205
|
+
if (!key) {
|
|
1206
|
+
// A directory that cannot say what it holds is from before this layer,
|
|
1207
|
+
// or its key file did not survive. Nothing can be served out of it,
|
|
1208
|
+
// because nothing can match it to a request.
|
|
1209
|
+
unidentified += 1;
|
|
1210
|
+
}
|
|
1211
|
+
found.push({ key, dir, segments, bytes });
|
|
1212
|
+
}
|
|
1213
|
+
const totals = found.reduce(
|
|
1214
|
+
(sum, entry) => ({ segments: sum.segments + entry.segments, bytes: sum.bytes + entry.bytes }),
|
|
1215
|
+
{ segments: 0, bytes: 0 }
|
|
1216
|
+
);
|
|
1217
|
+
if (found.length > 0) {
|
|
1218
|
+
this.#logger.info(
|
|
1219
|
+
`segment-store startup sweep: ${found.length} directories left by a previous run, ` +
|
|
1220
|
+
`${totals.segments} segments, ${(totals.bytes / 1048576).toFixed(1)}MB, ` +
|
|
1221
|
+
`${unidentified} of them unidentifiable — each one is an encoder that ended ` +
|
|
1222
|
+
"without anything recording why"
|
|
1223
|
+
);
|
|
1224
|
+
}
|
|
1225
|
+
return {
|
|
1226
|
+
directories: found.length,
|
|
1227
|
+
segments: totals.segments,
|
|
1228
|
+
bytes: totals.bytes,
|
|
1229
|
+
unidentified,
|
|
1230
|
+
found
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
/**
|
|
1235
|
+
* Take back what a previous life left: keep what is proven, remove the rest.
|
|
1236
|
+
*
|
|
1237
|
+
* Deliberately not "throw everything away". A killed process leaves material
|
|
1238
|
+
* that is valid by construction — a copied segment's bytes depend only on the
|
|
1239
|
+
* source — and re-encoding it costs the machine that is already known to be
|
|
1240
|
+
* short of processor. What cannot be kept is a directory that cannot name
|
|
1241
|
+
* itself, and the one file per directory whose closure nothing proves.
|
|
1242
|
+
*
|
|
1243
|
+
* @param {(key: string) => object | null} formatFor - How to read the file
|
|
1244
|
+
* names of an output, given its key. Null when this proxy cannot serve that
|
|
1245
|
+
* output at all, and then the directory goes.
|
|
1246
|
+
* @returns {{ adopted: number, dropped: number, unprovenRemoved: number }}
|
|
1247
|
+
*/
|
|
1248
|
+
adoptWhatSurvived(formatFor) {
|
|
1249
|
+
const swept = this.sweep();
|
|
1250
|
+
let adopted = 0;
|
|
1251
|
+
let dropped = 0;
|
|
1252
|
+
let unprovenRemoved = 0;
|
|
1253
|
+
for (const entry of swept.found) {
|
|
1254
|
+
const format = entry.key ? formatFor(entry.key) : null;
|
|
1255
|
+
if (!format) {
|
|
1256
|
+
try {
|
|
1257
|
+
rmSync(entry.dir, { recursive: true, force: true });
|
|
1258
|
+
} catch {
|
|
1259
|
+
// Leave it; the next sweep reports it again.
|
|
1260
|
+
}
|
|
1261
|
+
dropped += 1;
|
|
1262
|
+
this.#logger.info(
|
|
1263
|
+
`segment-store discarded ${path.basename(entry.dir)}: ` +
|
|
1264
|
+
(entry.key ? "this proxy cannot serve that output" : "it does not say what it holds")
|
|
1265
|
+
);
|
|
1266
|
+
continue;
|
|
1267
|
+
}
|
|
1268
|
+
this.#formats.set(entry.key, format);
|
|
1269
|
+
// EVERY SEGMENT FOUND IS COMPLETE, because a piece is given its served
|
|
1270
|
+
// name only once the encoder has said it is closed. So there is nothing to
|
|
1271
|
+
// prove here and nothing to un-prove: what the directory holds under
|
|
1272
|
+
// served names is what a killed process finished.
|
|
1273
|
+
//
|
|
1274
|
+
// What it may also hold is pieces it was in the middle of, under their
|
|
1275
|
+
// working names, and those are swept — the file a run was writing when the
|
|
1276
|
+
// kernel took the process is exactly this.
|
|
1277
|
+
unprovenRemoved += this.#sweepUnfinished(entry.key, entry.dir, format);
|
|
1278
|
+
adopted += 1;
|
|
1279
|
+
this.#logger.info(
|
|
1280
|
+
`segment-store adopted ${path.basename(entry.dir)}: ${this.provenNumbers(entry.key).length} ` +
|
|
1281
|
+
"segments a killed process had already finished"
|
|
1282
|
+
);
|
|
1283
|
+
}
|
|
1284
|
+
return { adopted, dropped, unprovenRemoved };
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/**
|
|
1289
|
+
* @param {number} bytes
|
|
1290
|
+
* @returns {string}
|
|
1291
|
+
*/
|
|
1292
|
+
function megabytes(bytes) {
|
|
1293
|
+
return `${Math.round(Math.max(0, bytes) / (1024 * 1024))}MB`;
|
|
1294
|
+
}
|