@torrent-tv/proxy 2.107.57 → 2.107.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +24 -15
- package/package.json +1 -1
- package/services/encode/AdmittedInput.js +6 -6
- package/services/encode/EncodeInputs.js +4 -2
- package/services/encode/LocalObservations.js +20 -2
- package/services/encode/MatroskaInput.js +10 -23
- package/services/encode/decode-cost-fit.js +1 -1
- package/services/identity/installation-id.js +17 -6
- package/services/media/container/AviContainer.js +4 -4
- package/services/media/playback-declarations.js +2 -15
- package/services/media/tracks/AudioTrack.js +2 -47
- package/services/media/tracks/codec-names.js +83 -0
- package/services/server/wire-outputs.js +7 -2
- package/services/storage/files/CompletedFiles.js +6 -1
- package/services/storage/memory-report.js +5 -5
- package/services/storage/segment-store/SegmentStore.js +2 -1
- package/services/torrent/worker/worker.js +1 -6
- package/utils/atomic-json.js +46 -0
package/bin/cli.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
// for the measurement that made it necessary.
|
|
15
15
|
import "./thread-pool.js";
|
|
16
16
|
import { Command } from "commander";
|
|
17
|
+
import { mkdirSync } from "node:fs";
|
|
17
18
|
import os from "node:os";
|
|
18
19
|
import { spawnSync } from "node:child_process";
|
|
19
20
|
import { createRequire } from "node:module";
|
|
@@ -35,6 +36,7 @@ import { collectHealthMetrics } from "../services/transport/health-collector.js"
|
|
|
35
36
|
import { LOAD_REFRESH_MS, ProxyStateReporter } from "../services/transport/proxy-state.js";
|
|
36
37
|
import { createPortMapper } from "../services/transport/port-mapper.js";
|
|
37
38
|
import { installationId } from "../services/identity/installation-id.js";
|
|
39
|
+
import { PROXY_ROOT } from "../services/encode/LocalObservations.js";
|
|
38
40
|
import { classifyNat } from "../services/transport/nat-classifier.js";
|
|
39
41
|
import { DEFAULT_SEGMENT_FORMAT_ID, SEGMENT_FORMAT_IDS } from "../services/encode/segment-formats/index.js";
|
|
40
42
|
import { logToFile, logger } from "../utils/logger.js";
|
|
@@ -100,7 +102,7 @@ program
|
|
|
100
102
|
"--usrsctp-state",
|
|
101
103
|
"Read usrsctp's association state with gdb when a wedge is declared. OFF by default: gdb attaches to THIS process and stops every thread of it while it works."
|
|
102
104
|
)
|
|
103
|
-
.option("--memory-bytes <bytes>", "Per-torrent budget for pieces kept in memory before spilling to disk (default
|
|
105
|
+
.option("--memory-bytes <bytes>", "Per-torrent budget for pieces kept in memory before spilling to disk (default: derived from the memory available when the torrent opens, shared equally between open torrents)")
|
|
104
106
|
.option("--budget <kind>", "How much of this machine the proxy may take: adaptive (default), share or fixed")
|
|
105
107
|
.option("--budget-share <fraction>", "With --budget share: the fraction of what is free, 0..1")
|
|
106
108
|
.option("--budget-bytes <bytes>", "With --budget fixed: the ceiling, per resource")
|
|
@@ -109,7 +111,7 @@ program
|
|
|
109
111
|
.option("--ffmpeg-bin <path>", "Path to ffmpeg binary")
|
|
110
112
|
.option(
|
|
111
113
|
"--state-dir <path>",
|
|
112
|
-
"Where to keep what this host has measured about itself (default:
|
|
114
|
+
"Where to keep this installation's id, what this host has measured about itself, and diagnostic dumps (default: the installed proxy's own directory; pass an empty value to keep nothing)"
|
|
113
115
|
)
|
|
114
116
|
.option(
|
|
115
117
|
"--log-file <path>",
|
|
@@ -137,7 +139,19 @@ const bindHost = String(options.host);
|
|
|
137
139
|
const explicitBaseUrl = options.publicBaseUrl
|
|
138
140
|
? String(options.publicBaseUrl).replace(/\/+$/, "")
|
|
139
141
|
: "";
|
|
140
|
-
|
|
142
|
+
// THE STATE DIRECTORY, resolved once: the one named, the installed proxy's own
|
|
143
|
+
// directory when none is named, or "" when the operator passes an empty value
|
|
144
|
+
// (nothing is kept). Every consumer below is handed this value and none reads
|
|
145
|
+
// the option again.
|
|
146
|
+
const stateDir = options.stateDir === undefined ? PROXY_ROOT : String(options.stateDir);
|
|
147
|
+
if (stateDir) {
|
|
148
|
+
try {
|
|
149
|
+
mkdirSync(stateDir, { recursive: true });
|
|
150
|
+
} catch (error) {
|
|
151
|
+
logger.warn(`Could not create the state directory ${stateDir}: ${error instanceof Error ? error.message : String(error)}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const clientId = installationId({ stateDir, explicitId: options.id });
|
|
141
155
|
const clientName = options.name ? String(options.name) : `proxy-${clientId.slice(0, 8)}`;
|
|
142
156
|
const token = String(options.token ?? "");
|
|
143
157
|
const transcodeAudio = options.transcodeAudio !== false;
|
|
@@ -347,15 +361,15 @@ async function shutdown(signal) {
|
|
|
347
361
|
const diagnostics = new Diagnostics({
|
|
348
362
|
logger,
|
|
349
363
|
kinds: [
|
|
350
|
-
{ name: "core dumps", directory: () =>
|
|
364
|
+
{ name: "core dumps", directory: () => stateDir, matches: (name) => name.startsWith("core.") },
|
|
351
365
|
{
|
|
352
366
|
name: "heap snapshots",
|
|
353
|
-
directory: () =>
|
|
367
|
+
directory: () => stateDir,
|
|
354
368
|
matches: (name) => name.endsWith(".heapsnapshot")
|
|
355
369
|
},
|
|
356
370
|
{
|
|
357
371
|
name: "packet captures",
|
|
358
|
-
directory: () =>
|
|
372
|
+
directory: () => stateDir,
|
|
359
373
|
matches: (name) => name.startsWith("packet-witness")
|
|
360
374
|
}
|
|
361
375
|
]
|
|
@@ -391,7 +405,7 @@ try {
|
|
|
391
405
|
ffmpegBin,
|
|
392
406
|
memoryBytes,
|
|
393
407
|
segmentFormat: options.segmentFormat,
|
|
394
|
-
stateDir
|
|
408
|
+
stateDir,
|
|
395
409
|
diagnostics,
|
|
396
410
|
budgetPolicy,
|
|
397
411
|
logFile: options.logFile ?? "",
|
|
@@ -444,11 +458,11 @@ try {
|
|
|
444
458
|
// A native fault writes the whole address space out — 4.18 GB each on the
|
|
445
459
|
// field host, and four of them nearly filled a 235 GB disk. Keep the newest
|
|
446
460
|
// two, which are the evidence for the fault still open, and drop the rest.
|
|
447
|
-
void pruneCoreDumps(
|
|
461
|
+
void pruneCoreDumps(stateDir);
|
|
448
462
|
// Same policy for the packet captures the send-queue witness writes.
|
|
449
463
|
packetWitness = createPacketWitness({
|
|
450
464
|
log: (message) => logger.info(message),
|
|
451
|
-
dir:
|
|
465
|
+
dir: stateDir,
|
|
452
466
|
port: actualPort,
|
|
453
467
|
// Evidence takes room from the product, so a capture is asked for rather
|
|
454
468
|
// than taken. Refused, it is a line and not a silence.
|
|
@@ -517,8 +531,6 @@ try {
|
|
|
517
531
|
// read stayed behind the decision to write, so a quiet second costs one
|
|
518
532
|
// call to `process.memoryUsage()`.
|
|
519
533
|
intervalMs: 1_000,
|
|
520
|
-
quietMs: 60_000,
|
|
521
|
-
changeBytes: 25 * 1024 * 1024,
|
|
522
534
|
diskPath: os.tmpdir(),
|
|
523
535
|
// WHERE A SNAPSHOT SURVIVES THE PROCESS IT IS ABOUT, and how many are kept
|
|
524
536
|
// — the same answer the torrent worker already had. This scope had neither:
|
|
@@ -526,11 +538,8 @@ try {
|
|
|
526
538
|
// on every restart, and all of them were kept. Fifty-four were written on
|
|
527
539
|
// 2026-09-07 and every one was gone by the time the process that wrote them
|
|
528
540
|
// died of a heap it had never been possible to look at.
|
|
529
|
-
snapshotDir:
|
|
541
|
+
snapshotDir: stateDir || undefined,
|
|
530
542
|
heapSnapshots: options.heapSnapshots === true,
|
|
531
|
-
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
532
|
-
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
533
|
-
keepSnapshots: 3
|
|
534
543
|
});
|
|
535
544
|
|
|
536
545
|
logger.info(`Starting @torrent-tv/proxy v${PROXY_VERSION}`);
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@ import { pcmNormalization, normalizePcmSlices } from "./pcm-normalization.js";
|
|
|
5
5
|
const INPUT_WRITE_BATCH_BYTES = 64 * 1024;
|
|
6
6
|
|
|
7
7
|
/** Acquire complete source ranges before any process can consume the input. */
|
|
8
|
-
export async function admitInput({ sources, reserve, readRanges }) {
|
|
8
|
+
export async function admitInput({ sources, reserve, readRanges, matroskaIdOf }) {
|
|
9
9
|
if (!Array.isArray(sources) || sources.length === 0 || typeof reserve !== "function" || typeof readRanges !== "function") {
|
|
10
10
|
throw new TypeError("Admission requires resolved source inputs, memory reservation and an available-only reader.");
|
|
11
11
|
}
|
|
@@ -41,7 +41,7 @@ export async function admitInput({ sources, reserve, readRanges }) {
|
|
|
41
41
|
pts: packet.pts - (source.timeShiftSeconds ?? 0),
|
|
42
42
|
...(packet.dts === undefined ? {} : { dts: packet.dts - (source.timeShiftSeconds ?? 0) }) })) };
|
|
43
43
|
tracks.push(shifted);
|
|
44
|
-
hash.update(matroskaTrackIdentity(input.track));
|
|
44
|
+
hash.update(matroskaTrackIdentity(input.track, matroskaIdOf));
|
|
45
45
|
hash.update(JSON.stringify(shifted.packets.map(packet => [packet.pts, packet.dts ?? null,
|
|
46
46
|
packet.duration, packet.keyframe === true, packet.ranges, packet.discardPaddingSeconds ?? 0,
|
|
47
47
|
packet.bitOffset ?? null, packet.bitLength ?? null])));
|
|
@@ -82,8 +82,8 @@ export async function admitInput({ sources, reserve, readRanges }) {
|
|
|
82
82
|
}
|
|
83
83
|
retained = true;
|
|
84
84
|
return {
|
|
85
|
-
kind: "result", bytes, fingerprint: hash.digest("hex"), originSeconds, tracks, readPacket,
|
|
86
|
-
stream: options => matroskaInput({ tracks,
|
|
85
|
+
kind: "result", matroskaIdOf, bytes, fingerprint: hash.digest("hex"), originSeconds, tracks, readPacket,
|
|
86
|
+
stream: options => matroskaInput({ tracks, matroskaIdOf,
|
|
87
87
|
readPacket: (input, packet) => normalized.has(packet) ? [normalized.get(packet)] : packetSlices(input, packet),
|
|
88
88
|
originSeconds, ...options }),
|
|
89
89
|
release: () => {
|
|
@@ -112,13 +112,13 @@ export async function admitInput({ sources, reserve, readRanges }) {
|
|
|
112
112
|
export async function writeAdmittedInput(input, stdin, { next = null, waiting = null, onFirstInputBytes = null } = {}) {
|
|
113
113
|
if (input?.kind !== "result") throw new TypeError("Only complete input may be written to an encoder.");
|
|
114
114
|
const originSeconds = input.originSeconds;
|
|
115
|
-
const declarations = JSON.stringify(input.tracks.map(
|
|
115
|
+
const declarations = JSON.stringify(input.tracks.map(track => matroskaTrackIdentity(track.track, input.matroskaIdOf)));
|
|
116
116
|
const seen = new Map();
|
|
117
117
|
let firstInputBytesNoted = false;
|
|
118
118
|
let current = input, includeHeader = true;
|
|
119
119
|
try {
|
|
120
120
|
while (current) {
|
|
121
|
-
if (JSON.stringify(current.tracks.map(
|
|
121
|
+
if (JSON.stringify(current.tracks.map(track => matroskaTrackIdentity(track.track, current.matroskaIdOf))) !== declarations) {
|
|
122
122
|
throw new Error("Consecutive encoder input changed its declared tracks.");
|
|
123
123
|
}
|
|
124
124
|
for (let index = 0; index < current.tracks.length; index++) {
|
|
@@ -67,10 +67,12 @@ export class EncodeInputs {
|
|
|
67
67
|
*/
|
|
68
68
|
#stretchCopied;
|
|
69
69
|
#observeWait;
|
|
70
|
+
#matroskaIdOf;
|
|
70
71
|
|
|
71
72
|
constructor({ resolve, readRanges, openWholeFile = null, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
|
|
72
73
|
rankAt = () => null, demandChanged = () => {}, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {},
|
|
73
|
-
observeWait = null }) {
|
|
74
|
+
observeWait = null, matroskaIdOf }) {
|
|
75
|
+
this.#matroskaIdOf = matroskaIdOf;
|
|
74
76
|
this.#stretchCopied = stretchCopied;
|
|
75
77
|
this.#observeWait = observeWait;
|
|
76
78
|
this.#openOf = openOf;
|
|
@@ -461,7 +463,7 @@ export class EncodeInputs {
|
|
|
461
463
|
}
|
|
462
464
|
if (!result) return;
|
|
463
465
|
} else {
|
|
464
|
-
result = await admitInput({ sources: resolved.sources, readRanges: this.#read, reserve: this.#reserveFor(request) });
|
|
466
|
+
result = await admitInput({ sources: resolved.sources, readRanges: this.#read, matroskaIdOf: this.#matroskaIdOf, reserve: this.#reserveFor(request) });
|
|
465
467
|
if (result.kind === "result") Object.assign(result, { from: request.from, to: request.from });
|
|
466
468
|
}
|
|
467
469
|
if (this.#requests.get(request.key) !== request) { result.release?.(); return; }
|
|
@@ -48,6 +48,19 @@ import path from "node:path";
|
|
|
48
48
|
import { fileURLToPath } from "node:url";
|
|
49
49
|
import { configurationKeyOf, kindOfEncoderName } from "./fingerprint.js";
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* The writer used when none is handed in (a check that only wants the file).
|
|
53
|
+
* The proxy always hands in the atomic one from `utils/atomic-json.js`, which
|
|
54
|
+
* this layer may not import.
|
|
55
|
+
*
|
|
56
|
+
* @param {string} filePath
|
|
57
|
+
* @param {unknown} value
|
|
58
|
+
* @returns {void}
|
|
59
|
+
*/
|
|
60
|
+
function writeInPlace(filePath, value) {
|
|
61
|
+
writeFileSync(filePath, JSON.stringify(value));
|
|
62
|
+
}
|
|
63
|
+
|
|
51
64
|
/**
|
|
52
65
|
* The directory of the installed proxy, where these observations are kept when
|
|
53
66
|
* the deployment names no state directory. Deliberately not the working
|
|
@@ -84,6 +97,8 @@ export class LocalObservations {
|
|
|
84
97
|
|
|
85
98
|
/** @type {{ info: (m: string) => void, warn: (m: string) => void }} */
|
|
86
99
|
#logger;
|
|
100
|
+
/** @type {(filePath: string, value: unknown) => void} */
|
|
101
|
+
#writeJson;
|
|
87
102
|
|
|
88
103
|
/** @type {() => number} */
|
|
89
104
|
#now;
|
|
@@ -99,16 +114,19 @@ export class LocalObservations {
|
|
|
99
114
|
* @param {import("./fingerprint.js").HostFingerprint} params.fingerprint
|
|
100
115
|
* @param {string} params.filePath - Where the readings are kept; empty keeps
|
|
101
116
|
* them for this process only.
|
|
117
|
+
* @param {(filePath: string, value: unknown) => void} [params.writeJson] - How
|
|
118
|
+
* the file is saved; throws on failure. The atomic writer is handed in.
|
|
102
119
|
* @param {{ info: (m: string) => void, warn: (m: string) => void }} [params.logger]
|
|
103
120
|
* @param {() => number} [params.now]
|
|
104
121
|
*/
|
|
105
|
-
constructor({ fingerprint, filePath, logger = null, now = Date.now }) {
|
|
122
|
+
constructor({ fingerprint, filePath, writeJson = writeInPlace, logger = null, now = Date.now }) {
|
|
106
123
|
if (!fingerprint || typeof fingerprint !== "object") {
|
|
107
124
|
throw new TypeError("LocalObservations requires the host's fingerprint");
|
|
108
125
|
}
|
|
109
126
|
this.#fingerprint = fingerprint;
|
|
110
127
|
this.#filePath = typeof filePath === "string" ? filePath : "";
|
|
111
128
|
this.#logger = logger ?? { info: () => {}, warn: () => {} };
|
|
129
|
+
this.#writeJson = writeJson;
|
|
112
130
|
this.#now = now;
|
|
113
131
|
}
|
|
114
132
|
|
|
@@ -367,7 +385,7 @@ export class LocalObservations {
|
|
|
367
385
|
return;
|
|
368
386
|
}
|
|
369
387
|
try {
|
|
370
|
-
|
|
388
|
+
this.#writeJson(this.#filePath, { version: FILE_VERSION, entries: this.#entries });
|
|
371
389
|
} catch (error) {
|
|
372
390
|
// A read-only install keeps what it learned for this process only.
|
|
373
391
|
this.#logger.warn(
|
|
@@ -2,12 +2,12 @@ import { avcPacketSlices } from "./annex-b-input.js";
|
|
|
2
2
|
import { bitPayloadSlices } from "./bit-payload.js";
|
|
3
3
|
|
|
4
4
|
/** Emit already admitted packets as one timestamped Matroska input. */
|
|
5
|
-
export async function* matroskaInput({ tracks, readPacket, originSeconds = 0, includeHeader = true, seen = new Map() }) {
|
|
5
|
+
export async function* matroskaInput({ tracks, readPacket, originSeconds = 0, includeHeader = true, seen = new Map(), matroskaIdOf }) {
|
|
6
6
|
if (!Array.isArray(tracks) || tracks.length === 0 || typeof readPacket !== "function" || !Number.isFinite(originSeconds)) {
|
|
7
7
|
throw new TypeError("A Matroska input requires tracks, admitted packets and a timeline origin.");
|
|
8
8
|
}
|
|
9
9
|
if (includeHeader) {
|
|
10
|
-
const declarations = tracks.map((input, position) => trackElement(input.track, position + 1));
|
|
10
|
+
const declarations = tracks.map((input, position) => trackElement(input.track, position + 1, matroskaIdOf));
|
|
11
11
|
yield element("1a45dfa3", Buffer.concat([
|
|
12
12
|
uintElement("4286", 1), uintElement("42f7", 1), uintElement("42f2", 4), uintElement("42f3", 8),
|
|
13
13
|
textElement("4282", "matroska"), uintElement("4287", 4), uintElement("4285", 2)
|
|
@@ -107,10 +107,10 @@ export async function* matroskaInput({ tracks, readPacket, originSeconds = 0, in
|
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
-
function trackElement(track, number) {
|
|
110
|
+
function trackElement(track, number, matroskaIdOf) {
|
|
111
111
|
const type = track.type === "video" ? 1 : track.type === "audio" ? 2 : null;
|
|
112
112
|
if (type === null) throw new Error("Admitted input carries an unsupported track type.");
|
|
113
|
-
const codec = track.matroskaCodecId ?? matroskaCodec(track.codecId);
|
|
113
|
+
const codec = track.matroskaCodecId ?? matroskaCodec(track.codecId, matroskaIdOf);
|
|
114
114
|
const fields = [uintElement("d7", number), uintElement("73c5", number), uintElement("83", type),
|
|
115
115
|
uintElement("9c", 0), textElement("86", codec)];
|
|
116
116
|
const privateBytes = track.matroskaCodecPrivateB64 ?? track.codecPrivateB64;
|
|
@@ -133,9 +133,10 @@ function trackElement(track, number) {
|
|
|
133
133
|
}
|
|
134
134
|
|
|
135
135
|
/** Stable identity of every declaration emitted into the input stream. */
|
|
136
|
-
|
|
136
|
+
/** @param {(codec: string) => string | undefined} matroskaIdOf - Names a codec's Matroska id; handed in, since this layer reads no other. */
|
|
137
|
+
export function matroskaTrackIdentity(track, matroskaIdOf) {
|
|
137
138
|
return JSON.stringify([
|
|
138
|
-
track.type, track.matroskaCodecId ?? matroskaCodec(track.codecId),
|
|
139
|
+
track.type, track.matroskaCodecId ?? matroskaCodec(track.codecId, matroskaIdOf),
|
|
139
140
|
track.matroskaCodecPrivateB64 ?? track.codecPrivateB64 ?? "",
|
|
140
141
|
track.language ?? "", track.packetFraming ?? "", track.width ?? null, track.height ?? null,
|
|
141
142
|
track.samplingFrequency ?? null, track.channels ?? null, track.bitDepth ?? null,
|
|
@@ -143,23 +144,9 @@ export function matroskaTrackIdentity(track) {
|
|
|
143
144
|
]);
|
|
144
145
|
}
|
|
145
146
|
|
|
146
|
-
function matroskaCodec(codec) {
|
|
147
|
-
if (
|
|
148
|
-
|
|
149
|
-
if (/^pcm_[su](?:16|24|32|64)be$/.test(codec)) return "A_PCM/INT/BIG";
|
|
150
|
-
if (/^pcm_f(?:32|64)(?:le|be)$/.test(codec)) return "A_PCM/FLOAT/IEEE";
|
|
151
|
-
const declared = new Map([
|
|
152
|
-
["avc1", "V_MPEG4/ISO/AVC"], ["avc3", "V_MPEG4/ISO/AVC"], ["h264", "V_MPEG4/ISO/AVC"],
|
|
153
|
-
["hvc1", "V_MPEGH/ISO/HEVC"], ["hev1", "V_MPEGH/ISO/HEVC"], ["hevc", "V_MPEGH/ISO/HEVC"],
|
|
154
|
-
["av01", "V_AV1"], ["av1", "V_AV1"], ["vp9", "V_VP9"], ["vp8", "V_VP8"],
|
|
155
|
-
["mjpeg", "V_MJPEG"], ["ffv1", "V_FFV1"], ["theora", "V_THEORA"], ["jpeg2000", "V_JPEG2000"],
|
|
156
|
-
["mpeg1video", "V_MPEG1"], ["mpeg2video", "V_MPEG2"], ["mpeg4", "V_MPEG4/ISO/ASP"],
|
|
157
|
-
["FMP4", "V_MPEG4/ISO/ASP"], ["XVID", "V_MPEG4/ISO/ASP"], ["DIVX", "V_MPEG4/ISO/ASP"],
|
|
158
|
-
["mp4a", "A_AAC"], ["aac", "A_AAC"], ["mp3", "A_MPEG/L3"], ["mp2", "A_MPEG/L2"], ["mp1", "A_MPEG/L1"],
|
|
159
|
-
["ac-3", "A_AC3"], ["ac3", "A_AC3"], ["ec-3", "A_EAC3"], ["eac3", "A_EAC3"],
|
|
160
|
-
["Opus", "A_OPUS"], ["opus", "A_OPUS"], ["fLaC", "A_FLAC"], ["flac", "A_FLAC"], ["dts", "A_DTS"],
|
|
161
|
-
["vorbis", "A_VORBIS"], ["alac", "A_ALAC"], ["truehd", "A_TRUEHD"]
|
|
162
|
-
]).get(codec);
|
|
147
|
+
function matroskaCodec(codec, matroskaIdOf) {
|
|
148
|
+
if (typeof matroskaIdOf !== "function") throw new TypeError("A Matroska input is given the lookup that names codec ids.");
|
|
149
|
+
const declared = matroskaIdOf(codec);
|
|
163
150
|
if (!declared) throw new Error(`Admitted codec ${codec} has no Matroska declaration.`);
|
|
164
151
|
return declared;
|
|
165
152
|
}
|
|
@@ -334,7 +334,7 @@ export function decodeFamilyOf(source) {
|
|
|
334
334
|
const codec = typeof source?.codec === "string" ? source.codec.trim().toLowerCase() : "";
|
|
335
335
|
const depth = Number(source?.bitDepth);
|
|
336
336
|
const tenBit = Number.isFinite(depth) && depth >= 10;
|
|
337
|
-
if (codec === "hevc"
|
|
337
|
+
if (codec === "hevc") {
|
|
338
338
|
return tenBit ? "hevc10" : "hevc";
|
|
339
339
|
}
|
|
340
340
|
// ffmpeg's own names. Only what has a clip set: `msmpeg4v3` (DivX 3) is a
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, readFileSync
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { writeJsonAtomic } from "../../utils/atomic-json.js";
|
|
5
|
+
import { logger } from "../../utils/logger.js";
|
|
4
6
|
|
|
5
7
|
const ID_FILE = "proxy-identity.json";
|
|
6
8
|
|
|
9
|
+
/** Ids that could not be written, by identity file: kept so the process keeps one id. */
|
|
10
|
+
const unsavedIds = new Map();
|
|
11
|
+
|
|
7
12
|
/**
|
|
8
13
|
* Resolve this installation's registry id. Without a state directory the id
|
|
9
14
|
* remains ephemeral; with one, the id survives process restarts.
|
|
@@ -27,12 +32,18 @@ export function installationId({ stateDir, explicitId, makeId = randomUUID } = {
|
|
|
27
32
|
}
|
|
28
33
|
}
|
|
29
34
|
|
|
30
|
-
const id = requestedId || savedId || makeId();
|
|
35
|
+
const id = requestedId || savedId || unsavedIds.get(file) || makeId();
|
|
31
36
|
if (id !== savedId) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
37
|
+
try {
|
|
38
|
+
mkdirSync(stateDir, { recursive: true });
|
|
39
|
+
writeJsonAtomic(file, { id }, { mode: 0o600 });
|
|
40
|
+
unsavedIds.delete(file);
|
|
41
|
+
} catch (error) {
|
|
42
|
+
if (unsavedIds.get(file) !== id) {
|
|
43
|
+
logger.warn(`Cannot save the proxy identity file ${file} (${error.message}); the installation id lasts only until this process exits`);
|
|
44
|
+
}
|
|
45
|
+
unsavedIds.set(file, id);
|
|
46
|
+
}
|
|
36
47
|
}
|
|
37
48
|
return id;
|
|
38
49
|
}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { FFMPEG_STREAM_SEARCH_SECONDS } from "./ffmpeg-stream-search.js";
|
|
11
11
|
import { Container } from "./Container.js";
|
|
12
|
+
import { MPEG4_FOURCCS } from "../tracks/codec-names.js";
|
|
12
13
|
import { VideoTrack } from "../tracks/VideoTrack.js";
|
|
13
14
|
import { AudioTrack } from "../tracks/AudioTrack.js";
|
|
14
15
|
import { ContainerTrack } from "../tracks/ContainerTrack.js";
|
|
@@ -22,7 +23,6 @@ import { ac3Frame } from "./ac3-frame.js";
|
|
|
22
23
|
import { RetainedReads } from "./RetainedReads.js";
|
|
23
24
|
import { OutsideReadableEdges, edgeReader, emptyWorkTags, text, textList, yearOf } from "./work-tags.js";
|
|
24
25
|
|
|
25
|
-
const MPEG4_CODECS = new Set(["FMP4", "XVID", "DIVX", "DX50", "MP4V", "M4S2", "MP4S"]);
|
|
26
26
|
/** FFmpeg's input buffer, filled ahead of what its demuxer parses (`IO_BUFFER_SIZE`, libavformat/aviobuf.c n8.1). */
|
|
27
27
|
const FFMPEG_INPUT_BUFFER_BYTES = 32768;
|
|
28
28
|
/** Picture codecs whose every picture stands alone, so decoding order is showing order. */
|
|
@@ -473,7 +473,7 @@ export class AviContainer extends Container {
|
|
|
473
473
|
const clocks = new Map();
|
|
474
474
|
for (const track of tracks.filter(track => ["video", "audio"].includes(track.type))) {
|
|
475
475
|
if (!(track.timeBase > 0)) throw new Error("AVI stream time base is invalid.");
|
|
476
|
-
if (track.type === "video" && !["h264", "hevc"].includes(track.codecId) && !
|
|
476
|
+
if (track.type === "video" && !["h264", "hevc"].includes(track.codecId) && !MPEG4_FOURCCS.has(track.codecId.toUpperCase()) && !INTRA_PICTURE_CODECS.has(track.codecId.toUpperCase())) {
|
|
477
477
|
throw new Error("AVI video without an index requires elementary picture timing.");
|
|
478
478
|
}
|
|
479
479
|
if (track.type === "audio" && !(track.sampleSize > 0) && !MPEG_AUDIO_CODECS.has(track.codecId)) throw new Error("AVI variable-size audio requires elementary frame timing.");
|
|
@@ -651,7 +651,7 @@ export class AviContainer extends Container {
|
|
|
651
651
|
if (length) await this.#appendPacket(state.index, state.timings, track, { pts, duration, keyframe: true, ranges: [[start, end - 1]] });
|
|
652
652
|
else if (track.type === "video") state.index.extendLastPresentation(trackId, track.timeBase);
|
|
653
653
|
state.clocks.set(trackId, pts + duration);
|
|
654
|
-
if (!
|
|
654
|
+
if (!MPEG4_FOURCCS.has(track.codecId.toUpperCase()) && !["h264", "hevc"].includes(track.codecId) && !MPEG_AUDIO_CODECS.has(track.codecId) && (track.type === "audio" || length)) state.index.coverThrough(trackId, Math.max(0, pts + duration));
|
|
655
655
|
}
|
|
656
656
|
state.at = next;
|
|
657
657
|
state.index.flushPending();
|
|
@@ -687,7 +687,7 @@ export class AviContainer extends Container {
|
|
|
687
687
|
}
|
|
688
688
|
return;
|
|
689
689
|
}
|
|
690
|
-
if (track.type !== "video" || !
|
|
690
|
+
if (track.type !== "video" || !MPEG4_FOURCCS.has(track.codecId.toUpperCase())) {
|
|
691
691
|
index.append(track.trackNumber, packet);
|
|
692
692
|
return;
|
|
693
693
|
}
|
|
@@ -1,33 +1,20 @@
|
|
|
1
1
|
import { AudioTrack } from "./tracks/AudioTrack.js";
|
|
2
2
|
import { TextSubtitleTrack } from "./tracks/TextSubtitleTrack.js";
|
|
3
3
|
import { ContainerTrack } from "./tracks/ContainerTrack.js";
|
|
4
|
+
import { videoCodecName } from "./tracks/codec-names.js";
|
|
4
5
|
|
|
5
|
-
const VIDEO_CODECS = new Map([
|
|
6
|
-
["V_MPEG4/ISO/AVC", "h264"], ["avc1", "h264"], ["avc3", "h264"],
|
|
7
|
-
["V_MPEGH/ISO/HEVC", "hevc"], ["hvc1", "hevc"], ["hev1", "hevc"],
|
|
8
|
-
["V_MPEG1", "mpeg1video"], ["V_MPEG2", "mpeg2video"],
|
|
9
|
-
["V_MPEG4/ISO/ASP", "mpeg4"], ["V_MPEG4/ISO/SP", "mpeg4"], ["mp4v", "mpeg4"],
|
|
10
|
-
["V_VP8", "vp8"], ["V_VP9", "vp9"], ["vp09", "vp9"],
|
|
11
|
-
["V_AV1", "av1"], ["av01", "av1"], ["V_MJPEG", "mjpeg"],
|
|
12
|
-
["V_FFV1", "ffv1"], ["V_THEORA", "theora"]
|
|
13
|
-
]);
|
|
14
6
|
const SUBTITLE_CODECS = new Map([
|
|
15
7
|
["S_TEXT/UTF8", "subrip"], ["S_TEXT/ASS", "ass"], ["S_TEXT/SSA", "ssa"],
|
|
16
8
|
["S_TEXT/WEBVTT", "webvtt"], ["wvtt", "webvtt"], ["tx3g", "mov_text"],
|
|
17
9
|
["S_HDMV/PGS", "hdmv_pgs_subtitle"], ["S_VOBSUB", "dvd_subtitle"]
|
|
18
10
|
]);
|
|
19
|
-
const FOURCC_CODECS = new Map([
|
|
20
|
-
["mjpg", "mjpeg"], ["jpeg", "mjpeg"], ["xvid", "mpeg4"],
|
|
21
|
-
["divx", "mpeg4"], ["dx50", "mpeg4"], ["mpg2", "mpeg2video"]
|
|
22
|
-
]);
|
|
23
|
-
|
|
24
11
|
function videoCodecOf(track) {
|
|
25
12
|
let id = track.codecId;
|
|
26
13
|
if (id === "V_MS/VFW/FOURCC" && track.codecPrivateB64) {
|
|
27
14
|
const header = Buffer.from(track.codecPrivateB64, "base64");
|
|
28
15
|
if (header.length >= 20) id = header.toString("ascii", 16, 20);
|
|
29
16
|
}
|
|
30
|
-
return
|
|
17
|
+
return videoCodecName(id);
|
|
31
18
|
}
|
|
32
19
|
|
|
33
20
|
/** One playback inventory, taken from the same declarations used for input. */
|
|
@@ -11,40 +11,9 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { codecParametersOf, peakKbpsOf } from "./audio-bound.js";
|
|
14
|
+
import { audioCodecName } from "./codec-names.js";
|
|
14
15
|
import { ContainerTrack } from "./ContainerTrack.js";
|
|
15
16
|
|
|
16
|
-
/**
|
|
17
|
-
* Codec identifiers as containers write them, against the name ffmpeg prints.
|
|
18
|
-
*
|
|
19
|
-
* Needed because the browser decides whether it can play a soundtrack from that
|
|
20
|
-
* name, and for a sidecar file there is no ffmpeg banner to read it from — the
|
|
21
|
-
* track came from the container's own table, where Matroska writes `A_AC3` and
|
|
22
|
-
* MP4 writes `ac-3` for the thing ffmpeg calls `ac3`. Only what a soundtrack can
|
|
23
|
-
* actually be is listed; an identifier not here is reported as it was written,
|
|
24
|
-
* which the browser treats as one it does not know and therefore transcodes.
|
|
25
|
-
*/
|
|
26
|
-
const CODEC_NAMES = new Map([
|
|
27
|
-
["A_AAC", "aac"],
|
|
28
|
-
["A_AC3", "ac3"],
|
|
29
|
-
["A_EAC3", "eac3"],
|
|
30
|
-
["A_DTS", "dts"],
|
|
31
|
-
["A_FLAC", "flac"],
|
|
32
|
-
["A_OPUS", "opus"],
|
|
33
|
-
["A_VORBIS", "vorbis"],
|
|
34
|
-
["A_TRUEHD", "truehd"],
|
|
35
|
-
["A_MPEG/L3", "mp3"],
|
|
36
|
-
["A_MPEG/L2", "mp2"],
|
|
37
|
-
["A_ALAC", "alac"],
|
|
38
|
-
["mp4a", "aac"],
|
|
39
|
-
["ac-3", "ac3"],
|
|
40
|
-
["ec-3", "eac3"],
|
|
41
|
-
["alac", "alac"],
|
|
42
|
-
["opus", "opus"],
|
|
43
|
-
["Opus", "opus"],
|
|
44
|
-
["fLaC", "flac"],
|
|
45
|
-
["flac", "flac"]
|
|
46
|
-
]);
|
|
47
|
-
|
|
48
17
|
/**
|
|
49
18
|
* Extensions of raw elementary streams, against the codec they carry.
|
|
50
19
|
*
|
|
@@ -126,21 +95,7 @@ export class AudioTrack extends ContainerTrack {
|
|
|
126
95
|
}
|
|
127
96
|
const codecId = typeof track?.codecId === "string" ? track.codecId.trim() : "";
|
|
128
97
|
if (codecId.length > 0) {
|
|
129
|
-
|
|
130
|
-
if (known) {
|
|
131
|
-
return known;
|
|
132
|
-
}
|
|
133
|
-
// Matroska allows a suffix — `A_AAC/MPEG4/LC`, `A_PCM/INT/LIT` — so the
|
|
134
|
-
// family is what the first two segments say.
|
|
135
|
-
const family = codecId.split("/").slice(0, 2).join("/");
|
|
136
|
-
const byFamily = CODEC_NAMES.get(family) ?? CODEC_NAMES.get(codecId.split("/")[0]);
|
|
137
|
-
if (byFamily) {
|
|
138
|
-
return byFamily;
|
|
139
|
-
}
|
|
140
|
-
if (codecId.startsWith("A_PCM")) {
|
|
141
|
-
return "pcm";
|
|
142
|
-
}
|
|
143
|
-
return codecId.toLowerCase();
|
|
98
|
+
return audioCodecName(codecId);
|
|
144
99
|
}
|
|
145
100
|
return CODEC_BY_EXTENSION.get(extension) ?? "";
|
|
146
101
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The one table that names codecs.
|
|
3
|
+
*
|
|
4
|
+
* Containers write a codec as a Matroska id (`V_MPEG4/ISO/AVC`), an MP4 sample
|
|
5
|
+
* entry (`avc1`) or an AVI FOURCC (`XVID`); the rest of the proxy speaks the
|
|
6
|
+
* names ffmpeg prints (`h264`, `mpeg4`). Every translation lives here, so a
|
|
7
|
+
* FOURCC cannot be named one thing for the inventory and another for the
|
|
8
|
+
* container that reads it (torrent-tv/meta#284). Pure: imports nothing.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** FOURCCs that are MPEG-4 Part 2 (ASP/SP) whatever their case. */
|
|
12
|
+
export const MPEG4_FOURCCS = new Set(["FMP4", "XVID", "DIVX", "DX50", "MP4V", "M4S2", "MP4S"]);
|
|
13
|
+
|
|
14
|
+
const VIDEO_NAMES = new Map([
|
|
15
|
+
["V_MPEG4/ISO/AVC", "h264"], ["avc1", "h264"], ["avc3", "h264"],
|
|
16
|
+
["V_MPEGH/ISO/HEVC", "hevc"], ["hvc1", "hevc"], ["hev1", "hevc"],
|
|
17
|
+
["V_MPEG1", "mpeg1video"], ["V_MPEG2", "mpeg2video"], ["mpg2", "mpeg2video"],
|
|
18
|
+
["V_MPEG4/ISO/ASP", "mpeg4"], ["V_MPEG4/ISO/SP", "mpeg4"],
|
|
19
|
+
["V_VP8", "vp8"], ["V_VP9", "vp9"], ["vp09", "vp9"],
|
|
20
|
+
["V_AV1", "av1"], ["av01", "av1"], ["V_MJPEG", "mjpeg"], ["mjpg", "mjpeg"], ["jpeg", "mjpeg"],
|
|
21
|
+
["V_FFV1", "ffv1"], ["V_THEORA", "theora"]
|
|
22
|
+
].map(([id, name]) => [id.toLowerCase(), name]));
|
|
23
|
+
|
|
24
|
+
/** The ffmpeg name of a video track from its codec id or FOURCC, in any case; unknown ids come back lower-cased. */
|
|
25
|
+
export function videoCodecName(id) {
|
|
26
|
+
const key = String(id ?? "").toLowerCase();
|
|
27
|
+
if (MPEG4_FOURCCS.has(key.toUpperCase())) return "mpeg4";
|
|
28
|
+
return VIDEO_NAMES.get(key) ?? key;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const AUDIO_NAMES = new Map([
|
|
32
|
+
["A_AAC", "aac"], ["A_AC3", "ac3"], ["A_EAC3", "eac3"], ["A_DTS", "dts"], ["A_FLAC", "flac"],
|
|
33
|
+
["A_OPUS", "opus"], ["A_VORBIS", "vorbis"], ["A_TRUEHD", "truehd"], ["A_MPEG/L3", "mp3"],
|
|
34
|
+
["A_MPEG/L2", "mp2"], ["A_ALAC", "alac"],
|
|
35
|
+
["mp4a", "aac"], ["ac-3", "ac3"], ["ec-3", "eac3"], ["alac", "alac"],
|
|
36
|
+
["opus", "opus"], ["Opus", "opus"], ["fLaC", "flac"], ["flac", "flac"]
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The ffmpeg name of a soundtrack from its container id. Matroska allows a
|
|
41
|
+
* suffix (`A_AAC/MPEG4/LC`, `A_PCM/INT/LIT`), so the family is what the first
|
|
42
|
+
* two segments say. An id not listed is reported as written, lower-cased.
|
|
43
|
+
*/
|
|
44
|
+
export function audioCodecName(codecId) {
|
|
45
|
+
const id = String(codecId ?? "").trim();
|
|
46
|
+
const known = AUDIO_NAMES.get(id);
|
|
47
|
+
if (known) return known;
|
|
48
|
+
const family = id.split("/").slice(0, 2).join("/");
|
|
49
|
+
const byFamily = AUDIO_NAMES.get(family) ?? AUDIO_NAMES.get(id.split("/")[0]);
|
|
50
|
+
if (byFamily) return byFamily;
|
|
51
|
+
if (id.startsWith("A_PCM")) return "pcm";
|
|
52
|
+
return id.toLowerCase();
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const MATROSKA_IDS = new Map([
|
|
56
|
+
["h264", "V_MPEG4/ISO/AVC"], ["hevc", "V_MPEGH/ISO/HEVC"], ["av1", "V_AV1"], ["vp9", "V_VP9"],
|
|
57
|
+
["vp8", "V_VP8"], ["mjpeg", "V_MJPEG"], ["ffv1", "V_FFV1"], ["theora", "V_THEORA"],
|
|
58
|
+
["jpeg2000", "V_JPEG2000"], ["mpeg1video", "V_MPEG1"], ["mpeg2video", "V_MPEG2"], ["mpeg4", "V_MPEG4/ISO/ASP"],
|
|
59
|
+
["aac", "A_AAC"], ["mp3", "A_MPEG/L3"], ["mp2", "A_MPEG/L2"], ["mp1", "A_MPEG/L1"],
|
|
60
|
+
["ac3", "A_AC3"], ["eac3", "A_EAC3"], ["opus", "A_OPUS"], ["flac", "A_FLAC"], ["dts", "A_DTS"],
|
|
61
|
+
["vorbis", "A_VORBIS"], ["alac", "A_ALAC"], ["truehd", "A_TRUEHD"]
|
|
62
|
+
]);
|
|
63
|
+
// Spellings an input may carry besides the canonical name, taken exactly: a
|
|
64
|
+
// codec outside this list has no Matroska declaration, and `M4S2` or `xvid`
|
|
65
|
+
// stay refused because an input's key follows its declared track identity.
|
|
66
|
+
const MATROSKA_SPELLINGS = [
|
|
67
|
+
...["avc1", "avc3", "hvc1", "hev1", "av01"].map(spelling => [spelling, videoCodecName(spelling)]),
|
|
68
|
+
...["FMP4", "XVID", "DIVX"].map(spelling => [spelling, "mpeg4"]),
|
|
69
|
+
...["mp4a", "ac-3", "ec-3", "Opus", "fLaC"].map(spelling => [spelling, audioCodecName(spelling)])
|
|
70
|
+
];
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The exact Matroska CodecID a canonical codec name is declared as, or
|
|
74
|
+
* `undefined` when there is none. Reverse of the names above, never wider.
|
|
75
|
+
*/
|
|
76
|
+
export function matroskaIdOf(codec) {
|
|
77
|
+
if (/^[VA]_/.test(codec)) return codec;
|
|
78
|
+
if (/^pcm_[su](?:16|24|32|64)le$/.test(codec) || /^pcm_[su]8(?:le|be)?$/.test(codec)) return "A_PCM/INT/LIT";
|
|
79
|
+
if (/^pcm_[su](?:16|24|32|64)be$/.test(codec)) return "A_PCM/INT/BIG";
|
|
80
|
+
if (/^pcm_f(?:32|64)(?:le|be)$/.test(codec)) return "A_PCM/FLOAT/IEEE";
|
|
81
|
+
const spelled = MATROSKA_SPELLINGS.find(([spelling]) => spelling === codec);
|
|
82
|
+
return MATROSKA_IDS.get(spelled ? spelled[1] : codec);
|
|
83
|
+
}
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { EventEmitter } from "node:events";
|
|
12
|
+
import { writeJsonAtomic } from "../../utils/atomic-json.js";
|
|
12
13
|
import { logger } from "../../utils/logger.js";
|
|
13
14
|
import { KeyframeTables } from "../media/KeyframeTables.js";
|
|
15
|
+
import { matroskaIdOf } from "../media/tracks/codec-names.js";
|
|
14
16
|
import { bandOf, WaitLedger } from "../encode/OutputWaitLedger.js";
|
|
15
17
|
import { readEndedChildrenCpuSeconds, readMachineState, readProcessCpuSeconds, readProxyCpuSeconds, readSystemCpu, shareOfMachine } from "../encode/host-load.js";
|
|
16
18
|
import { minimumBufferFrom } from "../torrent/supply-margin.js";
|
|
@@ -100,7 +102,7 @@ import { freeBytesFor } from "../storage/free.js";
|
|
|
100
102
|
import { PlaybackStorageDemand } from "../storage/PlaybackStorageDemand.js";
|
|
101
103
|
import { inputCacheRoot, probeInputCache } from "../encode/input-cache.js";
|
|
102
104
|
import path from "node:path";
|
|
103
|
-
import { contentOf, LocalObservations
|
|
105
|
+
import { contentOf, LocalObservations } from "../encode/LocalObservations.js";
|
|
104
106
|
import { configurationKeyOf } from "../encode/fingerprint.js";
|
|
105
107
|
import { qualityStateOf } from "../encode/quality/OutputQualityState.js";
|
|
106
108
|
import { preparedAddresses, watchedAddresses } from "../viewer/watched-addresses.js";
|
|
@@ -875,7 +877,9 @@ export function wireOutputs({
|
|
|
875
877
|
if (calibration?.fingerprint) {
|
|
876
878
|
parts.localObservations = new LocalObservations({
|
|
877
879
|
fingerprint: calibration.fingerprint,
|
|
878
|
-
|
|
880
|
+
// The state directory arrives resolved; none keeps the observations in memory only.
|
|
881
|
+
filePath: typeof stateDir === "string" && stateDir.length > 0 ? path.join(stateDir, "local-observations.json") : "",
|
|
882
|
+
writeJson: writeJsonAtomic,
|
|
879
883
|
logger
|
|
880
884
|
});
|
|
881
885
|
parts.localObservations.load(calibration.kinds().map((kind) => configurationKeyOf(calibration.fingerprint, {
|
|
@@ -1194,6 +1198,7 @@ export function wireOutputs({
|
|
|
1194
1198
|
parts.encodeInputs = encodeInputs ?? (typeof encodeInputResolution?.resolve === "function" ? new EncodeInputs({
|
|
1195
1199
|
resolve: (...args) => encodeInputResolution.resolve(...args),
|
|
1196
1200
|
readRanges: readEncodeRanges,
|
|
1201
|
+
matroskaIdOf,
|
|
1197
1202
|
openWholeFile: openEncodeWholeFile,
|
|
1198
1203
|
heldRanges: readEncodeHeldRanges,
|
|
1199
1204
|
stretchCopied: (...args) => encodeInputResolution.stretchCopied(...args),
|
|
@@ -27,6 +27,7 @@ import fs from "node:fs/promises";
|
|
|
27
27
|
import { randomUUID } from "node:crypto";
|
|
28
28
|
import os from "node:os";
|
|
29
29
|
import path from "node:path";
|
|
30
|
+
import { writeJsonAtomic } from "../../../utils/atomic-json.js";
|
|
30
31
|
import { IDLE_KEEP_MS } from "../keep.js";
|
|
31
32
|
|
|
32
33
|
/**
|
|
@@ -541,7 +542,11 @@ export class CompletedFiles {
|
|
|
541
542
|
await fs.rm(directory, { ...REMOVAL, recursive: true }).catch(() => undefined);
|
|
542
543
|
return;
|
|
543
544
|
}
|
|
544
|
-
|
|
545
|
+
try {
|
|
546
|
+
writeJsonAtomic(path.join(directory, "manifest.json"), named);
|
|
547
|
+
} catch {
|
|
548
|
+
// The manifest is a convenience for the next start; a failed write is not a failed keep.
|
|
549
|
+
}
|
|
545
550
|
}
|
|
546
551
|
|
|
547
552
|
/** Remove an empty or partial-only directory discovered during startup. */
|
|
@@ -459,13 +459,13 @@ export function startMemoryReport({
|
|
|
459
459
|
label = "",
|
|
460
460
|
diskPath = "",
|
|
461
461
|
intervalMs = MEMORY_REPORT_INTERVAL_MS,
|
|
462
|
-
quietMs =
|
|
463
|
-
changeBytes =
|
|
462
|
+
quietMs = 60_000,
|
|
463
|
+
changeBytes = 25 * 1024 * 1024,
|
|
464
464
|
snapshotDir = "",
|
|
465
465
|
mayKeep = null,
|
|
466
|
-
snapshotFloorBytes =
|
|
467
|
-
snapshotGrowthBytes =
|
|
468
|
-
keepSnapshots =
|
|
466
|
+
snapshotFloorBytes = 400 * 1024 * 1024,
|
|
467
|
+
snapshotGrowthBytes = 400 * 1024 * 1024,
|
|
468
|
+
keepSnapshots = 3,
|
|
469
469
|
heapSnapshots = false,
|
|
470
470
|
bufferSnapshotFloorBytes = 0,
|
|
471
471
|
bufferSnapshotGrowthBytes = 500 * 1024 * 1024
|
|
@@ -44,6 +44,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync,
|
|
|
44
44
|
import { rmSync } from "node:fs";
|
|
45
45
|
import os from "node:os";
|
|
46
46
|
import path from "node:path";
|
|
47
|
+
import { writeFileAtomic } from "../../../utils/atomic-json.js";
|
|
47
48
|
|
|
48
49
|
/** Where every output's segments live. One root for the process. */
|
|
49
50
|
const DEFAULT_STORE_ROOT = path.join(os.tmpdir(), "torrent-tv-hls");
|
|
@@ -235,7 +236,7 @@ export class SegmentStore {
|
|
|
235
236
|
mkdirSync(dir, { recursive: true });
|
|
236
237
|
// What this directory is, for whoever finds it after this process has
|
|
237
238
|
// been killed. Without it the sweep can only throw everything away.
|
|
238
|
-
|
|
239
|
+
writeFileAtomic(path.join(dir, KEY_FILE), `${key}\n`);
|
|
239
240
|
}
|
|
240
241
|
this.#touched.set(key, this.#now());
|
|
241
242
|
// Production is about to begin here, and a piece closing in this directory
|
|
@@ -793,20 +793,15 @@ startMemoryReport({
|
|
|
793
793
|
scope: "thread",
|
|
794
794
|
label: "torrent worker",
|
|
795
795
|
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
796
|
-
quietMs: 60_000,
|
|
797
|
-
changeBytes: 25 * 1024 * 1024,
|
|
798
796
|
snapshotDir: workerData?.stateDir || undefined,
|
|
799
797
|
heapSnapshots: workerData?.heapSnapshots === true,
|
|
800
|
-
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
801
|
-
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
802
798
|
// CHOSEN, not derived: no healthy session has recorded this figure yet, so
|
|
803
799
|
// there is no measured normal to stand above. It is set well below the
|
|
804
800
|
// 2.1-4.3 GB the worker held in the minutes before the kills of 2026-10-01,
|
|
805
801
|
// while the machine still has room to write a snapshot. The line prints what the
|
|
806
802
|
// stores hold beside it, which is what a derived floor will come from.
|
|
807
803
|
bufferSnapshotFloorBytes: 500 * 1024 * 1024,
|
|
808
|
-
bufferSnapshotGrowthBytes: 500 * 1024 * 1024
|
|
809
|
-
keepSnapshots: 3
|
|
804
|
+
bufferSnapshotGrowthBytes: 500 * 1024 * 1024
|
|
810
805
|
});
|
|
811
806
|
|
|
812
807
|
const STORE_REPORT_INTERVAL_MS = 60_000;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import nodeFs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
let writes = 0;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Write a file so that a reader (or a restart after a crash) sees either the
|
|
8
|
+
* previous content or the new content, never a half-written file: the text goes
|
|
9
|
+
* to a temporary file beside the target, then is renamed over it. Synchronous;
|
|
10
|
+
* throws on failure and removes its temporary file when it can. It never writes
|
|
11
|
+
* `filePath` itself.
|
|
12
|
+
*
|
|
13
|
+
* @param {string} filePath
|
|
14
|
+
* @param {string | Uint8Array} data
|
|
15
|
+
* @param {{ fs?: Partial<typeof nodeFs>, mode?: number }} [options] - `fs`
|
|
16
|
+
* overrides individual `node:fs` functions (a check stops the write between
|
|
17
|
+
* the two steps with it).
|
|
18
|
+
* @returns {void}
|
|
19
|
+
*/
|
|
20
|
+
export function writeFileAtomic(filePath, data, { fs = {}, mode } = {}) {
|
|
21
|
+
const io = { ...nodeFs, ...fs };
|
|
22
|
+
const temporary = path.join(path.dirname(filePath), `${path.basename(filePath)}.${process.pid}.${++writes}.tmp`);
|
|
23
|
+
try {
|
|
24
|
+
io.writeFileSync(temporary, data, mode === undefined ? undefined : { mode });
|
|
25
|
+
io.renameSync(temporary, filePath);
|
|
26
|
+
} catch (error) {
|
|
27
|
+
try {
|
|
28
|
+
nodeFs.rmSync(temporary, { force: true });
|
|
29
|
+
} catch {
|
|
30
|
+
// The write already failed; a temporary file that cannot be removed is the lesser fault.
|
|
31
|
+
}
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Serialise `value` as JSON and write it with {@link writeFileAtomic}.
|
|
38
|
+
*
|
|
39
|
+
* @param {string} filePath
|
|
40
|
+
* @param {unknown} value
|
|
41
|
+
* @param {{ fs?: Partial<typeof nodeFs>, mode?: number }} [options]
|
|
42
|
+
* @returns {void}
|
|
43
|
+
*/
|
|
44
|
+
export function writeJsonAtomic(filePath, value, options = {}) {
|
|
45
|
+
writeFileAtomic(filePath, `${JSON.stringify(value)}\n`, options);
|
|
46
|
+
}
|