@torrent-tv/proxy 2.96.14 → 2.97.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +2 -0
- package/package.json +1 -1
- package/routes/encode-input/get.js +13 -0
- package/server.js +25 -3
- package/services/encode/EncodeInputs.js +60 -47
- package/services/encode/EncodeOrchestrator.js +29 -38
- package/services/encode/EncodeRun.js +26 -5
- package/services/encode/EncodeRuns.js +88 -48
- package/services/encode/OriginalInput.js +45 -1
- package/services/encode/RunClock.js +27 -0
- package/services/encode/run-costs.js +47 -2
- package/services/server/wire-outputs.js +3 -0
- package/services/storage/MachineBudget.js +19 -1
- package/services/storage/cpu-profile.js +68 -0
- package/services/storage/processor-report.js +186 -0
- package/services/torrent/worker/client.js +3 -2
- package/services/torrent/worker/pool-adapter.js +2 -2
- package/services/torrent/worker/worker.js +6 -2
- package/services/viewer/DownloadMaps.js +53 -6
|
@@ -20,6 +20,7 @@ import { writeAdmittedInput } from "./AdmittedInput.js";
|
|
|
20
20
|
import { cutOf, cutShiftOf, judgeNeighbors, judgePiece } from "./piece-completeness.js";
|
|
21
21
|
import { presentationSegment } from "./segment-formats/presentation-segment.js";
|
|
22
22
|
import { InputFailures, failedAdmittedInput } from "./InputFailures.js";
|
|
23
|
+
import { readProcessCpuSeconds } from "./host-load.js";
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -161,6 +162,15 @@ export class EncodeRuns {
|
|
|
161
162
|
* @type {Map<number, import("./EncodeRun.js").EncodeRun>}
|
|
162
163
|
*/
|
|
163
164
|
#runsByInputToken = new Map();
|
|
165
|
+
/**
|
|
166
|
+
* Each live run's ffmpeg CPU seconds against the film it had made, as of its
|
|
167
|
+
* last progress report: what one second of film costs this machine in
|
|
168
|
+
* processor time, which the speed in x does not say once other work shares
|
|
169
|
+
* the processor (torrent-tv/meta#177). Said when the run ends.
|
|
170
|
+
*
|
|
171
|
+
* @type {WeakMap<object, { cpuSeconds: number, filmSeconds: number, reading: boolean, beside: number }>}
|
|
172
|
+
*/
|
|
173
|
+
#cpuOfRun = new WeakMap();
|
|
164
174
|
|
|
165
175
|
#lastInputToken = 0;
|
|
166
176
|
#inputFailures = new InputFailures();
|
|
@@ -572,6 +582,7 @@ export class EncodeRuns {
|
|
|
572
582
|
this.#host.encodeInputs?.forget(output);
|
|
573
583
|
if (this.#host.outputs.outputsOn(output.outputKey).length === 0) {
|
|
574
584
|
this.#inputFailures.forget(output.outputKey);
|
|
585
|
+
this.#host.encodeOrchestrator.forgetOpens?.(output.outputKey);
|
|
575
586
|
this.#host.encodeOrchestrator.forgetOutput(output.outputKey);
|
|
576
587
|
}
|
|
577
588
|
}
|
|
@@ -783,40 +794,6 @@ export class EncodeRuns {
|
|
|
783
794
|
);
|
|
784
795
|
}
|
|
785
796
|
|
|
786
|
-
/**
|
|
787
|
-
* How far a run starting here may work before it meets somebody else's
|
|
788
|
-
* material — read off the ONE coverage map, never worked out again here.
|
|
789
|
-
*
|
|
790
|
-
* This replaced `planRunInterval`, which was a second authority over the same
|
|
791
|
-
* question and answered it by different rules: it walked the whole track for
|
|
792
|
-
* the first free number, MOVED the start there itself, and counted every live
|
|
793
|
-
* run as claiming up to `head + look-ahead`. Measured in the field
|
|
794
|
-
* 2026-09-05: the plan commanded a start at #46, this moved it to #78 — the
|
|
795
|
-
* "run moved forward from #46 to #78" line — and the plan, seeing a run
|
|
796
|
-
* outside the window it had asked for, killed it as unwanted and commanded
|
|
797
|
-
* the same start again, 350-700ms per cycle, dozens of times, the viewer's
|
|
798
|
-
* picture stopped for 125 seconds.
|
|
799
|
-
*
|
|
800
|
-
* Where a run STARTS is the plan's decision and arrives here as an argument.
|
|
801
|
-
* All this answers is where it must stop, and the answer is a fact of the
|
|
802
|
-
* map: the free stretch from that number on. `-1` means the end of the track,
|
|
803
|
-
* which is what "no end" is written as everywhere here.
|
|
804
|
-
*
|
|
805
|
-
* @param {object} session
|
|
806
|
-
* @param {number} startIndex
|
|
807
|
-
* @param {object | null} exceptRun - The run being replaced, whose own claim
|
|
808
|
-
* is not somebody else's material.
|
|
809
|
-
* @returns {number} The last number to work through, or `-1` for the end.
|
|
810
|
-
*/
|
|
811
|
-
#runEndFrom(session, startIndex, exceptRun = null) {
|
|
812
|
-
return this.#host.encodeOrchestrator.freeStretchEnd({
|
|
813
|
-
address: session.outputKey ?? "",
|
|
814
|
-
from: startIndex,
|
|
815
|
-
exceptRun,
|
|
816
|
-
segmentCount: Number(session.timeline?.segmentCount) || 0
|
|
817
|
-
});
|
|
818
|
-
}
|
|
819
|
-
|
|
820
797
|
// Returns the encoder it built, or nothing when there was nothing to build.
|
|
821
798
|
// It waits for nothing: the one thing it used to wait for was the death of
|
|
822
799
|
// the run it was replacing, and that killing is gone.
|
|
@@ -824,7 +801,10 @@ export class EncodeRuns {
|
|
|
824
801
|
if (!this.isLive(session)) {
|
|
825
802
|
return null;
|
|
826
803
|
}
|
|
827
|
-
|
|
804
|
+
// A copy held for a start that no live run ends before and that is not
|
|
805
|
+
// being started now goes back (torrent-tv/meta#184).
|
|
806
|
+
this.#host.encodeInputs?.retain(session, this.#host.encodeOrchestrator.wantedSegmentsOn(session.outputKey),
|
|
807
|
+
new Set([startIndex, ...this.liveRunsOf(session).map((run) => run.to + 1)]));
|
|
828
808
|
// THE LAST NUMBER THIS RUN MAY REACH is the plan's; how far its input
|
|
829
809
|
// really reaches is the input's answer. An original input is one copied
|
|
830
810
|
// stretch, and the run is given exactly that stretch (torrent-tv/meta#158).
|
|
@@ -837,13 +817,13 @@ export class EncodeRuns {
|
|
|
837
817
|
// The stretch is part of an original input's identity: two stretches from
|
|
838
818
|
// one start can hold the same padded bytes, and a stretch that failed is
|
|
839
819
|
// refused only while it is the same stretch.
|
|
840
|
-
const inputParameters =
|
|
820
|
+
const inputParameters = { outputKey: session.outputKey, encoder: this.#host.videoEncoder.name,
|
|
841
821
|
width: session.output.encodeWidth, height: session.output.encodeHeight, fps: session.output.outputFps,
|
|
842
822
|
preset: session.output.softwarePreset, tonemap: session.output.applyTonemap,
|
|
843
823
|
rateControl: session.spec.video?.encode?.rateControl ?? null,
|
|
844
|
-
...(admittedInput.original ? { interval: [startIndex, inputEnd] } : {}) }
|
|
845
|
-
const inputKey =
|
|
846
|
-
if (
|
|
824
|
+
...(admittedInput.original ? { interval: [startIndex, inputEnd] } : {}) };
|
|
825
|
+
const inputKey = this.#inputFailures.key(admittedInput, inputParameters);
|
|
826
|
+
if (this.#inputFailures.failure(session.outputKey, startIndex, inputKey)) {
|
|
847
827
|
admittedInput.release();
|
|
848
828
|
return null;
|
|
849
829
|
}
|
|
@@ -869,9 +849,10 @@ export class EncodeRuns {
|
|
|
869
849
|
// an argument. Reading it off the coverage map a second time was the last
|
|
870
850
|
// remaining second answer to that question: the plan computed `to`, passed
|
|
871
851
|
// it, and the parameter list did not name it.
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
852
|
+
// A run ends where its input does: an original input is one copied
|
|
853
|
+
// stretch, and a packet input grows one segment at a time from its first
|
|
854
|
+
// (`run.to` follows it below).
|
|
855
|
+
const runEnd = inputEnd;
|
|
875
856
|
// The restart backs off a segment or two from what was asked for, so the
|
|
876
857
|
// request that prompted it is recorded under a HIGHER index than the run
|
|
877
858
|
// starts at. Looking it up by the start index alone found nothing and the
|
|
@@ -1046,11 +1027,12 @@ export class EncodeRuns {
|
|
|
1046
1027
|
const since = (at) => at ? `${at - run.startedAt}ms` : "never";
|
|
1047
1028
|
this.#host.logger.info(`encode input of run #${run.from}..#${run.to} on ${session.outputKey}: ` +
|
|
1048
1029
|
`${reads.count} read(s), ${reads.bytes} bytes, first at ${since(reads.firstAt)}, last at ${since(reads.lastAt)}, ` +
|
|
1049
|
-
`ended at ${Date.now() - run.startedAt}ms`);
|
|
1030
|
+
`ended at ${Date.now() - run.startedAt}ms` + describeTaken(reads.answers, run.startedAt, ended.firstNamedWallMs));
|
|
1050
1031
|
}
|
|
1051
1032
|
this.#noteOpen(session, run, ended);
|
|
1052
1033
|
admittedInput.release();
|
|
1053
1034
|
}
|
|
1035
|
+
this.#sayRunCpu(session, run);
|
|
1054
1036
|
this.noteRunEnded(session, run, ended);
|
|
1055
1037
|
this.#host.encodersChanged?.();
|
|
1056
1038
|
}
|
|
@@ -1061,10 +1043,6 @@ export class EncodeRuns {
|
|
|
1061
1043
|
// Every number of the stretch answers to its one key, so a failure
|
|
1062
1044
|
// anywhere inside it is recorded against the stretch that was commanded.
|
|
1063
1045
|
run.admittedInputKeys = new Map(Array.from({ length: inputEnd - safeIndex + 1 }, (_, offset) => [safeIndex + offset, inputKey]));
|
|
1064
|
-
// What this run can make is what its input holds, and that is what it
|
|
1065
|
-
// claims; the plan's longer bound would read as made by a run that
|
|
1066
|
-
// cannot reach it.
|
|
1067
|
-
run.inputThrough = inputEnd;
|
|
1068
1046
|
run.process.stdin.end();
|
|
1069
1047
|
// THE NEXT STRETCH IS PREPARED WHILE THIS ONE IS BEING MADE. Asked for
|
|
1070
1048
|
// only when this run had ended, its copy fell between the two runs and
|
|
@@ -1148,7 +1126,7 @@ export class EncodeRuns {
|
|
|
1148
1126
|
const speed = run.speedX > 0 ? run.speedX : this.#host.encodeOrchestrator.speedOn?.(session.outputKey) ?? 0;
|
|
1149
1127
|
if (!Number.isFinite(ended.firstNamedMs) || !(pieceSeconds > 0) || !(speed > 0)) return;
|
|
1150
1128
|
const openSeconds = Math.max(0, ended.firstNamedMs / 1000 - pieceSeconds / speed);
|
|
1151
|
-
this.#host.
|
|
1129
|
+
this.#host.encodeOrchestrator.noteOpen?.(session.outputKey ?? "", openSeconds);
|
|
1152
1130
|
this.#host.logger.info(`encode input of run #${run.from}..#${run.to} on ${session.outputKey}: opened in ` +
|
|
1153
1131
|
`${openSeconds.toFixed(2)}s (first piece named at ${ended.firstNamedMs}ms, ${pieceSeconds.toFixed(3)}s of film at ${speed.toFixed(2)}x)`);
|
|
1154
1132
|
}
|
|
@@ -1167,6 +1145,7 @@ export class EncodeRuns {
|
|
|
1167
1145
|
if (!this.isLive(session) || !this.runsOf(session).includes(run)) {
|
|
1168
1146
|
return;
|
|
1169
1147
|
}
|
|
1148
|
+
this.#readRunCpu(run, progress);
|
|
1170
1149
|
if (run.progress.shouldLog(PROGRESS_LOG_INTERVAL_MS)) {
|
|
1171
1150
|
this.#host.logger.info(
|
|
1172
1151
|
`transcode ${session.id} "${session.file.name}" ${progress.percent.toFixed(1)}% ` +
|
|
@@ -1176,6 +1155,44 @@ export class EncodeRuns {
|
|
|
1176
1155
|
}
|
|
1177
1156
|
}
|
|
1178
1157
|
|
|
1158
|
+
/**
|
|
1159
|
+
* Read the run's ffmpeg CPU beside the film it has made, one reading at a
|
|
1160
|
+
* time; a reading that cannot be taken (no `/proc`, the process gone)
|
|
1161
|
+
* leaves the last one.
|
|
1162
|
+
*
|
|
1163
|
+
* @param {import("./EncodeRun.js").EncodeRun} run
|
|
1164
|
+
* @param {{ processedSeconds?: number, startPositionSeconds?: number }} progress
|
|
1165
|
+
*/
|
|
1166
|
+
#readRunCpu(run, progress) {
|
|
1167
|
+
const pid = run.process?.pid;
|
|
1168
|
+
const filmSeconds = Number(progress?.processedSeconds) - Number(progress?.startPositionSeconds ?? 0);
|
|
1169
|
+
const known = this.#cpuOfRun.get(run) ?? { cpuSeconds: 0, filmSeconds: 0, reading: false, beside: 0 };
|
|
1170
|
+
this.#cpuOfRun.set(run, known);
|
|
1171
|
+
if (known.reading || !Number.isInteger(pid) || !(filmSeconds > 0)) return;
|
|
1172
|
+
known.reading = true;
|
|
1173
|
+
void readProcessCpuSeconds(pid).then((cpuSeconds) => {
|
|
1174
|
+
if (Number.isFinite(cpuSeconds)) {
|
|
1175
|
+
known.cpuSeconds = cpuSeconds;
|
|
1176
|
+
known.filmSeconds = filmSeconds;
|
|
1177
|
+
known.beside = Math.max(known.beside, this.#runsByInputToken.size - 1);
|
|
1178
|
+
}
|
|
1179
|
+
}).finally(() => { known.reading = false; });
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* What the run's ffmpeg spent per second of film, said once when it ends.
|
|
1184
|
+
*
|
|
1185
|
+
* @param {object} session
|
|
1186
|
+
* @param {import("./EncodeRun.js").EncodeRun} run
|
|
1187
|
+
*/
|
|
1188
|
+
#sayRunCpu(session, run) {
|
|
1189
|
+
const known = this.#cpuOfRun.get(run);
|
|
1190
|
+
if (!known || !(known.filmSeconds > 0) || !(known.cpuSeconds > 0)) return;
|
|
1191
|
+
this.#host.logger.info(`encode-run #${run.from}..#${run.to} on ${session.outputKey}: ffmpeg used ` +
|
|
1192
|
+
`${known.cpuSeconds.toFixed(2)}s of CPU for ${known.filmSeconds.toFixed(2)}s of film, ` +
|
|
1193
|
+
`${(known.cpuSeconds / known.filmSeconds).toFixed(3)}s per second of film; at most ${known.beside} other encoder(s) alive beside it`);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1179
1196
|
/**
|
|
1180
1197
|
* What to do about a run that has ended.
|
|
1181
1198
|
*
|
|
@@ -1531,3 +1548,26 @@ export class EncodeRuns {
|
|
|
1531
1548
|
});
|
|
1532
1549
|
}
|
|
1533
1550
|
}
|
|
1551
|
+
|
|
1552
|
+
/**
|
|
1553
|
+
* What FFmpeg was handed of an original input, as a clause of the run's input
|
|
1554
|
+
* line (torrent-tv/meta#177): bytes handed to the system (at most what it
|
|
1555
|
+
* read), connections closed before an answer was all handed over,
|
|
1556
|
+
* the share of reads that went forward, the largest jump back, bytes taken
|
|
1557
|
+
* twice, and what it read before it named its first piece — its open.
|
|
1558
|
+
*
|
|
1559
|
+
* @param {{ count: number, written: number, unfinished: number, forwardShare: number | null, largestBackBytes: number, rewritten: number, list: { at: number, written: number }[] } | undefined} answers
|
|
1560
|
+
* @param {number} startedAt
|
|
1561
|
+
* @param {number | null | undefined} firstNamedMs
|
|
1562
|
+
* @returns {string}
|
|
1563
|
+
*/
|
|
1564
|
+
function describeTaken(answers, startedAt, firstNamedMs) {
|
|
1565
|
+
if (!answers || answers.count === 0) return "";
|
|
1566
|
+
const opened = Number.isFinite(firstNamedMs)
|
|
1567
|
+
? answers.list.filter((answer) => answer.at - startedAt <= firstNamedMs) : null;
|
|
1568
|
+
return `; handed over: ${answers.written} bytes in ${answers.count} answer(s), ${answers.unfinished} closed before all was handed over, ` +
|
|
1569
|
+
`${answers.forwardShare === null ? "n/a" : `${Math.round(answers.forwardShare * 100)}%`} forward, ` +
|
|
1570
|
+
`largest jump back ${answers.largestBackBytes} bytes, ${answers.rewritten} bytes taken twice` +
|
|
1571
|
+
(opened ? `; before the first piece was named ${opened.length} answer(s), ` +
|
|
1572
|
+
`${opened.reduce((sum, answer) => sum + answer.written, 0)} bytes` : "");
|
|
1573
|
+
}
|
|
@@ -35,6 +35,11 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
35
35
|
// time went: field 2026-10-08, a run took 6-9 s for a piece FFmpeg makes in
|
|
36
36
|
// 1.6 s from a file, and nothing said whether it was waiting on its input.
|
|
37
37
|
const reads = { count: 0, bytes: 0, firstAt: 0, lastAt: 0 };
|
|
38
|
+
// Each answer FFmpeg was sent, in order: where it began, where its range
|
|
39
|
+
// ended, what was handed to the system before the connection closed, and
|
|
40
|
+
// whether all of it was (torrent-tv/meta#177).
|
|
41
|
+
/** @type {{ fileIndex: number, start: number, end: number, written: number, handedOver: boolean, at: number }[]} */
|
|
42
|
+
const answers = [];
|
|
38
43
|
return {
|
|
39
44
|
kind: "result", original: true, sources, bytes, copyMs, fingerprint: hash.digest("hex"),
|
|
40
45
|
// No track's own end: the container names the bytes an interval needs,
|
|
@@ -59,7 +64,10 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
|
|
|
59
64
|
return { length: fileLength, end: stop, bytes: entry.buffers[position].subarray(start - ranges[position][0], stop - ranges[position][0] + 1) };
|
|
60
65
|
},
|
|
61
66
|
lengthOf: fileIndex => held.get(fileIndex)?.source.input.fileLength ?? null,
|
|
62
|
-
reads: () => ({ ...reads }),
|
|
67
|
+
reads: () => ({ ...reads, answers: describeAnswers(answers) }),
|
|
68
|
+
noteAnswer(answer) {
|
|
69
|
+
if (!released) answers.push({ ...answer, at: Date.now() });
|
|
70
|
+
},
|
|
63
71
|
release() {
|
|
64
72
|
if (released) return;
|
|
65
73
|
released = true;
|
|
@@ -77,3 +85,39 @@ export function originalInputBytes(sources) {
|
|
|
77
85
|
return sources.reduce((total, source) => total + source.input.ranges
|
|
78
86
|
.reduce((sum, [start, end]) => sum + end - start + 1, 0), 0);
|
|
79
87
|
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The shape of FFmpeg's reads of one input, from the answers it was sent:
|
|
91
|
+
* how much was handed to the system (at most what FFmpeg read), how many
|
|
92
|
+
* connections closed before all of an answer was handed over, what share began
|
|
93
|
+
* at or after the end of the previous one (a forward read), the largest jump
|
|
94
|
+
* back, and the bytes written again that an earlier answer had already
|
|
95
|
+
* written. Per file, since two files are read independently.
|
|
96
|
+
*
|
|
97
|
+
* @param {{ fileIndex: number, start: number, end: number, written: number, handedOver: boolean, at: number }[]} answers
|
|
98
|
+
* @returns {{ count: number, written: number, unfinished: number, forwardShare: number | null, largestBackBytes: number, rewritten: number, list: object[] }}
|
|
99
|
+
*/
|
|
100
|
+
export function describeAnswers(answers) {
|
|
101
|
+
let written = 0, unfinished = 0, forward = 0, largestBackBytes = 0, rewritten = 0;
|
|
102
|
+
const reachedOf = new Map();
|
|
103
|
+
const coveredOf = new Map();
|
|
104
|
+
for (const answer of answers) {
|
|
105
|
+
written += answer.written;
|
|
106
|
+
if (!answer.handedOver) unfinished += 1;
|
|
107
|
+
const reached = reachedOf.get(answer.fileIndex);
|
|
108
|
+
if (reached === undefined || answer.start >= reached) forward += 1;
|
|
109
|
+
else largestBackBytes = Math.max(largestBackBytes, reached - answer.start);
|
|
110
|
+
// Bytes actually taken by this answer, as a range from its start.
|
|
111
|
+
const takenEnd = answer.start + Math.max(0, answer.written) - 1;
|
|
112
|
+
const covered = coveredOf.get(answer.fileIndex) ?? [];
|
|
113
|
+
for (const [from, to] of covered) {
|
|
114
|
+
const overlap = Math.min(to, takenEnd) - Math.max(from, answer.start) + 1;
|
|
115
|
+
if (overlap > 0) rewritten += overlap;
|
|
116
|
+
}
|
|
117
|
+
if (takenEnd >= answer.start) covered.push([answer.start, takenEnd]);
|
|
118
|
+
coveredOf.set(answer.fileIndex, covered);
|
|
119
|
+
reachedOf.set(answer.fileIndex, Math.max(reached ?? 0, takenEnd + 1));
|
|
120
|
+
}
|
|
121
|
+
return { count: answers.length, written, unfinished,
|
|
122
|
+
forwardShare: answers.length ? forward / answers.length : null, largestBackBytes, rewritten, list: answers };
|
|
123
|
+
}
|
|
@@ -28,6 +28,15 @@ export class RunClock {
|
|
|
28
28
|
#idleCauses = 0;
|
|
29
29
|
#idleSince = 0;
|
|
30
30
|
#idleMs = 0;
|
|
31
|
+
/**
|
|
32
|
+
* Stopped time alone, apart from input waits: a pause is a decision of
|
|
33
|
+
* ours, while a wait for input is part of what starting a run costs, so the
|
|
34
|
+
* readings of a start leave out the first and keep the second
|
|
35
|
+
* (torrent-tv/meta#184).
|
|
36
|
+
*/
|
|
37
|
+
#stoppedSince = 0;
|
|
38
|
+
#stoppedMs = 0;
|
|
39
|
+
#isStopped = false;
|
|
31
40
|
|
|
32
41
|
/**
|
|
33
42
|
* @param {{ now?: () => number }} [options]
|
|
@@ -67,11 +76,29 @@ export class RunClock {
|
|
|
67
76
|
/** The process is stopped. */
|
|
68
77
|
stopped() {
|
|
69
78
|
this.#idleBegins();
|
|
79
|
+
if (!this.#isStopped) {
|
|
80
|
+
this.#isStopped = true;
|
|
81
|
+
this.#stoppedSince = this.#now();
|
|
82
|
+
}
|
|
70
83
|
}
|
|
71
84
|
|
|
72
85
|
/** The process runs again. */
|
|
73
86
|
continued() {
|
|
74
87
|
this.#idleEnds();
|
|
88
|
+
if (this.#isStopped) {
|
|
89
|
+
this.#isStopped = false;
|
|
90
|
+
this.#stoppedMs += this.#now() - this.#stoppedSince;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Milliseconds the process has been stopped so far.
|
|
96
|
+
*
|
|
97
|
+
* @param {number} [at]
|
|
98
|
+
* @returns {number}
|
|
99
|
+
*/
|
|
100
|
+
stoppedMs(at = this.#now()) {
|
|
101
|
+
return this.#stoppedMs + (this.#isStopped ? Math.max(0, at - this.#stoppedSince) : 0);
|
|
75
102
|
}
|
|
76
103
|
|
|
77
104
|
/**
|
|
@@ -100,8 +100,10 @@ export class RunCosts {
|
|
|
100
100
|
// Without it the figure that prices a move could only ever be learned from
|
|
101
101
|
// runs that survived, so the state in which moves are ruinous was exactly
|
|
102
102
|
// the state in which their cost stayed unknown.
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
// Its pauses are left out: a pause is our decision, not what a start costs.
|
|
104
|
+
const livedWorking = Number(ended?.livedMs) - Math.max(0, Number(ended?.stoppedMs) || 0);
|
|
105
|
+
if (!Number.isFinite(ended?.firstOutputMs) && Number.isFinite(livedWorking) && livedWorking > 0) {
|
|
106
|
+
RunCosts.#keep(this.#firstOutput, livedWorking);
|
|
105
107
|
}
|
|
106
108
|
if (Number.isFinite(ended?.dyingMs)) {
|
|
107
109
|
RunCosts.#keep(this.#dying, /** @type {number} */ (ended.dyingMs));
|
|
@@ -111,6 +113,49 @@ export class RunCosts {
|
|
|
111
113
|
}
|
|
112
114
|
}
|
|
113
115
|
|
|
116
|
+
/**
|
|
117
|
+
* How long an encoder took to open one output's original input, in seconds:
|
|
118
|
+
* from its spawn to the name of its first closed piece, less that piece's
|
|
119
|
+
* encoding and any pause (torrent-tv/meta#184). Kept per output, beside the
|
|
120
|
+
* host's time to first output above and apart from it: the two are related
|
|
121
|
+
* and not the same quantity, and the stretch of an original input is sized
|
|
122
|
+
* from this one.
|
|
123
|
+
*
|
|
124
|
+
* @type {Map<string, number[]>}
|
|
125
|
+
*/
|
|
126
|
+
#opens = new Map();
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* @param {string} address
|
|
130
|
+
* @param {number} seconds
|
|
131
|
+
*/
|
|
132
|
+
noteOpen(address, seconds) {
|
|
133
|
+
if (!address || !Number.isFinite(seconds) || seconds < 0) return;
|
|
134
|
+
const readings = this.#opens.get(address) ?? [];
|
|
135
|
+
RunCosts.#keep(readings, seconds);
|
|
136
|
+
this.#opens.set(address, readings);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The open of this output: its own readings' median where it has any; else
|
|
141
|
+
* the host's startup reading of spawn to first piece, said to be an
|
|
142
|
+
* approximation — it opened a generated picture, not this file; else null.
|
|
143
|
+
*
|
|
144
|
+
* @param {string} address
|
|
145
|
+
* @returns {{ seconds: number, measured: boolean } | null}
|
|
146
|
+
*/
|
|
147
|
+
openOf(address) {
|
|
148
|
+
const own = middleOf(this.#opens.get(address) ?? []);
|
|
149
|
+
if (own !== null) return { seconds: own, measured: true };
|
|
150
|
+
const startup = this.#atStartup?.firstByteWaitSec;
|
|
151
|
+
return Number.isFinite(startup) && startup > 0 ? { seconds: startup, measured: false } : null;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** @param {string} address */
|
|
155
|
+
forgetOpens(address) {
|
|
156
|
+
this.#opens.delete(address);
|
|
157
|
+
}
|
|
158
|
+
|
|
114
159
|
/**
|
|
115
160
|
* @param {number[]} readings
|
|
116
161
|
* @param {number} value
|
|
@@ -975,6 +975,9 @@ export function wireOutputs({
|
|
|
975
975
|
heldRanges: readEncodeHeldRanges,
|
|
976
976
|
reviseBudget: () => parts.machineBudget.revise(),
|
|
977
977
|
capacity: () => parts.machineBudget.capacityOf("memory"),
|
|
978
|
+
// How long this output's input takes to open, from the one owner of start
|
|
979
|
+
// readings (torrent-tv/meta#184).
|
|
980
|
+
openOf: (output) => parts.encodeOrchestrator?.openOf?.(output.outputKey ?? "") ?? null,
|
|
978
981
|
urgent: (output, index) => parts.encodeOrchestrator.wantedSegmentsOn(output.outputKey).some(window =>
|
|
979
982
|
index >= window.from && index <= window.to && window.urgent === true),
|
|
980
983
|
log: line => logger.info(line),
|
|
@@ -77,6 +77,13 @@ import { OtherDemand, divideAllowance } from "./allowance.js";
|
|
|
77
77
|
export class MachineBudget {
|
|
78
78
|
#revisionPromise = null;
|
|
79
79
|
#reviseAgain = false;
|
|
80
|
+
/**
|
|
81
|
+
* The revision in hand, for its own line: when it was first asked for, how
|
|
82
|
+
* many asks joined it, and which pass this is. At a cold start on the addon
|
|
83
|
+
* host a caller waited 20.2 s inside `revise()` and nothing said whether it
|
|
84
|
+
* waited for passes, for a reading, or for the thread (torrent-tv/meta#177).
|
|
85
|
+
*/
|
|
86
|
+
#revision = null;
|
|
80
87
|
/** Resource name → how to read what is free, and what others have needed. */
|
|
81
88
|
#resources = new Map();
|
|
82
89
|
|
|
@@ -151,11 +158,14 @@ export class MachineBudget {
|
|
|
151
158
|
revise() {
|
|
152
159
|
if (this.#revisionPromise) {
|
|
153
160
|
this.#reviseAgain = true;
|
|
161
|
+
this.#revision.asks += 1;
|
|
154
162
|
return this.#revisionPromise;
|
|
155
163
|
}
|
|
164
|
+
this.#revision = { askedAt: Date.now(), asks: 1, pass: 0 };
|
|
156
165
|
this.#revisionPromise = (async () => {
|
|
157
166
|
do {
|
|
158
167
|
this.#reviseAgain = false;
|
|
168
|
+
this.#revision.pass += 1;
|
|
159
169
|
await this.#readRevision();
|
|
160
170
|
} while (this.#reviseAgain);
|
|
161
171
|
return this.#last;
|
|
@@ -164,13 +174,17 @@ export class MachineBudget {
|
|
|
164
174
|
}
|
|
165
175
|
|
|
166
176
|
async #readRevision() {
|
|
177
|
+
const passFrom = Date.now();
|
|
178
|
+
let readingMs = 0;
|
|
167
179
|
for (const [name, resource] of this.#resources) {
|
|
168
180
|
const claimants = this.#claimants.filter((one) => one.resource === name);
|
|
169
181
|
if (claimants.length === 0) {
|
|
170
182
|
continue;
|
|
171
183
|
}
|
|
172
184
|
const held = claimants.reduce((sum, one) => sum + Math.max(0, one.held()), 0);
|
|
185
|
+
const readFrom = Date.now();
|
|
173
186
|
const free = await resource.readFree();
|
|
187
|
+
readingMs += Date.now() - readFrom;
|
|
174
188
|
// WITHOUT A READING, NOTHING IS ALLOWED TO GROW. A resource whose free
|
|
175
189
|
// space cannot be read is not a resource with room; answering "unbounded"
|
|
176
190
|
// there is how the spill file came to have no limit in the first place.
|
|
@@ -197,7 +211,11 @@ export class MachineBudget {
|
|
|
197
211
|
);
|
|
198
212
|
}
|
|
199
213
|
}
|
|
200
|
-
this.#
|
|
214
|
+
const revision = this.#revision;
|
|
215
|
+
this.#logger?.info?.(`${this.describe()}` + (revision
|
|
216
|
+
? ` (pass ${revision.pass} of a revision asked ${revision.asks} time(s), first ${Date.now() - revision.askedAt}ms ago; ` +
|
|
217
|
+
`this pass ${Date.now() - passFrom}ms, ${readingMs}ms of it reading what is free)`
|
|
218
|
+
: ""));
|
|
201
219
|
return this.#last;
|
|
202
220
|
}
|
|
203
221
|
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file A CPU profile of the main thread at a cold start, asked for by the
|
|
3
|
+
* operator and never taken otherwise (torrent-tv/meta#177).
|
|
4
|
+
*
|
|
5
|
+
* At a cold start on the addon host (#166 run 12) the picture's first input
|
|
6
|
+
* waited 29.5 s, 20.2 s of it inside a budget revision, while the main thread
|
|
7
|
+
* ran at 83-108 % of a core; what it was doing is not in any log. A profile of
|
|
8
|
+
* that thread names it.
|
|
9
|
+
*
|
|
10
|
+
* In-process, through `node:inspector`: the sampling profiler runs beside the
|
|
11
|
+
* thread and does not stop it — unlike the debugger that stopped every thread
|
|
12
|
+
* of this process on 2026-09-05 (torrent-tv/meta#77). It begins at the first
|
|
13
|
+
* source opened, which is where a cold start begins, runs for the seconds the
|
|
14
|
+
* operator named, and writes `cpu-start-<time>.cpuprofile` beside the log,
|
|
15
|
+
* readable in Chrome's DevTools (Performance, "Load profile").
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { Session } from "node:inspector/promises";
|
|
19
|
+
import { writeFile } from "node:fs/promises";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @param {object} params
|
|
24
|
+
* @param {number} params.seconds - How long to profile; nothing happens unless positive.
|
|
25
|
+
* @param {string} params.dir - Where to write the profile.
|
|
26
|
+
* @param {(line: string) => void} params.log
|
|
27
|
+
* @param {(bytes: number) => boolean} [params.mayKeep] - Whether the storage budget lets it be kept.
|
|
28
|
+
* @returns {{ begin: () => void, active: () => boolean }}
|
|
29
|
+
*/
|
|
30
|
+
export function startupCpuProfile({ seconds, dir, log, mayKeep = () => true }) {
|
|
31
|
+
let begun = false;
|
|
32
|
+
let running = false;
|
|
33
|
+
const wanted = Number.isFinite(seconds) && seconds > 0 && typeof dir === "string" && dir.length > 0;
|
|
34
|
+
const take = async () => {
|
|
35
|
+
const session = new Session();
|
|
36
|
+
session.connect();
|
|
37
|
+
try {
|
|
38
|
+
await session.post("Profiler.enable");
|
|
39
|
+
await session.post("Profiler.start");
|
|
40
|
+
running = true;
|
|
41
|
+
log(`cpu profile: the main thread is profiled for ${seconds}s from the first source opened`);
|
|
42
|
+
await new Promise((resolve) => {
|
|
43
|
+
const timer = setTimeout(resolve, seconds * 1000);
|
|
44
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
45
|
+
});
|
|
46
|
+
const { profile } = await session.post("Profiler.stop");
|
|
47
|
+
const text = JSON.stringify(profile);
|
|
48
|
+
if (!mayKeep(Buffer.byteLength(text))) {
|
|
49
|
+
log(`cpu profile: not kept, ${Buffer.byteLength(text)} bytes refused by the storage budget`);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const file = path.join(dir, `cpu-start-${new Date().toISOString().replace(/[:.]/g, "-")}.cpuprofile`);
|
|
53
|
+
await writeFile(file, text);
|
|
54
|
+
log(`cpu profile: written to ${file} (${Buffer.byteLength(text)} bytes)`);
|
|
55
|
+
} finally {
|
|
56
|
+
running = false;
|
|
57
|
+
session.disconnect();
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
return {
|
|
61
|
+
begin() {
|
|
62
|
+
if (!wanted || begun) return;
|
|
63
|
+
begun = true;
|
|
64
|
+
void take().catch((error) => log(`cpu profile: failed — ${error.message}`));
|
|
65
|
+
},
|
|
66
|
+
active: () => running
|
|
67
|
+
};
|
|
68
|
+
}
|