@torrent-tv/proxy 2.22.0 → 2.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/docs/encode-run-state.md +96 -0
- package/package.json +2 -1
- package/scripts/render-run-graph.js +173 -0
- package/services/encode-exit.js +80 -0
- package/services/encode-run-state.js +488 -0
- package/services/hls-session-manager.js +627 -85
- package/test/concurrent-cost.test.js +138 -82
- package/test/encode-exit.test.js +93 -0
- package/test/encode-run-state.test.js +314 -0
- package/test/quality-variants.test.js +67 -1
- package/test/run-graph-drift.test.js +37 -0
- package/test/segment-serve-wiring.test.js +46 -0
|
@@ -20,6 +20,8 @@ import { logger } from "../utils/logger.js";
|
|
|
20
20
|
import { readKeyframeIndex } from "./container-index/index.js";
|
|
21
21
|
import { readMachineState, readProcessCpuSeconds, readProxyCpuSeconds, readSystemCpu, shareOfMachine } from "./host-load.js";
|
|
22
22
|
import { speedFromReadings } from "./encoder-readings.js";
|
|
23
|
+
import { ENCODE_RUN_EVENT, ENCODE_RUN_STATE, INITIAL_RUN_STATE, nextState } from "./encode-run-state.js";
|
|
24
|
+
import { ENCODE_EXIT, classifyEncodeExit } from "./encode-exit.js";
|
|
23
25
|
|
|
24
26
|
/** Own package version, stamped onto session-start log lines. */
|
|
25
27
|
const PROXY_VERSION = createRequire(import.meta.url)("../package.json").version;
|
|
@@ -1176,6 +1178,7 @@ function normalizeLogFileName(fileName, fileIndex) {
|
|
|
1176
1178
|
* @param {number | null} fileLengthBytes
|
|
1177
1179
|
* @returns {number | null}
|
|
1178
1180
|
*/
|
|
1181
|
+
|
|
1179
1182
|
function sourceMegabytesPerSecond(session, fileLengthBytes) {
|
|
1180
1183
|
// The FILE's rate, not the video stream's. What the torrent moves is the
|
|
1181
1184
|
// container: on the releases this serves, two or three AC-3 tracks add 10-25 %
|
|
@@ -1190,6 +1193,29 @@ function sourceMegabytesPerSecond(session, fileLengthBytes) {
|
|
|
1190
1193
|
return null;
|
|
1191
1194
|
}
|
|
1192
1195
|
|
|
1196
|
+
/**
|
|
1197
|
+
* Which cost a speed reading from this session is a measurement OF.
|
|
1198
|
+
*
|
|
1199
|
+
* Three encodes share one reading path and price three different things: a
|
|
1200
|
+
* soundtrack published on its own, a picture being re-encoded (whose reading
|
|
1201
|
+
* prices this source's DECODING, the encode half being known from the startup
|
|
1202
|
+
* benchmark), and a picture being copied (which prices copying).
|
|
1203
|
+
*
|
|
1204
|
+
* Exported because the routing is where the fault was: a rendition was refused
|
|
1205
|
+
* a reading by one guard while the call that would have priced it sat behind
|
|
1206
|
+
* another, so the soundtrack was charged at nothing no matter how long it ran.
|
|
1207
|
+
* A pure function makes that a test rather than a field session.
|
|
1208
|
+
*
|
|
1209
|
+
* @param {{ audioOnly?: boolean, transcodeVideo?: boolean }} session
|
|
1210
|
+
* @returns {"audio" | "decode" | "copy"}
|
|
1211
|
+
*/
|
|
1212
|
+
export function costKindForSession(session) {
|
|
1213
|
+
if (session?.audioOnly === true) {
|
|
1214
|
+
return "audio";
|
|
1215
|
+
}
|
|
1216
|
+
return session?.transcodeVideo === true ? "decode" : "copy";
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1193
1219
|
export class HlsSessionManager {
|
|
1194
1220
|
/**
|
|
1195
1221
|
* Recent times from session-create to a servable first segment, in ms.
|
|
@@ -1232,6 +1258,28 @@ export class HlsSessionManager {
|
|
|
1232
1258
|
* @type {Map<string, { costSec: number, readings: number[], version: number }>}
|
|
1233
1259
|
*/
|
|
1234
1260
|
#observedCopyCost = new Map();
|
|
1261
|
+
|
|
1262
|
+
/**
|
|
1263
|
+
* What encoding one audio TRACK of one file costs this host, in seconds of
|
|
1264
|
+
* work per second of video. Keyed by source, file and track, because two
|
|
1265
|
+
* tracks of one film are not the same encode: a 5.1 AC-3 dub and a stereo
|
|
1266
|
+
* AAC original decode and mix differently.
|
|
1267
|
+
*
|
|
1268
|
+
* Measured exactly as the copy's price is — from an audio-only session's own
|
|
1269
|
+
* reported speed, past its start, alone on the machine, and never while the
|
|
1270
|
+
* torrent is what is short.
|
|
1271
|
+
*
|
|
1272
|
+
* @type {Map<string, { costSec: number, readings: number[], version: number }>}
|
|
1273
|
+
*/
|
|
1274
|
+
#observedAudioCost = new Map();
|
|
1275
|
+
|
|
1276
|
+
/**
|
|
1277
|
+
* The last "not offering" line written, so the same one is not written again.
|
|
1278
|
+
* See the end of {@link HlsSessionManager##sustainableHeights}.
|
|
1279
|
+
*
|
|
1280
|
+
* @type {string}
|
|
1281
|
+
*/
|
|
1282
|
+
#lastOfferLine = "";
|
|
1235
1283
|
/** The previous reading of the machine, to compare the next one against. */
|
|
1236
1284
|
#hostLoadSample = null;
|
|
1237
1285
|
/** The previous reading taken while nothing was encoding, for the torrent's own cost. */
|
|
@@ -1746,6 +1794,13 @@ export class HlsSessionManager {
|
|
|
1746
1794
|
lastAccessedAt: Date.now(),
|
|
1747
1795
|
ffmpeg: null,
|
|
1748
1796
|
encodeRunGeneration: 0,
|
|
1797
|
+
// What the ENCODER RUN is doing, as one control state from the table in
|
|
1798
|
+
// `encode-run-state.js`. Written at every event today and read by nothing
|
|
1799
|
+
// yet: a refused pair in the log is the model disagreeing with reality,
|
|
1800
|
+
// and that disagreement is the measurement this release exists to take.
|
|
1801
|
+
// The fields it will replace — `state`, `progress.state`, `encoderPaused`
|
|
1802
|
+
// and the repeated liveness checks — keep their current writes meanwhile.
|
|
1803
|
+
runState: INITIAL_RUN_STATE,
|
|
1749
1804
|
lastError: "",
|
|
1750
1805
|
// Cold-start timing: entry timestamp + a once-guard so the first servable
|
|
1751
1806
|
// segment logs its latency exactly once.
|
|
@@ -2600,8 +2655,8 @@ export class HlsSessionManager {
|
|
|
2600
2655
|
if (aheadSeconds === null) {
|
|
2601
2656
|
// The segment the viewer needs does not exist. Whatever else is on disk,
|
|
2602
2657
|
// this encoder has work to do right now.
|
|
2603
|
-
if (session.encoderPaused) {
|
|
2604
|
-
this.#
|
|
2658
|
+
if (session.encoderPaused && this.#resumeEncoder(session, "the viewer needs a segment nobody has made")) {
|
|
2659
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.RESUME_ORDERED);
|
|
2605
2660
|
}
|
|
2606
2661
|
return;
|
|
2607
2662
|
}
|
|
@@ -2643,7 +2698,9 @@ export class HlsSessionManager {
|
|
|
2643
2698
|
`${reading.total} segment file(s) present)`
|
|
2644
2699
|
);
|
|
2645
2700
|
} else if (session.encoderPaused && aheadSeconds <= LOOKAHEAD_RESUME_SECONDS) {
|
|
2646
|
-
this.#resumeEncoder(session, `${Math.round(aheadSeconds)}s ahead of the viewer`)
|
|
2701
|
+
if (this.#resumeEncoder(session, `${Math.round(aheadSeconds)}s ahead of the viewer`)) {
|
|
2702
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.RESUME_ORDERED);
|
|
2703
|
+
}
|
|
2647
2704
|
}
|
|
2648
2705
|
}
|
|
2649
2706
|
|
|
@@ -2688,6 +2745,91 @@ export class HlsSessionManager {
|
|
|
2688
2745
|
return { seconds: Math.max(0, to - from), lastCovered, total: present.size };
|
|
2689
2746
|
}
|
|
2690
2747
|
|
|
2748
|
+
/**
|
|
2749
|
+
* Move this session's encoder run to the state the table says an event leads
|
|
2750
|
+
* to, and say so in the log.
|
|
2751
|
+
*
|
|
2752
|
+
* The log line is the point of it. Every transition a real run makes is
|
|
2753
|
+
* printed as state, event and target, so a session can be checked against the
|
|
2754
|
+
* specification after the fact — and a pair the table does not declare prints
|
|
2755
|
+
* as a refusal, which is a cell nobody considered rather than a line nobody
|
|
2756
|
+
* wrote. Five field failures in a row were exactly that.
|
|
2757
|
+
*
|
|
2758
|
+
* Refusing changes nothing and never throws: an event that means nothing here
|
|
2759
|
+
* is ignored, and the caller's own work goes on. The machine this pattern
|
|
2760
|
+
* replaces threw from inside a handler, so a refused transition abandoned the
|
|
2761
|
+
* rest of it and left the app describing a state it was no longer in.
|
|
2762
|
+
*
|
|
2763
|
+
* @param {HlsSession} session
|
|
2764
|
+
* @param {string} event - One of {@link ENCODE_RUN_EVENT}.
|
|
2765
|
+
* @returns {string} The state now in force.
|
|
2766
|
+
*/
|
|
2767
|
+
#transitionRun(session, event) {
|
|
2768
|
+
const from = session.runState ?? INITIAL_RUN_STATE;
|
|
2769
|
+
const to = nextState(from, event);
|
|
2770
|
+
if (to === null) {
|
|
2771
|
+
logger.warn(`run-state ${session.id} ${from} + ${event} — no such edge; ignored`);
|
|
2772
|
+
return from;
|
|
2773
|
+
}
|
|
2774
|
+
session.runState = to;
|
|
2775
|
+
logger.info(`run-state ${session.id} ${from} --${event}--> ${to}`);
|
|
2776
|
+
return to;
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2779
|
+
/**
|
|
2780
|
+
* Does this process's exit still say anything about the session?
|
|
2781
|
+
*
|
|
2782
|
+
* Two conditions, and the second is why this is a method rather than a
|
|
2783
|
+
* comparison. A process is not the session's own run once a NEWER one has
|
|
2784
|
+
* been installed — and also once it has been marked for replacement, which
|
|
2785
|
+
* happens BEFORE the replacement exists. Between those two moments the field
|
|
2786
|
+
* still names the doomed process, so comparing against it alone answers
|
|
2787
|
+
* "yes, this is the current run" about a process we have just killed.
|
|
2788
|
+
*
|
|
2789
|
+
* @param {HlsSession} session
|
|
2790
|
+
* @param {import("node:child_process").ChildProcess} ffmpeg
|
|
2791
|
+
* @returns {boolean}
|
|
2792
|
+
*/
|
|
2793
|
+
#isCurrentRun(session, ffmpeg) {
|
|
2794
|
+
if (session.ffmpeg !== ffmpeg) {
|
|
2795
|
+
return false;
|
|
2796
|
+
}
|
|
2797
|
+
return session.supersededRuns?.has(ffmpeg) !== true;
|
|
2798
|
+
}
|
|
2799
|
+
|
|
2800
|
+
/**
|
|
2801
|
+
* A segment this run made has just been served.
|
|
2802
|
+
*
|
|
2803
|
+
* The one event whose raising is guarded by the state, and the guard is a
|
|
2804
|
+
* READ of the state rather than a second copy of it: "something has been
|
|
2805
|
+
* produced" is a level, not an edge, so without the guard every served
|
|
2806
|
+
* segment would raise it and the log would fill with refusals.
|
|
2807
|
+
*
|
|
2808
|
+
* Whose file it is decides the rest, and the directory answers that outright:
|
|
2809
|
+
* runs write into one each, and serving searches them newest-first, so a
|
|
2810
|
+
* segment left by an EARLIER run is served routinely. Comparing indices
|
|
2811
|
+
* instead would have been fooled by the ordinary case of a backward seek —
|
|
2812
|
+
* the new run starts at #10, the old one left #50 on disk, and #50 is above
|
|
2813
|
+
* the new run's start index while saying nothing about it.
|
|
2814
|
+
*
|
|
2815
|
+
* @param {HlsSession} session
|
|
2816
|
+
* @param {string} filePath - Where the served segment was actually found.
|
|
2817
|
+
* @returns {void}
|
|
2818
|
+
*/
|
|
2819
|
+
#noteRunProducedSegment(session, filePath) {
|
|
2820
|
+
if (session.runState !== ENCODE_RUN_STATE.STARTING) {
|
|
2821
|
+
return;
|
|
2822
|
+
}
|
|
2823
|
+
const runDir = session.runDirPath;
|
|
2824
|
+
if (typeof runDir !== "string" || runDir.length === 0 || typeof filePath !== "string") {
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
if (path.dirname(filePath) !== runDir) {
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.FIRST_SEGMENT);
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2691
2833
|
/**
|
|
2692
2834
|
* Suspend a session's encoder. No-op when already paused or unsupported here.
|
|
2693
2835
|
*
|
|
@@ -2712,6 +2854,7 @@ export class HlsSessionManager {
|
|
|
2712
2854
|
return;
|
|
2713
2855
|
}
|
|
2714
2856
|
session.encoderPaused = true;
|
|
2857
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.SUSPEND_ORDERED);
|
|
2715
2858
|
logger.info(
|
|
2716
2859
|
`transcode ${session.id} encoder suspended — ${reason} ` +
|
|
2717
2860
|
`"${session.fileName}"`
|
|
@@ -2721,23 +2864,40 @@ export class HlsSessionManager {
|
|
|
2721
2864
|
/**
|
|
2722
2865
|
* Let a suspended encoder run again.
|
|
2723
2866
|
*
|
|
2867
|
+
* Answers whether a process was actually continued, because three of the four
|
|
2868
|
+
* callers do this in order to KILL it — a suspended process ignores SIGTERM
|
|
2869
|
+
* until it is running — and only the two look-ahead callers mean "carry on".
|
|
2870
|
+
* The run's state is theirs to move; a continue-then-kill is not a resume.
|
|
2871
|
+
*
|
|
2724
2872
|
* @param {HlsSession} session
|
|
2725
2873
|
* @param {string} reason
|
|
2726
|
-
* @returns {
|
|
2874
|
+
* @returns {boolean} True when a live process was continued.
|
|
2727
2875
|
*/
|
|
2728
2876
|
#resumeEncoder(session, reason) {
|
|
2729
2877
|
// Any pair spanning this would count a stopped encoder as slow.
|
|
2730
2878
|
session.learnSample = null;
|
|
2731
2879
|
if (!session.encoderPaused || !session.ffmpeg?.pid) {
|
|
2732
|
-
return;
|
|
2880
|
+
return false;
|
|
2733
2881
|
}
|
|
2882
|
+
let continued = true;
|
|
2734
2883
|
try {
|
|
2735
2884
|
process.kill(session.ffmpeg.pid, "SIGCONT");
|
|
2736
2885
|
} catch {
|
|
2737
|
-
// The process is gone; the exit handler will deal with it.
|
|
2886
|
+
// The process is gone; the exit handler will deal with it. The flag is
|
|
2887
|
+
// cleared either way — but nothing was resumed, and saying so is what
|
|
2888
|
+
// stops a dead run being reported as producing again.
|
|
2889
|
+
continued = false;
|
|
2738
2890
|
}
|
|
2739
2891
|
session.encoderPaused = false;
|
|
2740
|
-
|
|
2892
|
+
// Two records of one moment must not contradict each other: a line saying
|
|
2893
|
+
// the encoder resumed, beside a return value saying nothing was resumed, is
|
|
2894
|
+
// the sort of pair that costs an hour of reading a field log.
|
|
2895
|
+
logger.info(
|
|
2896
|
+
continued
|
|
2897
|
+
? `transcode ${session.id} encoder resumed — ${reason} "${session.fileName}"`
|
|
2898
|
+
: `transcode ${session.id} could not resume the encoder (the process is gone) — ${reason} "${session.fileName}"`
|
|
2899
|
+
);
|
|
2900
|
+
return continued;
|
|
2741
2901
|
}
|
|
2742
2902
|
|
|
2743
2903
|
/**
|
|
@@ -2833,10 +2993,13 @@ export class HlsSessionManager {
|
|
|
2833
2993
|
const encoding = [...this.sessionsById.values()].filter(
|
|
2834
2994
|
(session) => session?.ffmpeg != null && !hasChildExited(session.ffmpeg) && session.state !== "disposed"
|
|
2835
2995
|
);
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
//
|
|
2839
|
-
//
|
|
2996
|
+
const runningNow = encoding.filter((session) => session.encoderPaused !== true);
|
|
2997
|
+
if (runningNow.length === 0) {
|
|
2998
|
+
// No encoder is RUNNING. A suspended one costs nothing, and counting it
|
|
2999
|
+
// as work meant this was never reached: measured 2026-08-15, four minutes
|
|
3000
|
+
// in which every encoder was suspended, the torrent's price could have
|
|
3001
|
+
// been taken, and none was. What this process spends now is the download,
|
|
3002
|
+
// the verification, the piece store and the delivery. Item 7.
|
|
2840
3003
|
await this.#learnTorrentCost();
|
|
2841
3004
|
this.#hostLoadSample = null;
|
|
2842
3005
|
return;
|
|
@@ -3080,6 +3243,12 @@ export class HlsSessionManager {
|
|
|
3080
3243
|
session.budgetSlowSince = 0;
|
|
3081
3244
|
session.encodeWidth = rung.width;
|
|
3082
3245
|
session.encodeHeight = rung.height;
|
|
3246
|
+
// What this session was last seen doing described the encode it is no
|
|
3247
|
+
// longer running. Kept, it prices the new, cheaper picture at the old one's
|
|
3248
|
+
// cost, and it keeps the step it was measured on withdrawn from the offer
|
|
3249
|
+
// although nothing is producing that step any more. The next reading of the
|
|
3250
|
+
// new encode replaces it.
|
|
3251
|
+
session.lastAloneSpeed = null;
|
|
3083
3252
|
// Priced the same way the offer and the starting rung are. Choosing the
|
|
3084
3253
|
// preset on the encoder alone treats decoding as free, which is how the
|
|
3085
3254
|
// check and the encode came to disagree in the first place — and here it
|
|
@@ -3189,6 +3358,27 @@ export class HlsSessionManager {
|
|
|
3189
3358
|
session.behindHeadAsks = new Map();
|
|
3190
3359
|
const generation = ++session.encodeRunGeneration;
|
|
3191
3360
|
const previousFfmpeg = session.ffmpeg;
|
|
3361
|
+
// Marked BEFORE it is killed, and this is not a formality.
|
|
3362
|
+
//
|
|
3363
|
+
// The exit handler decides whether an exit belongs to the current run by
|
|
3364
|
+
// comparing against `session.ffmpeg` — and that field still names the
|
|
3365
|
+
// PREVIOUS process here, because the new one is not spawned for another
|
|
3366
|
+
// few hundred lines. So a predecessor killed for a seek passed the identity
|
|
3367
|
+
// check and was handled as though the session's own run had died: exit code
|
|
3368
|
+
// null with a signal, i.e. the "ffmpeg failed" branch. Consequences, in
|
|
3369
|
+
// rising order of cost — a spurious `state = "failed"` for the moment
|
|
3370
|
+
// between the kill and the spawn, which a segment request landing in that
|
|
3371
|
+
// window is answered 500 for; a fast-failure tally against a target that
|
|
3372
|
+
// never failed; and on any host with a hardware encoder, the runtime safety
|
|
3373
|
+
// net firing on every seek: the proxy downgraded itself to libx264 for good
|
|
3374
|
+
// and started an extra run at the OLD index, which then took the generation
|
|
3375
|
+
// and made the real restart abort. The comment further down claiming this
|
|
3376
|
+
// could not happen ("the old process's exit handler no-ops") described an
|
|
3377
|
+
// earlier arrangement where the field was already reassigned.
|
|
3378
|
+
session.supersededRuns ??= new WeakSet();
|
|
3379
|
+
if (previousFfmpeg) {
|
|
3380
|
+
session.supersededRuns.add(previousFfmpeg);
|
|
3381
|
+
}
|
|
3192
3382
|
// A suspended process does not act on SIGTERM until it is continued, so the
|
|
3193
3383
|
// wait below would never end. Let it run before asking it to stop.
|
|
3194
3384
|
this.#resumeEncoder(session, "terminating for a new run");
|
|
@@ -3246,11 +3436,13 @@ export class HlsSessionManager {
|
|
|
3246
3436
|
? segmentCutTimesFrom(session.segmentBoundaries, safeIndex)
|
|
3247
3437
|
: null;
|
|
3248
3438
|
|
|
3249
|
-
//
|
|
3250
|
-
//
|
|
3251
|
-
//
|
|
3439
|
+
// A second chance for a predecessor that survived the escalation above —
|
|
3440
|
+
// the first block is the one that does the work. Its exit is ignored
|
|
3441
|
+
// because it was marked superseded there, not because the field it is
|
|
3442
|
+
// compared against has changed: the spawn is still below this line.
|
|
3252
3443
|
this.#resumeEncoder(session, "terminating");
|
|
3253
3444
|
if (session.ffmpeg && !session.ffmpeg.killed) {
|
|
3445
|
+
session.supersededRuns.add(session.ffmpeg);
|
|
3254
3446
|
try {
|
|
3255
3447
|
session.ffmpeg.kill("SIGTERM");
|
|
3256
3448
|
} catch (_error) {
|
|
@@ -3467,6 +3659,7 @@ export class HlsSessionManager {
|
|
|
3467
3659
|
stdio: ["ignore", "pipe", "pipe"]
|
|
3468
3660
|
});
|
|
3469
3661
|
session.ffmpeg = ffmpeg;
|
|
3662
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.SPAWNED);
|
|
3470
3663
|
// Whether this run cuts at times we gave it. Decides how a segment is
|
|
3471
3664
|
// judged finished — see getFileStream.
|
|
3472
3665
|
session.usesExplicitCuts = Boolean(cutTimes && cutTimes.length > 0);
|
|
@@ -3607,25 +3800,42 @@ export class HlsSessionManager {
|
|
|
3607
3800
|
});
|
|
3608
3801
|
|
|
3609
3802
|
ffmpeg.on("error", (error) => {
|
|
3610
|
-
if (session
|
|
3803
|
+
if (!this.#isCurrentRun(session, ffmpeg)) {
|
|
3611
3804
|
return;
|
|
3612
3805
|
}
|
|
3613
3806
|
session.state = "failed";
|
|
3614
3807
|
session.lastError = error instanceof Error ? error.message : String(error);
|
|
3615
3808
|
session.progress.state = "failed";
|
|
3616
3809
|
session.progress.updatedAt = Date.now();
|
|
3810
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_FAILED);
|
|
3617
3811
|
logger.error(`ffmpeg ${session.id} process error: ${session.lastError}`);
|
|
3618
3812
|
});
|
|
3619
3813
|
|
|
3620
3814
|
ffmpeg.on("exit", (code, signal) => {
|
|
3621
|
-
//
|
|
3622
|
-
|
|
3815
|
+
// Asked FIRST, before anything is written or read. An exit that belongs
|
|
3816
|
+
// to a replaced run must not touch the session's error, which the viewer
|
|
3817
|
+
// is shown and the next real exit is classified by, and must not send the
|
|
3818
|
+
// handler reading directories on its behalf.
|
|
3819
|
+
if (!this.#isCurrentRun(session, ffmpeg) || session.state === "disposed") {
|
|
3623
3820
|
return;
|
|
3624
3821
|
}
|
|
3625
|
-
|
|
3822
|
+
const producedThrough = code === 0 ? this.#latestProducedSegment(session) : null;
|
|
3823
|
+
const expectedLast = session.segmentCount > 0 ? session.segmentCount - 1 : null;
|
|
3824
|
+
if (!session.lastError && code !== 0) {
|
|
3825
|
+
session.lastError = `ffmpeg exited with code ${code ?? -1}${signal ? ` (signal ${signal})` : ""}`;
|
|
3826
|
+
}
|
|
3827
|
+
// What this exit means is decided in one place, from facts, and the
|
|
3828
|
+
// reasoning behind each answer lives with it in `encode-exit.js`.
|
|
3829
|
+
const outcome = classifyEncodeExit({
|
|
3830
|
+
code,
|
|
3831
|
+
producedThrough,
|
|
3832
|
+
lastSegmentIndex: expectedLast,
|
|
3833
|
+
inputUnavailable: isInputUnavailable(session.lastError)
|
|
3834
|
+
});
|
|
3835
|
+
if (outcome === ENCODE_EXIT.IGNORED) {
|
|
3626
3836
|
return;
|
|
3627
3837
|
}
|
|
3628
|
-
if (
|
|
3838
|
+
if (outcome === ENCODE_EXIT.SHORT) {
|
|
3629
3839
|
// ffmpeg exits 0 both when it reaches the end of the file and when its
|
|
3630
3840
|
// input simply stops producing bytes — over HTTP the two look identical
|
|
3631
3841
|
// to it. Field 2026-08-05: the torrent's download died, the read ended,
|
|
@@ -3634,35 +3844,41 @@ export class HlsSessionManager {
|
|
|
3634
3844
|
// segment nobody was making. So the claim is checked against the
|
|
3635
3845
|
// playlist we published, and a run that stopped short is a FAILURE that
|
|
3636
3846
|
// can be restarted, not a finished file.
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
}
|
|
3847
|
+
session.state = "failed";
|
|
3848
|
+
session.progress.state = "failed";
|
|
3849
|
+
session.progress.updatedAt = Date.now();
|
|
3850
|
+
session.lastError =
|
|
3851
|
+
`input ended after segment #${producedThrough} of ${expectedLast} — ` +
|
|
3852
|
+
"the source stopped delivering data";
|
|
3853
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_SHORT);
|
|
3854
|
+
logger.error(
|
|
3855
|
+
`transcode ${session.id} ${session.runLabel ?? "run#?"} encode-run ended early: ` +
|
|
3856
|
+
`${session.lastError} "${session.fileName}"`
|
|
3857
|
+
);
|
|
3858
|
+
return;
|
|
3859
|
+
}
|
|
3860
|
+
if (outcome === ENCODE_EXIT.COMPLETE) {
|
|
3652
3861
|
session.state = "ready";
|
|
3653
3862
|
session.progress.state = "ready";
|
|
3654
3863
|
session.progress.updatedAt = Date.now();
|
|
3864
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_COMPLETE);
|
|
3655
3865
|
logger.info(`transcode ${session.id} encode-run complete "${session.fileName}"`);
|
|
3656
3866
|
return;
|
|
3657
3867
|
}
|
|
3658
|
-
if (!session.lastError) {
|
|
3659
|
-
session.lastError = `ffmpeg exited with code ${code ?? -1}${signal ? ` (signal ${signal})` : ""}`;
|
|
3660
|
-
}
|
|
3661
3868
|
// Runtime safety net: if a hardware encode fails, downgrade this proxy to
|
|
3662
3869
|
// software encoding for all sessions and restart this one, so playback is
|
|
3663
3870
|
// never permanently broken by a hardware/driver issue.
|
|
3664
|
-
|
|
3871
|
+
//
|
|
3872
|
+
// Asked only of a genuine encoder failure. It used to be asked of every
|
|
3873
|
+
// non-zero exit, so a run whose TORRENT DATA went away — which says
|
|
3874
|
+
// nothing whatever about the encoder — condemned a working NVENC or
|
|
3875
|
+
// QuickSync to software for the life of the process, and started an extra
|
|
3876
|
+
// run at the old index while it was at it.
|
|
3877
|
+
if (outcome === ENCODE_EXIT.FAILED && session.transcodeVideo && this.videoEncoder.kind !== "software") {
|
|
3665
3878
|
const failedEncoder = this.videoEncoder.name;
|
|
3879
|
+
// The run died before the software one takes its place: the failure is
|
|
3880
|
+
// an event of its own, and the restart below is a separate spawn.
|
|
3881
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_FAILED);
|
|
3666
3882
|
this.videoEncoder = softwareDescriptor();
|
|
3667
3883
|
logger.warn(
|
|
3668
3884
|
`transcode ${session.id} hardware encoder ${failedEncoder} failed ` +
|
|
@@ -3704,13 +3920,14 @@ export class HlsSessionManager {
|
|
|
3704
3920
|
// the data would have come back in seconds. The circuit breaker below
|
|
3705
3921
|
// stays for what it was built for, a target that genuinely cannot be
|
|
3706
3922
|
// encoded; it must not condemn a session whose data merely went away.
|
|
3707
|
-
if (
|
|
3923
|
+
if (outcome === ENCODE_EXIT.INPUT_LOST) {
|
|
3708
3924
|
session.state = "recovering";
|
|
3709
3925
|
// On the wire it is simply "not ready yet" — a state the browser has
|
|
3710
3926
|
// always known how to wait through. Only the proxy needs the
|
|
3711
3927
|
// distinction between waiting for data and having given up.
|
|
3712
3928
|
session.progress.state = "starting";
|
|
3713
3929
|
session.progress.updatedAt = Date.now();
|
|
3930
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_INPUT_LOST);
|
|
3714
3931
|
session.inputRetryCount = (session.inputRetryCount ?? 0) + 1;
|
|
3715
3932
|
const delayMs = Math.min(
|
|
3716
3933
|
INPUT_RETRY_MAX_MS,
|
|
@@ -3726,6 +3943,7 @@ export class HlsSessionManager {
|
|
|
3726
3943
|
if (session.state !== "recovering") {
|
|
3727
3944
|
return;
|
|
3728
3945
|
}
|
|
3946
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.RETRY_DUE);
|
|
3729
3947
|
const at = Number.isInteger(session.lastRequestedSegment)
|
|
3730
3948
|
? session.lastRequestedSegment
|
|
3731
3949
|
: (session.encodeStartIndex ?? 0);
|
|
@@ -3737,6 +3955,7 @@ export class HlsSessionManager {
|
|
|
3737
3955
|
session.state = "failed";
|
|
3738
3956
|
session.progress.state = "failed";
|
|
3739
3957
|
session.progress.updatedAt = Date.now();
|
|
3958
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.EXITED_FAILED);
|
|
3740
3959
|
logger.error(
|
|
3741
3960
|
`transcode ${session.id} ${session.runLabel ?? "run#?"} encode-run failed: ${session.lastError}`
|
|
3742
3961
|
);
|
|
@@ -4608,6 +4827,19 @@ export class HlsSessionManager {
|
|
|
4608
4827
|
}
|
|
4609
4828
|
}
|
|
4610
4829
|
}
|
|
4830
|
+
// The soundtracks published separately. They are encoders of this family
|
|
4831
|
+
// exactly as the quality steps are — one runs for as long as the picture
|
|
4832
|
+
// does — and they were reachable only through `audioRenditionSessions`,
|
|
4833
|
+
// which nothing here walked. So a family never contained one, and every
|
|
4834
|
+
// sum taken over the family priced the soundtrack at nothing.
|
|
4835
|
+
if (root.audioRenditionSessions instanceof Map) {
|
|
4836
|
+
for (const renditionId of root.audioRenditionSessions.values()) {
|
|
4837
|
+
const rendition = this.sessionsById.get(renditionId);
|
|
4838
|
+
if (rendition) {
|
|
4839
|
+
family.add(rendition);
|
|
4840
|
+
}
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4611
4843
|
}
|
|
4612
4844
|
return [...family];
|
|
4613
4845
|
}
|
|
@@ -4730,7 +4962,43 @@ export class HlsSessionManager {
|
|
|
4730
4962
|
// never moves at all, so the cache would never be recomputed.
|
|
4731
4963
|
const copyVersion = this.#observedCopyCost.get(`${owner.sourceKey}:${owner.fileIndex}`)?.version ?? 0;
|
|
4732
4964
|
const torrentCost = this.#observedTorrentCostPerMegabyte ?? 0;
|
|
4733
|
-
|
|
4965
|
+
// The soundtrack's price is an input too, and so is how many encoders of
|
|
4966
|
+
// this family are running: both move the answer, and an answer cached
|
|
4967
|
+
// across them is the stale menu this key exists to prevent.
|
|
4968
|
+
const audioVersion = [...this.#familyOf(owner)]
|
|
4969
|
+
.filter((member) => member.audioOnly === true)
|
|
4970
|
+
.map((member) => this.#observedAudioCost.get(this.#audioCostKey(member))?.version ?? 0)
|
|
4971
|
+
.reduce((total, one) => total + one, 0);
|
|
4972
|
+
const running = [...this.#familyOf(owner)]
|
|
4973
|
+
.filter((member) => member.ffmpeg != null && !hasChildExited(member.ffmpeg)).length;
|
|
4974
|
+
// What each running encode was last seen doing, which is BOTH an input to
|
|
4975
|
+
// the answer twice over — it withdraws a step measured below realtime, and
|
|
4976
|
+
// it prices every running picture in the committed total — and a figure
|
|
4977
|
+
// rewritten every five seconds. Left out of the key, the menu could be
|
|
4978
|
+
// pinned to what was computed before anything had been measured: on a
|
|
4979
|
+
// COPIED picture the decode version never moves at all, so nothing else in
|
|
4980
|
+
// the key would ever have recomputed it.
|
|
4981
|
+
// Encoded for the DECISIONS it feeds, not as a raw figure. Two of them: is
|
|
4982
|
+
// this encode below realtime (which withdraws its own step outright), and
|
|
4983
|
+
// what does it cost (which is charged against every other step). A raw
|
|
4984
|
+
// speed at two decimals moves on nearly every five-second reading, so the
|
|
4985
|
+
// menu would be recomputed — and its "not offering" line written — for the
|
|
4986
|
+
// whole film; while rounding alone would hide the 0.995-1.005 crossing,
|
|
4987
|
+
// which is exactly the band a step spends its time in when the host is
|
|
4988
|
+
// marginal. The flag carries the crossing, the rounded cost carries the
|
|
4989
|
+
// rest.
|
|
4990
|
+
const measured = this.#familyOf(owner)
|
|
4991
|
+
.map((member) => {
|
|
4992
|
+
const speed = member.lastAloneSpeed;
|
|
4993
|
+
if (!Number.isFinite(speed) || !(speed > 0)) {
|
|
4994
|
+
return "-";
|
|
4995
|
+
}
|
|
4996
|
+
return `${speed < 1 ? "slow" : "ok"}${(1 / speed).toFixed(2)}`;
|
|
4997
|
+
})
|
|
4998
|
+
.join(",");
|
|
4999
|
+
const version =
|
|
5000
|
+
`${observed?.version ?? 0}:${playing}:${copyVersion}:${torrentCost.toFixed(6)}:` +
|
|
5001
|
+
`${audioVersion}:${running}:${measured}`;
|
|
4734
5002
|
if (Array.isArray(owner.offeredHeightsCache) && owner.offeredHeightsVersion === version) {
|
|
4735
5003
|
return owner.offeredHeightsCache;
|
|
4736
5004
|
}
|
|
@@ -4757,6 +5025,9 @@ export class HlsSessionManager {
|
|
|
4757
5025
|
// picture being COPIED is the common case and used to be priced at
|
|
4758
5026
|
// nothing; measured, it is about an eighth of the machine.
|
|
4759
5027
|
concurrentCostSec: this.#committedCostOf(owner),
|
|
5028
|
+
// So a height already being produced is not charged for itself when it is
|
|
5029
|
+
// judged. See the subtraction in #sustainableHeights.
|
|
5030
|
+
runningCostByHeight: this.#runningCostByHeight(owner),
|
|
4760
5031
|
sourceWidth: Number(owner.sourceWidth) || 0,
|
|
4761
5032
|
sourceHeight: Math.round(Number(owner.sourceHeight) || 0),
|
|
4762
5033
|
fps: Number(owner.outputFps) || TRANSCODE_FPS,
|
|
@@ -4871,13 +5142,8 @@ export class HlsSessionManager {
|
|
|
4871
5142
|
session.state === "disposed" ||
|
|
4872
5143
|
session.state === "failed" ||
|
|
4873
5144
|
!session.ffmpeg ||
|
|
4874
|
-
session.encoderPaused === true
|
|
4875
|
-
session.audioOnly === true
|
|
5145
|
+
session.encoderPaused === true
|
|
4876
5146
|
) {
|
|
4877
|
-
// An audio rendition is excluded because its speed is the price of a
|
|
4878
|
-
// soundtrack, which neither decoding nor copying the picture is. A COPY
|
|
4879
|
-
// is NOT excluded any more: it says what copying costs, and that used to
|
|
4880
|
-
// be counted as nothing.
|
|
4881
5147
|
session.learnSample = null;
|
|
4882
5148
|
return;
|
|
4883
5149
|
}
|
|
@@ -4900,24 +5166,60 @@ export class HlsSessionManager {
|
|
|
4900
5166
|
if (speed === null) {
|
|
4901
5167
|
return;
|
|
4902
5168
|
}
|
|
4903
|
-
|
|
4904
|
-
// another encoder
|
|
4905
|
-
// ADDS the same work again when it predicts
|
|
4906
|
-
// and grows with every reading.
|
|
4907
|
-
// whose truth is 7.9x, was
|
|
4908
|
-
// 2.6x, as 0.87x. Every
|
|
4909
|
-
//
|
|
5169
|
+
const kind = costKindForSession(session);
|
|
5170
|
+
// A reading taken beside another encoder contains that other encoder's
|
|
5171
|
+
// work, and the budget ADDS the same work again when it predicts — so filed
|
|
5172
|
+
// as it stands the price is counted twice and grows with every reading.
|
|
5173
|
+
// Measured 2026-08-15 in the field: copying, whose truth is 7.9x, was
|
|
5174
|
+
// learned as 2.03x, and decoding, whose clips say 2.6x, as 0.87x. Every
|
|
5175
|
+
// step was then refused, the offer collapsed to the one copied height, and
|
|
5176
|
+
// the viewer lost the quality menu altogether.
|
|
5177
|
+
//
|
|
5178
|
+
// For a picture the answer is to wait for a moment alone, which comes often
|
|
5179
|
+
// enough. For a SOUNDTRACK it never comes: a rendition runs for exactly as
|
|
5180
|
+
// long as the picture it accompanies, so "alone" is a state it is never in,
|
|
5181
|
+
// and the price stayed unmeasured for ever — the hole this was meant to
|
|
5182
|
+
// close. Its share is instead recovered by subtracting what the machine is
|
|
5183
|
+
// already known to be spending, which is the same arithmetic that recovers
|
|
5184
|
+
// this source's decoding from a running encoder, and it is only done when
|
|
5185
|
+
// every other running encode HAS a price. Otherwise the unpriced work would
|
|
5186
|
+
// land in the soundtrack's account and refuse steps on it.
|
|
5187
|
+
let othersCostSec = 0;
|
|
4910
5188
|
if (this.#runningEncoders() > 1) {
|
|
4911
|
-
|
|
5189
|
+
if (kind !== "audio") {
|
|
5190
|
+
return;
|
|
5191
|
+
}
|
|
5192
|
+
const others = this.#pricedConcurrentCost(session);
|
|
5193
|
+
if (others === null) {
|
|
5194
|
+
return; // something running has no price; nothing can be attributed
|
|
5195
|
+
}
|
|
5196
|
+
othersCostSec = others;
|
|
4912
5197
|
}
|
|
4913
|
-
// What this rung did with the machine to itself — the one figure a live
|
|
4914
|
-
// reading is authority on, and what withdraws a rung that has been seen
|
|
4915
|
-
// failing without letting it speak for rungs nobody has run.
|
|
4916
|
-
session.lastAloneSpeed = speed;
|
|
4917
5198
|
if (speed < BUDGET_SPEED_OK && await this.#classifyTranscodeBound(session) === "download") {
|
|
4918
5199
|
return; // the torrent is what is short; this says nothing about the host
|
|
4919
5200
|
}
|
|
4920
|
-
|
|
5201
|
+
// What this encode did with the machine to itself — the one figure a live
|
|
5202
|
+
// reading is authority on, and what withdraws a quality step that has been
|
|
5203
|
+
// seen failing without letting it speak for steps nobody has run.
|
|
5204
|
+
//
|
|
5205
|
+
// Recorded only AFTER the download-bound check, and that order is the whole
|
|
5206
|
+
// point: a run starved of torrent data reports a speed that measures the
|
|
5207
|
+
// swarm. Stored first, as it was, that figure became this encode's price —
|
|
5208
|
+
// 0.3x reads as 3.33 s of work per second of video, more than the machine
|
|
5209
|
+
// has — and every other quality step was refused on the download's account.
|
|
5210
|
+
session.lastAloneSpeed = speed;
|
|
5211
|
+
if (kind === "audio") {
|
|
5212
|
+
// What is left after the work that was already accounted for. `null` when
|
|
5213
|
+
// the subtraction leaves nothing positive, which means the reading says
|
|
5214
|
+
// less than the noise in it.
|
|
5215
|
+
const ownCostSec = 1 / speed - othersCostSec;
|
|
5216
|
+
if (!(ownCostSec > 0) || !Number.isFinite(ownCostSec)) {
|
|
5217
|
+
return;
|
|
5218
|
+
}
|
|
5219
|
+
await this.#learnAudioCost(session, 1 / ownCostSec);
|
|
5220
|
+
return;
|
|
5221
|
+
}
|
|
5222
|
+
if (kind === "decode") {
|
|
4921
5223
|
this.#learnDecodeCost(session, speed);
|
|
4922
5224
|
return;
|
|
4923
5225
|
}
|
|
@@ -4976,23 +5278,70 @@ export class HlsSessionManager {
|
|
|
4976
5278
|
);
|
|
4977
5279
|
}
|
|
4978
5280
|
|
|
5281
|
+
/**
|
|
5282
|
+
* What this file's audio track costs to encode on this host.
|
|
5283
|
+
*
|
|
5284
|
+
* A rendition is a second encoder, running for as long as the picture does,
|
|
5285
|
+
* and the budget counted it at nothing — which is half of what roadmap item 6
|
|
5286
|
+
* was left owing. It is small beside a picture, and small is not zero: on a
|
|
5287
|
+
* host where a rung needs almost the whole machine, a soundtrack is the
|
|
5288
|
+
* difference between offering it and refusing it.
|
|
5289
|
+
*
|
|
5290
|
+
* Same rules as {@link #learnCopyCost}, for the same reasons: past the run's
|
|
5291
|
+
* own start, never suspended, never while the torrent is what is short.
|
|
5292
|
+
*
|
|
5293
|
+
* @param {HlsSession} session
|
|
5294
|
+
* @param {number} speed
|
|
5295
|
+
*/
|
|
5296
|
+
async #learnAudioCost(session, speed) {
|
|
5297
|
+
const runStartedAt = Number(session.encodeRunStartedAt);
|
|
5298
|
+
if (!Number.isFinite(runStartedAt) || Date.now() - runStartedAt < DECODE_LEARNING_SETTLE_MS) {
|
|
5299
|
+
return;
|
|
5300
|
+
}
|
|
5301
|
+
if (session.encoderPaused === true) {
|
|
5302
|
+
return;
|
|
5303
|
+
}
|
|
5304
|
+
if (await this.#classifyTranscodeBound(session) === "download") {
|
|
5305
|
+
return;
|
|
5306
|
+
}
|
|
5307
|
+
const costSec = 1 / speed;
|
|
5308
|
+
if (!(costSec > 0) || !Number.isFinite(costSec)) {
|
|
5309
|
+
return;
|
|
5310
|
+
}
|
|
5311
|
+
const key = this.#audioCostKey(session);
|
|
5312
|
+
const known = this.#observedAudioCost.get(key);
|
|
5313
|
+
const readings = [...(known?.readings ?? []), costSec].slice(-DECODE_LEARNING_READINGS);
|
|
5314
|
+
const sorted = [...readings].sort((left, right) => left - right);
|
|
5315
|
+
const median = sorted[Math.floor(sorted.length / 2)];
|
|
5316
|
+
if (known && Math.abs(median - known.costSec) / known.costSec < DECODE_LEARNING_CHANGE) {
|
|
5317
|
+
this.#observedAudioCost.set(key, { ...known, readings });
|
|
5318
|
+
return;
|
|
5319
|
+
}
|
|
5320
|
+
this.#observedAudioCost.set(key, { costSec: median, readings, version: (known?.version ?? 0) + 1 });
|
|
5321
|
+
logger.info(
|
|
5322
|
+
`transcode: ${session.fileName} encodes audio track ${session.audioTrackIndex ?? 0} at ` +
|
|
5323
|
+
`${(1 / median).toFixed(2)}x on this host (median of ${readings.length}, latest ${speed.toFixed(2)}x)`
|
|
5324
|
+
);
|
|
5325
|
+
}
|
|
5326
|
+
|
|
5327
|
+
/**
|
|
5328
|
+
* @param {HlsSession} session
|
|
5329
|
+
* @returns {string}
|
|
5330
|
+
*/
|
|
5331
|
+
#audioCostKey(session) {
|
|
5332
|
+
return `${session.sourceKey}:${session.fileIndex}:${session.audioTrackIndex ?? 0}`;
|
|
5333
|
+
}
|
|
5334
|
+
|
|
4979
5335
|
#learnDecodeCost(session, speed) {
|
|
4980
5336
|
if (!(speed > 0)) {
|
|
4981
5337
|
return;
|
|
4982
5338
|
}
|
|
4983
5339
|
if (session.transcodeVideo !== true) {
|
|
4984
|
-
//
|
|
4985
|
-
//
|
|
4986
|
-
//
|
|
4987
|
-
//
|
|
4988
|
-
//
|
|
4989
|
-
//
|
|
4990
|
-
// An audio rendition also carries no video, and its speed is the cost of
|
|
4991
|
-
// encoding a soundtrack — a different quantity that must not be filed
|
|
4992
|
-
// under what copying the picture costs.
|
|
4993
|
-
if (session.audioOnly !== true) {
|
|
4994
|
-
void this.#learnCopyCost(session, speed);
|
|
4995
|
-
}
|
|
5340
|
+
// Nothing to learn about decoding here, and nothing else either: the
|
|
5341
|
+
// caller routes a copy to #learnCopyCost and a rendition to
|
|
5342
|
+
// #learnAudioCost before this is ever reached. Routing them from here as
|
|
5343
|
+
// well put both calls behind a guard the caller had already made
|
|
5344
|
+
// (`transcodeVideo === true`), so neither could run.
|
|
4996
5345
|
return;
|
|
4997
5346
|
}
|
|
4998
5347
|
const runStartedAt = Number(session.encodeRunStartedAt);
|
|
@@ -5140,25 +5489,178 @@ export class HlsSessionManager {
|
|
|
5140
5489
|
* @param {{ heights: number[], ownHeight: number, sourceWidth: number, sourceHeight: number, fps: number, source: { megapixelsPerSecond: number, megabitsPerSecond: number } | null, transcodeVideo: boolean }} params
|
|
5141
5490
|
* @returns {number[]}
|
|
5142
5491
|
*/
|
|
5492
|
+
/**
|
|
5493
|
+
* What a running re-encode of the picture costs, in seconds of work per
|
|
5494
|
+
* second of video.
|
|
5495
|
+
*
|
|
5496
|
+
* Measured first: `lastAloneSpeed` is what this very rung did with the
|
|
5497
|
+
* machine to itself. Failing that, the encode model that decides every rung —
|
|
5498
|
+
* the same benchmark, the same decode term — applied to this rung's own pixel
|
|
5499
|
+
* rate. There is no third answer: a rung whose cost cannot be derived at all
|
|
5500
|
+
* contributes nothing rather than a number somebody invented.
|
|
5501
|
+
*
|
|
5502
|
+
* @param {HlsSession} session
|
|
5503
|
+
* @returns {number}
|
|
5504
|
+
*/
|
|
5505
|
+
#pictureCostOf(session) {
|
|
5506
|
+
if (Number.isFinite(session.lastAloneSpeed) && session.lastAloneSpeed > 0) {
|
|
5507
|
+
return 1 / session.lastAloneSpeed;
|
|
5508
|
+
}
|
|
5509
|
+
const benchmark = this.softwarePresetBenchmark;
|
|
5510
|
+
const width = Number(session.encodeWidth) || 0;
|
|
5511
|
+
const height = Number(session.encodeHeight) || 0;
|
|
5512
|
+
const fps = Number(session.outputFps) || TRANSCODE_FPS;
|
|
5513
|
+
if (!Array.isArray(benchmark) || benchmark.length === 0 || width <= 0 || height <= 0) {
|
|
5514
|
+
return 0;
|
|
5515
|
+
}
|
|
5516
|
+
const { speed } = canSustainOutput({
|
|
5517
|
+
benchmark,
|
|
5518
|
+
decodeModel: this.decodeCostModel,
|
|
5519
|
+
source: session.sourceDecode ?? null,
|
|
5520
|
+
outputPixelsPerSec: width * height * fps,
|
|
5521
|
+
observedDecodeCostSec: null,
|
|
5522
|
+
concurrentCostSec: 0
|
|
5523
|
+
});
|
|
5524
|
+
return Number.isFinite(speed) && speed > 0 ? 1 / speed : 0;
|
|
5525
|
+
}
|
|
5526
|
+
|
|
5527
|
+
/**
|
|
5528
|
+
* What everything OTHER than this session is costing right now, or null when
|
|
5529
|
+
* any of it is unpriced.
|
|
5530
|
+
*
|
|
5531
|
+
* Used to recover a soundtrack's own share from a reading taken beside the
|
|
5532
|
+
* picture — the only kind of reading a rendition ever gives, since it runs
|
|
5533
|
+
* exactly as long as the picture does. Refusing to answer when something
|
|
5534
|
+
* running has no price is the point: unpriced work would otherwise be
|
|
5535
|
+
* attributed to the soundtrack, and an overpriced soundtrack refuses quality
|
|
5536
|
+
* steps the host could actually hold.
|
|
5537
|
+
*
|
|
5538
|
+
* @param {HlsSession} session
|
|
5539
|
+
* @returns {number | null}
|
|
5540
|
+
*/
|
|
5541
|
+
#pricedConcurrentCost(session) {
|
|
5542
|
+
let cost = 0;
|
|
5543
|
+
for (const member of this.#familyOf(session)) {
|
|
5544
|
+
if (member === session || member.ffmpeg == null || hasChildExited(member.ffmpeg)) {
|
|
5545
|
+
continue;
|
|
5546
|
+
}
|
|
5547
|
+
if (member.audioOnly === true) {
|
|
5548
|
+
const audio = this.#observedAudioCost.get(this.#audioCostKey(member));
|
|
5549
|
+
if (!audio || !(audio.costSec > 0)) {
|
|
5550
|
+
return null;
|
|
5551
|
+
}
|
|
5552
|
+
cost += audio.costSec;
|
|
5553
|
+
continue;
|
|
5554
|
+
}
|
|
5555
|
+
if (member.transcodeVideo !== true) {
|
|
5556
|
+
const copy = this.#observedCopyCost.get(`${member.sourceKey}:${member.fileIndex}`);
|
|
5557
|
+
if (!copy || !(copy.costSec > 0)) {
|
|
5558
|
+
return null;
|
|
5559
|
+
}
|
|
5560
|
+
cost += copy.costSec;
|
|
5561
|
+
continue;
|
|
5562
|
+
}
|
|
5563
|
+
const picture = this.#pictureCostOf(member);
|
|
5564
|
+
if (!(picture > 0)) {
|
|
5565
|
+
return null;
|
|
5566
|
+
}
|
|
5567
|
+
cost += picture;
|
|
5568
|
+
}
|
|
5569
|
+
// Encoders outside this family are counted by number only — there is no
|
|
5570
|
+
// price to look up for another film's session — so a reading taken while
|
|
5571
|
+
// one is running cannot be attributed either.
|
|
5572
|
+
return this.#runningEncoders() > this.#familyOf(session).filter(
|
|
5573
|
+
(member) => member.ffmpeg != null && !hasChildExited(member.ffmpeg)
|
|
5574
|
+
).length
|
|
5575
|
+
? null
|
|
5576
|
+
: cost;
|
|
5577
|
+
}
|
|
5578
|
+
|
|
5579
|
+
/**
|
|
5580
|
+
* What each height of this family is costing RIGHT NOW, for the heights an
|
|
5581
|
+
* encoder is actually running at.
|
|
5582
|
+
*
|
|
5583
|
+
* Exists so a height can be judged against what the machine spends on
|
|
5584
|
+
* everything else — a step being warmed is running while it is judged, and
|
|
5585
|
+
* charged its own cost it refuses itself.
|
|
5586
|
+
*
|
|
5587
|
+
* @param {HlsSession} session
|
|
5588
|
+
* @returns {Map<number, number>}
|
|
5589
|
+
*/
|
|
5590
|
+
#runningCostByHeight(session) {
|
|
5591
|
+
/** @type {Map<number, number>} */
|
|
5592
|
+
const byHeight = new Map();
|
|
5593
|
+
for (const member of this.#familyOf(session)) {
|
|
5594
|
+
if (member.audioOnly === true || member.transcodeVideo !== true) {
|
|
5595
|
+
continue;
|
|
5596
|
+
}
|
|
5597
|
+
if (member.ffmpeg == null || hasChildExited(member.ffmpeg)) {
|
|
5598
|
+
continue;
|
|
5599
|
+
}
|
|
5600
|
+
const height = this.variantHeightOf(member);
|
|
5601
|
+
if (height > 0) {
|
|
5602
|
+
byHeight.set(height, (byHeight.get(height) ?? 0) + this.#pictureCostOf(member));
|
|
5603
|
+
}
|
|
5604
|
+
}
|
|
5605
|
+
return byHeight;
|
|
5606
|
+
}
|
|
5607
|
+
|
|
5143
5608
|
/**
|
|
5144
5609
|
* Seconds of work per second of video this family is ALREADY committed to,
|
|
5145
5610
|
* beside any rung being considered.
|
|
5146
5611
|
*
|
|
5147
|
-
*
|
|
5148
|
-
*
|
|
5149
|
-
*
|
|
5150
|
-
*
|
|
5151
|
-
*
|
|
5152
|
-
*
|
|
5612
|
+
* Every encoder of the family that is actually running: the picture, whether
|
|
5613
|
+
* it is copied or re-encoded, and each audio rendition. The rung the viewer
|
|
5614
|
+
* is watching and the source's own copied height are never withdrawn by the
|
|
5615
|
+
* caller, so charging for the encoder that serves them cannot strand anyone —
|
|
5616
|
+
* what it does is stop the NEXT rung being offered as though the machine were
|
|
5617
|
+
* idle, which is what the field disproved on 2026-08-15.
|
|
5618
|
+
*
|
|
5619
|
+
* Anything whose cost is neither measured nor derivable contributes nothing.
|
|
5620
|
+
* A guess here would refuse rungs on arithmetic nobody performed.
|
|
5153
5621
|
*
|
|
5154
5622
|
* @param {HlsSession} session
|
|
5155
5623
|
* @returns {number}
|
|
5156
5624
|
*/
|
|
5157
5625
|
#committedCostOf(session) {
|
|
5158
5626
|
let cost = 0;
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5627
|
+
for (const member of this.#familyOf(session)) {
|
|
5628
|
+
// Only what still HAS an encoder. A quality step the viewer left keeps
|
|
5629
|
+
// its session and its segments but not a process, and it produces nothing
|
|
5630
|
+
// for anybody — charging the machine for it would refuse steps on work
|
|
5631
|
+
// nobody is doing.
|
|
5632
|
+
//
|
|
5633
|
+
// A SUSPENDED encoder is charged, deliberately, and this is not the same
|
|
5634
|
+
// question. The unit here is seconds of work per second of VIDEO, not per
|
|
5635
|
+
// second of wall clock: a copy running at 8x costs 0.125 s/s whether it
|
|
5636
|
+
// is producing right now or parked by the look-ahead cap, because over an
|
|
5637
|
+
// hour of watching it still produces an hour of video. Suspension is how
|
|
5638
|
+
// that cost is spread, not a discount on it — and pricing a parked
|
|
5639
|
+
// encoder at zero would offer a step on the strength of a pause that ends
|
|
5640
|
+
// the moment the viewer catches up.
|
|
5641
|
+
if (member.ffmpeg == null || hasChildExited(member.ffmpeg)) {
|
|
5642
|
+
continue;
|
|
5643
|
+
}
|
|
5644
|
+
if (member.audioOnly === true) {
|
|
5645
|
+
// A soundtrack encoder, priced from its own measured speed. Nothing is
|
|
5646
|
+
// charged for a track nobody has measured: a guess here refuses rungs
|
|
5647
|
+
// on arithmetic no one performed.
|
|
5648
|
+
const audio = this.#observedAudioCost.get(this.#audioCostKey(member));
|
|
5649
|
+
cost += audio && audio.costSec > 0 ? audio.costSec : 0;
|
|
5650
|
+
continue;
|
|
5651
|
+
}
|
|
5652
|
+
if (member.transcodeVideo !== true) {
|
|
5653
|
+
const observed = this.#observedCopyCost.get(`${member.sourceKey}:${member.fileIndex}`);
|
|
5654
|
+
cost += observed && observed.costSec > 0 ? observed.costSec : 0;
|
|
5655
|
+
continue;
|
|
5656
|
+
}
|
|
5657
|
+
// A picture being RE-ENCODED beside the rung being judged — the warm-up
|
|
5658
|
+
// that makes a quality switch seamless is two encoders by design, and
|
|
5659
|
+
// that overlap is exactly where the field measured 0.504x on a rung
|
|
5660
|
+
// predicted at 1.58x (2026-08-15). Priced by what it has been SEEN doing
|
|
5661
|
+
// when it had the machine to itself, and otherwise by the same model that
|
|
5662
|
+
// judges every rung — which is a prediction, not a guess.
|
|
5663
|
+
cost += this.#pictureCostOf(member);
|
|
5162
5664
|
}
|
|
5163
5665
|
// And what the FILE costs simply by being fetched and delivered while it is
|
|
5164
5666
|
// watched: a viewer consumes it at its own byte rate, and every one of
|
|
@@ -5213,6 +5715,7 @@ export class HlsSessionManager {
|
|
|
5213
5715
|
transcodeVideo,
|
|
5214
5716
|
observedDecodeCostSec = null,
|
|
5215
5717
|
concurrentCostSec = 0,
|
|
5718
|
+
runningCostByHeight = null,
|
|
5216
5719
|
measuredHeights = null
|
|
5217
5720
|
}) {
|
|
5218
5721
|
const benchmark = this.softwarePresetBenchmark;
|
|
@@ -5254,13 +5757,24 @@ export class HlsSessionManager {
|
|
|
5254
5757
|
continue;
|
|
5255
5758
|
}
|
|
5256
5759
|
const width = Math.round(((sourceWidth / sourceHeight) * height) / 2) * 2;
|
|
5760
|
+
// What the machine is spending on everything EXCEPT this height. A step
|
|
5761
|
+
// being warmed for a switch is already running while it is judged, so its
|
|
5762
|
+
// own cost is inside the committed total — and charged against itself it
|
|
5763
|
+
// is counted twice. Measured against the field figures of 2026-08-15
|
|
5764
|
+
// that is 1.83x against 1.03x: below the margin, so the step the viewer
|
|
5765
|
+
// had just asked for was dropped from the offer by the act of warming it,
|
|
5766
|
+
// and its next segment answered 404 on a stream that was playing.
|
|
5767
|
+
const concurrentBesideThis = Math.max(
|
|
5768
|
+
0,
|
|
5769
|
+
concurrentCostSec - (runningCostByHeight?.get(height) ?? 0)
|
|
5770
|
+
);
|
|
5257
5771
|
const { speed, sustainable } = canSustainOutput({
|
|
5258
5772
|
benchmark,
|
|
5259
5773
|
decodeModel: this.decodeCostModel,
|
|
5260
5774
|
source,
|
|
5261
5775
|
outputPixelsPerSec: width * height * fps,
|
|
5262
5776
|
observedDecodeCostSec,
|
|
5263
|
-
concurrentCostSec
|
|
5777
|
+
concurrentCostSec: concurrentBesideThis
|
|
5264
5778
|
});
|
|
5265
5779
|
if (sustainable) {
|
|
5266
5780
|
kept.push(height);
|
|
@@ -5268,11 +5782,21 @@ export class HlsSessionManager {
|
|
|
5268
5782
|
}
|
|
5269
5783
|
dropped.push(`${height}p=${speed === null ? "n/a" : `${speed.toFixed(2)}x`}`);
|
|
5270
5784
|
}
|
|
5785
|
+
// Written when the ANSWER changes, not when the answer is recomputed. This
|
|
5786
|
+
// is asked on the path that serves every playlist, init and segment, and
|
|
5787
|
+
// the figures behind it move every five seconds — so an unconditional line
|
|
5788
|
+
// here is roughly seven hundred identical lines an hour into a forwarder
|
|
5789
|
+
// that holds five hundred, which buries whatever is worth reading.
|
|
5271
5790
|
if (dropped.length > 0) {
|
|
5272
|
-
|
|
5791
|
+
const line =
|
|
5273
5792
|
`transcode: not offering ${dropped.join(" ")} — below realtime × ${REALTIME_SPEED_MARGIN} ` +
|
|
5274
|
-
|
|
5275
|
-
)
|
|
5793
|
+
`(offering ${kept.map((height) => `${height}p`).join(" ")})`;
|
|
5794
|
+
if (line !== this.#lastOfferLine) {
|
|
5795
|
+
this.#lastOfferLine = line;
|
|
5796
|
+
logger.info(line);
|
|
5797
|
+
}
|
|
5798
|
+
} else {
|
|
5799
|
+
this.#lastOfferLine = "";
|
|
5276
5800
|
}
|
|
5277
5801
|
return kept;
|
|
5278
5802
|
}
|
|
@@ -5415,7 +5939,12 @@ export class HlsSessionManager {
|
|
|
5415
5939
|
// Cleared BEFORE the signal: the exit handler checks identity against this
|
|
5416
5940
|
// field, so a deliberate stop must not read as a run that failed.
|
|
5417
5941
|
session.ffmpeg = null;
|
|
5942
|
+
// Only a run that was still going is being STOPPED. The handle outlives the
|
|
5943
|
+
// process — nothing nulls it when a run ends — so a rung that had already
|
|
5944
|
+
// finished or failed reaches here too, and calling that a stop would erase
|
|
5945
|
+
// how it actually ended.
|
|
5418
5946
|
if (!hasChildExited(ffmpeg)) {
|
|
5947
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.STOP_ORDERED);
|
|
5419
5948
|
try {
|
|
5420
5949
|
ffmpeg.kill("SIGTERM");
|
|
5421
5950
|
} catch {
|
|
@@ -6435,6 +6964,7 @@ export class HlsSessionManager {
|
|
|
6435
6964
|
startSeconds: trueStart ?? declaredStart,
|
|
6436
6965
|
initBytes: session.initBytes ?? null
|
|
6437
6966
|
});
|
|
6967
|
+
this.#noteRunProducedSegment(session, filePath);
|
|
6438
6968
|
return {
|
|
6439
6969
|
kind: "file",
|
|
6440
6970
|
stream: Readable.from([prepared]),
|
|
@@ -6442,6 +6972,9 @@ export class HlsSessionManager {
|
|
|
6442
6972
|
isPlaylist: false
|
|
6443
6973
|
};
|
|
6444
6974
|
}
|
|
6975
|
+
if (!isPlaylist) {
|
|
6976
|
+
this.#noteRunProducedSegment(session, filePath);
|
|
6977
|
+
}
|
|
6445
6978
|
return {
|
|
6446
6979
|
kind: "file",
|
|
6447
6980
|
stream: isPlaylist
|
|
@@ -6855,7 +7388,16 @@ export class HlsSessionManager {
|
|
|
6855
7388
|
}
|
|
6856
7389
|
|
|
6857
7390
|
this.#resumeEncoder(session, "session disposed");
|
|
6858
|
-
|
|
7391
|
+
// Whether the process is still RUNNING, not whether anyone has called kill
|
|
7392
|
+
// on it: `.killed` means only that a signal was sent, and a run that ended
|
|
7393
|
+
// by itself — the file watched through, or a failure — was never killed at
|
|
7394
|
+
// all. Asked the old way, every idle session on disposal signalled a dead
|
|
7395
|
+
// pid and claimed to be stopping a run that had already ended.
|
|
7396
|
+
if (session.ffmpeg && !hasChildExited(session.ffmpeg)) {
|
|
7397
|
+
// The run ends with the session, and the state must say so: left where it
|
|
7398
|
+
// was, it would go on claiming a process that can be signalled and an
|
|
7399
|
+
// input that is being read, about a session that no longer exists.
|
|
7400
|
+
this.#transitionRun(session, ENCODE_RUN_EVENT.STOP_ORDERED);
|
|
6859
7401
|
session.ffmpeg.kill("SIGTERM");
|
|
6860
7402
|
await waitForChildExit(session.ffmpeg);
|
|
6861
7403
|
}
|