@torrent-tv/proxy 2.76.6 → 2.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1558 -1538
- package/bin/cli.js +14 -1
- package/package.json +1 -1
- package/services/data-channel-handler.js +83 -3
- package/services/encode/DemandMap.js +187 -0
- package/services/encode/EncodePlan.js +283 -272
- package/services/encode/SegmentDemand.js +0 -0
- package/services/hls-session-manager.js +476 -179
- package/services/orchestrators/EncodeOrchestrator.js +13 -3
- package/services/viewer/Viewer.js +214 -145
- package/services/viewer/Viewers.js +213 -124
- package/test/behind-head-repair.test.js +2 -2
- package/test/demand-map.test.js +102 -0
- package/test/encode-plan.test.js +44 -2
- package/test/failed-start-breaker.test.js +151 -0
- package/test/orchestrator-wired.test.js +190 -164
- package/test/quality-variants.test.js +2 -2
- package/test/run-intervals.test.js +67 -303
- package/test/seek-target-not-superseded.test.js +2 -2
- package/test/sidecar-naming.test.js +120 -120
- package/test/stale-request-after-seek.test.js +2 -2
- package/test/two-viewers-one-picture.test.js +2 -2
- package/test/viewer-outputs.test.js +7 -7
- package/test/viewer-presence.test.js +119 -0
- package/test/viewer.test.js +27 -10
|
@@ -23,6 +23,7 @@ import { speedFromReadings } from "./encoder-readings.js";
|
|
|
23
23
|
import { availableShareFrom } from "./available-share.js";
|
|
24
24
|
import { contentionPenalty } from "./contention.js";
|
|
25
25
|
import { minimumBufferFrom } from "./supply-margin.js";
|
|
26
|
+
import { mapForViewer } from "./encode/DemandMap.js";
|
|
26
27
|
import { baseDrawFrom, costPerMegabyteFrom } from "./torrent-cost.js";
|
|
27
28
|
import { medianOf, movedBeyondScatter, scatterOf } from "./learned-median.js";
|
|
28
29
|
import {
|
|
@@ -70,7 +71,6 @@ import { SourceFiles, sourceDecodeCharacteristics } from "./source/SourceFile.js
|
|
|
70
71
|
import { ProducedIndex } from "./produced-index.js";
|
|
71
72
|
import { SegmentStore } from "./encode/SegmentStore.js";
|
|
72
73
|
import { EncodeCost } from "./quality/EncodeCost.js";
|
|
73
|
-
import { endOfRun } from "./encode/EncodeRun.js";
|
|
74
74
|
import {
|
|
75
75
|
buildRunCommand,
|
|
76
76
|
ffmpegSeconds,
|
|
@@ -411,6 +411,22 @@ export function variantConsumerId(baseSessionId) {
|
|
|
411
411
|
return `variant-of:${baseSessionId}`;
|
|
412
412
|
}
|
|
413
413
|
|
|
414
|
+
/**
|
|
415
|
+
* Whether this name belongs to a person or to the family bookkeeping.
|
|
416
|
+
*
|
|
417
|
+
* An output made on behalf of a picture — a quality step, a soundtrack — is
|
|
418
|
+
* created under a made-up name so that the picture ending can let it go. That
|
|
419
|
+
* name is not somebody watching, and it must not enter the viewer registry: a
|
|
420
|
+
* viewer is placed the moment they arrive and counts as present until something
|
|
421
|
+
* says otherwise, so a made-up one would keep its output producing for ever.
|
|
422
|
+
*
|
|
423
|
+
* @param {string} consumerId
|
|
424
|
+
* @returns {boolean}
|
|
425
|
+
*/
|
|
426
|
+
export function isFamilyConsumerId(consumerId) {
|
|
427
|
+
return typeof consumerId === "string" && consumerId.startsWith("variant-of:");
|
|
428
|
+
}
|
|
429
|
+
|
|
414
430
|
const CLEANUP_INTERVAL_MS = 30_000;
|
|
415
431
|
|
|
416
432
|
// How long a session waits for the file's keyframe table before giving up on
|
|
@@ -482,7 +498,6 @@ const CUSHION_REPORT_MS = 30_000;
|
|
|
482
498
|
// is sent every 10 s, and a seek in between moves them somewhere this cannot
|
|
483
499
|
// predict — so anything older is treated as no report at all.
|
|
484
500
|
const NET_REPORT_FRESH_MS = 15_000;
|
|
485
|
-
const LOOKAHEAD_RESUME_SECONDS = 60;
|
|
486
501
|
// Seek debounce. A far (out-of-window) segment request is a server-side seek.
|
|
487
502
|
// Rather than restart ffmpeg on the first one, wait a short quiet period:
|
|
488
503
|
// further far requests re-arm it and update the target to the latest index, so
|
|
@@ -557,7 +572,7 @@ const ENCODE_RUN_TERMINATE_GRACE_MS = 2_000;
|
|
|
557
572
|
// the seek/open step itself (container demux error, bad audio frame boundary,
|
|
558
573
|
// etc.), not mid-stream. Used to tell a genuine seek failure apart from a
|
|
559
574
|
// later, unrelated crash so the circuit breaker below only counts the former.
|
|
560
|
-
const
|
|
575
|
+
const START_FAST_FAIL_MS = 2_000;
|
|
561
576
|
// Circuit breaker: consecutive fast failures AT THE SAME target before we stop
|
|
562
577
|
// auto-retrying and leave the session in its terminal "failed" state (surfaced
|
|
563
578
|
// to the client as a clean, retryable error) instead of looping forever. The
|
|
@@ -565,7 +580,7 @@ const SEEK_FAST_FAIL_MS = 2_000;
|
|
|
565
580
|
// mode (an unreliable container-computed seek position); this is a safety net
|
|
566
581
|
// for whatever residual case still fails — not a second competing "fix" that
|
|
567
582
|
// blindly retries the identical command hoping for a different result.
|
|
568
|
-
const
|
|
583
|
+
const MAX_FAILED_STARTS = 3;
|
|
569
584
|
// A run that lost its INPUT is retried rather than condemned: the torrent can
|
|
570
585
|
// be added again and the pieces downloaded again, so the data being gone is a
|
|
571
586
|
// wait, not a verdict. Backed off so a source that is truly unavailable costs a
|
|
@@ -1452,9 +1467,9 @@ function isWarmupTimeoutError(error) {
|
|
|
1452
1467
|
* @property {number[] | null} keyframeTimes - Real source keyframe times
|
|
1453
1468
|
* (sorted seconds), or null when the probe failed/timed out. Used to snap a
|
|
1454
1469
|
* source seek onto a known-valid position (see #startEncodeRun).
|
|
1455
|
-
* @property {number}
|
|
1456
|
-
* failure, for the consecutive-failure circuit breaker (see
|
|
1457
|
-
* @property {number}
|
|
1470
|
+
* @property {number} failedStartAt - Segment index of the last fast seek
|
|
1471
|
+
* failure, for the consecutive-failure circuit breaker (see MAX_FAILED_STARTS).
|
|
1472
|
+
* @property {number} failedStartCount - Consecutive fast failures at failedStartAt.
|
|
1458
1473
|
*/
|
|
1459
1474
|
|
|
1460
1475
|
/**
|
|
@@ -1494,6 +1509,13 @@ export class HlsSessionManager {
|
|
|
1494
1509
|
*
|
|
1495
1510
|
* @type {number[]}
|
|
1496
1511
|
*/
|
|
1512
|
+
/**
|
|
1513
|
+
* Whether a re-decision of what encoders should exist is already queued for
|
|
1514
|
+
* the end of this turn. See `planEncodersSoon`.
|
|
1515
|
+
* @type {boolean}
|
|
1516
|
+
*/
|
|
1517
|
+
#planScheduled = false;
|
|
1518
|
+
|
|
1497
1519
|
#firstSegmentLatencies = [];
|
|
1498
1520
|
|
|
1499
1521
|
/**
|
|
@@ -1671,7 +1693,11 @@ export class HlsSessionManager {
|
|
|
1671
1693
|
// person per output. What a viewer chose, where they are and which outputs
|
|
1672
1694
|
// they are watching are facts about the person; kept per output they were
|
|
1673
1695
|
// three copies of which two were always stale.
|
|
1674
|
-
|
|
1696
|
+
// Every change to who is watching what re-decides which encoders should
|
|
1697
|
+
// exist, because that decision reads nothing else about viewers. It used to
|
|
1698
|
+
// be re-taken on a five-second timer instead, which made a just-created
|
|
1699
|
+
// output wait up to five seconds before anything noticed it had a viewer.
|
|
1700
|
+
this.viewers = new Viewers({ onChange: () => this.planEncodersSoon() });
|
|
1675
1701
|
// Which outputs of one file exist right now, and what each of them is: the
|
|
1676
1702
|
// picture a step belongs to, the steps, the soundtracks, the height a
|
|
1677
1703
|
// session is named by. Read-only over the register above, and the layer the
|
|
@@ -1753,7 +1779,7 @@ export class HlsSessionManager {
|
|
|
1753
1779
|
// benchmark (the only path that can pick/step resolution). Cheap no-op scan
|
|
1754
1780
|
// otherwise.
|
|
1755
1781
|
this.budgetTimer = setInterval(() => {
|
|
1756
|
-
this.#
|
|
1782
|
+
this.#reportCushions();
|
|
1757
1783
|
void this.runQualityBudgetOnce();
|
|
1758
1784
|
}, BUDGET_CHECK_INTERVAL_MS);
|
|
1759
1785
|
this.budgetTimer.unref();
|
|
@@ -1949,10 +1975,18 @@ export class HlsSessionManager {
|
|
|
1949
1975
|
// joining knows nothing about: they may have chosen another language,
|
|
1950
1976
|
// and their browser may need a track re-encoded that the first
|
|
1951
1977
|
// viewer's could decode as it stands.
|
|
1952
|
-
this.viewers.of(existing, consumerId)
|
|
1978
|
+
const joining = this.viewers.of(existing, consumerId);
|
|
1979
|
+
joining.audio = {
|
|
1953
1980
|
trackIndex: normalizedAudioTrack,
|
|
1954
1981
|
transcode: transcodeAudio === true
|
|
1955
1982
|
};
|
|
1983
|
+
// And WHERE they are, which their own request names and this session
|
|
1984
|
+
// cannot guess: a viewer joining a session already playing at 40:00
|
|
1985
|
+
// may be opening the film from a link that carries 05:00. Placed now,
|
|
1986
|
+
// because a viewer who has not yet been placed states no want and an
|
|
1987
|
+
// output all of whose viewers state nothing has every encoder on it
|
|
1988
|
+
// stopped.
|
|
1989
|
+
this.#placeViewer(existing, joining, startPositionSeconds);
|
|
1956
1990
|
}
|
|
1957
1991
|
// Reuse said nothing at all before this, so a session serving two
|
|
1958
1992
|
// viewers looked exactly like a session serving one — and the whole
|
|
@@ -2630,7 +2664,7 @@ export class HlsSessionManager {
|
|
|
2630
2664
|
waitEpoch: 0,
|
|
2631
2665
|
// Highest segment the viewer has actually asked for, and whether the
|
|
2632
2666
|
// encoder is currently suspended for running too far past it.
|
|
2633
|
-
// See #
|
|
2667
|
+
// See #reportCushions. With several viewers on one session it is the
|
|
2634
2668
|
// FURTHEST of them, derived from the viewers below.
|
|
2635
2669
|
lastRequestedSegment: null,
|
|
2636
2670
|
// Where each viewer of this session is, separately: the segment they last
|
|
@@ -2650,12 +2684,12 @@ export class HlsSessionManager {
|
|
|
2650
2684
|
viewers: new Map(),
|
|
2651
2685
|
encoderPauseUnsupported: false,
|
|
2652
2686
|
seekFirstFarAt: 0,
|
|
2653
|
-
// Circuit breaker: consecutive FAST failures (see
|
|
2654
|
-
//
|
|
2687
|
+
// Circuit breaker: consecutive FAST failures (see START_FAST_FAIL_MS) at
|
|
2688
|
+
// failedStartAt. Reset whenever a run starts at a DIFFERENT target or
|
|
2655
2689
|
// survives past the fast-fail window. See the exit handler in
|
|
2656
|
-
// the run ending handler and
|
|
2657
|
-
|
|
2658
|
-
|
|
2690
|
+
// the run ending handler and MAX_FAILED_STARTS.
|
|
2691
|
+
failedStartAt: -1,
|
|
2692
|
+
failedStartCount: 0,
|
|
2659
2693
|
progress: {
|
|
2660
2694
|
// No `state` here. What the browser is told is `wireState(runState)`,
|
|
2661
2695
|
// computed where it is sent — a Moore output rather than a field
|
|
@@ -2707,12 +2741,21 @@ export class HlsSessionManager {
|
|
|
2707
2741
|
}
|
|
2708
2742
|
// The viewer who asked for this session, so a browser that names itself
|
|
2709
2743
|
// never has to have requested a segment first for its own soundtrack choice
|
|
2710
|
-
// to be known
|
|
2711
|
-
|
|
2712
|
-
|
|
2744
|
+
// to be known — nor for its own POSITION to be known, which is the same
|
|
2745
|
+
// request's `startPositionSeconds` and is therefore knowledge this process
|
|
2746
|
+
// already has before a single byte is encoded.
|
|
2747
|
+
//
|
|
2748
|
+
// A session made on behalf of the family — a quality step, a soundtrack —
|
|
2749
|
+
// is created under a made-up name, and that name is not a person. It stays
|
|
2750
|
+
// out of the viewer registry: given a position it would count as present
|
|
2751
|
+
// for ever, and nothing would ever stop the output it was created for.
|
|
2752
|
+
if (consumerId && !isFamilyConsumerId(consumerId)) {
|
|
2753
|
+
const first = this.viewers.of(session, consumerId);
|
|
2754
|
+
first.audio = {
|
|
2713
2755
|
trackIndex: normalizedAudioTrack,
|
|
2714
2756
|
transcode: transcodeAudio === true
|
|
2715
2757
|
};
|
|
2758
|
+
this.#placeViewer(session, first, startPositionSeconds);
|
|
2716
2759
|
}
|
|
2717
2760
|
this.sessionsById.set(sessionId, session);
|
|
2718
2761
|
this.sessionIdBySource.set(sourceMapKey, sessionId);
|
|
@@ -3353,23 +3396,18 @@ export class HlsSessionManager {
|
|
|
3353
3396
|
Number.isFinite(positionSeconds) && positionSeconds >= 0 ? positionSeconds : null,
|
|
3354
3397
|
at: now
|
|
3355
3398
|
};
|
|
3356
|
-
// A
|
|
3357
|
-
//
|
|
3358
|
-
//
|
|
3359
|
-
|
|
3399
|
+
// A stale reading must not go on deciding for the viewers still here: a
|
|
3400
|
+
// report describes a link at a moment, and a viewer who seeked since then
|
|
3401
|
+
// is somewhere else entirely.
|
|
3402
|
+
//
|
|
3403
|
+
// Only the READING expires. Whether the person is still watching is a
|
|
3404
|
+
// different question with its own answer — their connection — and a viewer
|
|
3405
|
+
// who has simply stopped reporting is not thereby gone. Answering both from
|
|
3406
|
+
// this one place is what stopped a soundtrack's encoder on 2026-09-05.
|
|
3407
|
+
for (const viewer of viewersOf(session).values()) {
|
|
3360
3408
|
const report = viewer.netReport;
|
|
3361
|
-
if (report
|
|
3362
|
-
continue;
|
|
3363
|
-
}
|
|
3364
|
-
if (now - report.at > LINK_REPORT_FRESH_MS) {
|
|
3409
|
+
if (report !== null && now - report.at > LINK_REPORT_FRESH_MS) {
|
|
3365
3410
|
viewer.netReport = null;
|
|
3366
|
-
// A viewer with nothing left to say about themselves is a viewer this
|
|
3367
|
-
// session has not met: one object goes, where six maps each had to be
|
|
3368
|
-
// emptied and none of them was. Through the one exit, so that what they
|
|
3369
|
-
// had claimed of production goes with them.
|
|
3370
|
-
if (viewer.head === null && viewer.activeVariantId === null) {
|
|
3371
|
-
this.#viewerLeaves(session, key);
|
|
3372
|
-
}
|
|
3373
3411
|
}
|
|
3374
3412
|
}
|
|
3375
3413
|
return true;
|
|
@@ -3406,25 +3444,23 @@ export class HlsSessionManager {
|
|
|
3406
3444
|
// where they are, and the two answer different questions — see
|
|
3407
3445
|
// `viewerPositionSource`. Only a seek writes it, so a request does not erase
|
|
3408
3446
|
// it.
|
|
3409
|
-
const viewer = this.viewers.of(session, consumerId);
|
|
3410
|
-
const seeked = viewer.
|
|
3411
|
-
viewer.
|
|
3412
|
-
|
|
3447
|
+
const viewer = this.viewers.of(session, consumerId, now);
|
|
3448
|
+
const seeked = viewer.position?.seeked ?? null;
|
|
3449
|
+
viewer.position = { segment, seconds, at: now, seeked };
|
|
3450
|
+
this.planEncodersSoon();
|
|
3451
|
+
const staleAfterMs = this.presenceStaleAfterMs();
|
|
3413
3452
|
let furthest = { segment, seconds };
|
|
3414
3453
|
for (const [key, other] of heads) {
|
|
3415
|
-
if (other.
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
//
|
|
3420
|
-
// One object goes, where six parallel maps each had to be remembered —
|
|
3421
|
-
// and it goes through the one exit, which also releases what they had
|
|
3422
|
-
// claimed of production.
|
|
3454
|
+
if (!other.isPresent(now, staleAfterMs)) {
|
|
3455
|
+
// Nothing has been heard from them for longer than any silence a
|
|
3456
|
+
// watching viewer can produce — not merely longer than a segment. They
|
|
3457
|
+
// go through the one exit, which also releases what they had claimed of
|
|
3458
|
+
// production.
|
|
3423
3459
|
this.#viewerLeaves(session, key);
|
|
3424
3460
|
continue;
|
|
3425
3461
|
}
|
|
3426
|
-
if (other.
|
|
3427
|
-
furthest = { segment: other.
|
|
3462
|
+
if (other.position !== null && other.position.segment > furthest.segment) {
|
|
3463
|
+
furthest = { segment: other.position.segment, seconds: other.position.seconds };
|
|
3428
3464
|
}
|
|
3429
3465
|
}
|
|
3430
3466
|
return furthest;
|
|
@@ -3443,7 +3479,7 @@ export class HlsSessionManager {
|
|
|
3443
3479
|
if (!consumerId) {
|
|
3444
3480
|
return null;
|
|
3445
3481
|
}
|
|
3446
|
-
const head = session.viewers?.get(consumerId)?.
|
|
3482
|
+
const head = session.viewers?.get(consumerId)?.position;
|
|
3447
3483
|
return head && Number.isFinite(head.seconds) ? head.seconds : null;
|
|
3448
3484
|
}
|
|
3449
3485
|
|
|
@@ -3735,11 +3771,175 @@ export class HlsSessionManager {
|
|
|
3735
3771
|
return readers;
|
|
3736
3772
|
}
|
|
3737
3773
|
|
|
3738
|
-
|
|
3739
|
-
|
|
3774
|
+
/**
|
|
3775
|
+
* One viewer's demand map, translated into this output's segment numbers.
|
|
3776
|
+
*
|
|
3777
|
+
* The map itself is seconds of film and knows nothing about cut grids
|
|
3778
|
+
* (`services/encode/DemandMap.js`). The translation is this output's own
|
|
3779
|
+
* business, and it is exact: the timeline holds the boundaries.
|
|
3780
|
+
*
|
|
3781
|
+
* Two measurements feed it, and neither is chosen here:
|
|
3782
|
+
*
|
|
3783
|
+
* 1. the allowance below which an interruption reaches the viewer, from this
|
|
3784
|
+
* file's own recent interruptions (`minimumBufferFrom`). Null until the
|
|
3785
|
+
* reader has seen two of them, and then only the segment itself counts;
|
|
3786
|
+
* 2. how fast this machine encodes THIS track, from ffmpeg's own progress.
|
|
3787
|
+
* Below realtime it decides how much has to exist before playback starts;
|
|
3788
|
+
* above it, nothing beyond the allowance is needed.
|
|
3789
|
+
*
|
|
3790
|
+
* @param {object} session
|
|
3791
|
+
* @param {number} atSegment - Where the viewer is.
|
|
3792
|
+
* @returns {{from: number, to: number, priority: number}[]} In segment
|
|
3793
|
+
* numbers, both ends inclusive.
|
|
3794
|
+
*/
|
|
3795
|
+
#demandZonesFor(session, atSegment) {
|
|
3796
|
+
const boundaries = session.timeline?.boundaries ?? [];
|
|
3797
|
+
const segmentCount = Number(session.timeline?.segmentCount) || 0;
|
|
3798
|
+
if (segmentCount <= 0) {
|
|
3799
|
+
// No playlist yet: the only thing that can be said is that they want
|
|
3800
|
+
// where they are.
|
|
3801
|
+
return [{ from: atSegment, to: atSegment, priority: 3 }];
|
|
3802
|
+
}
|
|
3803
|
+
const durationSeconds = Number(boundaries[boundaries.length - 1]) ||
|
|
3804
|
+
segmentCount * this.segmentDurationSec;
|
|
3805
|
+
const zones = mapForViewer({
|
|
3806
|
+
atSeconds: this.#segmentStartTime(session, atSegment),
|
|
3807
|
+
durationSeconds,
|
|
3808
|
+
allowanceSeconds: minimumBufferFrom({
|
|
3809
|
+
segmentSeconds: this.segmentDurationSec,
|
|
3810
|
+
worstSupplyWaitSec: session.supplyFigures?.worstWaitSec
|
|
3811
|
+
})?.seconds ?? this.segmentDurationSec,
|
|
3812
|
+
// The slope between two progress reports, which is what this class
|
|
3813
|
+
// measures everywhere else it prices an encode. `progress.speed` is
|
|
3814
|
+
// ffmpeg's own cumulative figure and includes the run's start, so it
|
|
3815
|
+
// reads low for the first seconds of every run; `recentSpeed` is the
|
|
3816
|
+
// reading taken over a window and is the one the budget already trusts.
|
|
3817
|
+
encodeSpeedX: Number(session.recentSpeed?.speed) || Number(session.progress?.speed) || 0
|
|
3818
|
+
});
|
|
3819
|
+
/** @type {{from: number, to: number, priority: number}[]} */
|
|
3820
|
+
const inSegments = [];
|
|
3821
|
+
for (const zone of zones) {
|
|
3822
|
+
const from = Math.max(atSegment, this.#segmentIndexForTime(session, zone.from));
|
|
3823
|
+
const to = Math.min(segmentCount - 1, this.#segmentIndexForTime(session, zone.to));
|
|
3824
|
+
if (to >= from) {
|
|
3825
|
+
inSegments.push({ from, to, priority: zone.priority });
|
|
3826
|
+
}
|
|
3827
|
+
}
|
|
3828
|
+
return inSegments.length > 0
|
|
3829
|
+
? inSegments
|
|
3830
|
+
: [{ from: atSegment, to: atSegment, priority: 3 }];
|
|
3831
|
+
}
|
|
3832
|
+
|
|
3833
|
+
/**
|
|
3834
|
+
* Say what the cushion is, for every session.
|
|
3835
|
+
*
|
|
3836
|
+
* This is all that is left of `#enforceLookAhead`, which also SUSPENDED a run
|
|
3837
|
+
* once it was `LOOKAHEAD_PAUSE_SECONDS` in front of the viewer and woke it at
|
|
3838
|
+
* `LOOKAHEAD_RESUME_SECONDS` — two chosen numbers, and a second authority
|
|
3839
|
+
* over the encoders beside the plan. The two contradicted each other
|
|
3840
|
+
* directly: this one deliberately pushed a run past the window the plan was
|
|
3841
|
+
* asking about, and the plan then killed it for standing there. Measured in
|
|
3842
|
+
* the field 2026-09-05, 350-700ms per cycle, the viewer's picture stopped for
|
|
3843
|
+
* 125 seconds.
|
|
3844
|
+
*
|
|
3845
|
+
* How far ahead a run may get is now a question for the plan alone, which
|
|
3846
|
+
* answers it from the demand map. What remains here is a READING — how much
|
|
3847
|
+
* film is ready in front of the earliest viewer — and a reading commands
|
|
3848
|
+
* nothing.
|
|
3849
|
+
*/
|
|
3850
|
+
#reportCushions() {
|
|
3740
3851
|
for (const session of this.sessionsById.values()) {
|
|
3741
|
-
this.#
|
|
3852
|
+
this.#reportCushionFor(session);
|
|
3853
|
+
}
|
|
3854
|
+
}
|
|
3855
|
+
|
|
3856
|
+
/**
|
|
3857
|
+
* Put a viewer where their own request says they are.
|
|
3858
|
+
*
|
|
3859
|
+
* A viewer arrives by asking for a POSITION — zero, or the time an address
|
|
3860
|
+
* bar carried — so "we do not know where they are" is not a state a viewer
|
|
3861
|
+
* can be in. Before 2026-09-05 it was: position was written only by a segment
|
|
3862
|
+
* request, so a viewer counted as placeless until they had asked for a
|
|
3863
|
+
* segment, and an output whose viewers were all placeless had every encoder
|
|
3864
|
+
* on it stopped for having nobody. The soundtrack that failed that day could
|
|
3865
|
+
* not have asked: the segment it would have asked for needed an `init.mp4`
|
|
3866
|
+
* that the stopped encoder was going to make.
|
|
3867
|
+
*
|
|
3868
|
+
* Only ever places a viewer who has none. A viewer already placed is being
|
|
3869
|
+
* kept current by their own requests and seeks, and a fresh create request
|
|
3870
|
+
* carries a default of zero that must not drag them back to the beginning.
|
|
3871
|
+
*
|
|
3872
|
+
* @param {HlsSession} session
|
|
3873
|
+
* @param {import("./viewer/Viewer.js").Viewer} viewer
|
|
3874
|
+
* @param {number} positionSeconds
|
|
3875
|
+
* @returns {void}
|
|
3876
|
+
*/
|
|
3877
|
+
#placeViewer(session, viewer, positionSeconds) {
|
|
3878
|
+
if (viewer.position !== null) {
|
|
3879
|
+
return;
|
|
3880
|
+
}
|
|
3881
|
+
const seconds = Number.isFinite(positionSeconds) && positionSeconds > 0 ? positionSeconds : 0;
|
|
3882
|
+
let segment = 0;
|
|
3883
|
+
try {
|
|
3884
|
+
const index = this.#segmentIndexForTime(session, seconds);
|
|
3885
|
+
if (Number.isInteger(index) && index >= 0) {
|
|
3886
|
+
segment = index;
|
|
3887
|
+
}
|
|
3888
|
+
} catch {
|
|
3889
|
+
// A session whose cut table is not built yet places its viewer at the
|
|
3890
|
+
// beginning, which is where the run starts anyway.
|
|
3891
|
+
}
|
|
3892
|
+
viewer.position = { segment, seconds, at: Date.now(), seeked: null };
|
|
3893
|
+
this.planEncodersSoon();
|
|
3894
|
+
}
|
|
3895
|
+
|
|
3896
|
+
/**
|
|
3897
|
+
* How long nothing may be heard from a viewer before this process concludes
|
|
3898
|
+
* they are gone.
|
|
3899
|
+
*
|
|
3900
|
+
* A backstop and nothing more. A viewer leaves by SAYING so — the browser
|
|
3901
|
+
* releases the session, or their connection closes — and this covers only the
|
|
3902
|
+
* case where nothing said it: a data channel's close event does not always
|
|
3903
|
+
* come, and a transport that is not a data channel may have nothing to say at
|
|
3904
|
+
* all.
|
|
3905
|
+
*
|
|
3906
|
+
* The figure is the proxy's own cushion plus a segment, which is the longest
|
|
3907
|
+
* silence a watching viewer can produce: one holding a full cushion asks for
|
|
3908
|
+
* nothing until it has drained, and one playing asks once a segment.
|
|
3909
|
+
*
|
|
3910
|
+
* @returns {number}
|
|
3911
|
+
*/
|
|
3912
|
+
presenceStaleAfterMs() {
|
|
3913
|
+
return (this.lookaheadSeconds + this.segmentDurationSec) * 1000;
|
|
3914
|
+
}
|
|
3915
|
+
|
|
3916
|
+
/**
|
|
3917
|
+
* Re-decide what encoders should exist, once, after the change that is being
|
|
3918
|
+
* made now.
|
|
3919
|
+
*
|
|
3920
|
+
* COALESCED, NOT DELAYED. One turn of the event loop may carry several
|
|
3921
|
+
* changes — a viewer arrives and is placed and states their soundtrack — and
|
|
3922
|
+
* each of them is a reason to re-decide, while re-deciding three times in a
|
|
3923
|
+
* row would give the same answer three times and could act on a half-built
|
|
3924
|
+
* state. So the decision is taken once, after the current turn, and no
|
|
3925
|
+
* interval is involved: there is nothing to choose and nothing to tune.
|
|
3926
|
+
*
|
|
3927
|
+
* @returns {void}
|
|
3928
|
+
*/
|
|
3929
|
+
planEncodersSoon() {
|
|
3930
|
+
if (this.#planScheduled) {
|
|
3931
|
+
return;
|
|
3742
3932
|
}
|
|
3933
|
+
this.#planScheduled = true;
|
|
3934
|
+
queueMicrotask(() => {
|
|
3935
|
+
this.#planScheduled = false;
|
|
3936
|
+
try {
|
|
3937
|
+
this.planEncodersNow();
|
|
3938
|
+
} catch (error) {
|
|
3939
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3940
|
+
logger.warn(`transcode could not re-plan encoders: ${message}`);
|
|
3941
|
+
}
|
|
3942
|
+
});
|
|
3743
3943
|
}
|
|
3744
3944
|
|
|
3745
3945
|
/**
|
|
@@ -3768,8 +3968,7 @@ export class HlsSessionManager {
|
|
|
3768
3968
|
}
|
|
3769
3969
|
byOutput.set(address, [...(byOutput.get(address) ?? []), session]);
|
|
3770
3970
|
}
|
|
3771
|
-
const staleAfterMs =
|
|
3772
|
-
const lookaheadSegments = Math.ceil(this.lookaheadSeconds / this.segmentDurationSec);
|
|
3971
|
+
const staleAfterMs = this.presenceStaleAfterMs();
|
|
3773
3972
|
const now = Date.now();
|
|
3774
3973
|
for (const [address, sessions] of byOutput) {
|
|
3775
3974
|
const coverage = this.encodeOrchestrator.coverageOf(address);
|
|
@@ -3791,17 +3990,45 @@ export class HlsSessionManager {
|
|
|
3791
3990
|
// What the viewers of this session are waiting for, as spans. A viewer
|
|
3792
3991
|
// wants the segment they are at and the cushion in front of it, which
|
|
3793
3992
|
// is what the encoder is steered by everywhere else in this class.
|
|
3993
|
+
//
|
|
3994
|
+
// PRESENCE AND POSITION ARE ASKED SEPARATELY, and that is the whole of
|
|
3995
|
+
// the 2026-09-05 fix. Presence decides whether this viewer states a
|
|
3996
|
+
// want at all; position decides what the want is. Asked as one question
|
|
3997
|
+
// — which is what a single field written only by segment requests
|
|
3998
|
+
// amounted to — a viewer who had just arrived answered "absent", every
|
|
3999
|
+
// encoder on their output was stopped for having nobody, and the
|
|
4000
|
+
// `init.mp4` they were waiting for in order to request their first
|
|
4001
|
+
// segment was therefore never made.
|
|
3794
4002
|
for (const [consumerId, viewer] of viewersOf(session)) {
|
|
3795
|
-
if (!viewer.
|
|
4003
|
+
if (!viewer.isPresent(now, staleAfterMs)) {
|
|
3796
4004
|
this.encodeOrchestrator.release(`${session.id}:${consumerId}`);
|
|
3797
4005
|
continue;
|
|
3798
4006
|
}
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
4007
|
+
// Placed when they arrived, from the position their own request
|
|
4008
|
+
// named. A viewer with no position is one assembled by hand outside
|
|
4009
|
+
// this class; they want the beginning, which is where an output
|
|
4010
|
+
// starts when nobody says otherwise.
|
|
4011
|
+
const at = viewer.position?.segment ?? 0;
|
|
4012
|
+
// Their own map, in seconds of film, from measurements: how much must
|
|
4013
|
+
// be ready before they set off so that they never stop — the observed
|
|
4014
|
+
// allowance for this file plus what an encoder at THIS machine's
|
|
4015
|
+
// measured speed will fail to deliver in time — then what it reaches
|
|
4016
|
+
// while they watch that, then the rest of the track. No constant is
|
|
4017
|
+
// consulted: the 120 seconds that used to size this window were the
|
|
4018
|
+
// suspended encoder's threshold, one chosen number answering seven
|
|
4019
|
+
// different questions.
|
|
4020
|
+
for (const zone of this.#demandZonesFor(session, at)) {
|
|
4021
|
+
this.encodeOrchestrator.want({
|
|
4022
|
+
// The claimant is the PERSON, without the priority in it: their
|
|
4023
|
+
// zones are separate windows, but they leave together, and
|
|
4024
|
+
// `release` matches on this name.
|
|
4025
|
+
claimant: `${session.id}:${consumerId}`,
|
|
4026
|
+
address,
|
|
4027
|
+
from: zone.from,
|
|
4028
|
+
to: zone.to,
|
|
4029
|
+
priority: zone.priority
|
|
4030
|
+
});
|
|
4031
|
+
}
|
|
3805
4032
|
}
|
|
3806
4033
|
}
|
|
3807
4034
|
}
|
|
@@ -3832,6 +4059,26 @@ export class HlsSessionManager {
|
|
|
3832
4059
|
if (!base) {
|
|
3833
4060
|
return null;
|
|
3834
4061
|
}
|
|
4062
|
+
// A position that has already failed to start, this many times running,
|
|
4063
|
+
// will fail again: nothing about it has changed between the attempts, which
|
|
4064
|
+
// is exactly why the attempts keep taking the same fraction of a second and
|
|
4065
|
+
// ending the same way.
|
|
4066
|
+
//
|
|
4067
|
+
// The plan is arithmetic over coverage and demand, and neither of them
|
|
4068
|
+
// knows that a process refused to start — so without asking here, the plan
|
|
4069
|
+
// commands the same start, the run ends, the ended stretch goes back to the
|
|
4070
|
+
// map, and the plan commands it again. Measured 2026-09-05 with ffmpeg
|
|
4071
|
+
// absent: fifty passes of the plan in the time a probe took to notice, as
|
|
4072
|
+
// fast as spawning could fail. The five-second timer this decision used to
|
|
4073
|
+
// sit on hid that, at the price of hiding it in the field too — the loop
|
|
4074
|
+
// seen there ran for sixteen minutes and read as "a restart every five
|
|
4075
|
+
// seconds".
|
|
4076
|
+
//
|
|
4077
|
+
// A DIFFERENT position is unaffected and gets its own budget, and the count
|
|
4078
|
+
// resets the moment a run at this one does real work.
|
|
4079
|
+
if (base.failedStartAt === from && base.failedStartCount >= MAX_FAILED_STARTS) {
|
|
4080
|
+
return null;
|
|
4081
|
+
}
|
|
3835
4082
|
// Answered with nothing straight away and the run started behind it,
|
|
3836
4083
|
// because the plan is arithmetic and must not wait on a spawn.
|
|
3837
4084
|
void this.#startEncodeRun(base, from, undefined, "the plan asked for an encoder here").catch((error) => {
|
|
@@ -3939,7 +4186,7 @@ export class HlsSessionManager {
|
|
|
3939
4186
|
* @param {HlsSession} session
|
|
3940
4187
|
* @returns {void}
|
|
3941
4188
|
*/
|
|
3942
|
-
#
|
|
4189
|
+
#reportCushionFor(session) {
|
|
3943
4190
|
if (!session || session.state === "disposed" || liveRunsOf(session).length === 0) {
|
|
3944
4191
|
return;
|
|
3945
4192
|
}
|
|
@@ -3971,9 +4218,10 @@ export class HlsSessionManager {
|
|
|
3971
4218
|
const reading = this.#contiguousAheadSeconds(session, viewerSegment);
|
|
3972
4219
|
const aheadSeconds = reading === null ? null : reading.seconds;
|
|
3973
4220
|
if (aheadSeconds === null) {
|
|
3974
|
-
// The segment the viewer needs does not exist
|
|
3975
|
-
//
|
|
3976
|
-
|
|
4221
|
+
// The segment the viewer needs does not exist, so there is no cushion to
|
|
4222
|
+
// report. Nothing is commanded here any more: whether an encoder should
|
|
4223
|
+
// be working on it is the plan's question, and it is asked the moment
|
|
4224
|
+
// anything the plan depends on changes.
|
|
3977
4225
|
return;
|
|
3978
4226
|
}
|
|
3979
4227
|
|
|
@@ -4000,23 +4248,6 @@ export class HlsSessionManager {
|
|
|
4000
4248
|
);
|
|
4001
4249
|
}
|
|
4002
4250
|
|
|
4003
|
-
if (runStateOf(session) !== ENCODE_RUN_STATE.SUSPENDED && aheadSeconds > LOOKAHEAD_PAUSE_SECONDS) {
|
|
4004
|
-
// The decision names what it was taken on. Suspending the encoder stops
|
|
4005
|
-
// the only thing that reads the input, so a wrong reading here stops the
|
|
4006
|
-
// download too — measured 2026-08-06: the log said "135s ahead" while
|
|
4007
|
-
// three segments totalling 31 s lay on disk, and neither figure could be
|
|
4008
|
-
// checked against the other because the line carried no evidence. The
|
|
4009
|
-
// directory holds segments from every run this session has had, so which
|
|
4010
|
-
// ones were counted is the whole question.
|
|
4011
|
-
this.#pauseEncoder(
|
|
4012
|
-
session,
|
|
4013
|
-
`${Math.round(aheadSeconds)}s ahead of the viewer ` +
|
|
4014
|
-
`(viewer at #${viewerSegment}, unbroken through #${reading.lastCovered}, ` +
|
|
4015
|
-
`${reading.total} segment file(s) present)`
|
|
4016
|
-
);
|
|
4017
|
-
} else if (aheadSeconds <= LOOKAHEAD_RESUME_SECONDS) {
|
|
4018
|
-
this.#resumeEncoder(session, `${Math.round(aheadSeconds)}s ahead of the viewer`);
|
|
4019
|
-
}
|
|
4020
4251
|
}
|
|
4021
4252
|
|
|
4022
4253
|
/**
|
|
@@ -5382,54 +5613,48 @@ export class HlsSessionManager {
|
|
|
5382
5613
|
return runs;
|
|
5383
5614
|
}
|
|
5384
5615
|
|
|
5385
|
-
|
|
5616
|
+
/**
|
|
5617
|
+
* How far a run starting here may work before it meets somebody else's
|
|
5618
|
+
* material — read off the ONE coverage map, never worked out again here.
|
|
5619
|
+
*
|
|
5620
|
+
* This replaced `planRunInterval`, which was a second authority over the same
|
|
5621
|
+
* question and answered it by different rules: it walked the whole track for
|
|
5622
|
+
* the first free number, MOVED the start there itself, and counted every live
|
|
5623
|
+
* run as claiming up to `head + look-ahead`. Measured in the field
|
|
5624
|
+
* 2026-09-05: the plan commanded a start at #46, this moved it to #78 — the
|
|
5625
|
+
* "run moved forward from #46 to #78" line — and the plan, seeing a run
|
|
5626
|
+
* outside the window it had asked for, killed it as unwanted and commanded
|
|
5627
|
+
* the same start again, 350-700ms per cycle, dozens of times, the viewer's
|
|
5628
|
+
* picture stopped for 125 seconds.
|
|
5629
|
+
*
|
|
5630
|
+
* Where a run STARTS is the plan's decision and arrives here as an argument.
|
|
5631
|
+
* All this answers is where it must stop, and the answer is a fact of the
|
|
5632
|
+
* map: the free stretch from that number on. `-1` means the end of the track,
|
|
5633
|
+
* which is what "no end" is written as everywhere here.
|
|
5634
|
+
*
|
|
5635
|
+
* @param {object} session
|
|
5636
|
+
* @param {number} startIndex
|
|
5637
|
+
* @param {object | null} exceptRun - The run being replaced, whose own claim
|
|
5638
|
+
* is not somebody else's material.
|
|
5639
|
+
* @returns {number} The last number to work through, or `-1` for the end.
|
|
5640
|
+
*/
|
|
5641
|
+
#runEndFrom(session, startIndex, exceptRun = null) {
|
|
5386
5642
|
const key = session.outputKey ?? "";
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
// Nothing to plan against: no address, or no playlist yet. The run keeps
|
|
5390
|
-
// the shape it has always had — start here, no end.
|
|
5391
|
-
return { from: Math.max(0, startIndex), to: -1 };
|
|
5392
|
-
}
|
|
5393
|
-
const ready = new Set(this.segmentStore.provenNumbers(key));
|
|
5394
|
-
const lookaheadSegments = Math.ceil(this.lookaheadSeconds / this.segmentDurationSec);
|
|
5395
|
-
/** @type {{from: number, to: number}[]} */
|
|
5396
|
-
const claims = [];
|
|
5397
|
-
// Every live run of this output, whichever session started it — a session
|
|
5398
|
-
// holds as many as the machine affords, so its OWN runs are claims too. The
|
|
5399
|
-
// one being replaced is not: this is the plan for its replacement.
|
|
5400
|
-
for (const run of this.#runsOnOutput(key)) {
|
|
5401
|
-
if (run === exceptRun) {
|
|
5402
|
-
continue;
|
|
5403
|
-
}
|
|
5404
|
-
// How far this run will ACTUALLY get, which is not the same as how far it
|
|
5405
|
-
// was allowed to go. A run without an end used to claim the whole film,
|
|
5406
|
-
// and a second viewer opening the same film at another place then found
|
|
5407
|
-
// every number taken and got no encoder at all — they would have waited
|
|
5408
|
-
// for the first run to encode its way there, which on a long film is an
|
|
5409
|
-
// hour. What bounds a run in practice is the look-ahead: it is suspended
|
|
5410
|
-
// once it is that far in front of the segment its viewer last asked for,
|
|
5411
|
-
// and past that point it produces nothing until somebody asks. So that is
|
|
5412
|
-
// the honest extent of its claim, and it is a measured figure rather than
|
|
5413
|
-
// a chosen one — the same allowance the browser sizes its cushion from.
|
|
5414
|
-
const willReach = run.head + lookaheadSegments;
|
|
5415
|
-
const allowed = Number.isFinite(endOfRun(run)) ? run.to : lastIndex;
|
|
5416
|
-
claims.push({ from: run.from, to: Math.min(allowed, willReach) });
|
|
5417
|
-
}
|
|
5418
|
-
const takenAt = (index) =>
|
|
5419
|
-
ready.has(index) || claims.some((span) => index >= span.from && index <= span.to);
|
|
5420
|
-
|
|
5421
|
-
let from = Math.max(0, startIndex);
|
|
5422
|
-
while (from <= lastIndex && takenAt(from)) {
|
|
5423
|
-
from += 1;
|
|
5424
|
-
}
|
|
5425
|
-
if (from > lastIndex) {
|
|
5426
|
-
return null;
|
|
5643
|
+
if (!key) {
|
|
5644
|
+
return -1;
|
|
5427
5645
|
}
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5646
|
+
const coverage = this.encodeOrchestrator.coverageOf(key);
|
|
5647
|
+
const segmentCount = Number(session.timeline?.segmentCount) || 0;
|
|
5648
|
+
if (segmentCount > 0) {
|
|
5649
|
+
coverage.setSegmentCount(segmentCount);
|
|
5431
5650
|
}
|
|
5432
|
-
|
|
5651
|
+
coverage.markReadyAll(this.segmentStore.provenNumbers(key));
|
|
5652
|
+
const free = coverage.freeRunFrom(Math.max(0, startIndex), exceptRun);
|
|
5653
|
+
if (!Number.isFinite(free)) {
|
|
5654
|
+
return -1;
|
|
5655
|
+
}
|
|
5656
|
+
const end = Math.max(0, startIndex) + Math.max(1, free) - 1;
|
|
5657
|
+
return segmentCount > 0 && end >= segmentCount - 1 ? -1 : end;
|
|
5433
5658
|
}
|
|
5434
5659
|
|
|
5435
5660
|
async #startEncodeRun(session, startIndex, positionSecondsOverride, because = "a viewer needs it") {
|
|
@@ -5459,21 +5684,11 @@ export class HlsSessionManager {
|
|
|
5459
5684
|
from: Number.isInteger(previousRun?.from) ? previousRun.from : 0,
|
|
5460
5685
|
to: Number.isInteger(previousRun?.to) ? previousRun.to : -1
|
|
5461
5686
|
};
|
|
5462
|
-
|
|
5463
|
-
|
|
5464
|
-
|
|
5465
|
-
|
|
5466
|
-
|
|
5467
|
-
);
|
|
5468
|
-
return;
|
|
5469
|
-
}
|
|
5470
|
-
if (interval.from !== startIndex) {
|
|
5471
|
-
logger.info(
|
|
5472
|
-
`transcode ${session.id} run moved forward from #${startIndex} to #${interval.from}: ` +
|
|
5473
|
-
`what lies between is already made "${session.file.name}"`
|
|
5474
|
-
);
|
|
5475
|
-
}
|
|
5476
|
-
startIndex = interval.from;
|
|
5687
|
+
// WHERE it starts was decided by whoever called, and is not touched here.
|
|
5688
|
+
// Only how far it may work is answered, and it is answered by reading the
|
|
5689
|
+
// one coverage map. Moving the start was the second authority's doing and
|
|
5690
|
+
// is gone with it.
|
|
5691
|
+
const runEnd = this.#runEndFrom(session, startIndex, previousRun);
|
|
5477
5692
|
// The restart backs off a segment or two from what was asked for, so the
|
|
5478
5693
|
// request that prompted it is recorded under a HIGHER index than the run
|
|
5479
5694
|
// starts at. Looking it up by the start index alone found nothing and the
|
|
@@ -5566,7 +5781,7 @@ export class HlsSessionManager {
|
|
|
5566
5781
|
audioSourceTrackIndex: session.audioSourceTrackIndex ?? session.audioTrackIndex ?? 0,
|
|
5567
5782
|
rateCapKbps: session.rateCapKbps ?? null,
|
|
5568
5783
|
startIndex,
|
|
5569
|
-
endIndex:
|
|
5784
|
+
endIndex: runEnd,
|
|
5570
5785
|
positionSecondsOverride,
|
|
5571
5786
|
videoEncoder: this.videoEncoder,
|
|
5572
5787
|
segmentDurationSec: this.segmentDurationSec
|
|
@@ -5600,7 +5815,7 @@ export class HlsSessionManager {
|
|
|
5600
5815
|
address: session.outputKey ?? session.id,
|
|
5601
5816
|
encoder: this.videoEncoder,
|
|
5602
5817
|
from: safeIndex,
|
|
5603
|
-
to:
|
|
5818
|
+
to: runEnd,
|
|
5604
5819
|
buildArgs: () => args,
|
|
5605
5820
|
argsDescribed: describeFfmpegArgs(args),
|
|
5606
5821
|
// Whether this run cuts at times we gave it. Decides how a segment is
|
|
@@ -5619,7 +5834,7 @@ export class HlsSessionManager {
|
|
|
5619
5834
|
session.timeline?.segmentCount > 0 ? session.timeline.segmentCount - 1 : null,
|
|
5620
5835
|
inputUnavailable: (message) => isInputUnavailable(message),
|
|
5621
5836
|
onProgress: (report) => this.#noteRunProgress(session, run, report),
|
|
5622
|
-
onEnded: (ended) => this
|
|
5837
|
+
onEnded: (ended) => this.noteRunEnded(session, run, ended)
|
|
5623
5838
|
});
|
|
5624
5839
|
session.runs.add(run);
|
|
5625
5840
|
session.pendingRestartIndex = -1;
|
|
@@ -5636,7 +5851,7 @@ export class HlsSessionManager {
|
|
|
5636
5851
|
run.start(because);
|
|
5637
5852
|
|
|
5638
5853
|
logger.info(
|
|
5639
|
-
`transcode ${session.id} encode-run #${safeIndex}..#${
|
|
5854
|
+
`transcode ${session.id} encode-run #${safeIndex}..#${runEnd} from segment #${safeIndex} ` +
|
|
5640
5855
|
`(+${Date.now() - restartEnteredAt}ms since the restart was asked for) ` +
|
|
5641
5856
|
`(${formatSeconds(startSeconds)}) "${session.file.name}"`
|
|
5642
5857
|
);
|
|
@@ -5646,7 +5861,7 @@ export class HlsSessionManager {
|
|
|
5646
5861
|
// difference, and nothing downstream can tell that from a bad index.
|
|
5647
5862
|
const liveStart = this.#segmentStartTime(session, safeIndex);
|
|
5648
5863
|
logger.info(
|
|
5649
|
-
`transcode ${session.id} run #${safeIndex}..#${
|
|
5864
|
+
`transcode ${session.id} run #${safeIndex}..#${runEnd} positioned at ${startSeconds.toFixed(3)}s ` +
|
|
5650
5865
|
`for boundary #${safeIndex} (published ${startSeconds.toFixed(3)}s, ` +
|
|
5651
5866
|
`live ${liveStart.toFixed(3)}s, apart ${(liveStart - startSeconds).toFixed(3)}s), ` +
|
|
5652
5867
|
`numbering from #${safeIndex}`
|
|
@@ -5726,18 +5941,34 @@ export class HlsSessionManager {
|
|
|
5726
5941
|
* @param {import("./encode/EncodeRun.js").RunEnded} ended
|
|
5727
5942
|
* @returns {void}
|
|
5728
5943
|
*/
|
|
5729
|
-
|
|
5944
|
+
noteRunEnded(session, run, ended) {
|
|
5730
5945
|
// First, and for every run whatever else follows: the stretch it held goes
|
|
5731
5946
|
// back to the map. A run whose claim is never released tells the plan that
|
|
5732
5947
|
// numbers nobody is making are being made, and nothing is ever started
|
|
5733
5948
|
// there again.
|
|
5734
5949
|
this.encodeOrchestrator.noteEnded(ended);
|
|
5950
|
+
// WAS this run still the session's when it ended? Asked before it is
|
|
5951
|
+
// removed, because after the removal the question always answers "no" —
|
|
5952
|
+
// and it was asked after, so every line below this point was unreachable.
|
|
5953
|
+
//
|
|
5954
|
+
// Measured 2026-09-05 over both of the field host's log files: zero
|
|
5955
|
+
// occurrences of this handler's own `encode-run #… failed:` line and zero
|
|
5956
|
+
// of `fast failure at segment`, across every session this proxy has ever
|
|
5957
|
+
// run. So the fallback from a failed hardware encoder to software, the
|
|
5958
|
+
// retry when the torrent data goes away, the limit on retrying a position
|
|
5959
|
+
// that keeps failing, and the error line naming the ffmpeg command have all
|
|
5960
|
+
// been dead code — which is also why nothing ever stopped the restart loop
|
|
5961
|
+
// recorded in the field the same day.
|
|
5962
|
+
const wasCurrent = session.runs?.has(run) ?? false;
|
|
5735
5963
|
session.runs?.delete(run);
|
|
5964
|
+
// A stretch went back to the map, so what should be running has changed.
|
|
5965
|
+
// Said here rather than waited for: this is the moment it became true.
|
|
5966
|
+
this.planEncodersSoon();
|
|
5736
5967
|
if (session.state === "disposed") {
|
|
5737
5968
|
return;
|
|
5738
5969
|
}
|
|
5739
|
-
if (!
|
|
5740
|
-
// A run
|
|
5970
|
+
if (!wasCurrent) {
|
|
5971
|
+
// A run the session had already replaced. It has logged its own ending;
|
|
5741
5972
|
// nothing about the session follows from it.
|
|
5742
5973
|
return;
|
|
5743
5974
|
}
|
|
@@ -5821,28 +6052,41 @@ export class HlsSessionManager {
|
|
|
5821
6052
|
session.inputRetryTimer.unref?.();
|
|
5822
6053
|
return;
|
|
5823
6054
|
}
|
|
5824
|
-
//
|
|
5825
|
-
//
|
|
5826
|
-
// mid-stream
|
|
5827
|
-
//
|
|
5828
|
-
//
|
|
5829
|
-
//
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
6055
|
+
// A run that exits THIS fast never did real work: it failed at the start
|
|
6056
|
+
// itself — opening the input, spawning the process — rather than
|
|
6057
|
+
// mid-stream. Consecutive fast failures at the SAME position are counted so
|
|
6058
|
+
// that whoever commands a start can stop commanding one that keeps failing.
|
|
6059
|
+
//
|
|
6060
|
+
// COUNTED AT EVERY POSITION, #0 included. It used to be counted only past
|
|
6061
|
+
// #0, because it was written for seek restarts and a seek is never to the
|
|
6062
|
+
// beginning. That left the one position the plan commands FIRST with no
|
|
6063
|
+
// count at all, and the plan re-commanding a start that cannot succeed is
|
|
6064
|
+
// an unbounded loop: measured 2026-09-05, ffmpeg failing to spawn produced
|
|
6065
|
+
// fifty passes of the plan before a probe stopped it, as fast as the
|
|
6066
|
+
// failures arrived.
|
|
6067
|
+
if (ended.livedMs < START_FAST_FAIL_MS) {
|
|
6068
|
+
if (session.failedStartAt === ended.from) {
|
|
6069
|
+
session.failedStartCount += 1;
|
|
5833
6070
|
} else {
|
|
5834
|
-
session.
|
|
5835
|
-
session.
|
|
6071
|
+
session.failedStartAt = ended.from;
|
|
6072
|
+
session.failedStartCount = 1;
|
|
5836
6073
|
}
|
|
5837
6074
|
logger.warn(
|
|
5838
6075
|
`transcode ${session.id} fast failure at segment #${ended.from} ` +
|
|
5839
|
-
`(${ended.livedMs}ms) — ${session.
|
|
6076
|
+
`(${ended.livedMs}ms) — ${session.failedStartCount}/${MAX_FAILED_STARTS} consecutive`
|
|
5840
6077
|
);
|
|
6078
|
+
if (session.failedStartCount >= MAX_FAILED_STARTS) {
|
|
6079
|
+
logger.error(
|
|
6080
|
+
`transcode ${session.id} will not be started at #${ended.from} again: ` +
|
|
6081
|
+
`${session.failedStartCount} starts there failed within ${START_FAST_FAIL_MS}ms each ` +
|
|
6082
|
+
`(${session.lastError || ended.because}) "${session.file.name}"`
|
|
6083
|
+
);
|
|
6084
|
+
}
|
|
5841
6085
|
} else {
|
|
5842
6086
|
// Real progress was made (or this was the very first run) — not a
|
|
5843
6087
|
// repeating seek failure. Reset the breaker.
|
|
5844
|
-
session.
|
|
5845
|
-
session.
|
|
6088
|
+
session.failedStartAt = -1;
|
|
6089
|
+
session.failedStartCount = 0;
|
|
5846
6090
|
}
|
|
5847
6091
|
logger.error(
|
|
5848
6092
|
`transcode ${session.id} encode-run #${ended.from}..#${ended.to} failed: ${session.lastError}` +
|
|
@@ -6010,13 +6254,13 @@ export class HlsSessionManager {
|
|
|
6010
6254
|
this.#repairBehindHead(session, index, head);
|
|
6011
6255
|
return;
|
|
6012
6256
|
}
|
|
6013
|
-
// Circuit breaker: this exact target has already failed
|
|
6014
|
-
// times in a row (fast failures — see
|
|
6257
|
+
// Circuit breaker: this exact target has already failed MAX_FAILED_STARTS
|
|
6258
|
+
// times in a row (fast failures — see noteRunEnded).
|
|
6015
6259
|
// Stop auto-retrying it; session.state stays "failed" so getFileStream
|
|
6016
6260
|
// reports a clean, retryable error instead of looping forever. A DIFFERENT
|
|
6017
6261
|
// target (the viewer seeking elsewhere) is unaffected — it gets its own
|
|
6018
6262
|
// fresh attempt budget.
|
|
6019
|
-
if (index === session.
|
|
6263
|
+
if (index === session.failedStartAt && session.failedStartCount >= MAX_FAILED_STARTS) {
|
|
6020
6264
|
return;
|
|
6021
6265
|
}
|
|
6022
6266
|
// A far request is NOT treated as a seek. Measured 2026-08-02: on a single
|
|
@@ -6140,7 +6384,7 @@ export class HlsSessionManager {
|
|
|
6140
6384
|
// The breaker has already refused this target repeatedly. Re-arming for it
|
|
6141
6385
|
// would log and re-arm on every poll for as long as the request is held,
|
|
6142
6386
|
// and move nothing.
|
|
6143
|
-
if (target === session.
|
|
6387
|
+
if (target === session.failedStartAt && session.failedStartCount >= MAX_FAILED_STARTS) {
|
|
6144
6388
|
return;
|
|
6145
6389
|
}
|
|
6146
6390
|
logger.warn(
|
|
@@ -6212,7 +6456,7 @@ export class HlsSessionManager {
|
|
|
6212
6456
|
// is the freeze of 2026-08-18; keeping per-viewer heads without moving them
|
|
6213
6457
|
// on a seek would bring it back one viewer at a time.
|
|
6214
6458
|
if (consumerId) {
|
|
6215
|
-
this.viewers.of(named, consumerId).
|
|
6459
|
+
this.viewers.of(named, consumerId).position = {
|
|
6216
6460
|
segment: this.#segmentIndexForTime(named, positionSeconds),
|
|
6217
6461
|
seconds: positionSeconds,
|
|
6218
6462
|
at: Date.now(),
|
|
@@ -6325,13 +6569,13 @@ export class HlsSessionManager {
|
|
|
6325
6569
|
if (!Number.isInteger(head) || target >= head) {
|
|
6326
6570
|
return false;
|
|
6327
6571
|
}
|
|
6328
|
-
const staleAfterMs =
|
|
6572
|
+
const staleAfterMs = this.presenceStaleAfterMs();
|
|
6329
6573
|
const now = Date.now();
|
|
6330
6574
|
for (const [otherId, viewer] of viewersOf(session)) {
|
|
6331
|
-
if (otherId === consumerId || !viewer.
|
|
6575
|
+
if (otherId === consumerId || !viewer.isPresent(now, staleAfterMs)) {
|
|
6332
6576
|
continue;
|
|
6333
6577
|
}
|
|
6334
|
-
if (viewer.
|
|
6578
|
+
if (viewer.position.segment > target) {
|
|
6335
6579
|
return true;
|
|
6336
6580
|
}
|
|
6337
6581
|
}
|
|
@@ -6424,7 +6668,7 @@ export class HlsSessionManager {
|
|
|
6424
6668
|
// Circuit breaker (defense in depth): a timer armed before the cap was hit
|
|
6425
6669
|
// could still be pending when it was reached — do not fire the restart it
|
|
6426
6670
|
// was going to make. See the matching check in #ensureEncodingFor.
|
|
6427
|
-
if (target === session.
|
|
6671
|
+
if (target === session.failedStartAt && session.failedStartCount >= MAX_FAILED_STARTS) {
|
|
6428
6672
|
session.seekTarget = null;
|
|
6429
6673
|
session.seekFirstFarAt = 0;
|
|
6430
6674
|
return;
|
|
@@ -8059,7 +8303,7 @@ export class HlsSessionManager {
|
|
|
8059
8303
|
* @returns {"seeked" | "requested" | "opened" | "none"}
|
|
8060
8304
|
*/
|
|
8061
8305
|
#viewerPositionSourceOf(session, consumerId = "") {
|
|
8062
|
-
const head = consumerId ? session.viewers?.get(consumerId)?.
|
|
8306
|
+
const head = consumerId ? session.viewers?.get(consumerId)?.position ?? null : null;
|
|
8063
8307
|
if (head) {
|
|
8064
8308
|
return viewerPositionSource({
|
|
8065
8309
|
seeked: head.seeked,
|
|
@@ -8090,7 +8334,7 @@ export class HlsSessionManager {
|
|
|
8090
8334
|
* @returns {number}
|
|
8091
8335
|
*/
|
|
8092
8336
|
#viewerPositionOf(session, consumerId = "") {
|
|
8093
|
-
const head = consumerId ? session.viewers?.get(consumerId)?.
|
|
8337
|
+
const head = consumerId ? session.viewers?.get(consumerId)?.position ?? null : null;
|
|
8094
8338
|
if (head && Number.isFinite(head.seconds)) {
|
|
8095
8339
|
return head.seconds;
|
|
8096
8340
|
}
|
|
@@ -8460,6 +8704,13 @@ export class HlsSessionManager {
|
|
|
8460
8704
|
if (!variant) {
|
|
8461
8705
|
return { sessionId: null };
|
|
8462
8706
|
}
|
|
8707
|
+
if (consumerId && !isFamilyConsumerId(consumerId)) {
|
|
8708
|
+
// The same circle as a soundtrack's: the init has to be made before a
|
|
8709
|
+
// segment can be asked for, and nothing is made for an output nobody is
|
|
8710
|
+
// watching. Asking for any of its files is watching it.
|
|
8711
|
+
const watcher = this.viewers.of(variant, consumerId);
|
|
8712
|
+
this.#placeViewer(variant, watcher, this.viewerPositionOf(baseSessionId, consumerId));
|
|
8713
|
+
}
|
|
8463
8714
|
// Only a SEGMENT says the viewer is watching this rung — and it says more
|
|
8464
8715
|
// than that: it names the exact segment the player wants from it.
|
|
8465
8716
|
if (isSegment) {
|
|
@@ -8833,6 +9084,19 @@ export class HlsSessionManager {
|
|
|
8833
9084
|
);
|
|
8834
9085
|
return { sessionId: null, error: message };
|
|
8835
9086
|
}
|
|
9087
|
+
if (rendition && consumerId && !isFamilyConsumerId(consumerId)) {
|
|
9088
|
+
// Asking for ANY file of this soundtrack is this viewer watching it, and
|
|
9089
|
+
// the init is the file they ask for first. Registered here rather than on
|
|
9090
|
+
// the segment alone, because the segment cannot be asked for until the
|
|
9091
|
+
// init has been served, and the init cannot be made unless somebody is
|
|
9092
|
+
// watching: that circle is what left a soundtrack with no encoder, no
|
|
9093
|
+
// init and a viewer waiting sixty seconds on 2026-09-05.
|
|
9094
|
+
//
|
|
9095
|
+
// WHERE they are on it is where they are on the picture: the two are
|
|
9096
|
+
// played together.
|
|
9097
|
+
const listener = this.viewers.of(rendition, consumerId);
|
|
9098
|
+
this.#placeViewer(rendition, listener, this.viewerPositionOf(base.id, consumerId));
|
|
9099
|
+
}
|
|
8836
9100
|
if (isSegment && rendition) {
|
|
8837
9101
|
this.#noteAudioTrackActive(base, trackIndex, consumerId);
|
|
8838
9102
|
}
|
|
@@ -8912,12 +9176,12 @@ export class HlsSessionManager {
|
|
|
8912
9176
|
* @returns {Set<string>}
|
|
8913
9177
|
*/
|
|
8914
9178
|
#liveConsumers(base) {
|
|
8915
|
-
const staleAfterMs =
|
|
9179
|
+
const staleAfterMs = this.presenceStaleAfterMs();
|
|
8916
9180
|
const now = Date.now();
|
|
8917
9181
|
const live = new Set();
|
|
8918
9182
|
for (const member of this.liveOutputs.familyOf(base)) {
|
|
8919
9183
|
for (const [consumerId, viewer] of member.viewers ?? []) {
|
|
8920
|
-
if (viewer.
|
|
9184
|
+
if (viewer.isPresent(now, staleAfterMs)) {
|
|
8921
9185
|
live.add(consumerId);
|
|
8922
9186
|
}
|
|
8923
9187
|
}
|
|
@@ -9511,7 +9775,7 @@ export class HlsSessionManager {
|
|
|
9511
9775
|
if (!isPlaylist) {
|
|
9512
9776
|
// Where the viewer actually is. Recorded for every segment request,
|
|
9513
9777
|
// served or not, because it is what bounds how far ahead the encoder is
|
|
9514
|
-
// allowed to run —
|
|
9778
|
+
// allowed to run — the plan decides that now.
|
|
9515
9779
|
const requested = session.segmentFormat.segmentIndexFromName(fileName);
|
|
9516
9780
|
if (requested >= 0) {
|
|
9517
9781
|
// This requester's own head, and with it the furthest any viewer of
|
|
@@ -9544,7 +9808,7 @@ export class HlsSessionManager {
|
|
|
9544
9808
|
// A viewer who has caught up must not wait out the monitor's interval —
|
|
9545
9809
|
// but only if they HAVE caught up, which is why this re-evaluates the
|
|
9546
9810
|
// same condition instead of resuming outright.
|
|
9547
|
-
this.#
|
|
9811
|
+
this.#reportCushionFor(session);
|
|
9548
9812
|
}
|
|
9549
9813
|
}
|
|
9550
9814
|
// Whether the file is there is asked on its own, and nothing else shares
|
|
@@ -10385,6 +10649,39 @@ export class HlsSessionManager {
|
|
|
10385
10649
|
};
|
|
10386
10650
|
}
|
|
10387
10651
|
|
|
10652
|
+
/**
|
|
10653
|
+
* This person has gone, and their connection is what said so.
|
|
10654
|
+
*
|
|
10655
|
+
* Departure is a fact about the PERSON, not about one of the three outputs
|
|
10656
|
+
* the browser happens to hold an id for, and their connection knows it before
|
|
10657
|
+
* any output does. Until 2026-09-05 nothing carried it: the only exits were
|
|
10658
|
+
* the browser's own `release`, which a killed tab never sends, and a silence
|
|
10659
|
+
* long enough to be called an absence, which a paused viewer produces without
|
|
10660
|
+
* having gone anywhere.
|
|
10661
|
+
*
|
|
10662
|
+
* Every output they were watching is told, and one with nobody left is
|
|
10663
|
+
* disposed by the same path a normal release takes.
|
|
10664
|
+
*
|
|
10665
|
+
* @param {string} consumerId
|
|
10666
|
+
* @param {string} [because]
|
|
10667
|
+
* @returns {Promise<number>} How many outputs they were let go of.
|
|
10668
|
+
*/
|
|
10669
|
+
async viewerHasGone(consumerId, because = "their connection closed") {
|
|
10670
|
+
if (typeof consumerId !== "string" || consumerId.length === 0) {
|
|
10671
|
+
return 0;
|
|
10672
|
+
}
|
|
10673
|
+
const watched = this.viewers.get(consumerId)?.outputs;
|
|
10674
|
+
if (!watched || watched.size === 0) {
|
|
10675
|
+
return 0;
|
|
10676
|
+
}
|
|
10677
|
+
// Copied before anything is released: releasing walks the same set.
|
|
10678
|
+
const outputs = [...watched];
|
|
10679
|
+
for (const outputId of outputs) {
|
|
10680
|
+
await this.releaseSessionConsumer(outputId, consumerId, because);
|
|
10681
|
+
}
|
|
10682
|
+
return outputs.length;
|
|
10683
|
+
}
|
|
10684
|
+
|
|
10388
10685
|
/**
|
|
10389
10686
|
* Remove a consumer from a session. Disposes the session when the last
|
|
10390
10687
|
* consumer leaves.
|