@torrent-tv/proxy 2.80.17 → 2.80.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/docs/encode-architecture.md +91 -7
- package/package.json +1 -1
- package/services/encode/EncodePlan.js +22 -12
- package/services/encode/EncodeRun.js +12 -6
- package/services/encode/SegmentStore.js +147 -223
- package/services/encode/run-command.js +16 -1
- package/services/encode/run-costs.js +15 -40
- package/services/hls-session-manager.js +5 -83
- package/services/orchestrators/EncodeOrchestrator.js +63 -36
- package/services/segment-formats/fmp4.js +54 -0
- package/services/segment-formats/mpegts.js +54 -0
- package/test/coverage-follows-the-disk.test.js +5 -1
- package/test/move-cost.test.js +109 -38
- package/test/orchestrator-wired.test.js +5 -1
- package/test/produced-copy-choice.test.js +258 -358
- package/test/segment-serve-wiring.test.js +8 -9
- package/test/segment-store.test.js +238 -216
- package/test/split-a-stretch.test.js +139 -0
- package/services/encode/open-piece.js +0 -135
- package/test/open-piece.test.js +0 -152
|
@@ -70,7 +70,6 @@ import { Output, Outputs } from "./output/Output.js";
|
|
|
70
70
|
import { masterPlaylistText, mediaPlaylistText, segmentIndexForTime } from "./output/playlists.js";
|
|
71
71
|
import { SourceFiles, sourceDecodeCharacteristics } from "./source/SourceFile.js";
|
|
72
72
|
import { ProducedIndex } from "./produced-index.js";
|
|
73
|
-
import { discardOpenPiece } from "./encode/open-piece.js";
|
|
74
73
|
import { SegmentStore } from "./encode/SegmentStore.js";
|
|
75
74
|
import { EncodeCost } from "./quality/EncodeCost.js";
|
|
76
75
|
import {
|
|
@@ -425,10 +424,6 @@ const TRUE_START_MEMORY = 200;
|
|
|
425
424
|
// twice. It runs on the restart path and a session an hour in has thousands of
|
|
426
425
|
// segments; the figure is for a comparison, not an inventory.
|
|
427
426
|
const BACKWARD_RESTART_SCAN_SEGMENTS = 300;
|
|
428
|
-
// Grace period to wait for the PREVIOUS ffmpeg process to exit (per signal
|
|
429
|
-
// escalation step: SIGTERM, then SIGKILL) before spawning its replacement into
|
|
430
|
-
// the same session directory. See #startEncodeRun.
|
|
431
|
-
const ENCODE_RUN_TERMINATE_GRACE_MS = 2_000;
|
|
432
427
|
// A seek-restart run that exits this fast never did real work — it failed at
|
|
433
428
|
// the seek/open step itself (container demux error, bad audio frame boundary,
|
|
434
429
|
// etc.), not mid-stream. Used to tell a genuine seek failure apart from a
|
|
@@ -5535,7 +5530,7 @@ export class HlsSessionManager {
|
|
|
5535
5530
|
inputUnavailable: (message) => isInputUnavailable(message),
|
|
5536
5531
|
onProgress: (report) => this.#noteRunProgress(session, run, report),
|
|
5537
5532
|
indexOfName: (name) => session.segmentFormat.segmentIndexFromName(name),
|
|
5538
|
-
onClosed: (name) => this.segmentStore.
|
|
5533
|
+
onClosed: (name) => this.segmentStore.publish(session.outputKey ?? "", name, session.segmentFormat),
|
|
5539
5534
|
onEnded: (ended) => this.noteRunEnded(session, run, ended)
|
|
5540
5535
|
});
|
|
5541
5536
|
session.runs.add(run);
|
|
@@ -7692,31 +7687,15 @@ export class HlsSessionManager {
|
|
|
7692
7687
|
// disposed. A run that had already finished or failed is not among them,
|
|
7693
7688
|
// which is what keeps a stop from erasing how it actually ended.
|
|
7694
7689
|
for (const run of running) {
|
|
7695
|
-
const ffmpeg = run.process;
|
|
7696
|
-
// The stretch it was given, read now rather than when the process finally
|
|
7697
|
-
// exits: by then the session may have started another run with another
|
|
7698
|
-
// stretch, and the piece to discard belongs to this one.
|
|
7699
|
-
const stoppedSpan = {
|
|
7700
|
-
from: Number.isInteger(run.from) ? run.from : 0,
|
|
7701
|
-
to: Number.isInteger(run.to) ? run.to : -1
|
|
7702
|
-
};
|
|
7703
7690
|
// The run resumes itself if it was suspended — a stopped process does not
|
|
7704
7691
|
// act on SIGTERM until it is continued — records the cause, and answers
|
|
7705
7692
|
// its own exit. Nothing here has to null a field so that the exit is read
|
|
7706
7693
|
// correctly, because there is no shared field left to misread.
|
|
7707
7694
|
run.stop(reason);
|
|
7708
|
-
//
|
|
7709
|
-
//
|
|
7710
|
-
//
|
|
7711
|
-
//
|
|
7712
|
-
// WHAT THE RUN ITSELF NAMED, which is the only thing that can say a piece
|
|
7713
|
-
// is whole — a piece cut short still decodes. Written 2026-09-06 and never
|
|
7714
|
-
// passed from here, so the clearing-up ran with nothing proven and fell
|
|
7715
|
-
// back to bounding itself by the run's declared stretch alone.
|
|
7716
|
-
const provenName = typeof run.provenName === "string" ? run.provenName : null;
|
|
7717
|
-
void waitForChildExit(ffmpeg, ENCODE_RUN_TERMINATE_GRACE_MS).then(() =>
|
|
7718
|
-
this.#discardUnfinishedPiece(session, session.dirPath, stoppedSpan, provenName)
|
|
7719
|
-
);
|
|
7695
|
+
// Nothing is cleared up from here. What this run left open is under its
|
|
7696
|
+
// own working name, and the encoding layer removes it when the run's
|
|
7697
|
+
// ending reaches it — one place, and it needs neither the stretch nor the
|
|
7698
|
+
// init bytes this method used to fetch to judge a file by its contents.
|
|
7720
7699
|
}
|
|
7721
7700
|
logger.info(`transcode ${session.id} ${running.length} encoder(s) stopped: ${reason}`);
|
|
7722
7701
|
}
|
|
@@ -9589,63 +9568,6 @@ export class HlsSessionManager {
|
|
|
9589
9568
|
return this.#producedIndex(session).pathOf(fileName);
|
|
9590
9569
|
}
|
|
9591
9570
|
|
|
9592
|
-
/**
|
|
9593
|
-
* Throw away the piece a run was in the middle of when it ended.
|
|
9594
|
-
*
|
|
9595
|
-
* The `segment` muxer creates its output file when it opens it and writes
|
|
9596
|
-
* into it until the next cut, so at any instant exactly one file in a run's
|
|
9597
|
-
* directory is unfinished: the highest-numbered one. A run that reaches the
|
|
9598
|
-
* end of its work closes that file properly and it is a good piece; a run
|
|
9599
|
-
* killed for a seek does not — measured 2026-09-03, ffmpeg exited 19 ms after
|
|
9600
|
-
* SIGTERM and left `segment-00025.mp4` at zero bytes.
|
|
9601
|
-
*
|
|
9602
|
-
* Leaving it is what created the deadlock this method exists to prevent: an
|
|
9603
|
-
* empty file has a name like any other, so it closed the only hole in the
|
|
9604
|
-
* numbering and the look-ahead kept the encoder stopped for having "produced"
|
|
9605
|
-
* it. Removing it at the moment the run ends means the question never has to
|
|
9606
|
-
* be asked again by anyone.
|
|
9607
|
-
*
|
|
9608
|
-
* A piece is removed only when it is unusable. A run that finished its last
|
|
9609
|
-
* file — the ordinary end of a file, or a stop that arrived between two cuts
|
|
9610
|
-
* — has nothing wrong with it, and deleting good output would cost the work
|
|
9611
|
-
* of making it twice.
|
|
9612
|
-
*
|
|
9613
|
-
* @param {HlsSession} session
|
|
9614
|
-
* @param {string | null | undefined} runDirPath
|
|
9615
|
-
* @returns {Promise<void>}
|
|
9616
|
-
*/
|
|
9617
|
-
async #discardUnfinishedPiece(session, runDirPath, within = null, provenName = null) {
|
|
9618
|
-
const canJudgeTracks =
|
|
9619
|
-
typeof session.segmentFormat?.hasEveryTrack === "function" &&
|
|
9620
|
-
session.initBytes &&
|
|
9621
|
-
session.initBytes.length > 0;
|
|
9622
|
-
const removed = await discardOpenPiece(
|
|
9623
|
-
runDirPath,
|
|
9624
|
-
session.segmentFormat,
|
|
9625
|
-
within,
|
|
9626
|
-
canJudgeTracks
|
|
9627
|
-
? (raw) => {
|
|
9628
|
-
const bytes = cutsAtGivenTimes(session) && session.segmentFormat.stripInit
|
|
9629
|
-
? session.segmentFormat.stripInit(raw)
|
|
9630
|
-
: raw;
|
|
9631
|
-
return session.segmentFormat.hasEveryTrack(bytes, session.initBytes);
|
|
9632
|
-
}
|
|
9633
|
-
: null,
|
|
9634
|
-
provenName
|
|
9635
|
-
);
|
|
9636
|
-
if (removed !== null) {
|
|
9637
|
-
// Removed on purpose, so the index must not go on answering with it.
|
|
9638
|
-
this.#producedIndex(session).invalidate();
|
|
9639
|
-
logger.info(
|
|
9640
|
-
`transcode ${session.id} discarded segment #${removed}: ` +
|
|
9641
|
-
"the run ended while it was open, so it holds no usable piece"
|
|
9642
|
-
);
|
|
9643
|
-
}
|
|
9644
|
-
}
|
|
9645
|
-
|
|
9646
|
-
|
|
9647
|
-
|
|
9648
|
-
|
|
9649
9571
|
#holdForProduction(session, fileName, isPlaylist, options) {
|
|
9650
9572
|
/** @type {{ address: string, rank: number, topRank: number } | null} */
|
|
9651
9573
|
let ranked = null;
|
|
@@ -273,11 +273,10 @@ export class EncodeOrchestrator {
|
|
|
273
273
|
* @param {number} index
|
|
274
274
|
*/
|
|
275
275
|
noteProduced(address, index) {
|
|
276
|
-
//
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
this.segmentStore?.markClosed(address, index);
|
|
276
|
+
// Nothing is told to the store: by the time this is called the piece is
|
|
277
|
+
// ALREADY under its served name, because the rename is what closing it
|
|
278
|
+
// means. A statement kept beside the disk would be a second owner of one
|
|
279
|
+
// fact, which is what item 87 removed from the coverage map.
|
|
281
280
|
this.coverageOf(address).markReady(index);
|
|
282
281
|
for (const run of this.runsOn(address)) {
|
|
283
282
|
run.noteProduced(index);
|
|
@@ -375,6 +374,23 @@ export class EncodeOrchestrator {
|
|
|
375
374
|
// when the reason it cuts the budget changes, so asking it three times in
|
|
376
375
|
// one pass is three chances to say a thing that happened once.
|
|
377
376
|
const maxRuns = this.#affordableOn(address, live);
|
|
377
|
+
// THE TERMS EVERY ARRIVAL IS COMPUTED FROM, named here so the line below can
|
|
378
|
+
// print them. A decision of this plan is `delay + (index - at) / rate +
|
|
379
|
+
// madeBetween * refetch` against a deadline, so without the rate and the two
|
|
380
|
+
// prices no recorded decision can be reproduced — which is what happened
|
|
381
|
+
// with the one-piece intervals of 2026-09-08: the rate was substituted six
|
|
382
|
+
// times from the speeds the session reported elsewhere and none of them gave
|
|
383
|
+
// the answer the plan had given.
|
|
384
|
+
const costs = this.#costs.seconds();
|
|
385
|
+
const refetchSecPerFilmSecond = this.refetchSecPerFilmSecond(address);
|
|
386
|
+
// The best figure this host has: what a run here is doing now, what one was
|
|
387
|
+
// last measured doing, or what the startup benchmark predicted. The first
|
|
388
|
+
// two are this output's own; the third exists before either.
|
|
389
|
+
const speedX = Math.max(
|
|
390
|
+
live.reduce((best, run) => Math.max(best, run.speedX || 0), 0),
|
|
391
|
+
this.#lastSpeed.get(address) ?? 0,
|
|
392
|
+
this.startingSpeedFor(address) || 0
|
|
393
|
+
);
|
|
378
394
|
const actions = planEncoders({
|
|
379
395
|
coverage,
|
|
380
396
|
windows,
|
|
@@ -387,26 +403,19 @@ export class EncodeOrchestrator {
|
|
|
387
403
|
// What a start and a kill cost, measured from this host's own runs rather
|
|
388
404
|
// than written into the code from one machine's reading. Zero until
|
|
389
405
|
// something has been measured, which is the same convention as the
|
|
390
|
-
// refetch price
|
|
391
|
-
...
|
|
406
|
+
// refetch price and is stated so the bias is known.
|
|
407
|
+
...costs,
|
|
392
408
|
// What a second of film costs to fetch again, in seconds of swarm time.
|
|
393
409
|
// Answered by whoever measures the film's own byte rate and the swarm's;
|
|
394
410
|
// zero until they have, which makes driving through look cheaper than it
|
|
395
411
|
// is and is stated here so the bias is known.
|
|
396
|
-
refetchSecPerFilmSecond
|
|
412
|
+
refetchSecPerFilmSecond,
|
|
397
413
|
// How much slower one encoder runs beside others, read off this host's own
|
|
398
414
|
// startup measurement. A pure function over a measured table: beyond what
|
|
399
415
|
// was measured it holds the largest reading rather than continuing a curve
|
|
400
416
|
// nothing observed.
|
|
401
417
|
contentionPenaltyFor: (others) => contentionPenalty(others, this.contentionPenalties).penalty,
|
|
402
|
-
|
|
403
|
-
// was last measured doing, or what the startup benchmark predicted. The
|
|
404
|
-
// first two are this output's own; the third exists before either.
|
|
405
|
-
speedX: Math.max(
|
|
406
|
-
live.reduce((best, run) => Math.max(best, run.speedX || 0), 0),
|
|
407
|
-
this.#lastSpeed.get(address) ?? 0,
|
|
408
|
-
this.startingSpeedFor(address) || 0
|
|
409
|
-
)
|
|
418
|
+
speedX
|
|
410
419
|
});
|
|
411
420
|
|
|
412
421
|
// A move is the plan taking a running encoder away from where it already
|
|
@@ -417,6 +426,33 @@ export class EncodeOrchestrator {
|
|
|
417
426
|
// happens — everything a rerun of the same decision needs: the windows
|
|
418
427
|
// this call saw (priority, the real time, which side of the viewers),
|
|
419
428
|
// the budget, and where every live run stood.
|
|
429
|
+
// WHAT WAS DECIDED AND WITH WHAT INTERVAL, said whenever anything is placed
|
|
430
|
+
// or taken away — not only on a move.
|
|
431
|
+
//
|
|
432
|
+
// Field 2026-09-08 could not be diagnosed from this line: it printed the
|
|
433
|
+
// windows, the budget and where the live runs stood, and NOT the intervals
|
|
434
|
+
// the actions carried. What the session actually did was give every encoder
|
|
435
|
+
// an interval of exactly ONE segment — 63 runs, each making one piece and
|
|
436
|
+
// exiting, twelve of them normally — and the reasons printed beside them
|
|
437
|
+
// read as moves, so the fault was diagnosed three times as something it was
|
|
438
|
+
// not. An interval is what a run is, and it was the one thing missing.
|
|
439
|
+
//
|
|
440
|
+
// AND THE TERMS, for the same reason one step further: an interval says what
|
|
441
|
+
// was decided and the terms say why. `speed` is what every arrival is
|
|
442
|
+
// divided by, so a decision recorded without it can be re-read and not
|
|
443
|
+
// recomputed; `firstByte` and `kill` are what a start and a stop cost here;
|
|
444
|
+
// `refetch` is what a second of film costs to fetch again. A zero in the
|
|
445
|
+
// last three is a measurement nobody has taken, not a free operation, and it
|
|
446
|
+
// is printed so that reading it as free is a choice rather than an accident.
|
|
447
|
+
if (actions.some((action) => action.type !== "keep")) {
|
|
448
|
+
this.logger.info(
|
|
449
|
+
`encode-plan on ${address}: ` +
|
|
450
|
+
`${actions.map((action) => `${action.type} #${action.from ?? "?"}..#${action.to ?? "?"}`).join(", ")}` +
|
|
451
|
+
` [speed=${speedX.toFixed(2)}x firstByte=${costs.firstByteWaitSec.toFixed(2)}s ` +
|
|
452
|
+
`kill=${costs.killCostSec.toFixed(2)}s refetch=${refetchSecPerFilmSecond.toFixed(3)}s/s ` +
|
|
453
|
+
`maxRuns=${maxRuns} live=${live.length}]`
|
|
454
|
+
);
|
|
455
|
+
}
|
|
420
456
|
if (actions.some((action) => action.type === "move")) {
|
|
421
457
|
this.logger.info(
|
|
422
458
|
`encode-plan move on ${address}: windows=${JSON.stringify(windows)} ` +
|
|
@@ -531,17 +567,12 @@ export class EncodeOrchestrator {
|
|
|
531
567
|
const onThisOutput = this.#runs.get(address) ?? [];
|
|
532
568
|
onThisOutput.push(run);
|
|
533
569
|
this.#runs.set(address, onThisOutput);
|
|
534
|
-
//
|
|
535
|
-
//
|
|
536
|
-
//
|
|
537
|
-
//
|
|
538
|
-
//
|
|
539
|
-
|
|
540
|
-
// and readiness is now a projection of what is proven — so a one-segment run
|
|
541
|
-
// at the beginning would have declared the rest of the output unmade.
|
|
542
|
-
const runsTo = endOfRun({ from, to });
|
|
543
|
-
this.segmentStore?.forgetClosed(address, from, runsTo);
|
|
544
|
-
this.coverageOf(address).claim(run, from, runsTo);
|
|
570
|
+
// Nothing has to be un-proved when a run takes a stretch that has already
|
|
571
|
+
// been written. A piece is only ever NAMED as served once it is closed, and
|
|
572
|
+
// a run writing that number again writes under a working name until it
|
|
573
|
+
// closes its own — so the file standing there is a complete piece made by
|
|
574
|
+
// somebody, and serving it is right until the newer one replaces it whole.
|
|
575
|
+
this.coverageOf(address).claim(run, from, endOfRun({ from, to }));
|
|
545
576
|
run.start(because);
|
|
546
577
|
}
|
|
547
578
|
|
|
@@ -638,15 +669,11 @@ export class EncodeOrchestrator {
|
|
|
638
669
|
this.#costs.note(ended);
|
|
639
670
|
// Exactly one ending is normal — the run reached the end of the stretch it
|
|
640
671
|
// was given and closed its last file. Every other leaves a piece open, and
|
|
641
|
-
// that
|
|
642
|
-
//
|
|
643
|
-
//
|
|
644
|
-
//
|
|
645
|
-
|
|
646
|
-
void this.segmentStore
|
|
647
|
-
.discardOpenPieceOf(ended.address, { from: ended.from, to: ended.to }, null, ended.provenName)
|
|
648
|
-
.catch(() => {});
|
|
649
|
-
}
|
|
672
|
+
// that piece is under this run's OWN working name, so clearing up after it
|
|
673
|
+
// is a name match: no stretch to search and no bytes to judge. Done for
|
|
674
|
+
// every ending, the normal one included, since a run that finished cleanly
|
|
675
|
+
// has nothing under a working name and the sweep then removes nothing.
|
|
676
|
+
this.segmentStore?.clearUpAfter(ended.address, ended.from);
|
|
650
677
|
this.coverageOf(ended.address).release(ended.run);
|
|
651
678
|
const remaining = this.runsOn(ended.address).filter((run) => run !== ended.run);
|
|
652
679
|
if (remaining.length === 0) {
|
|
@@ -76,6 +76,9 @@ function findFragmentBounds(bytes) {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
const INIT_FILE_NAME = "init.mp4";
|
|
79
|
+
// What a piece is called while it is still being written. See
|
|
80
|
+
// `makingFileNameTemplate`.
|
|
81
|
+
const MAKING_PATTERN = /^making-([0-9a-z]+)-(\d{5})\.mp4$/;
|
|
79
82
|
const SEGMENT_PATTERN = /^segment-(\d{5})\.mp4$/;
|
|
80
83
|
|
|
81
84
|
/**
|
|
@@ -140,6 +143,57 @@ export const fmp4Format = {
|
|
|
140
143
|
},
|
|
141
144
|
|
|
142
145
|
/** The output path template for the `segment` muxer. */
|
|
146
|
+
/**
|
|
147
|
+
* The name ffmpeg is told to write a piece under WHILE IT IS MAKING IT.
|
|
148
|
+
*
|
|
149
|
+
* Not the name it is served under. A piece being written is not a piece, and
|
|
150
|
+
* under its final name it is indistinguishable from one — which is how half a
|
|
151
|
+
* segment came to be served: the proof of completeness was "the next file
|
|
152
|
+
* exists", true of one writer walking forward and false the moment two runs
|
|
153
|
+
* share an output, because the next file is then written by another process.
|
|
154
|
+
* Field 2026-09-08: `segment-00057.mp4` served at 2 268 361 bytes and then at
|
|
155
|
+
* 4 510 940, and the browser refused the whole one for the rest of the session.
|
|
156
|
+
*
|
|
157
|
+
* With a working name, the served name appears only when the encoder has said
|
|
158
|
+
* the piece is closed, and existence under it IS the proof — one rule, the
|
|
159
|
+
* same for every branch, and true whether or not our own process is alive.
|
|
160
|
+
*
|
|
161
|
+
* @returns {string}
|
|
162
|
+
*/
|
|
163
|
+
makingFileNameTemplate(tag) {
|
|
164
|
+
return `making-${String(tag ?? "0").replace(/[^0-9a-z]/g, "")}-%05d.mp4`;
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Which run is writing this working name, or null when the name is not one.
|
|
169
|
+
*
|
|
170
|
+
* The tag is what makes clearing up after a dead run possible without
|
|
171
|
+
* guessing: several runs write into one directory, so "the unfinished pieces"
|
|
172
|
+
* is only a well-formed question per run. It used to be answered by looking
|
|
173
|
+
* for the highest SERVED name inside the stretch the ended run was given and
|
|
174
|
+
* judging its bytes — a guess, and under the naming rule above it would remove
|
|
175
|
+
* a complete piece somebody else had closed.
|
|
176
|
+
*
|
|
177
|
+
* @param {string} name
|
|
178
|
+
* @returns {string | null}
|
|
179
|
+
*/
|
|
180
|
+
makingTagOf(name) {
|
|
181
|
+
const match = MAKING_PATTERN.exec(String(name ?? "").trim());
|
|
182
|
+
return match ? match[1] : null;
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The name a piece just closed under a working name is served as, or null when
|
|
187
|
+
* the name is not one of ours.
|
|
188
|
+
*
|
|
189
|
+
* @param {string} makingName
|
|
190
|
+
* @returns {string | null}
|
|
191
|
+
*/
|
|
192
|
+
servedNameOf(makingName) {
|
|
193
|
+
const match = MAKING_PATTERN.exec(String(makingName ?? "").trim());
|
|
194
|
+
return match ? this.segmentFileName(Number(match[2])) : null;
|
|
195
|
+
},
|
|
196
|
+
|
|
143
197
|
segmentFileNameTemplate() {
|
|
144
198
|
return "segment-%05d.mp4";
|
|
145
199
|
},
|
|
@@ -14,6 +14,9 @@
|
|
|
14
14
|
* See {@link SegmentFormat} in `./index.js` for the interface contract.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
// What a piece is called while it is still being written. See
|
|
18
|
+
// `makingFileNameTemplate`.
|
|
19
|
+
const MAKING_PATTERN = /^making-([0-9a-z]+)-(\d{5})\.ts$/;
|
|
17
20
|
const SEGMENT_PATTERN = /^segment-(\d{5})\.ts$/;
|
|
18
21
|
|
|
19
22
|
/**
|
|
@@ -47,6 +50,57 @@ export const mpegtsFormat = {
|
|
|
47
50
|
},
|
|
48
51
|
|
|
49
52
|
/** The output path template for the `segment` muxer. */
|
|
53
|
+
/**
|
|
54
|
+
* The name ffmpeg is told to write a piece under WHILE IT IS MAKING IT.
|
|
55
|
+
*
|
|
56
|
+
* Not the name it is served under. A piece being written is not a piece, and
|
|
57
|
+
* under its final name it is indistinguishable from one — which is how half a
|
|
58
|
+
* segment came to be served: the proof of completeness was "the next file
|
|
59
|
+
* exists", true of one writer walking forward and false the moment two runs
|
|
60
|
+
* share an output, because the next file is then written by another process.
|
|
61
|
+
* Field 2026-09-08: `segment-00057.mp4` served at 2 268 361 bytes and then at
|
|
62
|
+
* 4 510 940, and the browser refused the whole one for the rest of the session.
|
|
63
|
+
*
|
|
64
|
+
* With a working name, the served name appears only when the encoder has said
|
|
65
|
+
* the piece is closed, and existence under it IS the proof — one rule, the
|
|
66
|
+
* same for every branch, and true whether or not our own process is alive.
|
|
67
|
+
*
|
|
68
|
+
* @returns {string}
|
|
69
|
+
*/
|
|
70
|
+
makingFileNameTemplate(tag) {
|
|
71
|
+
return `making-${String(tag ?? "0").replace(/[^0-9a-z]/g, "")}-%05d.ts`;
|
|
72
|
+
},
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Which run is writing this working name, or null when the name is not one.
|
|
76
|
+
*
|
|
77
|
+
* The tag is what makes clearing up after a dead run possible without
|
|
78
|
+
* guessing: several runs write into one directory, so "the unfinished pieces"
|
|
79
|
+
* is only a well-formed question per run. It used to be answered by looking
|
|
80
|
+
* for the highest SERVED name inside the stretch the ended run was given and
|
|
81
|
+
* judging its bytes — a guess, and under the naming rule above it would remove
|
|
82
|
+
* a complete piece somebody else had closed.
|
|
83
|
+
*
|
|
84
|
+
* @param {string} name
|
|
85
|
+
* @returns {string | null}
|
|
86
|
+
*/
|
|
87
|
+
makingTagOf(name) {
|
|
88
|
+
const match = MAKING_PATTERN.exec(String(name ?? "").trim());
|
|
89
|
+
return match ? match[1] : null;
|
|
90
|
+
},
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The name a piece just closed under a working name is served as, or null when
|
|
94
|
+
* the name is not one of ours.
|
|
95
|
+
*
|
|
96
|
+
* @param {string} makingName
|
|
97
|
+
* @returns {string | null}
|
|
98
|
+
*/
|
|
99
|
+
servedNameOf(makingName) {
|
|
100
|
+
const match = MAKING_PATTERN.exec(String(makingName ?? "").trim());
|
|
101
|
+
return match ? this.segmentFileName(Number(match[2])) : null;
|
|
102
|
+
},
|
|
103
|
+
|
|
50
104
|
segmentFileNameTemplate() {
|
|
51
105
|
return "segment-%05d.ts";
|
|
52
106
|
},
|
|
@@ -103,8 +103,12 @@ function aViewerAtTheStart(made) {
|
|
|
103
103
|
*/
|
|
104
104
|
function theWholeFilmIsOnDisk(store, dir) {
|
|
105
105
|
for (let index = 0; index < SEGMENTS; index += 1) {
|
|
106
|
+
// Under the served name, which is the whole of what says a piece is closed:
|
|
107
|
+
// it takes that name only when its writer has said so. There used to be a
|
|
108
|
+
// statement kept beside the disk as well, told by whoever noticed a piece
|
|
109
|
+
// being produced — a second owner of one fact, and the fault this file's own
|
|
110
|
+
// subject is.
|
|
106
111
|
writeFileSync(path.join(dir, fmp4Format.segmentFileName(index)), Buffer.alloc(16));
|
|
107
|
-
store.markClosed(PICTURE, index);
|
|
108
112
|
}
|
|
109
113
|
}
|
|
110
114
|
|
package/test/move-cost.test.js
CHANGED
|
@@ -41,73 +41,145 @@ import { RunCosts } from "../services/encode/run-costs.js";
|
|
|
41
41
|
import { planEncoders } from "../services/encode/EncodePlan.js";
|
|
42
42
|
import { CoverageMap } from "../services/encode/CoverageMap.js";
|
|
43
43
|
|
|
44
|
-
test("nothing measured
|
|
44
|
+
test("nothing measured is a plain zero, and the floor is derived where the arithmetic is", () => {
|
|
45
45
|
const costs = new RunCosts();
|
|
46
46
|
|
|
47
|
-
const {
|
|
48
|
-
assert.equal(
|
|
49
|
-
// Placing one where there is none is the OTHER question, and it has no
|
|
50
|
-
// alternative: the film gets made or it does not.
|
|
51
|
-
assert.equal(firstByteWaitSec, 0, "placing an encoder is not blocked by an unknown price");
|
|
47
|
+
const { firstByteWaitSec, killCostSec } = costs.seconds();
|
|
48
|
+
assert.equal(firstByteWaitSec, 0, "no reading is said as none, not as a guess");
|
|
52
49
|
assert.equal(killCostSec, 0);
|
|
50
|
+
// There was an `Infinity` here — the cost of a move, made unaffordable until
|
|
51
|
+
// something had been measured, on the reasoning that an unmeasured price must
|
|
52
|
+
// not license an irreversible act. It was an exception in a model that needs
|
|
53
|
+
// none: a first piece cannot appear faster than it takes to ENCODE one, and
|
|
54
|
+
// how fast this host encodes is measured before any viewer exists, so the
|
|
55
|
+
// floor belongs where the arithmetic is.
|
|
56
|
+
assert.equal("moveCostSec" in costs.seconds(), false, "no such figure any more");
|
|
53
57
|
});
|
|
54
58
|
|
|
55
59
|
test("a run killed before producing anything is a lower bound on the first output", () => {
|
|
56
60
|
const costs = new RunCosts();
|
|
57
61
|
|
|
58
62
|
// Exactly what a thrash supplies: a run that lived 800 ms and finished
|
|
59
|
-
// nothing. It says the first output takes AT LEAST that long, which is a fact
|
|
63
|
+
// nothing. It says the first output takes AT LEAST that long, which is a fact
|
|
64
|
+
// and the only reading a thrash can give — every run in one is killed before
|
|
65
|
+
// it produces.
|
|
60
66
|
costs.note({ livedMs: 800, dyingMs: 40 });
|
|
61
67
|
|
|
62
|
-
const {
|
|
63
|
-
assert.ok(
|
|
64
|
-
assert.ok(Math.abs(
|
|
68
|
+
const { firstByteWaitSec, killCostSec } = costs.seconds();
|
|
69
|
+
assert.ok(Math.abs(firstByteWaitSec - 0.8) < 0.001, `got ${firstByteWaitSec}`);
|
|
70
|
+
assert.ok(Math.abs(killCostSec - 0.04) < 0.001, `got ${killCostSec}`);
|
|
65
71
|
});
|
|
66
72
|
|
|
67
73
|
test("a run that produced something is measured by its first output, not its life", () => {
|
|
68
74
|
const costs = new RunCosts();
|
|
69
75
|
|
|
70
|
-
costs.note({ livedMs: 60_000, firstOutputMs:
|
|
76
|
+
costs.note({ livedMs: 60_000, firstOutputMs: 1260, dyingMs: 100 });
|
|
71
77
|
|
|
72
|
-
const {
|
|
73
|
-
assert.ok(Math.abs(firstByteWaitSec -
|
|
74
|
-
assert.ok(Math.abs(moveCostSec - 1.0) < 0.001, `got ${moveCostSec}`);
|
|
78
|
+
const { firstByteWaitSec } = costs.seconds();
|
|
79
|
+
assert.ok(Math.abs(firstByteWaitSec - 1.26) < 0.001, `got ${firstByteWaitSec}`);
|
|
75
80
|
});
|
|
76
81
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
/**
|
|
83
|
+
* The map's real shape: one segment at the viewer, doubling zones ahead down to
|
|
84
|
+
* p91, and everything behind them at p1 with no deadline. Written out because a
|
|
85
|
+
* fixture of two zones is not this, and the difference decides the answer: with
|
|
86
|
+
* nothing stated past the viewer's own zone, a run one segment behind it is
|
|
87
|
+
* compared on that zone alone and loses by a tenth of a second.
|
|
88
|
+
*
|
|
89
|
+
* @param {number} head - The segment the viewer is on.
|
|
90
|
+
* @param {number} count
|
|
91
|
+
* @returns {object[]}
|
|
92
|
+
*/
|
|
93
|
+
function mapAt(head, count) {
|
|
94
|
+
const zones = [];
|
|
95
|
+
if (head > 0) {
|
|
96
|
+
zones.push({ from: 0, to: head - 1, priority: 1, withinSeconds: null, behind: true });
|
|
97
|
+
}
|
|
98
|
+
let from = head;
|
|
99
|
+
let width = 1;
|
|
100
|
+
let rank = 100;
|
|
101
|
+
while (from < count && rank > 90) {
|
|
102
|
+
const to = Math.min(count - 1, from + width - 1);
|
|
103
|
+
zones.push({ from, to, priority: rank, withinSeconds: (from - head) * 4.2, behind: false });
|
|
104
|
+
from = to + 1;
|
|
105
|
+
width *= 2;
|
|
106
|
+
rank -= 1;
|
|
107
|
+
}
|
|
108
|
+
if (from < count) {
|
|
109
|
+
zones.push({ from, to: count - 1, priority: 90, withinSeconds: (from - head) * 4.2, behind: false });
|
|
110
|
+
}
|
|
111
|
+
return zones;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
test("an encoder is left alone while the viewer is still at or before it", () => {
|
|
115
|
+
// Every slide of the viewer's zone used to make standing one number behind it
|
|
116
|
+
// score worse than standing in it — by ten milliseconds, which is nothing but
|
|
117
|
+
// the double charge for a piece already being made. What holds now is the
|
|
118
|
+
// narrower and true statement: while the viewer's own zone still contains the
|
|
119
|
+
// encoder's position, it is left alone. A viewer BEFORE it is a different
|
|
120
|
+
// case entirely and correctly moves it back — encoders only go forward, so
|
|
121
|
+
// one standing past a viewer never reaches them.
|
|
122
|
+
//
|
|
123
|
+
// Once the viewer has PASSED it, moving forward is correct and happens once: a
|
|
124
|
+
// run that has produced nothing in 0.8 s of a 1.26 s warm-up owes 0.46 s
|
|
125
|
+
// before its piece exists, while a fresh one at the viewer's own number owes
|
|
126
|
+
// 0.32 s of spawn and then the piece — so the viewer is served sooner, and the
|
|
127
|
+
// number left behind is in nobody's zone.
|
|
81
128
|
const coverage = new CoverageMap();
|
|
82
129
|
coverage.setSegmentCount(482);
|
|
83
|
-
const run = { from: 58, to:
|
|
84
|
-
coverage.claim(run, 58,
|
|
130
|
+
const run = { from: 58, to: -1, head: 58, speedX: 4.45, isAlive: true, startedAt: 1_000_000 };
|
|
131
|
+
coverage.claim(run, 58, -1);
|
|
85
132
|
|
|
86
|
-
const
|
|
133
|
+
for (const viewerAt of [58]) {
|
|
134
|
+
const actions = planEncoders({
|
|
135
|
+
coverage,
|
|
136
|
+
windows: mapAt(viewerAt, 482),
|
|
137
|
+
runs: [run],
|
|
138
|
+
maxRuns: 3,
|
|
139
|
+
segmentSeconds: 4.2,
|
|
140
|
+
speedX: 4.45,
|
|
141
|
+
killCostSec: 0.04,
|
|
142
|
+
// Measured on the addon host: a run started at 15:50:15.521 and its first
|
|
143
|
+
// piece existed at 15:50:16.785.
|
|
144
|
+
firstByteWaitSec: 1.26,
|
|
145
|
+
refetchSecPerFilmSecond: 0,
|
|
146
|
+
// 1.98 at 1920x1080, measured: a second encoder takes very nearly all of
|
|
147
|
+
// the first's speed.
|
|
148
|
+
contentionPenaltyFor: (others) => (others <= 0 ? 1 : 1.98 ** others),
|
|
149
|
+
now: 1_000_000 + 800
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
assert.deepEqual(
|
|
153
|
+
actions.filter((one) => one.type === "move"),
|
|
154
|
+
[],
|
|
155
|
+
`the viewer at #${viewerAt} does not cost the encoder its place`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// And once they are past it, exactly one move — not one per slide.
|
|
160
|
+
const past = [59, 60, 61].map((viewerAt) => planEncoders({
|
|
87
161
|
coverage,
|
|
88
|
-
windows:
|
|
89
|
-
{ from: 0, to: 57, priority: 1, withinSeconds: null, behind: true },
|
|
90
|
-
{ from: 59, to: 60, priority: 100, withinSeconds: 0, behind: false }
|
|
91
|
-
],
|
|
162
|
+
windows: mapAt(viewerAt, 482),
|
|
92
163
|
runs: [run],
|
|
93
164
|
maxRuns: 3,
|
|
94
165
|
segmentSeconds: 4.2,
|
|
95
166
|
speedX: 4.45,
|
|
96
|
-
// Measured on this host: killing takes 40 ms, a fresh encoder's first piece
|
|
97
|
-
// 900 ms. Against that, driving one segment at 4.45x costs 0.94 s — so the
|
|
98
|
-
// two are close, and what settles it is that the move ALSO has to encode
|
|
99
|
-
// the same segment afterwards.
|
|
100
167
|
killCostSec: 0.04,
|
|
101
|
-
firstByteWaitSec:
|
|
102
|
-
moveCostSec: 0.94,
|
|
168
|
+
firstByteWaitSec: 1.26,
|
|
103
169
|
refetchSecPerFilmSecond: 0,
|
|
104
|
-
contentionPenaltyFor: () => 1
|
|
105
|
-
|
|
170
|
+
contentionPenaltyFor: (others) => (others <= 0 ? 1 : 1.98 ** others),
|
|
171
|
+
now: 1_000_000 + 800
|
|
172
|
+
}).filter((one) => one.type === "move"));
|
|
106
173
|
|
|
107
174
|
assert.deepEqual(
|
|
108
|
-
|
|
109
|
-
[],
|
|
110
|
-
"
|
|
175
|
+
past.map((moves) => moves.length),
|
|
176
|
+
[1, 1, 1],
|
|
177
|
+
"one move to where the viewer now is, whichever number that is"
|
|
178
|
+
);
|
|
179
|
+
assert.deepEqual(
|
|
180
|
+
past.map((moves) => moves[0].from),
|
|
181
|
+
[59, 60, 61],
|
|
182
|
+
"and it goes to the viewer's own number, not one past it"
|
|
111
183
|
);
|
|
112
184
|
});
|
|
113
185
|
|
|
@@ -128,8 +200,7 @@ test("a move that genuinely saves the viewer time still happens", () => {
|
|
|
128
200
|
segmentSeconds: 4.2,
|
|
129
201
|
speedX: 4.45,
|
|
130
202
|
killCostSec: 0.04,
|
|
131
|
-
firstByteWaitSec:
|
|
132
|
-
moveCostSec: 0.94,
|
|
203
|
+
firstByteWaitSec: 1.26,
|
|
133
204
|
refetchSecPerFilmSecond: 0,
|
|
134
205
|
contentionPenaltyFor: () => 1
|
|
135
206
|
});
|
|
@@ -190,6 +190,10 @@ test("segments already made are known to the plan, whoever made them", (t) => {
|
|
|
190
190
|
const coverage = manager.encodeOrchestrator.coverageOf(KEY);
|
|
191
191
|
assert.equal(coverage.isReady(0), true);
|
|
192
192
|
assert.equal(coverage.isReady(2), true);
|
|
193
|
-
|
|
193
|
+
// Including the highest. It used to be excluded for having no successor to
|
|
194
|
+
// prove it closed, which left the last piece of every run unprovable for ever
|
|
195
|
+
// — and proved a half-written one whenever the number above it was written by
|
|
196
|
+
// another run of the same output. A served name is the proof now.
|
|
197
|
+
assert.equal(coverage.isReady(3), true, "its name says it is closed");
|
|
194
198
|
void store;
|
|
195
199
|
});
|