@torrent-tv/proxy 2.85.1 → 2.87.0

package/services/{orchestrators → encode}/EncodeOrchestrator.js

@@ -23,15 +23,15 @@
  * result when it is ready.
  */
 
-import { CoverageMap } from "../encode/CoverageMap.js";
-import { firstUnmetWant, planEncoders } from "../encode/EncodePlan.js";
-import { endOfRun } from "../encode/EncodeRun.js";
-import { ENCODE_EXIT } from "../encode/encode-exit.js";
-import { affordableRuns } from "../encode/run-budget.js";
-import { RunCosts } from "../encode/run-costs.js";
-import { contentionPenalty } from "../encode/contention.js";
-import { waits } from "../priority/WaitLedger.js";
-import { SegmentDemand } from "../encode/SegmentDemand.js";
+import { CoverageMap } from "./CoverageMap.js";
+import { firstUnmetWant, planEncoders } from "./EncodePlan.js";
+import { endOfRun } from "./EncodeRun.js";
+import { ENCODE_EXIT } from "./encode-exit.js";
+import { affordableRuns } from "./run-budget.js";
+import { RunCosts } from "./run-costs.js";
+import { contentionPenalty } from "./contention.js";
+import { SegmentDemand } from "./SegmentDemand.js";
+import { runStateOf } from "./encode-run-state.js";
 
 /**
  * How long nothing is placed on an output whose input has just gone, and the
@@ -46,14 +46,32 @@ import { SegmentDemand } from "../encode/SegmentDemand.js";
  */
 const INPUT_QUIET_BASE_MS = 2_000;
 const INPUT_QUIET_MAX_MS = 15_000;
+const START_FAST_FAIL_MS = 2_000;
+const MAX_FAILED_STARTS = 3;
 
 export class EncodeOrchestrator {
+  /** Derived priority maps. Discardable and rebuilt from viewer state. */
+  #demand = new SegmentDemand();
+  /** @type {(address: string) => string} */
+  #describeWaits = () => "";
+
   /** Output address to what has been made of it. @type {Map<string, CoverageMap>} */
   #coverage = new Map();
 
   /** Output address to the runs on it. @type {Map<string, import("../encode/EncodeRun.js").EncodeRun[]>} */
   #runs = new Map();
 
+  /** Last state after an output has no registered run. @type {Map<string, string>} */
+  #lastState = new Map();
+
+  /** Last encoder error per output. @type {Map<string, string>} */
+  #lastError = new Map();
+
+  /** Last progress snapshot per output after its run ends. @type {Map<string, object>} */
+  #lastProgress = new Map();
+
+  /** Consecutive fast failures at one position, per output. @type {Map<string, { at: number, count: number }>} */
+  #failedStarts = new Map();
 
   /** The fastest speed measured on one output, kept across restarts. @type {Map<string, number>} */
   #lastSpeed = new Map();
@@ -93,12 +111,20 @@ export class EncodeOrchestrator {
     this.#costs.noteStartup(measured);
   }
 
+  /** Derived segment demand for one output, copied from the discardable cache. */
+  wantedSegmentsOn(address) {
+    return [...this.#demand.mapOn(address)];
+  }
+
   /** The last reason a budget was cut, so the same one is not said twice. */
   #lastBudgetReason = new Map();
 
   /** The last unmet want said out loud, so a stuck one is said once. */
   #lastUnmet = new Map();
 
+  /** The last time the machine bound an output's encoders, so it is said once. */
+  #lastMachineBound = new Map();
+
   /**
    * @param {object} params
    * @param {(address: string) => number} params.maxRunsFor - How many encoders
@@ -117,6 +143,11 @@ export class EncodeOrchestrator {
    *   encoding speed.
    * @param {{ info: (line: string) => void, warn: (line: string) => void }} params.logger
    * @param {() => number} [params.now]
+   * @param {(address: string) => string} [params.describeWaits] - What the
+   *   viewers actually waited for on this output, by band, for the state line.
+   *   HANDED IN: what anybody waited for belongs to the priority layer, and
+   *   this one may be asked about encoders with no ledger, no clock and no
+   *   viewer anywhere. Absent, the line simply does not carry it.
    */
   constructor({
     maxRunsFor,
@@ -127,15 +158,22 @@ export class EncodeOrchestrator {
     startingSpeedFor = () => 0,
     segmentStore = null,
     planSoon = null,
+    describeWaits = null,
+    admission = null,
     logger,
     now
   }) {
+    // HOW MANY ENCODERS THE WHOLE MACHINE HOLDS, across outputs. `maxRunsFor`
+    // answers for one output as though it were alone; this is what refuses the
+    // encoder that would slow everybody below realtime. Absent in a check that
+    // is about one output, and then only the per-output limit binds.
+    this.admission = admission && typeof admission.placesFor === "function" ? admission : null;
+    this.#describeWaits = typeof describeWaits === "function" ? describeWaits : () => "";
     // The store of produced segments — the layer below this one. It is asked to
     // clean up after a run that ended other than by reaching the end of its
     // stretch, which is the one thing an ending must not leave behind: a file
     // under a name that promises a whole segment.
     this.segmentStore = segmentStore;
-    this.demand = new SegmentDemand();
     this.maxRunsFor = maxRunsFor;
     // Seconds of swarm time per second of film: what re-encoding material that
     // already exists costs the download, over and above the encoder's own time.
@@ -249,6 +287,189 @@ export class EncodeOrchestrator {
     return this.#runs.get(address) ?? [];
   }
 
+  /**
+   * The aggregate state of this output's current runs, or its last ending when
+   * none remain.
+   *
+   * @param {string} address
+   * @returns {string}
+   */
+  stateOf(address) {
+    const runs = this.runsOn(address);
+    return runs.length > 0 ? runStateOf(runs) : (this.#lastState.get(address) ?? runStateOf([]));
+  }
+
+  /** @param {string} address @returns {string} */
+  errorOf(address) {
+    return this.#lastError.get(address) ?? "";
+  }
+
+  /**
+   * Progress of the live run that covers a position, or the most recently
+   * updated run when no position is supplied.
+   *
+   * @param {string} address
+   * @param {number | null} [index]
+   * @returns {object | null}
+   */
+  progressOf(address, index = null) {
+    const live = this.runsOn(address).filter((run) => run.isAlive);
+    const covering = Number.isInteger(index)
+      ? live.filter((run) => run.from <= index && endOfRun(run) >= index)
+      : live;
+    const candidates = covering.length > 0 ? covering : live;
+    const run = [...candidates].sort(
+      (left, right) => (right.progress?.updatedAt ?? 0) - (left.progress?.updatedAt ?? 0)
+    )[0];
+    return run?.progress?.snapshot?.() ?? this.#lastProgress.get(address) ?? null;
+  }
+
+  /**
+   * Whether another start at this position is allowed.
+   *
+   * @param {string} address
+   * @param {number} from
+   * @returns {boolean}
+   */
+  mayStartAt(address, from) {
+    const failed = this.#failedStarts.get(address);
+    return failed?.at !== from || failed.count < MAX_FAILED_STARTS;
+  }
+
+  /**
+   * Record whether a failed run died before it performed sustained work.
+   *
+   * @param {string} address
+   * @param {number} from
+   * @param {number} livedMs
+   * @returns {{ at: number, count: number, blocked: boolean, limit: number, fastMs: number }}
+   */
+  noteStartFailure(address, from, livedMs) {
+    if (!(livedMs < START_FAST_FAIL_MS)) {
+      this.#failedStarts.delete(address);
+      return { at: -1, count: 0, blocked: false, limit: MAX_FAILED_STARTS, fastMs: START_FAST_FAIL_MS };
+    }
+    const previous = this.#failedStarts.get(address);
+    const next = {
+      at: from,
+      count: previous?.at === from ? previous.count + 1 : 1
+    };
+    this.#failedStarts.set(address, next);
+    return {
+      ...next,
+      blocked: next.count >= MAX_FAILED_STARTS,
+      limit: MAX_FAILED_STARTS,
+      fastMs: START_FAST_FAIL_MS
+    };
+  }
+
+  /**
+   * An output that no longer exists and has no run left on it.
+   *
+   * How its last run ended, what went wrong and which position kept failing are
+   * facts about THAT output. The name follows from the key, so the next output
+   * with the same parameters has the same address; kept, those facts would
+   * greet it as its own — a failed ending makes its first wait throw before any
+   * encoder is placed, and nothing would ever clear it.
+   *
+   * @param {string} address
+   */
+  forgetOutput(address) {
+    if (this.runsOn(address).length > 0) {
+      return;
+    }
+    this.#lastState.delete(address);
+    this.#lastError.delete(address);
+    this.#lastProgress.delete(address);
+    this.#failedStarts.delete(address);
+  }
+
+  /** @param {string} address @returns {{ at: number, count: number, blocked: boolean, limit: number }} */
+  startFailureOf(address) {
+    const failure = this.#failedStarts.get(address) ?? { at: -1, count: 0 };
+    return { ...failure, blocked: failure.count >= MAX_FAILED_STARTS, limit: MAX_FAILED_STARTS };
+  }
+
+  /**
+   * What the map says about one segment of one output, and about the map it
+   * sits in.
+   *
+   * Asked by whoever is holding a request and measuring the wait: a wait means
+   * one thing at the top rank and another at the bottom, and telling them apart
+   * needs both the segment's own rank and the highest rank stated, since a rank
+   * is a position in this map and not an absolute number.
+   *
+   * AND AN EMPTY MAP IS NOT AN ANSWER. A top rank of zero says the map has not
+   * been built yet — a session created a moment ago, before the first pass —
+   * and that is a different thing from "nobody is coming". Conflated, every
+   * request behind the run reads as absent for as long as a fresh session has
+   * no map.
+   *
+   * It exists because the one caller reached through this object into the store
+   * behind it. A reach like that is how a store becomes the surface: the day it
+   * changes, a caller two layers away breaks for a reason it cannot see.
+   *
+   * @param {string} address
+   * @param {number} index
+   * @returns {{ rank: number, topRank: number }}
+   */
+  rankAt(address, index) {
+    return this.#demand.rankOf(address, index);
+  }
+
+  /**
+   * How many encoder processes are running on this machine right now, over
+   * every output.
+   *
+   * ONE COUNT, and it is this class's because this class is what makes and
+   * unmakes runs: adopted the moment one exists and forgotten the moment one
+   * ends, both synchronously. The session manager answered the same question
+   * twice by walking its own registry — once per SESSION and once per RUN —
+   * which agreed only while a session held at most one run. It has held several
+   * since the plan was allowed to place them, and the per-session count is the
+   * wrong one of the two wherever they differ: what a second encoder costs was
+   * measured between ffmpeg PROCESSES, not between outputs.
+   *
+   * A SUSPENDED run is not counted. It is stopped where it stands and is using
+   * nothing; a cost measured beside it is a cost measured alone.
+   *
+   * @returns {number}
+   */
+  /**
+   * How many encoders are alive on each output, suspended ones included.
+   *
+   * What the machine-wide admission counts as holding a place. A suspended
+   * encoder is counted here, unlike in {@link runningCount}: that one answers
+   * what is competing for the processor this instant, this one what the
+   * machine is committed to — and a parked encoder resumes the moment the
+   * viewer catches up.
+   *
+   * @returns {Map<string, number>}
+   */
+  liveRunsByAddress() {
+    /** @type {Map<string, number>} */
+    const byAddress = new Map();
+    for (const [address, runs] of this.#runs) {
+      const alive = runs.filter((run) => run.isAlive).length;
+      if (alive > 0) {
+        byAddress.set(address, alive);
+      }
+    }
+    return byAddress;
+  }
+
+  runningCount() {
+    let running = 0;
+    for (const runs of this.#runs.values()) {
+      for (const run of runs) {
+        if (run.isAlive && !run.isSuspended) {
+          running += 1;
+        }
+      }
+    }
+    return running;
+  }
+
   /**
    * How long an output is, once its playlist is known.
    *
@@ -307,7 +528,7 @@ export class EncodeOrchestrator {
    * @param {{ from: number, to: number, priority: number, withinSeconds: number }[]} zones
    */
   notePriorityMap(address, zones) {
-    this.demand.state(address, zones);
+    this.#demand.state(address, zones);
     // NOBODY IS COMING HERE, so what was remembered about this output's input
     // is about nobody. Kept, those three entries would stay for the life of the
     // process, and the wait they describe would greet whoever opens this output
@@ -330,6 +551,7 @@ export class EncodeOrchestrator {
     // means. A statement kept beside the disk would be a second owner of one
     // fact, which is what item 87 removed from the coverage map.
     this.coverageOf(address).markReady(index);
+    this.segmentStore?.announce?.(address, index);
     for (const run of this.runsOn(address)) {
       run.noteProduced(index);
     }
@@ -364,7 +586,7 @@ export class EncodeOrchestrator {
    * state, so a pass that finds nothing to change does nothing.
    */
   reconcile() {
-    const addresses = new Set([...this.demand.addresses(), ...this.#runs.keys()]);
+    const addresses = new Set([...this.#demand.addresses(), ...this.#runs.keys()]);
     for (const address of addresses) {
       this.#reconcileOne(address);
     }
@@ -433,12 +655,12 @@ export class EncodeOrchestrator {
     // used to reach into the layer that STATES them for the same arithmetic,
     // which is the coupling the layer rule forbids; the arithmetic itself now
     // lives where it belongs to nobody.
-    const windows = this.demand.mapOn(address);
+    const windows = this.#demand.mapOn(address);
     const live = this.runsOn(address).filter((run) => run.isAlive);
     // Asked ONCE. It is arithmetic over measurements, but it also says out loud
     // when the reason it cuts the budget changes, so asking it three times in
     // one pass is three chances to say a thing that happened once.
-    const maxRuns = this.#affordableOn(address, live);
+    const maxRuns = this.#withinMachine(address, live, this.#affordableOn(address, live));
     // THE TERMS EVERY ARRIVAL IS COMPUTED FROM, named here so the line below can
     // print them. A decision of this plan is `delay + (index - at) / rate +
     // madeBetween * refetch` against a deadline, so without the rate and the two
@@ -691,6 +913,42 @@ export class EncodeOrchestrator {
     return budget.runs;
   }
 
+  /**
+   * The per-output limit, bounded by what the whole machine can still hold.
+   *
+   * The per-output answer is at least one whatever else is running; the
+   * machine's may be none, and that is the refusal: the output gets no encoder
+   * until one elsewhere ends. It is never below what already runs here, so a
+   * refusal stops a start and never takes an encoder away.
+   *
+   * Said out loud when the machine is what bound it, once per change.
+   *
+   * @param {string} address
+   * @param {object[]} live
+   * @param {number} perOutput
+   * @returns {number}
+   */
+  #withinMachine(address, live, perOutput) {
+    if (!this.admission) {
+      return perOutput;
+    }
+    const machine = this.admission.placesFor(address, live.length);
+    const runs = Math.min(perOutput, machine.runs);
+    const said = runs < perOutput ? `${runs}:${machine.because}` : "";
+    if (said !== (this.#lastMachineBound.get(address) ?? "")) {
+      if (said) {
+        this.logger.info(
+          `encode: ${runs} encoder(s) on ${address} — the whole machine holds no more ` +
+          `(${machine.because}; this output alone would be allowed ${perOutput})`
+        );
+        this.#lastMachineBound.set(address, said);
+      } else {
+        this.#lastMachineBound.delete(address);
+      }
+    }
+    return runs;
+  }
+
   /**
    * Take charge of a run this class did not start.
    *
@@ -734,6 +992,16 @@ export class EncodeOrchestrator {
    * @param {import("../encode/EncodeRun.js").RunEnded} ended
    */
   noteEnded(ended) {
+    this.#lastState.set(ended.address, ended.run?.state ?? "IDLE");
+    if (ended.lastError) {
+      this.#lastError.set(ended.address, ended.lastError);
+    } else if (ended.because && ended.ending !== ENCODE_EXIT.COMPLETE) {
+      this.#lastError.set(ended.address, ended.because);
+    }
+    const progress = ended.run?.progress?.snapshot?.();
+    if (progress) {
+      this.#lastProgress.set(ended.address, progress);
+    }
     this.#costs.note(ended);
     // Exactly one ending is normal — the run reached the end of the stretch it
     // was given and closed its last file. Every other leaves a piece open, and
@@ -889,9 +1157,9 @@ export class EncodeOrchestrator {
    */
   describe() {
     const parts = [];
-    for (const address of new Set([...this.demand.addresses(), ...this.#runs.keys()])) {
+    for (const address of new Set([...this.#demand.addresses(), ...this.#runs.keys()])) {
       const coverage = this.coverageOf(address);
-      const stated = this.demand.mapOn(address);
+      const stated = this.#demand.mapOn(address);
       const windows = stated.map((zone) => ({ from: zone.from, to: zone.to }));
       const waiting = firstUnmetWant(coverage, windows);
       const runs = this.runsOn(address)
@@ -921,7 +1189,7 @@ export class EncodeOrchestrator {
       // existed there was no figure anywhere saying whether the urgent zone was
       // served first — the map could have been read backwards and every line
       // above would have looked the same.
-      const served = waits.describe(address);
+      const served = this.#describeWaits(address);
       parts.push(
         `${address} ready=${coverage.stats().ready}` +
         `${held < 0 ? "" : ` of ${held} file(s) on disk`} ` +

package/services/encode/EncodeRun.js

@@ -39,6 +39,7 @@
 
 import { ENCODE_RUN_EVENT, ENCODE_RUN_STATE, INITIAL_RUN_STATE, nextState } from "./encode-run-state.js";
 import { classifyEncodeExit, ENCODE_EXIT } from "./encode-exit.js";
+import { RunProgress } from "./RunProgress.js";
 
 /** Microseconds in a second, as ffmpeg's `out_time_ms` counts them. */
 const MICROSECONDS_PER_SECOND = 1_000_000;
@@ -186,12 +187,8 @@ export class EncodeRun {
    *   encoder itself names it on its own channel, and answers with the name that
    *   piece is served under — because making it servable is a rename, and only
    *   whoever owns the disk can perform one.
-   * @param {(progress: { processedSeconds: number | null, speed: string | null }) => void} [params.onProgress]
-   *   Called for every `-progress` report. Seconds count from the START OF THIS
-   *   RUN on both branches — neither `-output_ts_offset` nor `-copyts` changes
-   *   what `-progress` reports, both measured — so rebasing them onto the
-   *   source's timeline is the caller's, which is the only side that knows
-   *   where this run began.
+   * @param {(progress: object) => void} [params.onProgress] - Called with this
+   *   run's progress on the source timeline after every `-progress` report.
    * @param {() => number | null} [params.lastSegmentIndex] - The film's last
    *   segment number, for telling "reached the end" from "the input dried up".
    *   ffmpeg exits zero for both, and over a torrent it cannot tell them apart.
@@ -208,6 +205,8 @@ export class EncodeRun {
    *   a closed piece's name carries. How a piece is named belongs to the format
    *   that writes it, so it arrives as a plain function rather than this class
    *   knowing any naming.
+   * @param {number} [params.startSeconds] - Where this run begins on the source timeline.
+   * @param {number | null} [params.totalSeconds] - Source duration when known.
    */
   constructor({
     address,
@@ -226,6 +225,8 @@ export class EncodeRun {
     argsDescribed = "",
     usesExplicitCuts = false,
     indexOfName,
+    startSeconds = 0,
+    totalSeconds = null,
     because = "no reason was given"
   }) {
     this.address = address;
@@ -236,6 +237,7 @@ export class EncodeRun {
     this.spawnProcess = spawn;
     this.logger = logger;
     this.now = typeof now === "function" ? now : Date.now;
+    this.progress = new RunProgress({ startSeconds, totalSeconds, now: this.now });
     this.onEnded = typeof onEnded === "function" ? onEnded : () => {};
     this.onProgress = typeof onProgress === "function" ? onProgress : () => {};
     // Told the NAME of every piece the encoder has closed. The name is the
@@ -449,17 +451,22 @@ export class EncodeRun {
       if (key === "out_time_ms") {
         const numeric = Number(value);
         if (Number.isFinite(numeric) && numeric >= 0) {
-          this.onProgress({ processedSeconds: numeric / MICROSECONDS_PER_SECOND, speed: null });
+          this.#reportProgress({ processedSeconds: numeric / MICROSECONDS_PER_SECOND, speed: null });
         }
       } else if (key === "out_time") {
-        this.onProgress({ processedSeconds: null, speed: null, outTime: value });
+        this.#reportProgress({ processedSeconds: null, speed: null, outTime: value });
       } else if (key === "speed") {
         this.noteSpeed(Number.parseFloat(value));
-        this.onProgress({ processedSeconds: null, speed: value });
+        this.#reportProgress({ processedSeconds: null, speed: value });
       }
     }
   }
 
+  #reportProgress(report) {
+    this.progress.note(report);
+    this.onProgress(this.progress.snapshot());
+  }
+
   /**
    * A segment this run has finished.
    *