@torrent-tv/proxy 2.100.15 → 2.101.1

package/bin/cli.js

@@ -90,6 +90,7 @@ program
   .option("--no-port-mapping", "Disable automatic UPnP/NAT-PMP port mapping")
   .option("--delivery-sink", "Serve /api/delivery-sink, a torrent-free byte stream for delivery testing")
   .option("--pin-preset <name>", "Encode every software picture at this libx264 preset only, for runs that must be compared at one quality (off by default)")
+  .option("--input-stretch-bytes <bytes>", "Size every original-input stretch to this many bytes instead of copy rate times open, for runs that compare input sizes; memory, downloaded bytes and the plan still bound it (off by default)")
   .option("--heap-snapshots", "Write a heap snapshot of a thread whose memory reaches a new high-water; each one stops that thread while it is written (off by default)")
   .option("--profile-start <seconds>", "Profile the CPU of the main thread and the torrent thread for this many seconds from the first source opened, written beside the state (off by default)")
   .option(
@@ -399,6 +400,7 @@ try {
     profileStartSeconds: Number(options.profileStart) || 0,
     heapSnapshots: options.heapSnapshots === true,
     pinnedPreset: typeof options.pinPreset === "string" && options.pinPreset.trim() ? options.pinPreset.trim() : null,
+    inputStretchBytes: Number(options.inputStretchBytes) > 0 ? Math.floor(Number(options.inputStretchBytes)) : null,
     // Late-bound the same way `webRtcManager` is below: the torrent pool is
     // built inside `startProxyServer`, before `dataChannelHandler` — the
     // thing that actually owns a channel to push down — exists.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.100.15",
+  "version": "2.101.1",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -134,6 +134,9 @@ export async function startProxyServer({
   // The one software preset every picture is encoded at, for runs compared at
   // one quality; null leaves the choice to the machine.
   pinnedPreset = null,
+  // The size every original-input stretch is given in place of copy rate
+  // times open, for runs that compare input sizes; null keeps the rule.
+  inputStretchBytes = null,
   // What viewers' connections carry beyond the film, per byte of film; the
   // transport counts it and is built after this server, so it is late-bound.
   serviceShare = () => null,
@@ -854,6 +857,7 @@ export async function startProxyServer({
   let sourcePreparation;
   const outputParts = wireOutputs({
     pinnedPreset,
+    inputStretchBytes,
     indexMemory,
     serviceShare,
     readMetadataActivity: () => containerOrchestrator.activity(),
@@ -1147,10 +1151,13 @@ export async function startProxyServer({
   });
   startProcessorReport({
     log: (line) => logger.info(line),
-    ffmpegPids: () => [...outputParts.outputs.values()]
-      .flatMap((output) => outputParts.encodeRuns.liveRunsOf(output))
-      .map((run) => run.process?.pid)
-      .filter((pid) => Number.isInteger(pid)),
+    ffmpegRuns: () => [...outputParts.outputs.values()]
+      .flatMap((output) => outputParts.encodeRuns.liveRunsOf(output).map((run) => ({
+        pid: run.process?.pid,
+        label: `${output.outputKey} #${run.from}..#${run.to}`,
+        film: Number(run.progress?.processedSeconds) - Number(run.progress?.startPositionSeconds ?? 0)
+      })))
+      .filter((run) => Number.isInteger(run.pid)),
     detailed: () => startupProfile.active()
   });
   // What the last life of this process left on the disk. The kernel kills this

package/services/encode/EncodeInputs.js

@@ -62,10 +62,20 @@ export class EncodeInputs {
    * @type {(output: object, from: number, to: number, input: object) => void}
    */
   #stretchCopied;
+  /**
+   * A stretch size the operator named for comparison runs
+   * (`--input-stretch-bytes`), in place of copy rate times open; null keeps
+   * the measured rule. Only the target changes: memory, downloaded bytes and
+   * the plan's bound still end a stretch (torrent-tv/meta#182).
+   *
+   * @type {number | null}
+   */
+  #fixedStretchBytes;
 
   constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
-    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {} }) {
+    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {}, stretchBytes = null }) {
     this.#stretchCopied = stretchCopied;
+    this.#fixedStretchBytes = Number.isSafeInteger(stretchBytes) && stretchBytes > 0 ? stretchBytes : null;
     this.#openOf = openOf;
     this.#resolve = resolve;
     this.#read = readRanges;
@@ -344,6 +354,10 @@ export class EncodeInputs {
   #sizingFor(output) {
     const rate = middleOf(this.#copyRates);
     const open = this.#openOf(output);
+    if (this.#fixedStretchBytes !== null) {
+      return { bytes: this.#fixedStretchBytes, openMeasured: true, openSeconds: open?.seconds ?? null,
+        copyBytesPerMs: rate, fixed: true };
+    }
     return rate === null || !open ? null : { bytes: Math.floor(rate * open.seconds * 1000),
       openMeasured: open.measured, openSeconds: open.seconds, copyBytesPerMs: rate };
   }
@@ -607,7 +621,7 @@ export class EncodeInputs {
     if (result.kind === "result") {
       this.#log(`encode input selection of ${addressOf(output)}: ${JSON.stringify({
         from, requestedTo: request.to, serves, floor, targetEnd, chosenEnd: end,
-        targetBytes: target, minimumBytes: minimum, inputBytes: result.bytes,
+        targetBytes: target, targetFixed: sizing?.fixed === true, minimumBytes: minimum, inputBytes: result.bytes,
         copyBytesPerMs: sizing?.copyBytesPerMs ?? null, openSeconds: sizing?.openSeconds ?? null,
         openMeasured: sizing?.openMeasured ?? null, reservations, probes
       })}`);

package/services/encode/quality/HostLoad.js

@@ -16,6 +16,7 @@ import { baseDrawFrom, costPerMegabyteFrom } from "../torrent-cost.js";
 import { medianOf, movedBeyondScatter, READINGS_KEPT, scatterOf } from "../learned-median.js";
 import { ENCODE_RUN_STATE, processCanBeSignalled } from "../encode-run-state.js";
 import { SourceFiles } from "../../media/SourceFile.js";
+import { costKindForSession } from "./EncodeCost.js";
 
 /**
  * What the torrent thread says of itself over one interval: how late its event
@@ -537,7 +538,8 @@ export class HostLoad {
       this.#host.readSystemCpu?.() ?? null,
       this.#host.readEndedChildrenCpuSeconds?.() ?? null
     ]);
-    const sample = { takenAt: Date.now(), system, endedChildren, byPid: new Map(), proxyCpuSeconds: this.#host.readProxyCpuSeconds() };
+    const sample = { takenAt: Date.now(), system, endedChildren, byPid: new Map(), proxyCpuSeconds: this.#host.readProxyCpuSeconds(),
+      processor: this.#host.sampleProcessor?.() ?? null };
     const previous = this.#idleAvailabilitySample;
     this.#idleAvailabilitySample = sample;
     if (previous === null) {
@@ -556,7 +558,7 @@ export class HostLoad {
     // stands is seen beside what it is free for (torrent-tv/meta#178); a proxy
     // nobody watches stays quiet.
     if (share !== null && watched) {
-      await this.#sayMachine(share, proxyShare, 0, suspended);
+      await this.#sayMachine(share, proxyShare, 0, suspended, false, this.#processorLine(previous, sample, share, proxyShare));
     }
   }
 
@@ -566,7 +568,7 @@ export class HostLoad {
    * and the processor share divide; `free` is what nothing used over the
    * interval. Unknown stays `n/a`, never zero.
    */
-  async #sayMachine(share, proxyShare, running, suspended, incomplete = false) {
+  async #sayMachine(share, proxyShare, running, suspended, incomplete = false, processor = "") {
     const machine = await this.#host.readMachineState();
     // HOW LATE THIS THREAD'S LOOP RAN over the interval: every answer to FFmpeg
     // and every hand-over of a copy waits for it (torrent-tv/meta#178). Begun at
@@ -591,10 +593,74 @@ export class HostLoad {
       // What the offer will multiply a prediction by, in the same line as the
       // readings it comes from.
       ` available=${asPercent(this.hostAvailability.share)} for-encoders=${asPercent(this.hostAvailability.forEncoders)}` +
-      ` free=${asPercent(free)}` + late
+      ` free=${asPercent(free)}` + processor + late
     );
   }
 
+  /**
+   * The processor's upper bound and its measured free amount over the same
+   * interval, for the machine's line (torrent-tv/meta#180, phase 1). Read
+   * beside today's figures and used by no decision; empty where the host does
+   * not take these readings.
+   *
+   * @param {{ processor?: object | null }} previous
+   * @param {{ processor?: object | null }} sample
+   * @param {{ processShare: number | null, systemShare: number | null }} share
+   * @param {number | null} proxyShare
+   * @returns {string}
+   */
+  #processorLine(previous, sample, share, proxyShare) {
+    if (typeof this.#host.processorBetween !== "function" || typeof this.#host.describeProcessor !== "function") {
+      return "";
+    }
+    const ourShare = Number.isFinite(proxyShare) && Number.isFinite(share.processShare) ? proxyShare + share.processShare : null;
+    return this.#host.describeProcessor(this.#host.processorBetween(previous.processor, sample.processor, {
+      systemShare: share.systemShare,
+      ourShare,
+      hostCores: Math.max(1, os.cpus().length)
+    }));
+  }
+
+  /**
+   * What each run's FFmpeg spent per second of film over the interval between
+   * two readings, with the runs beside it (torrent-tv/meta#180, phase 1).
+   *
+   * In CPU seconds per second of film (`k`), which is not the unit of
+   * `EncodeCost` (seconds of the whole machine per second of film, from
+   * speed): a run that uses all it is given has `cost = k / cores`. A `k` read
+   * beside other runs already carries their contention, so it is said with
+   * them; a run is alone only when no other run was alive at either reading.
+   * Only runs read at both ends that made film in between are said. For the
+   * log only.
+   *
+   * @param {{ takenAt: number, byPid: Map<number, number>, runs?: Map<number, { film: number | null, label: string, kind: string }> }} previous
+   * @param {{ takenAt: number, byPid: Map<number, number>, runs?: Map<number, { film: number | null, label: string, kind: string }> }} sample
+   */
+  #sayRunCosts(previous, sample) {
+    if (!previous.runs || !sample.runs) return;
+    const elapsedSec = (sample.takenAt - previous.takenAt) / 1000;
+    if (!(elapsedSec > 0)) return;
+    const alive = new Map([...previous.runs, ...sample.runs]);
+    const said = [];
+    for (const [pid, run] of sample.runs) {
+      const before = previous.runs.get(pid);
+      const cpuNow = sample.byPid.get(pid);
+      const cpuBefore = previous.byPid.get(pid);
+      if (!before || !Number.isFinite(cpuNow) || !Number.isFinite(cpuBefore) || cpuNow < cpuBefore) continue;
+      if (!Number.isFinite(run.film) || !Number.isFinite(before.film)) continue;
+      const film = run.film - before.film;
+      if (!(film > 0)) continue;
+      const cpu = cpuNow - cpuBefore;
+      const beside = [...alive].filter(([other]) => other !== pid).map(([, other]) => other.kind);
+      said.push(`${run.label} ${run.kind} k=${(cpu / film).toFixed(3)} CPU-s per film-s ` +
+        `(${cpu.toFixed(2)} CPU-s, ${film.toFixed(2)} film-s, speed ${(film / elapsedSec).toFixed(2)}x, ` +
+        `${(cpu / elapsedSec).toFixed(2)} cores), beside ${beside.length}` + (beside.length > 0 ? ` [${beside.join(", ")}]` : ""));
+    }
+    if (said.length > 0) {
+      logger.info(`processor: run cost over ${elapsedSec.toFixed(1)}s: ${said.join("; ")}`);
+    }
+  }
+
   /**
    * @param {{ elapsedSec: number, processShare: number | null, systemShare: number | null, rejected?: string[] } | null} share
    * @param {{ proxyCpuSeconds: number | null }} previous
@@ -658,9 +724,17 @@ export class HostLoad {
     // that ends meanwhile drops its process, and read after the await it threw
     // `Cannot read properties of null (reading 'pid')`, seven times in one
     // field session, each losing that reading (r64, torrent-tv/meta#178).
-    const runs = encoding.flatMap((session) => this.#host.liveRunsOf(session))
+    // With the film each has made and what it is, for its cost over the
+    // interval (torrent-tv/meta#180).
+    const runs = encoding.flatMap((session) => this.#host.liveRunsOf(session)
       .filter((run) => Number.isInteger(run.process?.pid))
-      .map((run) => ({ pid: run.process.pid, startedAt: run.startedAt }));
+      .map((run) => ({
+        pid: run.process.pid,
+        startedAt: run.startedAt,
+        film: filmMadeBy(run),
+        label: `${session.outputKey ?? session.id} #${run.from}..#${run.to}`,
+        kind: costKindForSession(session)
+      })));
     const [system, endedChildren, ...cpuReadings] = await Promise.all([
       this.#host.readSystemCpu(),
       this.#host.readEndedChildrenCpuSeconds?.() ?? null,
@@ -670,12 +744,15 @@ export class HostLoad {
     const byPid = new Map();
     /** @type {Map<number, number>} */
     const startedAt = new Map();
+    /** @type {Map<number, { film: number | null, label: string, kind: string }>} */
+    const runsByPid = new Map();
     runs.forEach((run, index) => {
       const seconds = cpuReadings[index];
       if (seconds !== null) {
         byPid.set(run.pid, seconds);
         startedAt.set(run.pid, run.startedAt);
       }
+      runsByPid.set(run.pid, { film: run.film, label: run.label, kind: run.kind });
     });
     const sample = {
       takenAt: Date.now(),
@@ -687,7 +764,9 @@ export class HostLoad {
       // piece store and the delivery. None of it is in the encode budget, and
       // on the addon host it is most of what the machine does while encoders
       // are suspended.
-      proxyCpuSeconds: this.#host.readProxyCpuSeconds()
+      proxyCpuSeconds: this.#host.readProxyCpuSeconds(),
+      runs: runsByPid,
+      processor: this.#host.sampleProcessor?.() ?? null
     };
     const previous = this.#hostLoadSample;
     this.#hostLoadSample = sample;
@@ -699,6 +778,7 @@ export class HostLoad {
     // `/proc` there are no readings at all, and the share is null rather than a
     // confident zero.
     const encoders = encoderCpuBetween(previous, sample);
+    this.#sayRunCosts(previous, sample);
     const share = this.#host.shareOfMachine(
       { takenAt: previous.takenAt, processCpuSeconds: encoders === null ? null : 0, system: previous.system },
       { takenAt: sample.takenAt, processCpuSeconds: encoders?.seconds ?? null, system: sample.system }
@@ -718,7 +798,7 @@ export class HostLoad {
     if (encoders && !encoders.complete) {
       // A lower bound of our own load is not a reading of what other work left
       // us: the last availability stands, and the line says so.
-      await this.#sayMachine(share, null, running, suspended, true);
+      await this.#sayMachine(share, null, running, suspended, true, this.#processorLine(previous, sample, share, null));
       return false;
     }
     // Kept for the quality offer, which predicts from a benchmark taken on a
@@ -726,7 +806,7 @@ export class HostLoad {
     // machine a new encoder could actually have. Only what nobody has been
     // charged for is subtracted — see `encode/available-share.js`.
     const proxyShare = this.#setAvailability(share, previous, sample);
-    await this.#sayMachine(share, proxyShare, running, suspended);
+    await this.#sayMachine(share, proxyShare, running, suspended, false, this.#processorLine(previous, sample, share, proxyShare));
     return false;
   }
 
@@ -756,3 +836,15 @@ export class HostLoad {
       : 0;
   }
 }
+
+/**
+ * The film a run has made since it started, from its progress, or null before
+ * it has said any.
+ *
+ * @param {{ progress?: { processedSeconds?: number, startPositionSeconds?: number } }} run
+ * @returns {number | null}
+ */
+function filmMadeBy(run) {
+  const made = Number(run.progress?.processedSeconds) - Number(run.progress?.startPositionSeconds ?? 0);
+  return Number.isFinite(made) && made >= 0 ? made : null;
+}

package/services/server/wire-outputs.js

@@ -101,6 +101,7 @@ import { contentOf, LocalObservations, PROXY_ROOT } from "../encode/LocalObserva
 import { configurationKeyOf } from "../encode/fingerprint.js";
 import { qualityStateOf } from "../encode/quality/OutputQualityState.js";
 import { downloadShare } from "../storage/processor-share.js";
+import { describeProcessor, processorBetween, sampleProcessor } from "../storage/machine-processor.js";
 import { ENCODE_RUN_STATE, processCanBeSignalled } from "../encode/encode-run-state.js";
 import { secondsUntilClosed } from "../encode/piece-close-time.js";
 
@@ -195,6 +196,9 @@ export function wireOutputs({
     // A software preset every picture is encoded at, for runs compared at one
     // quality (`--pin-preset`); null leaves the choice to the machine.
     pinnedPreset = null,
+    // A size every original-input stretch is given in place of copy rate
+    // times open (`--input-stretch-bytes`); null keeps the rule.
+    inputStretchBytes = null,
     // What every encoder this host may use was qualified and measured to do at
     // startup, by kind (`encode/calibration.js`). Takes the place of the one
     // array above, which a test may still hand in on its own.
@@ -300,6 +304,11 @@ export function wireOutputs({
     readProxyCpuSeconds,
     readSystemCpu,
     shareOfMachine,
+    // The processor's upper bound and measured free amount, said beside the
+    // machine's line and used by no decision yet (torrent-tv/meta#180).
+    sampleProcessor: () => sampleProcessor(),
+    processorBetween,
+    describeProcessor,
     liveRunsOf: (...args) => parts.encodeRuns.liveRunsOf(...args),
     runStateOf: (...args) => parts.encodeRuns.runStateOf(...args),
     get getSourceStats() { return parts.getSourceStats; },
@@ -1097,6 +1106,7 @@ export function wireOutputs({
     readRanges: readEncodeRanges,
     heldRanges: readEncodeHeldRanges,
     stretchCopied,
+    stretchBytes: inputStretchBytes,
     reviseBudget: () => parts.machineBudget.revise("memory", "admitted encode inputs"),
     capacity: () => parts.machineBudget.capacityOf("memory"),
     // How long this output's input takes to open, from the one owner of start
@@ -1120,6 +1130,9 @@ export function wireOutputs({
       parts.serving.invalidateWaits(output);
     }
   }) : null);
+  if (parts.encodeInputs && inputStretchBytes) {
+    logger.info(`encode input: every original-input stretch sized to ${inputStretchBytes} bytes (--input-stretch-bytes)`);
+  }
   // One owner of the disk, and the list of what takes it lives with the owner.
   parts.machineBudget = wireMachineBudget({
     segmentStore: parts.segmentStore,

package/services/storage/machine-memory.js

@@ -163,7 +163,7 @@ function hostAvailable(readFile) {
 function containerRoom(readFile, hostTotal) {
   let membership;
   try {
-    membership = ownCgroup(readFile("/proc/self/cgroup"));
+    membership = cgroupOf(readFile("/proc/self/cgroup"), "memory");
   } catch {
     return null; // silent-ok: not Linux, or /proc is not readable.
   }
@@ -187,18 +187,19 @@ function containerRoom(readFile, hostTotal) {
 }
 
 /**
- * Which cgroup this process is in, for the memory controller.
+ * Which cgroup this process is in, for one controller (`memory`, `cpu`).
  *
- * A host in hybrid mode lists both versions; the memory controller is then on
- * version 1, named in the line that lists it.
+ * A host in hybrid mode lists both versions; a controller mounted on version 1
+ * is named in the line that lists it, and only then is version 2 the answer.
  *
  * @param {string} text - `/proc/self/cgroup`.
+ * @param {string} controller
  * @returns {{ version: 1 | 2, path: string } | null}
  */
-function ownCgroup(text) {
+export function cgroupOf(text, controller) {
   for (const line of text.split("\n")) {
     const match = /^\d+:([^:]*):(.*)$/.exec(line);
-    if (match && match[1].split(",").includes("memory")) {
+    if (match && match[1].split(",").includes(controller)) {
       return { version: 1, path: match[2] };
     }
   }
@@ -219,7 +220,7 @@ function ownCgroup(text) {
  * @param {string} path
  * @returns {string[]}
  */
-function levelsOf(mount, path) {
+export function levelsOf(mount, path) {
   const parts = path.split("/").filter(Boolean);
   if (parts.includes("..")) {
     return [];

package/services/storage/machine-processor.js

@@ -0,0 +1,395 @@
+/**
+ * @file How much processor this process could ever use, and how much of it was
+ * free over an interval (torrent-tv/meta#180, phase 1: readings only).
+ *
+ * Two quantities, never put in each other's place:
+ *
+ * 1. **The upper bound**, in cores: what the operating system lets this process
+ *    run on — the CPUs it may be scheduled on (`os.availableParallelism()`,
+ *    which follows the affinity mask on Linux) and the CPU quota of every cgroup
+ *    on its path (Docker's `--cpus`, systemd's `CPUQuota=`), the smallest of
+ *    them. A quota may be fractional: `cpu.max` of `150000 100000` is one and a
+ *    half cores. The bound says what can never be exceeded; it says nothing of
+ *    what is free.
+ * 2. **The measured free amount**, in cores: the idle time of the machine's
+ *    CPUs over the interval — from `/proc/stat` where the caller has it, from
+ *    `os.cpus()` times on every other platform — and, under a bound smaller
+ *    than the machine, no more than the bound less what this proxy and its
+ *    encoders used. Inside a container both readings describe the HOST's CPUs,
+ *    which is what the other containers left. Where neither reading exists, or
+ *    our own use is unknown under such a bound, it is unknown, and said so.
+ *
+ * A cgroup's own `cpu.stat` is NOT a reading of other work: its usage is what
+ * this cgroup and its descendants used, and its throttling says that the quota
+ * held our work back. Both are reported as that, and nothing else.
+ *
+ * Memory had the same defect this file answers for the processor — the host's
+ * figure where a container's limit decided — and was fixed in #153/#155
+ * (`machine-memory.js`); the cgroup is found the same way.
+ */
+
+import { readFileSync } from "node:fs";
+import os from "node:os";
+
+import { cgroupOf, levelsOf } from "./machine-memory.js";
+
+/**
+ * Where each cgroup version keeps its CPU controller, and under which names.
+ *
+ * Version 1 keeps the usage in a controller of its own, `cpuacct`, mounted
+ * either beside `cpu` (`cpu,cpuacct`, with `cpu` and `cpuacct` as links to
+ * it) or on its own; `/sys/fs/cgroup/cpuacct` names it in both cases. Its
+ * counters are in nanoseconds; version 2's in microseconds.
+ */
+const HIERARCHIES = {
+  1: { mount: "/sys/fs/cgroup/cpu", source: "cgroup v1 cfs", usageController: "cpuacct", usageMount: "/sys/fs/cgroup/cpuacct",
+    usage: "cpuacct.usage", usageUnit: 1e-9, throttledName: "throttled_time", throttledUnit: 1e-9 },
+  2: { mount: "/sys/fs/cgroup", source: "cgroup v2 cpu.max", usageController: "cpu", usageMount: "/sys/fs/cgroup",
+    usage: "cpu.stat", usageUnit: 1e-6, throttledName: "throttled_usec", throttledUnit: 1e-6 }
+};
+
+/**
+ * @typedef {object} ProcessorBound
+ * @property {number} cores - What this process can never exceed, in cores.
+ * @property {number} parallelism - The CPUs it may be scheduled on.
+ * @property {number | null} quotaCores - The smallest cgroup quota on its path, or null for none.
+ * @property {string} source - Which of the two decided `cores`.
+ */
+
+/**
+ * @typedef {object} CgroupCpu
+ * @property {number | null} usageSeconds - CPU seconds this cgroup and its descendants used.
+ * @property {number | null} throttledPeriods - Periods the binding quota held the cgroup back.
+ * @property {number | null} throttledSeconds - Time it was held back.
+ */
+
+/**
+ * @typedef {object} ProcessorSample
+ * @property {number} takenAt - Wall clock, ms.
+ * @property {ProcessorBound} bound
+ * @property {CgroupCpu | null} cgroup
+ * @property {number | null} idleSeconds - Idle time of every CPU since boot, from `os.cpus()`.
+ */
+
+/**
+ * What this process can never exceed.
+ *
+ * @param {object} [options]
+ * @param {(path: string) => string} [options.readFile] - Throws where a file is absent; injected by the checks.
+ * @param {number} [options.parallelism]
+ * @returns {ProcessorBound}
+ */
+export function processorBound({ readFile = readText, parallelism = os.availableParallelism() } = {}) {
+  const quota = quotaOnPath(readFile);
+  const cores = Math.max(1, parallelism);
+  if (quota !== null && quota.cores < cores) {
+    return { cores: quota.cores, parallelism: cores, quotaCores: quota.cores, source: HIERARCHIES[quota.version].source };
+  }
+  return { cores, parallelism: cores, quotaCores: quota?.cores ?? null, source: "cores" };
+}
+
+/**
+ * This cgroup's own CPU counters: its usage, and how often its binding quota
+ * held it back. Null where there is no cgroup to read (not Linux, no CPU
+ * controller); each counter null where its file is not there.
+ *
+ * @param {object} [options]
+ * @param {(path: string) => string} [options.readFile]
+ * @returns {CgroupCpu | null}
+ */
+export function cgroupCpu({ readFile = readText } = {}) {
+  const levels = cpuLevels(readFile);
+  if (levels === null) {
+    return null;
+  }
+  const hierarchy = HIERARCHIES[levels.version];
+  let usageSeconds = null;
+  for (const directory of usageLevels(readFile, levels.version)) {
+    const text = tryRead(readFile, `${directory}/${hierarchy.usage}`);
+    if (text === null) continue;
+    const value = levels.version === 2 ? statField(text, "usage_usec") : wholeNumber(text);
+    if (value !== null) {
+      usageSeconds = value * hierarchy.usageUnit;
+      break;
+    }
+  }
+  let throttledPeriods = null;
+  let throttledSeconds = null;
+  const binding = bindingLevel(readFile, levels);
+  const stat = binding === null ? null : tryRead(readFile, `${binding}/cpu.stat`);
+  if (stat !== null) {
+    throttledPeriods = statField(stat, "nr_throttled");
+    const time = statField(stat, hierarchy.throttledName);
+    throttledSeconds = time === null ? null : time * hierarchy.throttledUnit;
+  }
+  return { usageSeconds, throttledPeriods, throttledSeconds };
+}
+
+/**
+ * The idle time of every CPU since boot, from `os.cpus()`, which every
+ * platform answers. Null where it states no times.
+ *
+ * @param {Array<{ times?: { idle?: number } }>} [cpus]
+ * @returns {number | null} Seconds.
+ */
+export function cpusIdleSeconds(cpus = os.cpus()) {
+  if (!Array.isArray(cpus) || cpus.length === 0) {
+    return null;
+  }
+  let idleMs = 0;
+  for (const cpu of cpus) {
+    const idle = Number(cpu?.times?.idle);
+    if (!Number.isFinite(idle)) return null;
+    idleMs += idle;
+  }
+  return idleMs / 1000;
+}
+
+/**
+ * One reading of everything above, to be compared against the next.
+ *
+ * @param {object} [options]
+ * @param {(path: string) => string} [options.readFile]
+ * @param {() => Array<{ times?: { idle?: number } }>} [options.cpus]
+ * @param {number} [options.parallelism]
+ * @param {() => number} [options.now]
+ * @returns {ProcessorSample}
+ */
+export function sampleProcessor({ readFile = readText, cpus = () => os.cpus(), parallelism, now = Date.now } = {}) {
+  return {
+    takenAt: now(),
+    bound: processorBound({ readFile, ...(parallelism === undefined ? {} : { parallelism }) }),
+    cgroup: cgroupCpu({ readFile }),
+    idleSeconds: cpusIdleSeconds(cpus())
+  };
+}
+
+/**
+ * @typedef {object} ProcessorReading
+ * @property {ProcessorBound} bound
+ * @property {number | null} freeCores - Measured free, in cores; null when unknown.
+ * @property {string} freeSource - Where it came from, or why it is unknown.
+ * @property {number | null} cgroupUsedCores - This cgroup's own use over the interval.
+ * @property {number | null} throttledPeriods
+ * @property {number | null} throttledSeconds
+ */
+
+/**
+ * What two samples say about the interval between them.
+ *
+ * @param {ProcessorSample | null | undefined} before
+ * @param {ProcessorSample | null | undefined} after
+ * @param {object} measured - Shares of the WHOLE machine over the same interval.
+ * @param {number | null} measured.systemShare - Everything the machine did, from `/proc/stat`; null where unread.
+ * @param {number | null} measured.ourShare - The proxy and its encoders; null where either is unread.
+ * @param {number} measured.hostCores - The CPUs those shares are fractions of.
+ * @returns {ProcessorReading | null} Null without two samples.
+ */
+export function processorBetween(before, after, { systemShare, ourShare, hostCores }) {
+  if (!before || !after) {
+    return null;
+  }
+  const elapsedSec = (after.takenAt - before.takenAt) / 1000;
+  if (!(elapsedSec > 0)) {
+    return null;
+  }
+  const cores = Math.max(1, hostCores);
+  let idleCores = null;
+  let freeSource = "unknown: no idle reading";
+  if (Number.isFinite(systemShare)) {
+    idleCores = clamp((1 - systemShare) * cores, 0, cores);
+    freeSource = "/proc/stat";
+  } else if (Number.isFinite(before.idleSeconds) && Number.isFinite(after.idleSeconds) && after.idleSeconds >= before.idleSeconds) {
+    idleCores = clamp((after.idleSeconds - before.idleSeconds) / elapsedSec, 0, cores);
+    freeSource = "os.cpus";
+  }
+  let freeCores = idleCores;
+  const bound = after.bound;
+  if (idleCores !== null && bound.cores < cores) {
+    // Under a bound smaller than the machine, an idle CPU is ours only as far
+    // as the bound leaves room beside what we already use.
+    if (Number.isFinite(ourShare)) {
+      freeCores = Math.min(idleCores, Math.max(0, bound.cores - ourShare * cores));
+    } else {
+      freeCores = null;
+      freeSource = "unknown: our own use is unread under a limit";
+    }
+  }
+  const delta = (key) => {
+    const from = before.cgroup?.[key];
+    const to = after.cgroup?.[key];
+    return Number.isFinite(from) && Number.isFinite(to) && to >= from ? to - from : null;
+  };
+  const used = delta("usageSeconds");
+  return {
+    bound,
+    freeCores,
+    freeSource,
+    cgroupUsedCores: used === null ? null : used / elapsedSec,
+    throttledPeriods: delta("throttledPeriods"),
+    throttledSeconds: delta("throttledSeconds")
+  };
+}
+
+/**
+ * The reading as it is written into the machine's line of the log.
+ *
+ * @param {ProcessorReading | null} reading
+ * @returns {string}
+ */
+export function describeProcessor(reading) {
+  if (reading === null) {
+    return " bound=n/a free-cores=n/a";
+  }
+  const { bound } = reading;
+  const limit = bound.quotaCores === null ? "no quota" : `quota ${bound.quotaCores.toFixed(2)}`;
+  const free = reading.freeCores === null ? `unknown (${reading.freeSource.replace(/^unknown: /, "")})`
+    : `${reading.freeCores.toFixed(2)} (${reading.freeSource})`;
+  const used = reading.cgroupUsedCores === null ? "n/a" : `${reading.cgroupUsedCores.toFixed(2)} cores`;
+  const throttled = reading.throttledPeriods === null && reading.throttledSeconds === null ? "n/a"
+    : `${reading.throttledPeriods ?? "n/a"} periods ${reading.throttledSeconds === null ? "n/a" : `${reading.throttledSeconds.toFixed(2)}s`}`;
+  return ` bound=${bound.cores.toFixed(2)} cores (${bound.source}; ${bound.parallelism} cpus, ${limit})` +
+    ` free-cores=${free} cgroup-used=${used} throttled=${throttled}`;
+}
+
+/**
+ * This process's cgroup for the CPU controller and the directories to read,
+ * its own first, or null where there is none.
+ *
+ * @param {(path: string) => string} readFile
+ * @returns {{ version: 1 | 2, directories: string[] } | null}
+ */
+function cpuLevels(readFile) {
+  const text = tryRead(readFile, "/proc/self/cgroup");
+  const membership = text === null ? null : cgroupOf(text, "cpu");
+  if (membership === null) {
+    return null;
+  }
+  const directories = levelsOf(HIERARCHIES[membership.version].mount, membership.path);
+  return directories.length === 0 ? null : { version: membership.version, directories };
+}
+
+/**
+ * The directories that hold this process's CPU usage, its own first.
+ *
+ * @param {(path: string) => string} readFile
+ * @param {1 | 2} version
+ * @returns {string[]}
+ */
+function usageLevels(readFile, version) {
+  const hierarchy = HIERARCHIES[version];
+  const text = tryRead(readFile, "/proc/self/cgroup");
+  const membership = text === null ? null : cgroupOf(text, hierarchy.usageController);
+  return membership === null ? [] : levelsOf(hierarchy.usageMount, membership.path);
+}
+
+/**
+ * The quota of one cgroup directory, in cores, or null where it states none.
+ *
+ * @param {(path: string) => string} readFile
+ * @param {1 | 2} version
+ * @param {string} directory
+ * @returns {number | null}
+ */
+function quotaAt(readFile, version, directory) {
+  if (version === 2) {
+    const text = tryRead(readFile, `${directory}/cpu.max`);
+    const [quota, period] = (text ?? "").trim().split(/\s+/);
+    const cores = Number(quota) / Number(period);
+    return /^\d+$/.test(quota ?? "") && Number.isFinite(cores) && cores > 0 ? cores : null;
+  }
+  const quota = wholeNumber(tryRead(readFile, `${directory}/cpu.cfs_quota_us`) ?? "");
+  const period = wholeNumber(tryRead(readFile, `${directory}/cpu.cfs_period_us`) ?? "");
+  // `-1` is version 1's "no quota", which `wholeNumber` does not accept.
+  return quota !== null && period !== null && quota > 0 && period > 0 ? quota / period : null;
+}
+
+/**
+ * The smallest quota on this process's cgroup path, and its version.
+ *
+ * @param {(path: string) => string} readFile
+ * @returns {{ version: 1 | 2, cores: number } | null}
+ */
+function quotaOnPath(readFile) {
+  const levels = cpuLevels(readFile);
+  if (levels === null) {
+    return null;
+  }
+  const binding = bindingLevel(readFile, levels);
+  return binding === null ? null : { version: levels.version, cores: quotaAt(readFile, levels.version, binding) };
+}
+
+/**
+ * The directory whose quota is the smallest on the path, or null for none.
+ *
+ * @param {(path: string) => string} readFile
+ * @param {{ version: 1 | 2, directories: string[] }} levels
+ * @returns {string | null}
+ */
+function bindingLevel(readFile, levels) {
+  let binding = null;
+  let smallest = Infinity;
+  for (const directory of levels.directories) {
+    const cores = quotaAt(readFile, levels.version, directory);
+    if (cores !== null && cores < smallest) {
+      smallest = cores;
+      binding = directory;
+    }
+  }
+  return binding;
+}
+
+/**
+ * @param {string} path
+ * @returns {string}
+ */
+function readText(path) {
+  return readFileSync(path, "utf8");
+}
+
+/**
+ * @param {(path: string) => string} readFile
+ * @param {string} path
+ * @returns {string | null}
+ */
+function tryRead(readFile, path) {
+  try {
+    return readFile(path);
+  } catch {
+    return null; // silent-ok: not Linux, no such level, or no CPU controller on it.
+  }
+}
+
+/**
+ * @param {string} text
+ * @returns {number | null}
+ */
+function wholeNumber(text) {
+  const trimmed = text.trim();
+  return /^\d+$/.test(trimmed) ? Number(trimmed) : null;
+}
+
+/**
+ * One field of a `cpu.stat`.
+ *
+ * @param {string} text
+ * @param {string} name
+ * @returns {number | null}
+ */
+function statField(text, name) {
+  for (const line of text.split("\n")) {
+    const [key, value = ""] = line.trim().split(/\s+/);
+    if (key === name) return wholeNumber(value);
+  }
+  return null;
+}
+
+/**
+ * @param {number} value
+ * @param {number} low
+ * @param {number} high
+ * @returns {number}
+ */
+function clamp(value, low, high) {
+  return Math.min(high, Math.max(low, value));
+}

package/services/storage/processor-report.js

@@ -14,6 +14,12 @@
  * threads only says where that time went. Each ffmpeg is its own process and
  * is added beside it. Linux only for the split and for ffmpeg; elsewhere the
  * process figure alone is said.
+ *
+ * Each run is also said on its own once a minute: the middle of its seconds'
+ * processor and speed, and how far those seconds sit from it
+ * (torrent-tv/meta#180, phase 1). That scatter is what a slowdown of the most
+ * urgent run would have to exceed before it could be put down to another run
+ * beside it; nothing decides on it yet.
  */
 
 import { readFileSync, readdirSync } from "node:fs";
@@ -212,35 +218,108 @@ export function describeShares(seconds) {
   return `${said.join(", ")} of a core over ${seconds.length}s`;
 }
 
+/**
+ * One second of each run read at both ends of it: the cores its ffmpeg used
+ * and the film it made per second of wall clock. A run whose film is not
+ * known at either end says its processor alone.
+ *
+ * @param {{ at: number, ffmpeg: Map<number, number>, runs: Map<number, { label: string, film: number | null }> }} before
+ * @param {{ at: number, ffmpeg: Map<number, number>, runs: Map<number, { label: string, film: number | null }> }} after
+ * @returns {Map<number, { label: string, cores: number, speed: number | null }>}
+ */
+export function runSecondsBetween(before, after) {
+  /** @type {Map<number, { label: string, cores: number, speed: number | null }>} */
+  const seconds = new Map();
+  const elapsed = (after.at - before.at) / 1000;
+  if (!(elapsed > 0)) return seconds;
+  for (const [pid, run] of after.runs) {
+    const from = before.ffmpeg.get(pid);
+    const to = after.ffmpeg.get(pid);
+    const earlier = before.runs.get(pid);
+    if (!earlier || !Number.isFinite(from) || !Number.isFinite(to) || to < from) continue;
+    const made = Number.isFinite(run.film) && Number.isFinite(earlier.film) && run.film >= earlier.film ? run.film - earlier.film : null;
+    seconds.set(pid, { label: run.label, cores: (to - from) / elapsed, speed: made === null ? null : made / elapsed });
+  }
+  return seconds;
+}
+
+/**
+ * Per run, over the interval: the median of its seconds and the median of
+ * their distances to it, for its processor and its speed.
+ *
+ * @param {Map<number, { label: string, cores: number[], speed: number[] }>} runs
+ * @returns {string} Empty when no run was read for two seconds.
+ */
+export function describeRuns(runs) {
+  const said = [];
+  for (const run of runs.values()) {
+    if (run.cores.length < 2) continue;
+    const cores = middleAndScatter(run.cores);
+    const speed = middleAndScatter(run.speed);
+    said.push(`${run.label} cores ${cores.middle.toFixed(2)} (scatter ${cores.scatter.toFixed(2)})` +
+      (speed === null ? ", speed n/a" : `, speed ${speed.middle.toFixed(2)}x (scatter ${speed.scatter.toFixed(2)})`) +
+      ` over ${run.cores.length}s`);
+  }
+  return said.join("; ");
+}
+
+/**
+ * @param {number[]} values
+ * @returns {{ middle: number, scatter: number } | null}
+ */
+function middleAndScatter(values) {
+  if (values.length === 0) return null;
+  const median = (list) => {
+    const sorted = [...list].sort((a, b) => a - b);
+    const half = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0 ? (sorted[half - 1] + sorted[half]) / 2 : sorted[half];
+  };
+  const middle = median(values);
+  return { middle, scatter: median(values.map((value) => Math.abs(value - middle))) };
+}
+
 /**
  * Read every second, say once a minute, and every second while `detailed()`
  * is true.
  *
  * @param {object} params
  * @param {(line: string) => void} params.log
- * @param {() => number[]} params.ffmpegPids - The encoders running now.
+ * @param {() => Array<{ pid: number, label: string, film: number | null }>} params.ffmpegRuns - The encoders running now, with the film each has made.
  * @param {() => boolean} [params.detailed]
  * @param {number} [params.everyMs]
  * @param {number} [params.sayEveryMs]
  * @returns {{ stop: () => void }}
  */
-export function startProcessorReport({ log, ffmpegPids, detailed = () => false, everyMs = 1000, sayEveryMs = 60_000 }) {
+export function startProcessorReport({ log, ffmpegRuns, detailed = () => false, everyMs = 1000, sayEveryMs = 60_000 }) {
   let previous = null;
   let series = [];
+  /** @type {Map<number, { label: string, cores: number[], speed: number[] }>} */
+  let runSeries = new Map();
   let since = Date.now();
   const threads = threadReader();
   const tick = () => {
     const at = Date.now();
     const usage = process.cpuUsage();
-    const now = { at, process: (usage.user + usage.system) / 1e6, threads: threads.read(), ffmpeg: readProcessesSeconds(ffmpegPids()) };
+    const runs = new Map(ffmpegRuns().map((run) => [run.pid, { label: run.label, film: run.film }]));
+    const now = { at, process: (usage.user + usage.system) / 1e6, threads: threads.read(), ffmpeg: readProcessesSeconds([...runs.keys()]), runs };
     const shares = previous ? sharesBetween(previous, now) : null;
+    const perRun = previous ? runSecondsBetween(previous, now) : new Map();
     previous = now;
     if (!shares) return;
     series.push(shares);
+    for (const [pid, second] of perRun) {
+      const kept = runSeries.get(pid) ?? { label: second.label, cores: [], speed: [] };
+      kept.cores.push(second.cores);
+      if (second.speed !== null) kept.speed.push(second.speed);
+      runSeries.set(pid, kept);
+    }
     if (detailed()) log(`processor: ${describeShares([shares])}`);
     if (now.at - since >= sayEveryMs) {
       log(`processor: ${describeShares(series)}`);
+      const perRunLine = describeRuns(runSeries);
+      if (perRunLine) log(`processor: per run, by second: ${perRunLine}`);
       series = [];
+      runSeries = new Map();
       since = now.at;
       // A torrent thread made again since is counted as the rest until then.
       threads.rediscover();