@torrent-tv/proxy 2.107.56 → 2.107.58

package/services/server/wire-outputs.js

@@ -8,6 +8,8 @@
  * `server.js` to give each route the ones that route asks.
  */
 
+import { EventEmitter } from "node:events";
+import { writeJsonAtomic } from "../../utils/atomic-json.js";
 import { logger } from "../../utils/logger.js";
 import { KeyframeTables } from "../media/KeyframeTables.js";
 import { bandOf, WaitLedger } from "../encode/OutputWaitLedger.js";
@@ -15,6 +17,7 @@ import { readEndedChildrenCpuSeconds, readMachineState, readProcessCpuSeconds, r
 import { minimumBufferFrom } from "../torrent/supply-margin.js";
 import { PriorityOrchestrator } from "../viewer/PriorityOrchestrator.js";
 import { urgentOutputsReady } from "../viewer/urgent-output-ready.js";
+import { trackChoicesOf } from "../viewer/segment-demands.js";
 import { softwareDescriptor } from "../encode/hwaccel.js";
 import { resolveSegmentFormat } from "../encode/segment-formats/index.js";
 import { Timelines } from "../encode/output/Timeline.js";
@@ -50,7 +53,6 @@ import {
   noteServingVerdict,
   noteStepBeingWarmed,
   noteStepOnScreen,
-  outputsBeingPrepared,
   placeInitially,
   placeOn,
   presentOn,
@@ -99,10 +101,11 @@ import { freeBytesFor } from "../storage/free.js";
 import { PlaybackStorageDemand } from "../storage/PlaybackStorageDemand.js";
 import { inputCacheRoot, probeInputCache } from "../encode/input-cache.js";
 import path from "node:path";
-import { contentOf, LocalObservations, PROXY_ROOT } from "../encode/LocalObservations.js";
+import { contentOf, LocalObservations } from "../encode/LocalObservations.js";
 import { configurationKeyOf } from "../encode/fingerprint.js";
 import { qualityStateOf } from "../encode/quality/OutputQualityState.js";
-import { downloadShare } from "../storage/processor-share.js";
+import { preparedAddresses, watchedAddresses } from "../viewer/watched-addresses.js";
+import { downloadShare, watchedFiles } from "../storage/processor-share.js";
 import { describeProcessor, processorBetween, sampleProcessor } from "../storage/machine-processor.js";
 import { ProcessorBudget } from "../encode/ProcessorBudget.js";
 import { ENCODE_RUN_STATE, processCanBeSignalled } from "../encode/encode-run-state.js";
@@ -232,19 +235,17 @@ export function wireOutputs({
     diagnostics = null,
     diagnosticsRoot = "",
     memoryClaimant = null,
-    resolveEncodeInput = null,
+    encodeInputResolution = null,
     repriceEncodeDemand = null,
     readSourceMedia = null,
     readMetadataActivity = null,
     sourceInputsFor = null,
     readEncodeRanges = null,
     openEncodeWholeFile = null,
-    stretchCopied = () => {},
     readEncodeHeldRanges = null,
     encodeInputs = null,
     indexMemory = null,
     budgetPolicy = null,
-    onViewerChanged = null,
     // Told when an encoder starts or ends: the room this host has for one more
     // encode changes with it, and the server chooses proxies by that room.
     onEncodersChanged = null}) {
@@ -774,19 +775,15 @@ export function wireOutputs({
   // watched file of every torrent.
   parts.applyDownloadShare = () => {
     if (!parts.setDownloadRate || !parts.hostLoad || !parts.encodeCost) return;
-    const files = new Map();
-    for (const output of parts.outputs.values()) {
-      if (!(parts.viewers?.forOutput(output)?.size > 0)) continue;
-      const key = output.file.key;
-      const length = Number(output.file.lengthBytes ?? parts.hostLoad.fileLengthByKey.get(key));
-      const file = files.get(key) ?? { bytesPerFilmSecond: length / Number(output.file.durationSeconds), encodeCostSec: 0 };
-      const state = parts.encodeRuns.runStateOf(output);
-      if (processCanBeSignalled(state) && state !== ENCODE_RUN_STATE.SUSPENDED) {
-        file.encodeCostSec += parts.encodeCost.loadOfOutput(output.outputKey)?.costSec ?? 0;
+    const watched = watchedFiles(parts.outputs.values(), {
+      watched: (output) => parts.viewers?.forOutput(output)?.size > 0,
+      lengthOf: (output) => Number(output.file.lengthBytes ?? parts.hostLoad.fileLengthByKey.get(output.file.key)),
+      encodeCostOf: (output) => {
+        const state = parts.encodeRuns.runStateOf(output);
+        return processCanBeSignalled(state) && state !== ENCODE_RUN_STATE.SUSPENDED
+          ? parts.encodeCost.loadOfOutput(output.outputKey)?.costSec ?? 0 : 0;
       }
-      files.set(key, file);
-    }
-    const watched = [...files.values()].filter((file) => Number.isFinite(file.bytesPerFilmSecond) && file.bytesPerFilmSecond > 0);
+    });
     const share = downloadShare({ availableShare: parts.processor?.availability()?.share ?? 1,
       baseDraw: parts.hostLoad.baseDraw, costPerMegabyte: parts.hostLoad.observedTorrentCostPerMegabyte, files: watched });
     // Kept beside the readings it is judged against: what the torrents moved
@@ -879,7 +876,9 @@ export function wireOutputs({
   if (calibration?.fingerprint) {
     parts.localObservations = new LocalObservations({
       fingerprint: calibration.fingerprint,
-      filePath: path.join(typeof stateDir === "string" && stateDir.length > 0 ? stateDir : PROXY_ROOT, "local-observations.json"),
+      // The state directory arrives resolved; none keeps the observations in memory only.
+      filePath: typeof stateDir === "string" && stateDir.length > 0 ? path.join(stateDir, "local-observations.json") : "",
+      writeJson: writeJsonAtomic,
       logger
     });
     parts.localObservations.load(calibration.kinds().map((kind) => configurationKeyOf(calibration.fingerprint, {
@@ -979,10 +978,13 @@ export function wireOutputs({
   // be re-taken on a five-second timer instead, which made a just-created
   // output wait up to five seconds before anything noticed it had a viewer.
   parts.retention = new OutputRetention();
+  // Who listens to viewer changes subscribes here (`startPlayback`); a throw
+  // of a listener reaches the code that changed the viewers.
+  parts.viewerChanges = new EventEmitter();
   parts.viewers = new Viewers({ onChange: () => {
     parts.lifecycle.observeUse();
     parts.encodeRuns.planEncodersSoon();
-    onViewerChanged?.();
+    parts.viewerChanges.emit("change");
   } });
   // The outputs that exist, and every question about them: the picture a step
   // belongs to, the steps, the soundtracks, the height a step is named by.
@@ -1124,36 +1126,12 @@ export function wireOutputs({
   // cost) and how much of the machine nobody has priced (the host's reading).
   parts.admission = new EncodeAdmission({
     liveRunsByAddress: () => parts.encodeOrchestrator.liveRunsByAddress(),
-    preparedAddresses: () => {
-      const addresses = new Set();
-      for (const id of outputsBeingPrepared(parts.viewers)) {
-        const key = parts.outputs.get(id)?.outputKey;
-        if (key) {
-          addresses.add(key);
-        }
-      }
-      return addresses;
-    },
+    preparedAddresses: () => preparedAddresses(parts.viewers, parts.outputs),
     // Every output a present viewer RECEIVES — by the same rule the per-output
     // priority map is built by — that still has something left to make. The
     // place an output is opened on is held from the moment its viewer is put
     // on it, not from when the plan gets round to its first encoder.
-    watchedAddresses: () => {
-      const addresses = new Set();
-      for (const session of parts.outputs.values()) {
-        const address = session.outputKey;
-        if (!address || addresses.has(address) || finishedOutput(session)) {
-          continue;
-        }
-        for (const viewer of parts.viewers.forOutput(session).values()) {
-          if (viewer.isPresent() && !parts.outputs.supersededBy(session, viewer.activeVariantId ?? null)) {
-            addresses.add(address);
-            break;
-          }
-        }
-      }
-      return addresses;
-    },
+    watchedAddresses: () => watchedAddresses(parts.viewers, parts.outputs, finishedOutput),
     finished: (address) => parts.outputs.outputsOn(address).some((session) => finishedOutput(session)),
     loadOf: (address) => parts.encodeCost.loadOfOutput(address),
     loadForCandidate: (spec, file) => parts.encodeCost.loadForCandidate(spec, file),
@@ -1216,12 +1194,12 @@ export function wireOutputs({
       void parts.machineBudget.revise(parts.machineBudget.storageResource, "playback stock");
     }
   });
-  parts.encodeInputs = encodeInputs ?? (typeof resolveEncodeInput === "function" ? new EncodeInputs({
-    resolve: resolveEncodeInput,
+  parts.encodeInputs = encodeInputs ?? (typeof encodeInputResolution?.resolve === "function" ? new EncodeInputs({
+    resolve: (...args) => encodeInputResolution.resolve(...args),
     readRanges: readEncodeRanges,
     openWholeFile: openEncodeWholeFile,
     heldRanges: readEncodeHeldRanges,
-    stretchCopied,
+    stretchCopied: (...args) => encodeInputResolution.stretchCopied(...args),
     reviseBudget: () => parts.machineBudget.revise("memory", "admitted encode inputs"),
     capacity: () => parts.machineBudget.capacityOf("memory"),
     // How long this output's input takes to open, from the one owner of start
@@ -1376,5 +1354,40 @@ export function wireOutputs({
     void parts.quality.runQualityBudgetOnce();
   }, BUDGET_CHECK_INTERVAL_MS);
   parts.budgetTimer.unref();
+  /**
+   * What the download map's pass reads of the outputs: each output of a source
+   * that a present viewer watches and that reads the file, in catalog order,
+   * with what decides how long its segments take to make and deliver and what
+   * it has made (`PlaybackResolution`).
+   *
+   * @param {string} sourceKey
+   * @param {number} fileIndex
+   * @returns {Array<{ outputKey: string, timeline: object, choices: object[], revision: unknown,
+   *   sizes: () => Map<number, number>, encodeSpeed: number | undefined, linkMbps: number | null, target: number | null }>}
+   */
+  parts.playedOutputsOf = (sourceKey, fileIndex) => {
+    const played = [];
+    for (const output of parts.outputs.values()) {
+      if (output.file.sourceKey !== sourceKey) continue;
+      const consumers = [...parts.viewers.forOutput(output).values()].filter(viewer =>
+        viewer.isPresent() && !parts.outputs.supersededBy(output, viewer.activeVariantId ?? null));
+      if (consumers.length === 0) continue;
+      const choices = trackChoicesOf(output, fileIndex);
+      if (choices.length === 0) continue;
+      // An output with no cut grid is left out by the pass (`PlaybackResolution`);
+      // none of what follows is worth working out for it.
+      const timeline = output.timeline;
+      if (!timeline || typeof timeline !== "object" || !Array.isArray(timeline.published ?? timeline.boundaries)) continue;
+      const links = consumers.map(viewer => viewer.linkReading()?.linkMbps)
+        .filter(value => Number.isFinite(value) && value > 0);
+      played.push({ outputKey: output.outputKey, timeline: output.timeline, choices,
+        revision: parts.segmentStore.revisionOf(output.outputKey),
+        sizes: () => parts.segmentStore.sizesOf(output.outputKey),
+        encodeSpeed: parts.encodeCost.latestSpeedReadingOf(output)?.speed,
+        linkMbps: links.length ? Math.min(...links) : null,
+        target: parts.encodeInputs?.targetInputBytes?.(output) ?? null });
+    }
+    return played;
+  };
   return parts;
 }

package/services/storage/files/CompletedFiles.js

@@ -27,6 +27,7 @@ import fs from "node:fs/promises";
 import { randomUUID } from "node:crypto";
 import os from "node:os";
 import path from "node:path";
+import { writeJsonAtomic } from "../../../utils/atomic-json.js";
 import { IDLE_KEEP_MS } from "../keep.js";
 
 /**
@@ -541,7 +542,11 @@ export class CompletedFiles {
       await fs.rm(directory, { ...REMOVAL, recursive: true }).catch(() => undefined);
       return;
     }
-    await fs.writeFile(path.join(directory, "manifest.json"), JSON.stringify(named)).catch(() => undefined);
+    try {
+      writeJsonAtomic(path.join(directory, "manifest.json"), named);
+    } catch {
+      // The manifest is a convenience for the next start; a failed write is not a failed keep.
+    }
   }
 
   /** Remove an empty or partial-only directory discovered during startup. */

package/services/storage/memory-report.js

@@ -459,13 +459,13 @@ export function startMemoryReport({
   label = "",
   diskPath = "",
   intervalMs = MEMORY_REPORT_INTERVAL_MS,
-  quietMs = 0,
-  changeBytes = 0,
+  quietMs = 60_000,
+  changeBytes = 25 * 1024 * 1024,
   snapshotDir = "",
   mayKeep = null,
-  snapshotFloorBytes = 500 * 1024 * 1024,
-  snapshotGrowthBytes = 100 * 1024 * 1024,
-  keepSnapshots = 0,
+  snapshotFloorBytes = 400 * 1024 * 1024,
+  snapshotGrowthBytes = 400 * 1024 * 1024,
+  keepSnapshots = 3,
   heapSnapshots = false,
   bufferSnapshotFloorBytes = 0,
   bufferSnapshotGrowthBytes = 500 * 1024 * 1024

package/services/storage/processor-share.js

@@ -55,3 +55,27 @@ export function downloadShare({ availableShare, baseDraw, costPerMegabyte, files
   const bytesPerSecond = perByte > 0 ? Math.max(filmSpeed, 1) * bytes : Infinity;
   return { filmSpeed, capacity, bytesPerSecond: Number.isFinite(bytesPerSecond) && bytes > 0 ? bytesPerSecond : Infinity };
 }
+
+/**
+ * The watched outputs grouped by the file they read: what the share is worked
+ * out over. Moved out of `applyDownloadShare` verbatim (torrent-tv/meta#280).
+ *
+ * @param {Iterable<{ file: { key: string, durationSeconds: number } }>} outputs
+ * @param {object} ports
+ * @param {(output: object) => boolean} ports.watched - Whether a viewer receives the output.
+ * @param {(output: object) => number} ports.lengthOf - The file's length in bytes.
+ * @param {(output: object) => number} ports.encodeCostOf - What its encoder costs, 0 when none runs.
+ * @returns {{ bytesPerFilmSecond: number, encodeCostSec: number }[]}
+ *   One per file in the order first seen; only finite positive rates kept.
+ */
+export function watchedFiles(outputs, { watched, lengthOf, encodeCostOf }) {
+  const files = new Map();
+  for (const output of outputs) {
+    if (!watched(output)) continue;
+    const key = output.file.key;
+    const file = files.get(key) ?? { bytesPerFilmSecond: lengthOf(output) / Number(output.file.durationSeconds), encodeCostSec: 0 };
+    file.encodeCostSec += encodeCostOf(output);
+    files.set(key, file);
+  }
+  return [...files.values()].filter((file) => Number.isFinite(file.bytesPerFilmSecond) && file.bytesPerFilmSecond > 0);
+}

package/services/storage/segment-store/SegmentStore.js

@@ -44,6 +44,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync,
 import { rmSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { writeFileAtomic } from "../../../utils/atomic-json.js";
 
 /** Where every output's segments live. One root for the process. */
 const DEFAULT_STORE_ROOT = path.join(os.tmpdir(), "torrent-tv-hls");
@@ -235,7 +236,7 @@ export class SegmentStore {
       mkdirSync(dir, { recursive: true });
       // What this directory is, for whoever finds it after this process has
       // been killed. Without it the sweep can only throw everything away.
-      writeFileSync(path.join(dir, KEY_FILE), `${key}\n`, "utf8");
+      writeFileAtomic(path.join(dir, KEY_FILE), `${key}\n`);
     }
     this.#touched.set(key, this.#now());
     // Production is about to begin here, and a piece closing in this directory