@torrent-tv/proxy 2.62.0 → 2.64.0

package/services/memory-report.js

@@ -19,7 +19,7 @@
  * runtime already maintains.
  */
 
-import { readFile } from "node:fs/promises";
+import { readFile, statfs } from "node:fs/promises";
 import os from "node:os";
 
 /** How often the reading is taken and written. */
@@ -86,6 +86,55 @@ export async function availableMemory() {
   return { bytes: os.freemem(), measured: false };
 }
 
+/**
+ * Anonymous memory this process holds, from the kernel's own rollup.
+ *
+ * `process.memoryUsage()` sees what V8 knows about. It cannot see memory the
+ * allocator has taken and not returned, and on musl — which is what the addon
+ * runs on — a heavy churn of 8 MiB pieces leaves exactly that. The rollup's
+ * `Anonymous` counts every mapping backed by nothing but memory, which is
+ * where both a live `SharedArrayBuffer` and a freed-but-retained span sit, so
+ * the difference between it and what the isolates admit to is the size of what
+ * neither can explain (roadmap item 2, step 4).
+ *
+ * @returns {Promise<number | null>} Bytes, or null off Linux.
+ */
+export async function readAnonymousMemory() {
+  try {
+    const text = await readFile("/proc/self/smaps_rollup", "utf8");
+    const match = /^Anonymous:\s+(\d+)\s+kB$/m.exec(text);
+    if (match) {
+      return Number(match[1]) * 1024;
+    }
+  } catch {
+    // silent-ok: not Linux, or the kernel is too old for the rollup. The line
+    // simply leaves the term out rather than printing a worse one.
+  }
+  return null;
+}
+
+/**
+ * How much room is left where this proxy spills pieces and writes segments.
+ *
+ * A budget for memory alone is half a budget: pieces evicted from memory go to
+ * disk, so a store that is well behaved about RAM can still fill the card an
+ * addon host boots from. Both limits are the machine's, and neither was
+ * measured before (roadmap item 2).
+ *
+ * @param {string} directory
+ * @returns {Promise<number | null>} Bytes free, or null where it cannot be read.
+ */
+export async function readDiskFree(directory) {
+  try {
+    const stats = await statfs(directory);
+    return Number(stats.bavail) * Number(stats.bsize);
+  } catch {
+    // silent-ok: `statfs` is not everywhere, and a missing disk figure must not
+    // cost the memory reading beside it.
+  }
+  return null;
+}
+
 /**
  * Render a size the way a person reads one.
  *
@@ -104,26 +153,61 @@ function megabytes(bytes) {
  * count is meaningless without the piece size, and the piece size differs per
  * torrent — on the film this proxy died under, 63 pieces meant 504 MB.
  *
+ * Two scopes, because two things are being asked and only one of them is
+ * per-thread. `rss` and the kernel's rollup belong to the PROCESS, so they are
+ * read once, on the main thread. The heap, external and arrayBuffer figures
+ * belong to an ISOLATE, and the torrent worker's are the ones that matter —
+ * the piece pool is a `SharedArrayBuffer` allocated there, which the main
+ * isolate cannot see at all. Reporting only the main thread's was half the
+ * reason 650 MB of a 893 MB process had no explanation on 2026-08-28.
+ *
  * @param {Object} reading
+ * @param {"process" | "thread"} [reading.scope]
+ * @param {string} [reading.label] - Which thread the isolate figures are of.
  * @param {{ rss: number, heapUsed: number, heapTotal: number, external: number, arrayBuffers: number }} reading.process
- * @param {number} reading.availableBytes
- * @param {boolean} reading.availableMeasured
- * @param {{ name: string, residentBytes: number, budgetBytes: number }[]} [reading.stores]
+ * @param {number} [reading.availableBytes]
+ * @param {boolean} [reading.availableMeasured]
+ * @param {number | null} [reading.anonymousBytes]
+ * @param {number | null} [reading.diskFreeBytes]
+ * @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
  * @returns {string}
  */
-export function describeMemory({ process: usage, availableBytes, availableMeasured, stores = [] }) {
-  const storeResident = stores.reduce((total, store) => total + (store.residentBytes || 0), 0);
-  const storeBudget = stores.reduce((total, store) => total + (store.budgetBytes || 0), 0);
+export function describeMemory({
+  scope = "process",
+  label = "",
+  process: usage,
+  availableBytes,
+  availableMeasured,
+  anonymousBytes = null,
+  diskFreeBytes = null,
+  stores = []
+}) {
+  const total = (field) => stores.reduce((sum, store) => sum + (store[field] || 0), 0);
+  const storeResident = total("residentBytes");
+  const storeCommitted = total("committedBytes");
+  const storeSpilled = total("spilledBytes");
+  const storeBudget = total("budgetBytes");
+  // Holding and having taken are different quantities, and the gap between
+  // them is the whole of the growth this series exists to find: the pool only
+  // grows, so a spilled piece frees a slot and no memory.
   const storesPart = stores.length === 0
     ? "no torrent stores"
-    : `${stores.length} torrent store(s) holding ${megabytes(storeResident)} ` +
-      `of ${megabytes(storeBudget)} allowed`;
+    : `${stores.length} torrent store(s) holding ${megabytes(storeResident)}, ` +
+      `committed ${megabytes(storeCommitted)} of ${megabytes(storeBudget)} allowed, ` +
+      `${megabytes(storeSpilled)} spilled to disk`;
+  const isolate =
+    `heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)} ` +
+    `external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}`;
+  if (scope === "thread") {
+    return `memory (${label || "thread"}): ${isolate}; ${storesPart}`;
+  }
   return (
-    `memory: rss=${megabytes(usage.rss)} heap=${megabytes(usage.heapUsed)}/${megabytes(usage.heapTotal)} ` +
-    `external=${megabytes(usage.external)} arrayBuffers=${megabytes(usage.arrayBuffers)}; ` +
+    `memory: rss=${megabytes(usage.rss)} ${isolate}` +
+    `${anonymousBytes === null ? "" : ` anon=${megabytes(anonymousBytes)}`}; ` +
     `${storesPart}; ` +
-    `machine has ${megabytes(availableBytes)} available` +
-    `${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}`
+    `machine has ${megabytes(availableBytes ?? 0)} available` +
+    `${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
+    `${diskFreeBytes === null ? "" : `, ${megabytes(diskFreeBytes)} free on disk`}`
   );
 }
 
@@ -133,13 +217,25 @@ export function describeMemory({ process: usage, availableBytes, availableMeasur
  * @param {Object} options
  * @param {(message: string) => void} options.log
  * @param {() => { name: string, residentBytes: number, budgetBytes: number }[]} [options.readStores]
+ * @param {"process" | "thread"} [options.scope] - `thread` leaves out the
+ *   process-wide figures, so the torrent worker can report its own isolate
+ *   without reading /proc twice.
+ * @param {string} [options.label] - Which thread the isolate figures are of.
+ * @param {string} [options.diskPath] - Where pieces spill, for the free-space
+ *   reading. Omitted, the disk term is left out rather than guessed.
  * @param {number} [options.intervalMs]
  * @returns {{ stop: () => void }}
  */
-export function startMemoryReport({ log, readStores, intervalMs = MEMORY_REPORT_INTERVAL_MS }) {
+export function startMemoryReport({
+  log,
+  readStores,
+  scope = "process",
+  label = "",
+  diskPath = "",
+  intervalMs = MEMORY_REPORT_INTERVAL_MS
+}) {
   const tick = async () => {
     try {
-      const { bytes, measured } = await availableMemory();
       let stores = [];
       try {
         stores = typeof readStores === "function" ? readStores() ?? [] : [];
@@ -147,10 +243,19 @@ export function startMemoryReport({ log, readStores, intervalMs = MEMORY_REPORT_
         // silent-ok: a store list that cannot be read must not stop the reading
         // that matters, which is the process's own.
       }
+      if (scope === "thread") {
+        log(describeMemory({ scope, label, process: readProcessMemory(), stores }));
+        return;
+      }
+      const { bytes, measured } = await availableMemory();
       log(describeMemory({
+        scope,
+        label,
         process: readProcessMemory(),
         availableBytes: bytes,
         availableMeasured: measured,
+        anonymousBytes: await readAnonymousMemory(),
+        diskFreeBytes: diskPath ? await readDiskFree(diskPath) : null,
         stores
       }));
     } catch {

package/services/orchestrators/ContainerOrchestrator.js

@@ -0,0 +1,89 @@
+/**
+ * @file Container orchestrator — application layer over Container domain.
+ *
+ * Holds a per-file cache of Container instances (key sourceKey:fileIndex) so
+ * Tracks and keyframe index are read once per file, not per request.
+ * Delegates format detection to ContainerFactory. Transport-agnostic — takes
+ * readRange, knows nothing about torrents or HTTP.
+ */
+
+import { ContainerFactory } from "../container/ContainerFactory.js";
+import { logger } from "../../utils/logger.js";
+
+export class ContainerOrchestrator {
+  constructor() {
+    /** @type {Map<string, import("../container/Container.js").Container|null>} */
+    this.cache = new Map();
+    /** @type {Map<string, Promise<import("../container/Container.js").Container|null>>} */
+    this.pending = new Map();
+  }
+
+  /**
+   * @param {object} params
+   * @param {string} params.sourceKey
+   * @param {number} params.fileIndex
+   * @param {(start:number,end:number)=>Promise<Buffer|null>} params.readRange
+   * @param {number} params.fileSize
+   * @param {string} [params.label]
+   * @returns {Promise<import("../container/Container.js").Container|null>}
+   */
+  async getContainer({ sourceKey, fileIndex, readRange, fileSize, label = "" }) {
+    const key = `${sourceKey}:${fileIndex}`;
+    if (this.cache.has(key)) return this.cache.get(key);
+    if (this.pending.has(key)) return this.pending.get(key);
+    const p = ContainerFactory.create({ readRange, fileSize, label }).then((c) => {
+      this.cache.set(key, c);
+      this.pending.delete(key);
+      if (c) logger.info(`container: ${c.formatName} for "${label}"`);
+      else logger.info(`container: unknown for "${label}"`);
+      return c;
+    }).catch((e) => {
+      this.pending.delete(key);
+      logger.warn(`container: failed for "${label}": ${e?.message ?? e}`);
+      return null;
+    });
+    this.pending.set(key, p);
+    return p;
+  }
+
+  /**
+   * @param {object} params - same as getContainer
+   * @returns {Promise<import("../tracks/index.js").ContainerTrack[]>}
+   */
+  async getTracks(params) {
+    const container = await this.getContainer(params);
+    if (!container) return [];
+    try {
+      return await container.readTracks();
+    } catch (e) {
+      logger.warn(`container: readTracks failed for "${params.label}": ${e?.message ?? e}`);
+      return [];
+    }
+  }
+
+  /**
+   * @param {object} params - same as getContainer
+   * @returns {Promise<{times:number[],tolerance:number}|null>}
+   */
+  async getKeyframeIndex(params) {
+    const container = await this.getContainer(params);
+    if (!container) return null;
+    try {
+      return await container.readKeyframeIndex();
+    } catch {
+      return null;
+    }
+  }
+
+  forget(sourceKey, fileIndex) {
+    if (fileIndex === undefined) {
+      for (const k of [...this.cache.keys()]) if (k.startsWith(`${sourceKey}:`)) this.cache.delete(k);
+      for (const k of [...this.pending.keys()]) if (k.startsWith(`${sourceKey}:`)) this.pending.delete(k);
+      return;
+    }
+    this.cache.delete(`${sourceKey}:${fileIndex}`);
+    this.pending.delete(`${sourceKey}:${fileIndex}`);
+  }
+}
+
+export const containerOrchestrator = new ContainerOrchestrator();

package/services/orchestrators/SubtitleOrchestrator.js

@@ -0,0 +1,101 @@
+/**
+ * @file Subtitle orchestrator — application layer over subtitle domain.
+ *
+ * Wraps torrent-worker/subtitle-cues.js domain (planFor, cuesHeldFor,
+ * warmSubtitleCues, forgetSubtitles) behind Container/Track abstraction.
+ * Provides per-file track list and cue streaming, with the same "only already
+ * downloaded clusters" rule as before. Controllers (HTTP or data-channel)
+ * depend on this, not on the worker module directly.
+ *
+ * Delegates to ContainerOrchestrator for track enumeration so subtitle tracks
+ * and their flags come from the unified ContainerTrack hierarchy.
+ */
+
+import { containerOrchestrator } from "./ContainerOrchestrator.js";
+import {
+  cuesHeldFor as domainCuesHeldFor,
+  warmSubtitleCues as domainWarm,
+  subtitleTracksOf,
+  declaredSubtitleTracksOf,
+  forgetSubtitles as domainForget
+} from "../torrent-worker/subtitle-cues.js";
+import { logger } from "../../utils/logger.js";
+
+export class SubtitleOrchestrator {
+  /**
+   * @param {import("./ContainerOrchestrator.js").ContainerOrchestrator} containerOrchestrator
+   */
+  constructor(containerOrchestrator) {
+    this.containers = containerOrchestrator;
+  }
+
+  /**
+   * Tracks for menu — text tracks via domain, enriched with ContainerTrack flags.
+   * Falls back to container tracks when domain has no plan yet.
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @param {string} sourceKey
+   * @param {(start:number,end:number)=>Promise<Buffer|null>} [readRange]
+   * @param {number} [fileSize]
+   * @returns {Promise<import("../tracks/index.js").ContainerTrack[]>}
+   */
+  async getTracks(torrent, fileIndex, sourceKey, readRange, fileSize) {
+    try {
+      const domain = await subtitleTracksOf(torrent, fileIndex, sourceKey);
+      if (Array.isArray(domain) && domain.length > 0) return domain;
+    } catch {}
+    if (readRange && Number.isFinite(fileSize)) {
+      try {
+        const tracks = await this.containers.getTracks({ sourceKey, fileIndex, readRange, fileSize, label: torrent?.files?.[fileIndex]?.name ?? "" });
+        return tracks.filter((t) => t.type === "subtitle");
+      } catch {}
+    }
+    return [];
+  }
+
+  /**
+   * Declared subtitle tracks in container order (including image tracks) — for declaredIndex alignment.
+   */
+  async getDeclaredTracks(torrent, fileIndex, sourceKey) {
+    try {
+      return await declaredSubtitleTracksOf(torrent, fileIndex, sourceKey);
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Cues already downloaded for one track.
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @param {string} sourceKey
+   * @param {number} trackNumber - Container trackNumber
+   */
+  async getCues(torrent, fileIndex, sourceKey, trackNumber) {
+    try {
+      return await domainCuesHeldFor(torrent, fileIndex, sourceKey, trackNumber);
+    } catch (e) {
+      logger.warn(`subtitle-orchestrator: getCues failed: ${e?.message ?? e}`);
+      return { cues: [], coveredClusters: 0, indexedClusters: 0, track: null };
+    }
+  }
+
+  /**
+   * Warm all subtitle tracks of a file — called periodically and on verified pieces.
+   * Returns per-track fresh cues for push.
+   */
+  async warm(torrent, fileIndex, sourceKey) {
+    try {
+      return await domainWarm(torrent, fileIndex, sourceKey);
+    } catch {
+      return [];
+    }
+  }
+
+  forget(sourceKey, fileIndex) {
+    domainForget(sourceKey, fileIndex);
+    this.containers.forget(sourceKey, fileIndex);
+  }
+}
+
+export const subtitleOrchestrator = new SubtitleOrchestrator(containerOrchestrator);

package/services/orchestrators/index.js

@@ -0,0 +1,2 @@
+export { ContainerOrchestrator, containerOrchestrator } from "./ContainerOrchestrator.js";
+export { SubtitleOrchestrator, subtitleOrchestrator } from "./SubtitleOrchestrator.js";