@torrent-tv/proxy 2.97.26 → 2.97.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.26",
+  "version": "2.97.28",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -56,11 +56,12 @@ import { createPlaybackPlanner } from "./services/media/playback-planner.js";
 import { KeyframeTables } from "./services/media/KeyframeTables.js";
 import { contentsOf } from "./services/torrent/Contents.js";
 import { joinedWithinPieces } from "./services/torrent/demand/pieces.js";
-import { withoutRepeatedZones } from "./services/torrent/map-zones.js";
+import { coalescedZones, withoutRepeatedZones } from "./services/torrent/map-zones.js";
 import { SubtitleOrchestrator } from "./services/media/SubtitleOrchestrator.js";
 import { containerOrchestrator, CONTAINER_HEAD_BYTES, describeWorkTags } from "./services/media/ContainerOrchestrator.js";
 import { readPlaybackDeclarations } from "./services/media/read-playback-declarations.js";
 import { DownloadMaps } from "./services/viewer/DownloadMaps.js";
+import { passInputs, samePass } from "./services/viewer/playback-pass.js";
 import { nativeSourceMap, nativeOriginalSourceMap } from "./services/viewer/NativeSourceMap.js";
 import { MediaReadRequests } from "./services/media/MediaReadRequests.js";
 import { SegmentInputs } from "./services/media/SegmentInputs.js";
@@ -183,6 +184,7 @@ export async function startProxyServer({
       downloadMaps.retire(sourceKey);
       for (const key of segmentSourceRanges.keys()) if (key.startsWith(`${sourceKey}:`)) segmentSourceRanges.delete(key);
       for (const key of segmentReadings.keys()) if (key.startsWith(`${sourceKey}:`)) segmentReadings.delete(key);
+      for (const key of resolvedPasses.keys()) if (key.startsWith(`${sourceKey}:`)) resolvedPasses.delete(key);
       outputParts.encodeInputs?.bytesChanged();
     },
     // Pieces arriving is ANNOUNCED by the thread that owns the swarm; what is
@@ -436,10 +438,23 @@ export async function startProxyServer({
    * change; forgotten with the source.
    */
   const segmentReadings = new Map();
+  // Per output, then per segment: every segment of an output of one pass shares
+  // its list of track choices, so the output's key and the choices' key are
+  // worked out once for it. Built for each of a pass's segments, the two
+  // strings cost the main thread 13 s in ten minutes of a cold AVI on Home
+  // Assistant (torrent-tv/meta#178).
+  const readingsByChoices = new WeakMap();
   const segmentReadingOf = (zone, trackList, shift, map) => {
-    const choicesKey = zone.tracks.map(choice => `${choice.type}/${choice.index}/${choice.mode}`).join(",");
-    const key = `${map.sourceKey}:${map.fileIndex}:${zone.outputKey}:${zone.index}`;
-    const kept = segmentReadings.get(key);
+    let output = readingsByChoices.get(zone.tracks);
+    if (!output) {
+      const outputKey = `${map.sourceKey}:${map.fileIndex}:${zone.outputKey}`;
+      let byIndex = segmentReadings.get(outputKey);
+      if (!byIndex) segmentReadings.set(outputKey, byIndex = new Map());
+      output = { byIndex, choicesKey: zone.tracks.map(choice => `${choice.type}/${choice.index}/${choice.mode}`).join(",") };
+      readingsByChoices.set(zone.tracks, output);
+    }
+    const { byIndex, choicesKey } = output;
+    const kept = byIndex.get(zone.index);
     if (kept && kept.trackList === trackList && kept.shift === shift && kept.choicesKey === choicesKey &&
       kept.from === zone.sourceInterval.from && kept.to === zone.sourceInterval.to) return kept.reading;
     const selected = zone.tracks.map(choice => ({ choice, track: choice.type === "video"
@@ -453,9 +468,14 @@ export async function startProxyServer({
         modes: Object.fromEntries(wanted.map(track => [track.trackNumber, modes.get(track)])) };
       reading = { wanted, modes, interval, rangesKey: `${map.sourceKey}:${map.fileIndex}:${JSON.stringify(interval)}` };
     }
-    segmentReadings.set(key, { trackList, shift, choicesKey, from: zone.sourceInterval.from, to: zone.sourceInterval.to, reading });
+    byIndex.set(zone.index, { trackList, shift, choicesKey, from: zone.sourceInterval.from, to: zone.sourceInterval.to, reading });
     return reading;
   };
+  /**
+   * The last complete pass over each file, by file: what it was worked out
+   * from and what it resolved (`playback-pass.js`); forgotten with the source.
+   */
+  const resolvedPasses = new Map();
   /** A segment's ranges joined at the precision the torrent fetches, once per kept list and piece length. */
   const joinedRanges = new WeakMap();
   const joinedRangesOf = (ranges, pieceLength) => {
@@ -474,6 +494,10 @@ export async function startProxyServer({
     // same bytes — lost them with 2.96.11, and the start forecast answered
     // "source-input-ranges-unavailable" for ever (Home Assistant 2026-10-11).
     publish: (map) => torrentPool.setPriorityMap({ ...map, zones: withoutRepeatedZones(map.zones) }),
+    // And what is sent is packed: a stretch of one level and priority is one
+    // window, not one zone per range of each of its segments
+    // (`coalescedZones`, torrent-tv/meta#178).
+    pack: (zones) => coalescedZones(zones),
     log: (line) => logger.info(line),
     resolvePlayback: async (map) => {
       if (map.zones.length === 0) return [];
@@ -521,6 +545,17 @@ export async function startProxyServer({
         }
       }
       demands.sort((left, right) => left.deadlineAt - right.deadlineAt || right.priority - left.priority || left.index - right.index);
+      // WHAT THIS PASS IS WORKED OUT FROM, compared with the last complete pass
+      // over the file (`resolvedPasses`): the outputs and their targets, how the
+      // torrent fetches and where the file's clock starts, and every segment
+      // asked for, in the order of its deadline, with its level and priority.
+      const fileKey = `${map.sourceKey}:${map.fileIndex}`;
+      const pass = { tracks: tracks.value, ...passInputs({ pieceLength, shift, targets, demands }) };
+      const last = resolvedPasses.get(fileKey);
+      if (samePass(last, pass)) return last.result;
+      // Kept only from a pass that worked every segment out; one that met a
+      // reading not there yet is worked out again until it is.
+      let complete = true;
       // A container that names the bytes of an interval from an index it holds
       // answers every segment by arithmetic: once that index is read
       // ("source-navigation"), the ranges are computed here directly instead of
@@ -563,7 +598,7 @@ export async function startProxyServer({
         // were never asked for (Home Assistant 2026-10-08, torrent-tv/meta#151).
         const known = segmentSourceRanges.get(rangesKey);
         if (known) { convert(known); continue; }
-        if (waiting.has(zone.outputKey)) continue;
+        if (waiting.has(zone.outputKey)) { complete = false; continue; }
         // What is resolved so far is stated before a lower zone is worked out
         // (DownloadMaps, torrent-tv/meta#178).
         map.workingOut?.(zone.priority, expanded);
@@ -596,9 +631,10 @@ export async function startProxyServer({
           requestId: `download:${map.sourceKey}:${map.fileIndex}:${zone.outputKey}:${zone.index}`,
           demand: { ...zone.owner, leadSeconds: zone.leadSeconds } });
         if (!map.isCurrent()) return [];
-        if (!intervalParams) continue;
+        if (!intervalParams) { complete = false; continue; }
         const navigation = await containerOrchestrator.inspect(intervalParams, "source-navigation");
         if (navigation.kind !== "result") {
+          complete = false;
           if (navigation.kind === "needs-ranges" || navigation.kind === "needs-memory") waiting.add(zone.outputKey);
           continue;
         }
@@ -606,13 +642,16 @@ export async function startProxyServer({
         const packets = await containerOrchestrator.inspect(intervalParams, originalRanges ? "source-ranges" : "packets");
         if (!map.isCurrent()) return [];
         if (packets.kind === "needs-memory") waiting.add(zone.outputKey);
-        if (packets.kind !== "result") continue;
+        if (packets.kind !== "result") { complete = false; continue; }
         const input = originalRanges ? packets.value : new SegmentInputs({ index: packets.value, tracks: wanted }).forInterval({ ...interval, mode: track => modes.get(track) });
-        if (input.kind !== "result") continue;
+        if (input.kind !== "result") { complete = false; continue; }
         segmentSourceRanges.set(rangesKey, input.ranges);
         convert(input.ranges);
       }
-      return expanded();
+      const result = expanded();
+      if (complete) resolvedPasses.set(fileKey, { ...pass, result });
+      else resolvedPasses.delete(fileKey);
+      return result;
     }
   });
   const mediaReads = new MediaReadRequests({

package/services/encode/EncodePlan.js

@@ -720,6 +720,37 @@ export function planEncoders({
  * @returns {{ stall: number, ahead: number, whole: number }} Three counts of
  *   seconds, compared in that order by {@link cheaperThan}.
  */
+/** Each number's rank in the map, worked out once per map rather than per arrangement scored. */
+const ranksByMap = new WeakMap();
+
+/**
+ * The highest priority any span gives each number from `first` to `last`.
+ *
+ * Asked once per number of the film by every arrangement a plan scores, it
+ * walked every span each time: 19 s of the main thread in ten minutes of a cold
+ * AVI on Home Assistant (torrent-tv/meta#178). The map does not change between
+ * the arrangements of one plan, so the answer is kept with it.
+ *
+ * @param {WantedSpan[]} wanted
+ * @param {number} first
+ * @param {number} last
+ * @returns {(index: number) => number}
+ */
+function ranksOf(wanted, first, last) {
+  const kept = ranksByMap.get(wanted);
+  if (kept && kept.first === first && kept.last === last) return kept.rankAt;
+  const ranks = new Float64Array(Math.max(0, last - first + 1));
+  for (const span of wanted) {
+    const priority = Number(span.priority) || 0;
+    for (let index = Math.max(first, span.from); index <= Math.min(last, span.to); index += 1) {
+      if (priority > ranks[index - first]) ranks[index - first] = priority;
+    }
+  }
+  const rankAt = (index) => (index >= first && index <= last ? ranks[index - first] : 0);
+  ranksByMap.set(wanted, { first, last, rankAt });
+  return rankAt;
+}
+
 function latenessOf(encoders, coverage, wanted, untilNeeded, rate, refetchSecPerSegment, segmentSeconds) {
   const first = Math.min(...wanted.map((span) => span.from));
   const last = Math.max(...wanted.map((span) => span.to));
@@ -750,15 +781,7 @@ function latenessOf(encoders, coverage, wanted, untilNeeded, rate, refetchSecPer
   // that is 2024 s, which outvotes everything: field 2026-09-08, one viewer got
   // three encoders, two of them on film behind them, and the run serving them
   // was killed to make room for one.
-  const rankAt = (at) => {
-    let rank = 0;
-    for (const span of wanted) {
-      if (at >= span.from && at <= span.to) {
-        rank = Math.max(rank, Number(span.priority) || 0);
-      }
-    }
-    return rank;
-  };
+  const rankAt = ranksOf(wanted, first, last);
   // The ranks the map actually states, most urgent first. The comparison is over
   // these and nothing else, so a rank can never be outvoted by a lower one
   // however many seconds are at stake there.

package/services/encode/output/OutputSpec.js

@@ -196,6 +196,9 @@ const KEY_PATTERN =
  * packaged how.
  */
 export class OutputSpec {
+  /** The identity, once worked out. @type {string | null} */
+  #key = null;
+
   /**
    * @param {object} params
    * @param {string} params.sourceKey - `torrent:<infohash>`, the canonical
@@ -282,6 +285,13 @@ export class OutputSpec {
    * @returns {string}
    */
   toKey() {
+    // Worked out once: nothing that enters it changes after construction —
+    // every part copies what it is given — and it is asked for wherever an
+    // output is addressed, 5 s of the main thread in fifteen minutes on Home
+    // Assistant (r22, torrent-tv/meta#178).
+    if (this.#key !== null) {
+      return this.#key;
+    }
     const parts = [this.sourceKey, `fmt=${this.segmentFormatId}`, this.grid.toKey(), this.carries];
     if (this.video) {
       parts.push(this.video.toKey());
@@ -289,7 +299,8 @@ export class OutputSpec {
     if (this.audio) {
       parts.push(this.audio.toKey());
     }
-    return parts.join(":");
+    this.#key = parts.join(":");
+    return this.#key;
   }
 
   /**

package/services/storage/processor-report.js

@@ -74,31 +74,78 @@ function statOf(file) {
 
 /**
  * CPU seconds of this process's threads, by part, or null where `/proc` cannot
- * be read.
+ * be read. Each reading names which torrent threads it counted (`epoch`): the
+ * difference between two readings that counted different ones says nothing.
  *
  * READ SYNCHRONOUSLY, with the clock read beside it: a reading is only a share
  * when every counter in it and its time belong to one instant. Read as
  * promises on the field host (2.97.0), the counters were taken while the main
  * thread was busy and the time after, and a second read "process 1238 %" of a
- * core (torrent-tv/meta#177). About a hundred small files a second, kernel
- * memory, no disk.
+ * core (torrent-tv/meta#177).
  *
- * @param {number} [pid]
- * @returns {{ main: number, worker: number, other: number } | null}
+ * THREE OR FOUR FILES A SECOND, NOT EVERY THREAD'S. The line names the main
+ * thread, the torrent thread and the rest, and the rest is the whole process
+ * less the two: the process's own counter carries every thread it has had.
+ * Reading the stat of each of its threads every second, about a hundred
+ * files, took 12 s of the main thread in fifteen minutes on Home Assistant
+ * (r22, torrent-tv/meta#178). Which threads are the torrent thread is looked
+ * up again when one of them is gone and at every `rediscover()`.
+ *
+ * @param {object} [params]
+ * @param {number} [params.pid]
+ * @param {(file: string) => string} [params.readText]
+ * @param {(dir: string) => string[]} [params.list]
+ * @returns {{ read: () => ({ main: number, worker: number, other: number, epoch: number } | null), rediscover: () => void }}
  */
-export function readThreadSeconds(pid = process.pid) {
-  let tids;
-  try {
-    tids = readdirSync(`/proc/${pid}/task`);
-  } catch {
-    return null;
-  }
-  const parts = { main: 0, worker: 0, other: 0 };
-  for (const entry of tids) {
-    const stat = statOf(`/proc/${pid}/task/${entry}/stat`);
-    if (stat) parts[partOfThread(Number(entry), stat.name, pid)] += stat.seconds;
-  }
-  return parts;
+export function threadReader({ pid = process.pid, readText = (file) => readFileSync(file, "utf8"), list = readdirSync } = {}) {
+  const stat = (file) => {
+    try {
+      return parseStat(readText(file));
+    } catch {
+      return null;
+    }
+  };
+  /** The torrent thread's ids, or null until they are looked up. @type {number[] | null} */
+  let workers = null;
+  /** How many times they have been looked up. */
+  let epoch = 0;
+  const lookUp = () => {
+    let tids;
+    try {
+      tids = list(`/proc/${pid}/task`);
+    } catch {
+      return null;
+    }
+    return tids.map(Number).filter((tid) => {
+      const one = stat(`/proc/${pid}/task/${tid}/stat`);
+      return one && partOfThread(tid, one.name, pid) === "worker";
+    });
+  };
+  return {
+    read() {
+      if (workers === null) {
+        workers = lookUp();
+        epoch += 1;
+      }
+      if (workers === null) return null;
+      const total = stat(`/proc/${pid}/stat`);
+      const main = stat(`/proc/${pid}/task/${pid}/stat`);
+      if (!total || !main) return null;
+      let worker = 0;
+      for (const tid of workers) {
+        const one = stat(`/proc/${pid}/task/${tid}/stat`);
+        if (!one) {
+          workers = null;
+          return null;
+        }
+        worker += one.seconds;
+      }
+      return { main: main.seconds, worker, other: Math.max(0, total.seconds - main.seconds - worker), epoch };
+    },
+    rediscover() {
+      workers = null;
+    }
+  };
 }
 
 /**
@@ -129,6 +176,9 @@ export function sharesBetween(before, after) {
   const seconds = (after.at - before.at) / 1000;
   if (!(seconds > 0)) return null;
   const share = (from, to) => (Number.isFinite(from) && Number.isFinite(to) ? Math.max(0, to - from) / seconds : null);
+  // The torrent threads looked up again in between: their counters and the
+  // rest's are not comparable across that.
+  const threads = before.threads?.epoch === after.threads?.epoch;
   let ffmpeg = null;
   for (const [pid, value] of after.ffmpeg) {
     const earlier = before.ffmpeg.get(pid);
@@ -137,9 +187,9 @@ export function sharesBetween(before, after) {
   return {
     seconds,
     process: share(before.process, after.process),
-    main: share(before.threads?.main, after.threads?.main),
-    worker: share(before.threads?.worker, after.threads?.worker),
-    other: share(before.threads?.other, after.threads?.other),
+    main: threads ? share(before.threads?.main, after.threads?.main) : null,
+    worker: threads ? share(before.threads?.worker, after.threads?.worker) : null,
+    other: threads ? share(before.threads?.other, after.threads?.other) : null,
     ffmpeg
   };
 }
@@ -178,10 +228,11 @@ export function startProcessorReport({ log, ffmpegPids, detailed = () => false,
   let previous = null;
   let series = [];
   let since = Date.now();
+  const threads = threadReader();
   const tick = () => {
     const at = Date.now();
     const usage = process.cpuUsage();
-    const now = { at, process: (usage.user + usage.system) / 1e6, threads: readThreadSeconds(), ffmpeg: readProcessesSeconds(ffmpegPids()) };
+    const now = { at, process: (usage.user + usage.system) / 1e6, threads: threads.read(), ffmpeg: readProcessesSeconds(ffmpegPids()) };
     const shares = previous ? sharesBetween(previous, now) : null;
     previous = now;
     if (!shares) return;
@@ -191,6 +242,8 @@ export function startProcessorReport({ log, ffmpegPids, detailed = () => false,
       log(`processor: ${describeShares(series)}`);
       series = [];
       since = now.at;
+      // A torrent thread made again since is counted as the rest until then.
+      threads.rediscover();
     }
   };
   const timer = setInterval(tick, everyMs);

package/services/storage/segment-store/SegmentStore.js

@@ -66,6 +66,20 @@ export function directoryNameFor(key) {
   return createHash("sha256").update(String(key)).digest("hex").slice(0, 16);
 }
 
+/**
+ * The biggest of these pieces and its number.
+ *
+ * @param {Map<number, number>} sizes
+ * @returns {{ index: number, size: number }}
+ */
+function largestOf(sizes) {
+  let largest = { index: -1, size: 0 };
+  for (const [index, size] of sizes) {
+    if (size > largest.size) largest = { index, size };
+  }
+  return largest;
+}
+
 /**
  * What one output's directory holds, as last read.
  *
@@ -75,6 +89,9 @@ export function directoryNameFor(key) {
  * @property {number} bytes - What those files weigh.
  * @property {number} unproven - The highest number, whose closure nothing
  *   proves, or -1 when the directory holds no segments.
+ * @property {boolean} followed - Whether every change since the reading has
+ *   been told to this store: it was read while the directory was watched, and
+ *   each piece published, removed or seen arriving since was put into it.
  */
 
 export class SegmentStore {
@@ -226,21 +243,36 @@ export class SegmentStore {
   }
 
   /**
-   * Re-read this output's directory if it has moved since last time.
+   * What this output's directory holds: kept up to date piece by piece while
+   * the directory is watched, read whole otherwise.
+   *
+   * A WATCHED DIRECTORY IS READ ONCE. Every change to it after that is a piece
+   * published or removed here, or one the watch names, and each puts that one
+   * piece in or takes it out. It used to be read again — `readdir` and a `stat`
+   * of every piece — whenever its modification time moved, and the time moves
+   * with every working file an encoder opens: on Home Assistant 71 s of the
+   * main thread in fifteen minutes of a two-hour AVI, 2250 pieces of soundtrack
+   * listed again for each new one (r22, torrent-tv/meta#178).
    *
-   * A directory's modification time changes when an entry is added or removed,
-   * so a quiet request costs one `stat` rather than a listing.
+   * Unwatched, a directory's modification time still decides, so a quiet
+   * request costs one `stat` rather than a listing.
    *
    * @param {string} key
    * @returns {HeldContents}
    */
   refresh(key) {
     const format = this.#formats.get(key);
-    const empty = { readAt: 0, byNumber: new Map(), bytes: 0, unproven: -1, largest: { index: -1, size: 0 } };
+    const empty = { readAt: 0, byNumber: new Map(), bytes: 0, unproven: -1, largest: { index: -1, size: 0 }, followed: false };
     if (!format) {
       return this.#held.get(key) ?? empty;
     }
+    const followed = this.#held.get(key);
+    if (followed?.followed && this.#watchers.has(key)) {
+      return followed;
+    }
     const dir = path.join(this.#root, directoryNameFor(key));
+    // Watched from before the listing, so nothing that happens after it is missed.
+    const watched = this.#watchers.has(key);
     let mtime = 0;
     try {
       mtime = statSync(dir).mtimeMs;
@@ -300,11 +332,61 @@ export class SegmentStore {
       this.#held.delete(key);
       return empty;
     }
-    const contents = { readAt: mtime, byNumber, sizes, bytes, unproven: highest, largest };
+    const contents = { readAt: mtime, byNumber, sizes, bytes, unproven: highest, largest, followed: watched };
     this.#held.set(key, contents);
     return contents;
   }
 
+  /**
+   * Put one closed piece into what is held, from the file now under its served
+   * name. Nothing is done while nothing is held: the next question reads the
+   * directory whole.
+   *
+   * @param {string} key
+   * @param {number} index
+   * @returns {void}
+   */
+  #heldArrived(key, index) {
+    const contents = this.#held.get(key);
+    const format = this.#formats.get(key);
+    if (!contents || !format?.segmentFileName) {
+      return;
+    }
+    const full = path.join(this.#root, directoryNameFor(key), format.segmentFileName(index));
+    const size = statSync(full, { throwIfNoEntry: false })?.size ?? 0;
+    // Under the rule `refresh` applies to a listing: a file of no bytes is not
+    // a segment, whatever its name says.
+    if (!(size > 0)) {
+      this.#heldGone(key, index);
+      return;
+    }
+    contents.bytes += size - (contents.sizes.get(index) ?? 0);
+    contents.byNumber.set(index, full);
+    contents.sizes.set(index, size);
+    if (index > contents.unproven) contents.unproven = index;
+    if (size > contents.largest.size) contents.largest = { index, size };
+    else if (contents.largest.index === index) contents.largest = largestOf(contents.sizes);
+  }
+
+  /**
+   * Take one piece out of what is held.
+   *
+   * @param {string} key
+   * @param {number} index
+   * @returns {void}
+   */
+  #heldGone(key, index) {
+    const contents = this.#held.get(key);
+    if (!contents?.byNumber.has(index)) {
+      return;
+    }
+    contents.bytes -= contents.sizes.get(index) ?? 0;
+    contents.byNumber.delete(index);
+    contents.sizes.delete(index);
+    if (contents.unproven === index) contents.unproven = Math.max(-1, ...contents.byNumber.keys());
+    if (contents.largest.index === index) contents.largest = largestOf(contents.sizes);
+  }
+
   /**
    * The segment numbers this output holds that are finished.
    *
@@ -566,8 +648,9 @@ export class SegmentStore {
     }
     // Publication registers the material even after the previous directory was dropped.
     this.useFormat(key, format);
-    // What the directory holds has changed, so the memory of it is stale.
-    this.#held.delete(key);
+    // What the directory holds has changed by this one piece.
+    if (index >= 0) this.#heldArrived(key, index);
+    else this.#held.delete(key);
     if (index >= 0) {
       if (mediaRanges) {
         const byIndex = this.#mediaRanges.get(key) ?? new Map();
@@ -750,6 +833,8 @@ export class SegmentStore {
           end();
           return;
         }
+        // Nothing can be inferred from a change the platform did not name.
+        if (typeof name !== "string") this.#held.delete(key);
         this.#noticed(key, typeof name === "string" ? name : null);
         this.#changed(key);
       });
@@ -781,6 +866,9 @@ export class SegmentStore {
       return;
     }
     this.#watchers.delete(key);
+    // Changes are no longer told: what is held is checked against the disk again.
+    const held = this.#held.get(key);
+    if (held) held.followed = false;
     try {
       watcher.close();
     } catch {
@@ -800,19 +888,16 @@ export class SegmentStore {
    * @returns {void}
    */
   #noticed(key, name) {
-    // WHAT WAS READ OF THIS DIRECTORY IS STALE. Dropped rather than re-read:
-    // re-reading HERE is quadratic in the pieces of a film — one listing per
-    // file written, of a directory that grows with every file, measured at 110 s
-    // against 3 s over 482 pieces — while dropping costs one listing to whoever
-    // asks next, which is bounded by the questions and not by the writes.
-    //
-    // IT IS NOT ENOUGH TO LET `refresh` NOTICE BY ITSELF. It re-reads when the
-    // directory's modification time has moved, and that time is stated in
-    // milliseconds: a file written in the same millisecond as the last reading
-    // leaves it unmoved, and the piece is then reported absent although it is
-    // there. That is the hazard `refresh` documents and `publish` has always
-    // guarded against by dropping the memory.
-    this.#held.delete(key);
+    // A reading taken before the watch began is checked against the
+    // directory's modification time, which is stated in milliseconds: a file
+    // written in the same millisecond as that reading leaves it unmoved, and
+    // the piece would be reported absent although it is there. Such a reading
+    // is dropped, and the next question reads the directory once, while it is
+    // watched. A reading kept up piece by piece is told of this one piece below
+    // — not read again, which is a listing of a directory that grows with every
+    // file for every file written (71 s in fifteen minutes on Home Assistant,
+    // r22, torrent-tv/meta#178).
+    if (!this.#held.get(key)?.followed) this.#held.delete(key);
     const format = this.#formats.get(key);
     const dir = path.join(this.#root, directoryNameFor(key));
     const index = name && format?.isSegmentFileName?.(name)
@@ -823,8 +908,10 @@ export class SegmentStore {
       // before anything is announced — otherwise clearing up after a dead run
       // would read as that run's pieces arriving.
       if (!existsSync(path.join(dir, name))) {
+        this.#heldGone(key, index);
         return;
       }
+      this.#heldArrived(key, index);
       this.#announceArrival(key, index);
       return;
     }
@@ -839,6 +926,7 @@ export class SegmentStore {
     }
     for (const waiting of [...byIndex.keys()]) {
       if (existsSync(path.join(dir, format.segmentFileName(waiting)))) {
+        this.#heldArrived(key, waiting);
         this.#announceArrival(key, waiting);
       }
     }
@@ -1025,7 +1113,14 @@ export class SegmentStore {
    */
   pathOf(key, index) {
     this.#touched.set(key, this.#now());
-    return this.refresh(key).byNumber.get(index) ?? null;
+    const contents = this.refresh(key);
+    const found = contents.byNumber.get(index);
+    if (found || !contents.followed) return found ?? null;
+    // A reading kept up piece by piece answers from what it was told, and a
+    // change the watch failed to deliver would leave a closed piece absent for
+    // good. The one file asked about is looked at — a `stat`, not a listing.
+    this.#heldArrived(key, index);
+    return contents.byNumber.get(index) ?? null;
   }
 
   /**
@@ -1180,7 +1275,7 @@ export class SegmentStore {
       this.#logger.warn(`segment store: could not remove segment ${index}: ${error.message}`);
       return 0;
     }
-    this.#held.delete(key);
+    this.#heldGone(key, index);
     this.#mediaRanges.get(key)?.delete(index);
     this.#changed(key);
     return size;

package/services/torrent/download/FutureDownload.js

@@ -6,9 +6,12 @@ import { compareBands } from "./bands.js";
 const BLOCK_BYTES = 16 * 1024;
 
 /** Estimate completion from observed queues and the public selection bands.
- * The native picker decides the order within a band, so every unqueued piece
- * uses that band's completion time rather than an invented request order.
- * Pieces without a measured supplier remain unknown. No requests are changed.
+ * The native picker takes a band's pieces from its first up (the `sequential`
+ * strategy), so an unqueued piece arrives after the band's pieces before it.
+ * A piece whose every missing block is asked for arrives when its queues say,
+ * and is not the band's to share out. A supplier is a peer that can be asked
+ * for the piece now. Pieces without a measured supplier remain unknown. No
+ * requests are changed.
  * Times are conditional on the measured rate and successful verification.
  */
 export async function futureDownload(entries, { now = Date.now(), findStore = pieceStoreOf, isCurrent = () => true } = {}) {
@@ -78,9 +81,23 @@ export async function futureDownload(entries, { now = Date.now(), findStore = pi
         at = Math.max(at, arrival);
       }
       arrivals.set(index, at);
+      // EVERY MISSING BLOCK ASKED FOR: the queues holding them say when it
+      // arrives. Each block is reserved to the wire it was asked of, so a faster
+      // peer elsewhere does not bring it sooner, and its bytes are not the
+      // band's to share out (torrent-tv/meta#178 review, WebTorrent 2.8.5
+      // `_request`, torrent.js:1758).
+      if (at !== null) {
+        if (++steps % 256 === 0) {
+          await nextTurn();
+          if (!isCurrent()) return new Map();
+        }
+        continue;
+      }
       const remaining = piece?.missing;
       const speed = (torrent.wires ?? []).reduce((sum, wire) => {
-        if (wire.destroyed || !wire.peerPieces?.get(index)) return sum;
+        // Only a peer that can be asked for it now: one choking us is asked for
+        // nothing outside the pieces it allows, whatever it carried before.
+        if (wire.destroyed || !wire.peerPieces?.get(index) || !mayAsk(wire, index)) return sum;
         // What it carried with its queue kept full, where it has been seen so:
         // its speed while our selection gave it little measures our asking
         // (`SelectionLimit`, torrent-tv/meta#178).
@@ -97,21 +114,30 @@ export async function futureDownload(entries, { now = Date.now(), findStore = pi
     }
     forecasts.set(torrent, { measuredAt: now, arrivals });
   }
-  pending.sort(compareBands);
+  // WITHIN A BAND, IN THE ORDER THE PIECES COME. The picker walks each
+  // selection from its first piece up (WebTorrent 2.8.5's default `sequential`
+  // strategy, which this proxy keeps), so a piece of a band arrives once those
+  // before it have. Charged the whole band's end instead, a band of many
+  // segments — the map states one per stretch of a level since 2.97.27 — would
+  // put its nearest piece at its farthest piece's time (torrent-tv/meta#178).
+  pending.sort((left, right) => compareBands(left, right) || left.index - right.index);
   let finish = now;
-  for (let start = 0; start < pending.length;) {
-    let end = start;
-    const first = pending[start];
-    while (end < pending.length && compareBands(pending[end], first) === 0) {
-      const seconds = pending[end++].seconds;
-      finish = finish === null || seconds === null ? null : finish + seconds * 1000;
-    }
-    for (let i = start; i < end; i++) {
-      const { arrivals, index } = pending[i];
-      const queuedAt = arrivals.get(index);
-      if (finish !== null) arrivals.set(index, queuedAt === null ? finish : Math.min(queuedAt, finish));
-    }
-    start = end;
+  for (const { arrivals, index, seconds } of pending) {
+    finish = finish === null || seconds === null ? null : finish + seconds * 1000;
+    if (finish !== null) arrivals.set(index, finish);
   }
   return isCurrent() ? forecasts : new Map();
 }
+
+/**
+ * Whether this peer can be asked for this piece now: it is not choking us, or
+ * it allows this piece while it does (BEP 6, WebTorrent 2.8.5 `_updateWire`).
+ *
+ * @param {{ peerChoking?: boolean, hasFast?: boolean, peerAllowedFastSet?: number[] }} wire
+ * @param {number} index
+ * @returns {boolean}
+ */
+function mayAsk(wire, index) {
+  if (wire.peerChoking !== true) return true;
+  return wire.hasFast === true && Array.isArray(wire.peerAllowedFastSet) && wire.peerAllowedFastSet.includes(index);
+}

package/services/torrent/map-zones.js

@@ -80,12 +80,70 @@ export function deadlineOfMapZone(zone, now = Date.now()) {
  * @returns {Zone[]} In their order, without the repeats.
  */
 export function withoutRepeatedZones(zones, now = Date.now()) {
-  const bandOf = zone => ({ urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) });
+  const keys = zones.map(zone => `${zone.byteStart}:${zone.byteEnd}`);
   const best = new Map();
   zones.forEach((zone, index) => {
-    const key = `${zone.byteStart}:${zone.byteEnd}`;
-    const kept = best.get(key);
-    if (!kept || compareBands(bandOf(zone), kept.band) < 0) best.set(key, { index, band: bandOf(zone) });
+    const kept = best.get(keys[index]);
+    const band = { urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) };
+    if (!kept || compareBands(band, kept.band) < 0) best.set(keys[index], { index, band });
   });
-  return zones.filter((zone, index) => best.get(`${zone.byteStart}:${zone.byteEnd}`).index === index);
+  return zones.filter((_zone, index) => best.get(keys[index]).index === index);
+}
+
+/**
+ * What the torrent is told, as few windows as say the same thing: every stretch
+ * of bytes once, in the best band that names it, and the stretches of one
+ * level and priority that touch or overlap joined into one window, wanted by
+ * the earliest deadline among them.
+ *
+ * A map names every range of every segment — three of an AVI's segment, two of
+ * them the header and index all its segments read — and a film is thousands of
+ * segments: 10 000 zones, built, compared, copied to the torrent thread and
+ * stated there every three seconds a viewer plays on, about 70 s of the main
+ * thread in ten minutes on Home Assistant (torrent-tv/meta#178). The swarm acts
+ * on the band of a piece and nothing else, and within one band the pieces of
+ * consecutive segments are consecutive bytes. A zone a metadata read states
+ * keeps its own request and order, and is left as it is.
+ *
+ * @template {{ byteStart: number, byteEnd: number }} Zone
+ * @param {Zone[]} zones
+ * @param {number} [now]
+ * @returns {object[]}
+ */
+export function coalescedZones(zones, now = Date.now()) {
+  const kept = [];
+  const groups = new Map();
+  for (const zone of withoutRepeatedZones(zones, now)) {
+    if (zone.requestId !== undefined || zone.downloadOnly || (Number.isSafeInteger(zone.order) && zone.order > 0)) {
+      kept.push(zone);
+      continue;
+    }
+    const urgency = levelOfMapZone(zone);
+    const priority = priorityOfMapZone(zone);
+    const key = `${urgency}:${priority}`;
+    let group = groups.get(key);
+    if (!group) groups.set(key, group = { urgency, priority, parts: [] });
+    group.parts.push({ start: zone.byteStart, end: zone.byteEnd, deadlineAt: deadlineOfMapZone(zone, now) });
+  }
+  for (const { urgency, priority, parts } of groups.values()) {
+    parts.sort((left, right) => left.start - right.start);
+    let window = null;
+    const close = () => {
+      if (!window) return;
+      kept.push({ byteStart: window.start, byteEnd: window.end, priority, urgent: urgency === Urgency.NEAR,
+        behind: urgency === Urgency.BEHIND, deferred: urgency === Urgency.TAIL,
+        ...(Number.isFinite(window.deadlineAt) ? { deadlineAt: window.deadlineAt } : {}) });
+    };
+    for (const part of parts) {
+      if (window && part.start <= window.end + 1) {
+        window.end = Math.max(window.end, part.end);
+        window.deadlineAt = Math.min(window.deadlineAt, part.deadlineAt);
+      } else {
+        close();
+        window = { ...part };
+      }
+    }
+    close();
+  }
+  return kept;
 }

package/services/viewer/DownloadMaps.js

@@ -18,12 +18,17 @@ export class DownloadMaps {
    * thread's receipt included (torrent-tv/meta#177).
    */
   #clock;
+  /** What the torrent is told of the playback zones: as many windows as say the same thing. */
+  #pack;
+  /** What a list of zones says each segment reads, by the list. @type {WeakMap<object[], Map>} */
+  #inputsOf = new WeakMap();
 
   constructor({ publish, resolvePlayback = async map => map.zones, log = null, now = Date.now,
-    clock = () => performance.timeOrigin + performance.now() }) {
+    clock = () => performance.timeOrigin + performance.now(), pack = zones => zones }) {
     if (typeof publish !== "function") throw new TypeError("Download map publication is required.");
     this.#publish = publish;
     this.#resolvePlayback = resolvePlayback;
+    this.#pack = pack;
     this.#log = typeof log === "function" ? log : null;
     this.#now = now;
     this.#clock = clock;
@@ -74,7 +79,8 @@ export class DownloadMaps {
     }
     if (changed) await this.#emit(file);
     if (revision !== file.revision) return;
-    const isCurrent = () => this.#files.get(`${sourceKey}:${fileIndex}`) === file && revision === file.revision;
+    const fileKey = `${sourceKey}:${fileIndex}`;
+    const isCurrent = () => this.#files.get(fileKey) === file && revision === file.revision;
     // WHAT IS RESOLVED IS STATED WHILE THE REST IS STILL BEING RESOLVED. A pass
     // works the zones out in the order the swarm is to fetch them, and
     // published only at its end it kept the first bytes from the swarm for as
@@ -250,7 +256,16 @@ export class DownloadMaps {
     file.zones = zones;
     // What `#emit` reads has changed, whether or not it is published now.
     file.dirty = true;
+    // A pass that resolves what the pass before resolved hands back the same
+    // zones (`resolvedPasses` in server.js), and what they say of each segment
+    // is the same too.
+    const known = this.#inputsOf.get(zones);
+    if (known) {
+      file.inputs = known;
+      return;
+    }
     file.inputs = new Map();
+    this.#inputsOf.set(zones, file.inputs);
     for (const zone of zones) {
       if (typeof zone.outputKey !== "string" || !Number.isSafeInteger(zone.index)) continue;
       let output = file.inputs.get(zone.outputKey);
@@ -264,7 +279,9 @@ export class DownloadMaps {
   #emit(file) {
     file.dirty = false;
     const builtFrom = this.#clock();
-    const zones = [...file.zones, ...file.nativeZones];
+    // Compared and sent as the torrent is to act on it: the playback zones
+    // packed into as few windows as say the same (`coalescedZones`).
+    const zones = [...this.#packedPlayback(file)];
     for (const { ranges, requestId, priority, urgent, deadlineAt, order } of file.metadata.values()) {
       for (const [byteStart, byteEnd] of ranges) {
         zones.push({ byteStart, byteEnd, priority, urgent, downloadOnly: true, requestId, deadlineAt, ...(order ? { order } : {}) });
@@ -333,6 +350,27 @@ export class DownloadMaps {
     return pending;
   }
 
+  /**
+   * The playback and native zones packed, worked out again only when they
+   * have changed. Both are replaced whole, never edited in place, so the arrays
+   * themselves say whether they have. A metadata statement used to pack every
+   * zone of the film again before it added its own few ranges: 27 s of the main
+   * thread in fifteen minutes of a two-hour AVI on Home Assistant, the thread
+   * that also sends the viewer's segments (r22, torrent-tv/meta#178).
+   *
+   * Not kept when a zone is wanted "within N seconds": packing turns that into
+   * an instant on the clock it is packed at.
+   */
+  #packedPlayback(file) {
+    const kept = file.packed;
+    if (kept && kept.zones === file.zones && kept.nativeZones === file.nativeZones) return kept.result;
+    const input = [...file.zones, ...file.nativeZones];
+    const result = this.#pack(input);
+    const relative = input.some(zone => !Number.isFinite(zone.deadlineAt) && Number.isFinite(zone.withinSeconds));
+    file.packed = relative ? null : { zones: file.zones, nativeZones: file.nativeZones, result };
+    return result;
+  }
+
   #statsOf(file) {
     file.stats ??= emptyStats(this.#now());
     return file.stats;

package/services/viewer/PriorityOrchestrator.js

@@ -18,6 +18,40 @@
 
 import { emptyMap, mapForViewer, mergeMaps, runsOf, pauseCoefficient } from "./PriorityMap.js";
 
+/**
+ * What a map built from these viewers depends on, as values to compare: the
+ * same values build the same map (`mapForViewer`, `mergeMaps`).
+ *
+ * A viewer's map is a function of the second they stand at, not of the moment
+ * it is asked — `mapForViewer` floors the position — and of the time they have
+ * paused only while that pause is lowering their priorities. A map built on
+ * every pass of the encoders' plan, which is every piece a soundtrack copy
+ * closes, was the same map 2.4 times a second (Home Assistant r22,
+ * torrent-tv/meta#178).
+ *
+ * @param {{ durationSeconds: number, allowanceSeconds: number, viewers: object[], viewerCount?: number }} params
+ * @returns {number[]}
+ */
+export function mapInputsOf({ durationSeconds, allowanceSeconds, viewers, viewerCount }) {
+  const count = viewerCount ?? new Set((viewers ?? []).map((one, index) => one.id ?? index)).size;
+  const inputs = [Number(durationSeconds) || 0, Number(allowanceSeconds) || 0, count];
+  for (const viewer of viewers ?? []) {
+    const playing = viewer.playing !== false;
+    const urgentReady = viewer.urgentReady === true;
+    const at = Number.isFinite(viewer.atSeconds) && viewer.atSeconds > 0 ? Math.floor(viewer.atSeconds) : 0;
+    const attenuated = !playing && urgentReady && count > 1;
+    inputs.push(at, playing ? 1 : 0, urgentReady ? 1 : 0, attenuated ? Number(viewer.pauseSeconds) || 0 : 0);
+  }
+  return inputs;
+}
+
+/** Whether two lists hold the same values in the same order. */
+function sameValues(left, right) {
+  if (!left || left.length !== right.length) return false;
+  for (let index = 0; index < left.length; index++) if (!Object.is(left[index], right[index])) return false;
+  return true;
+}
+
 export class PriorityOrchestrator {
   /** Where the map goes once it is built. @type {(published: object) => void} */
   #publish;
@@ -57,6 +91,12 @@ export class PriorityOrchestrator {
    */
   #byOutput = new Map();
 
+  /** What the map of each file was built from, with its demands. @type {Map<string, { inputs: number[], demand: string }>} */
+  #builtFrom = new Map();
+
+  /** What the map of each output was built from. @type {Map<string, number[]>} */
+  #outputBuiltFrom = new Map();
+
   /** The registry of this component that knows who watches each output. @type {{ forOutput: (output: object) => Map<string, object> }} */
   #viewers;
 
@@ -155,8 +195,18 @@ export class PriorityOrchestrator {
    *   film, merged over everyone watching it.
    */
   build({ sourceKey, fileIndex, durationSeconds, allowanceSeconds, viewers, demandKey = null }) {
-    const map = this.#mapFrom({ durationSeconds, allowanceSeconds, viewers });
     const key = `${sourceKey}:${fileIndex}`;
+    // The same map and the same demands publish nothing new: neither is built
+    // again (`mapInputsOf`).
+    const inputs = mapInputsOf({ durationSeconds, allowanceSeconds, viewers });
+    const demand = JSON.stringify(demandKey);
+    const built = this.#builtFrom.get(key);
+    const kept = this.#maps.get(key);
+    if (kept && built?.demand === demand && sameValues(built.inputs, inputs)) {
+      return kept;
+    }
+    this.#builtFrom.set(key, { inputs, demand });
+    const map = this.#mapFrom({ durationSeconds, allowanceSeconds, viewers });
     this.#maps.set(key, map);
     // Unchanged maps are not republished: the downloading rebuilds what it asks
     // the swarm for on every one, and a viewer sitting still would otherwise
@@ -302,6 +352,7 @@ export class PriorityOrchestrator {
         const last = this.#last.get(key);
         this.#maps.delete(key);
         this.#last.delete(key);
+        this.#builtFrom.delete(key);
         if (last) {
           this.#publishNothingFor(last);
         }
@@ -310,6 +361,7 @@ export class PriorityOrchestrator {
     for (const address of [...this.#byOutput.keys()]) {
       if (!byOutput.has(address)) {
         this.#byOutput.delete(address);
+        this.#outputBuiltFrom.delete(address);
       }
     }
     for (const one of byFile.values()) {
@@ -326,6 +378,9 @@ export class PriorityOrchestrator {
     for (const [address, one] of byOutput) {
       // Competition is per file, even when two viewers use different outputs.
       one.viewerCount = new Set((byFile.get(one.fileKey)?.viewers ?? []).map((viewer, index) => viewer.id ?? index)).size;
+      const inputs = mapInputsOf(one);
+      if (this.#byOutput.has(address) && sameValues(this.#outputBuiltFrom.get(address), inputs)) continue;
+      this.#outputBuiltFrom.set(address, inputs);
       this.#byOutput.set(address, this.#mapFrom(one));
     }
   }

package/services/viewer/playback-pass.js

@@ -0,0 +1,58 @@
+/**
+ * @file What one pass that turns a file's playback map into bytes is worked out
+ * from, so a pass whose inputs have not changed hands back what the last one
+ * resolved (torrent-tv/meta#178).
+ *
+ * The viewer's map moves a second at a time while segments are seconds long,
+ * so most passes give the zones the pass before gave: on Home Assistant 30–110
+ * of 50–120 download maps a minute came out unchanged, and working them out
+ * took 72 s of the main thread in fifteen minutes (r22). The deadlines are left
+ * out of what is compared and their order is kept in it: a segment's deadline
+ * is when the viewer reaches it, which moves only when they stop or seek, and
+ * what is sent to the torrent is compared by the deadlines' order alone
+ * (`DownloadMaps` `shapeOf`).
+ */
+
+/**
+ * @param {object} params
+ * @param {number} params.pieceLength - The precision the torrent fetches at.
+ * @param {number} params.shift - Where the file's own clock starts.
+ * @param {Map<string, number | null>} params.targets - Each output's next input, by output.
+ * @param {Array<{ outputKey: string, index: number, priority: number, urgent?: boolean,
+ *   behind?: boolean, deferred?: boolean, from: number, to: number,
+ *   sourceInterval?: { from: number, to: number } }>} params.demands - In the
+ *   order of their deadlines.
+ * @returns {{ outputs: string[], inputs: number[] }}
+ */
+export function passInputs({ pieceLength, shift, targets, demands }) {
+  const outputs = [...targets.keys()];
+  const inputs = [pieceLength, shift];
+  for (const target of targets.values()) inputs.push(target ?? -1);
+  for (const demand of demands) {
+    // The segment's own interval too: a cut grid read again can give one
+    // number other bytes.
+    inputs.push(outputs.indexOf(demand.outputKey), demand.index, demand.priority,
+      (demand.urgent ? 1 : 0) + (demand.behind ? 2 : 0) + (demand.deferred ? 4 : 0),
+      demand.from, demand.to, demand.sourceInterval?.from ?? -1, demand.sourceInterval?.to ?? -1);
+  }
+  return { outputs, inputs };
+}
+
+/**
+ * Whether a pass is worked out from what an earlier one was.
+ *
+ * @param {{ tracks: unknown, outputs: string[], inputs: number[] } | undefined} last
+ * @param {{ tracks: unknown, outputs: string[], inputs: number[] }} next
+ * @returns {boolean}
+ */
+export function samePass(last, next) {
+  return Boolean(last) && last.tracks === next.tracks &&
+    sameValues(last.outputs, next.outputs) && sameValues(last.inputs, next.inputs);
+}
+
+/** Whether two lists hold the same values in the same order. */
+function sameValues(left, right) {
+  if (left.length !== right.length) return false;
+  for (let index = 0; index < left.length; index++) if (!Object.is(left[index], right[index])) return false;
+  return true;
+}