@torrent-tv/proxy 2.99.11 → 2.99.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.99.11",
+  "version": "2.99.13",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/services/media/container/MatroskaContainer.js

@@ -44,7 +44,8 @@ import { ContainerTrack } from "../tracks/ContainerTrack.js";
 import { iterateElements, readFloat, readUint } from "./ebml-reader.js";
 import { ElementReader } from "./ebml-stream.js";
 import { firstFrameOffset, readBlockHeader, walkHeldClusters } from "./matroska-clusters.js";
-import { readMatroskaPackets } from "./matroska-packets.js";
+import { buildMatroskaIndex, matroskaIntervalCovered, walkMatroskaPackets } from "./matroska-packets.js";
+import { PacketRecordsChain } from "./PacketRecordsChain.js";
 import { RetainedReads } from "./RetainedReads.js";
 import { isUnavailable } from "./unavailable.js";
 import {
@@ -153,7 +154,17 @@ export class MatroskaContainer extends Container {
   /** @type {Array<import("../tracks/ContainerTrack.js").ContainerTrack> | null} */
   #tracks = null;
   #packets = null;
-  #packetStates = new Map();
+  /**
+   * Per track, the readings of its packets in file order: each began where a
+   * request first needed it, records that track alone, and adjoins the next
+   * once it reaches it. One table per track per stretch of the file, and no
+   * packet in two of them (torrent-tv/meta#178).
+   *
+   * @type {Map<number, Array<{ startAt: number, at: number, cluster: object | null, packets: Map<number, import("./PacketRecords.js").PacketRecords>, next?: object, reachedEnd?: boolean }>>}
+   */
+  #packetReadings = new Map();
+  /** Per track, the walk under way: one at a time, so two cannot record one block. */
+  #packetWalks = new Map();
 
   /**
    * The Cues reading: undefined until read, null when the file has none.
@@ -175,8 +186,8 @@ export class MatroskaContainer extends Container {
   }
 
   packetIndexBytes() {
-    return [...this.#packetStates.values()].reduce((sum, state) => sum +
-      [...(state.packets?.values() ?? [])].reduce((bytes, records) => bytes + records.allocatedBytes, 0), 0);
+    return [...this.#packetReadings.values()].flat().reduce((sum, reading) => sum +
+      [...(reading.packets?.values() ?? [])].reduce((bytes, records) => bytes + records.allocatedBytes, 0), 0);
   }
 
   static detect(head) {
@@ -653,15 +664,19 @@ export class MatroskaContainer extends Container {
    * One preceding cluster retains preroll; one following cluster covers demux
    * lookahead beyond the presentation cut. No packet payload is parsed here.
    */
-  async supportsOriginalSourceRanges() {
+  async supportsOriginalSourceRanges(interval = null) {
     const cues = await this.readCues();
     if (!cues?.points.length) return false;
     const video = ContainerTrack.firstUsable(await this.readTracks(), "video");
-    // A soundtrack alone is read from the original file too, picture clusters
-    // included (38 MB for one piece of 171 KB of sound, r46). Read from its own
-    // packets (proxy 2.99.6), the soundtrack's packet table was refused memory
-    // on Home Assistant, its first run was given one piece and never closed
-    // it, and no Matroska film started (r47, torrent-tv/meta#178).
+    // Sound alone is read from its own packets, as an AVI's is
+    // (`AviContainer`). From the original file it was given whole clusters,
+    // picture included: 260 MB for 23 pieces of a soundtrack, 38 MB for one of
+    // 171 KB, on Home Assistant (r44, r46). Proxy 2.99.6 did so and no film
+    // started: every stretch the download map asked for kept a table for every
+    // track, 38.7 MB in a minute (r47); the soundtrack is now one chain of
+    // tables of its own (`#readTrackPackets`, torrent-tv/meta#178). Asked of
+    // the file as a whole, the answer is the picture's.
+    if (video && interval?.trackIds?.length && !interval.trackIds.includes(video.trackNumber)) return false;
     return cues.points.some(point => point.positions.some(position => !video || position.track === video.trackNumber));
   }
 
@@ -741,27 +756,110 @@ export class MatroskaContainer extends Container {
     if (!layout) throw new Error("Matroska Segment is absent.");
     const tracks = await this.readTracks();
     const info = await this.readMediaInfo();
-    const start = await this.#packetStart(layout, tracks, interval);
-    // Reuse a contiguous reading that has already reached this cue. A seek
-    // across an unindexed gap gets its own reading instead of walking the gap.
-    let state = [...this.#packetStates.values()]
-      .filter(one => one.startAt <= start && one.at >= start)
-      .sort((a, b) => b.startAt - a.startAt)[0];
-    if (!state) {
-      state = { startAt: start, at: start };
-      this.#packetStates.set(start, state);
-    }
-    const index = await readMatroskaPackets({ readRange: this.readRange,
-      fileSize: this.fileSize, portionBytes: this.portionBytes, layout, tracks,
-      // The last frame ends where the film does, on the packets' own clock.
-      endSeconds: info.durationSeconds === null ? null : info.durationSeconds + (info.startTimeSeconds ?? 0),
-      interval, state, packetMemory: this.packetMemory });
-    if (index.isComplete() && state.startAt === layout.firstClusterAt) {
-      this.#packets = index;
+    const declared = new Map(tracks.filter(track => ["video", "audio", "subtitle"].includes(track.type))
+      .map(track => [track.trackNumber, track]));
+    // The tracks this request reads: those it names, the picture and sound
+    // when it names none, and every track for the whole file.
+    const requested = [...declared.values()].filter(track => interval?.trackIds ? interval.trackIds.includes(track.trackNumber)
+      : !interval || ["video", "audio"].includes(track.type));
+    const start = interval ? await this.#packetStart(layout, tracks, interval) : (layout.firstClusterAt ?? layout.segmentEnd);
+    // The last frame ends where the film does, on the packets' own clock.
+    const endSeconds = info.durationSeconds === null ? null : info.durationSeconds + (info.startTimeSeconds ?? 0);
+    const streams = new Map();
+    const complete = new Set();
+    let whole = true;
+    for (const track of requested) {
+      const chain = await this.#readTrackPackets(layout, tracks, track, start, interval);
+      streams.set(track.trackNumber, chain.records);
+      if (chain.reachesEnd) complete.add(track.trackNumber);
+      if (!chain.reachesEnd || chain.startAt !== layout.firstClusterAt) whole = false;
     }
+    const index = buildMatroskaIndex({ declared, streams, endSeconds, complete, covered: interval ?? null,
+      packetMemory: this.packetMemory });
+    if (!interval && whole) this.#packets = index;
     return index;
   }
 
+  /**
+   * One track's packets from `start` on, read as far as the interval needs.
+   *
+   * The reading that holds `start` goes on; with none, one begins there. A
+   * reading that reaches the next begins nothing new: the two adjoin and the
+   * walk goes on where the next one stopped. Before this, every request the
+   * download map made for a stretch not yet read began a reading of its own
+   * with a table for every track of the file; a 25-minute film's soundtrack
+   * asked 147 of them and held 38.7 MB of mostly empty tables in its first
+   * minute, and the rest was refused memory (r47, torrent-tv/meta#178).
+   *
+   * @returns {Promise<{ records: PacketRecordsChain, startAt: number, reachesEnd: boolean }>}
+   */
+  async #readTrackPackets(layout, tracks, track, start, interval) {
+    const id = track.trackNumber;
+    const previous = this.#packetWalks.get(id) ?? Promise.resolve();
+    let finished;
+    const walk = previous.then(() => new Promise(resolve => { finished = resolve; }));
+    this.#packetWalks.set(id, walk);
+    await previous;
+    try {
+      const readings = this.#packetReadings.get(id) ?? [];
+      this.#packetReadings.set(id, readings);
+      const position = reading => reading.cluster ? reading.cluster.cursor : reading.at;
+      let first = readings.find(reading => reading.startAt <= start && position(reading) >= start);
+      if (!first) {
+        first = { startAt: start, at: start, cluster: null };
+        readings.push(first);
+        readings.sort((left, right) => left.startAt - right.startAt);
+      }
+      const chainOf = () => {
+        const parts = [first];
+        while (parts.at(-1).next) parts.push(parts.at(-1).next);
+        return parts;
+      };
+      const viewOf = parts => new PacketRecordsChain(parts.map(reading => reading.packets?.get(id))
+        .filter(Boolean));
+      // Asked after every block of a walk: the reading being extended answers
+      // alone once it holds the packets the question looks at.
+      const declared = new Map([[id, track]]);
+      const asked = interval ? { ...interval, trackIds: [id] } : null;
+      const depth = track.type === "video" ? track.reorderDepth : 0;
+      let current = null;
+      const covered = () => {
+        if (!asked) return false;
+        const own = current?.packets?.get(id);
+        if (own && Number.isSafeInteger(depth) && own.length > depth) {
+          return matroskaIntervalCovered(declared, new Map([[id, own]]), asked);
+        }
+        const parts = chainOf();
+        if (!parts.every(reading => reading.packets)) return false;
+        return matroskaIntervalCovered(declared, new Map([[id, viewOf(parts)]]), asked);
+      };
+      for (;;) {
+        current = chainOf().at(-1);
+        if (current.reachedEnd) break;
+        const following = readings.find(reading => reading.startAt > current.startAt);
+        const outcome = await walkMatroskaPackets({ readRange: this.readRange, fileSize: this.fileSize,
+          portionBytes: this.portionBytes, layout, tracks, trackIds: [id], interval, packetMemory: this.packetMemory,
+          state: current, until: following?.startAt ?? null, isCovered: covered });
+        if (outcome !== "until") break;
+        current.next = following;
+      }
+      // The request reads as far as its interval reaches, not every reading
+      // adjoined since: each of its packets is looked at once per request.
+      const parts = chainOf();
+      if (asked && Number.isSafeInteger(depth)) {
+        const reaching = parts.findIndex(reading => {
+          const own = reading.packets.get(id);
+          return own.length > depth && matroskaIntervalCovered(declared, new Map([[id, own]]), asked);
+        });
+        if (reaching >= 0) parts.length = reaching + 1;
+      }
+      return { records: viewOf(parts), startAt: first.startAt, reachesEnd: parts.at(-1).reachedEnd === true };
+    } finally {
+      finished();
+      if (this.#packetWalks.get(id) === walk) this.#packetWalks.delete(id);
+    }
+  }
+
   async #packetStart(layout, tracks, interval) {
     const first = layout.firstClusterAt ?? layout.segmentEnd;
     if (!Number.isFinite(interval?.from) || interval.from <= 0) return first;

package/services/media/container/PacketIndex.js

@@ -1,4 +1,5 @@
 import { PacketRecords } from "./PacketRecords.js";
+import { PacketRecordsChain } from "./PacketRecordsChain.js";
 import { IndexMemoryUnavailable } from "./memory-unavailable.js";
 import { isMp3Codec, mp3PacketFacts } from "./mp3-packet-dependencies.js";
 
@@ -95,13 +96,19 @@ export class PacketIndex {
     }
   }
 
-  /** The progressive parser and its interval views share one packet allocation. */
+  /**
+   * The progressive parser and its interval views share one packet allocation.
+   * A chain of a container's own tables was checked record by record as each
+   * was written; read again in full for every request, a two-hour soundtrack
+   * would be 340 thousand records per request (torrent-tv/meta#178).
+   */
   sharePackets(id, records) {
     const track = this.#track(id);
-    if (track.complete || track.packets.length || !(records instanceof PacketRecords)) {
+    const chain = records instanceof PacketRecordsChain;
+    if (track.complete || track.packets.length || !(records instanceof PacketRecords || chain)) {
       throw new TypeError("Only an empty unfinished track may adopt packet records.");
     }
-    for (const packet of records) validatePacket(packet);
+    if (!chain) for (const packet of records) validatePacket(packet);
     track.packets.dispose();
     track.packets = records;
   }

package/services/media/container/PacketRecords.js

@@ -3,6 +3,12 @@
 import { IndexMemoryUnavailable } from "./memory-unavailable.js";
 
 const BLOCK_BYTES = 65536;
+// A table's first blocks, of records and of their addresses, and each block
+// after it twice the one before, up to BLOCK_BYTES. Taken whole from the first
+// record, the 128 KiB of a table that held a few hundred records was most of a
+// Matroska soundtrack read in stretches: 144 tables of one 25-minute film held
+// 18.9 MB for 3.7 MB of records (r47 reproduced, torrent-tv/meta#178).
+const FIRST_BLOCK_BYTES = 4096;
 const ADDRESS_BYTES = 8;
 const HEADER_BYTES = 56;
 
@@ -78,6 +84,12 @@ export class PacketRecords {
       const block = this.#blocks[blockIndex];
       this.#blocks.length = blockIndex + 1;
       this.#addresses.length = Math.ceil(value * ADDRESS_BYTES / BLOCK_BYTES);
+      // The last address block back to the size its records need, as it grew.
+      const lastAddresses = this.#addresses.at(-1);
+      const neededAddresses = ((value - 1) % (BLOCK_BYTES / ADDRESS_BYTES) + 1) * ADDRESS_BYTES;
+      let addressBytes = FIRST_BLOCK_BYTES;
+      while (addressBytes < neededAddresses) addressBytes *= 2;
+      if (lastAddresses.length > addressBytes) this.#addresses[this.#addresses.length - 1] = Buffer.from(lastAddresses.subarray(0, addressBytes));
       const flags = block.readUInt32LE(at + 48);
       this.#used = at + HEADER_BYTES + block.readUInt32LE(at + 52) * 16 + ((flags & 2) ? 32 : 0) + ((flags & 8) ? 4 : 0) + ((flags & 16) ? 4 : 0);
       this.#bytes = [...this.#blocks, ...this.#addresses].reduce((sum, buffer) => sum + buffer.length, 0);
@@ -121,17 +133,25 @@ export class PacketRecords {
     const bytes = HEADER_BYTES + packet.ranges.length * 16 + (packet.expectedHash ? 32 : 0) + (packet.streamId !== undefined ? 4 : 0) + (packet.decodeFromIndex !== undefined ? 4 : 0);
     let block = this.#blocks.at(-1);
     const newBlock = !block || block.length - this.#used < bytes;
+    const blockBytes = newBlock ? Math.max(bytes, Math.min(BLOCK_BYTES, block ? block.length * 2 : FIRST_BLOCK_BYTES)) : 0;
     const perAddressBlock = BLOCK_BYTES / ADDRESS_BYTES;
     const addressBlockIndex = Math.floor(this.#length / perAddressBlock);
-    const newAddresses = !this.#addresses[addressBlockIndex];
-    const growth = (newBlock ? Math.max(BLOCK_BYTES, bytes) : 0) + (newAddresses ? BLOCK_BYTES : 0);
+    const addressAt = (this.#length % perAddressBlock) * ADDRESS_BYTES;
+    const heldAddresses = this.#addresses[addressBlockIndex];
+    // An address block is the same span of records at any size; a full one
+    // below BLOCK_BYTES is copied into one twice its size.
+    const addressBytes = !heldAddresses ? FIRST_BLOCK_BYTES
+      : addressAt >= heldAddresses.length ? Math.min(BLOCK_BYTES, heldAddresses.length * 2) : 0;
+    const newAddresses = addressBytes > 0;
+    const addressGrowth = newAddresses ? addressBytes - (heldAddresses?.length ?? 0) : 0;
+    const growth = blockBytes + addressGrowth;
     const prepaid = Math.min(growth, this.#prepaid);
     if (growth > prepaid && !this.#reserve(growth - prepaid)) throw new IndexMemoryUnavailable(growth - prepaid);
     this.#prepaid -= prepaid;
     let payloadAllocation, addressAllocation;
     try {
-      if (newBlock) payloadAllocation = Buffer.alloc(Math.max(BLOCK_BYTES, bytes));
-      if (newAddresses) addressAllocation = Buffer.alloc(BLOCK_BYTES);
+      if (newBlock) payloadAllocation = Buffer.alloc(blockBytes);
+      if (newAddresses) addressAllocation = Buffer.alloc(addressBytes);
     } catch (error) {
       if (growth) this.#release(growth);
       throw error;
@@ -143,11 +163,11 @@ export class PacketRecords {
       this.#used = 0;
     }
     if (newAddresses) {
-      this.#addresses.push(addressAllocation);
-      this.#bytes += BLOCK_BYTES;
+      heldAddresses?.copy(addressAllocation);
+      this.#addresses[addressBlockIndex] = addressAllocation;
+      this.#bytes += addressGrowth;
     }
     const address = this.#addresses[addressBlockIndex];
-    const addressAt = (this.#length % perAddressBlock) * ADDRESS_BYTES;
     address.writeUInt32LE(this.#blocks.length - 1, addressAt);
     address.writeUInt32LE(this.#used, addressAt + 4);
     const at = this.#used;

package/services/media/container/PacketRecordsChain.js

@@ -0,0 +1,132 @@
+/**
+ * @file Packet records of one track read as one list across adjoining tables.
+ *
+ * A Matroska track's packets are read in stretches: wherever a viewer is, the
+ * download map asks first, and a stretch begins at the cue before it. Each
+ * stretch keeps its own table, and when one reaches the next they adjoin. A
+ * request reads the stretches it spans as one list through this view; nothing
+ * is copied, and the tables stay the container's (torrent-tv/meta#178).
+ *
+ * Read-only apart from what the index derives per request: a duration worked
+ * out from the following packet is written to the table holding the packet,
+ * and an MP3 decoder dependency to the table holding it when its first packet
+ * is in the same table. One that reaches into an earlier table is kept in this
+ * view only, since a table states its dependencies by its own positions.
+ */
+export class PacketRecordsChain {
+  #parts;
+  #offsets;
+  #fixed;
+  #dependencies = new Map();
+
+  /** @param {import("./PacketRecords.js").PacketRecords[]} parts - In file order; every part but the last adjoins the next and no longer grows. */
+  constructor(parts) {
+    if (!Array.isArray(parts) || parts.length === 0) throw new TypeError("A packet chain needs at least one table.");
+    this.#parts = parts;
+    this.#offsets = [];
+    let offset = 0;
+    for (const part of parts) {
+      this.#offsets.push(offset);
+      offset += part.length;
+    }
+    this.#fixed = this.#offsets.at(-1);
+  }
+
+  get length() { return this.#fixed + this.#parts.at(-1).length; }
+  get allocatedBytes() { return 0; }
+
+  get ptsAscending() {
+    for (let part = 0; part < this.#parts.length; part++) {
+      const records = this.#parts[part];
+      if (!records.ptsAscending) return false;
+      const next = this.#parts[part + 1];
+      if (next?.length && records.length && next.ptsAt(0) < records.ptsAt(records.length - 1)) return false;
+    }
+    return true;
+  }
+
+  get longestDuration() { return Math.max(...this.#parts.map(part => part.longestDuration)); }
+
+  /** The table holding `index` and the position within it. */
+  #locate(index) {
+    if (!Number.isSafeInteger(index) || index < 0 || index >= this.length) throw new RangeError(`No packet record ${index}.`);
+    if (index >= this.#fixed) return [this.#parts.length - 1, index - this.#fixed];
+    let low = 0, high = this.#parts.length - 1;
+    while (low < high) {
+      const middle = (low + high + 1) >>> 1;
+      if (this.#offsets[middle] <= index) low = middle;
+      else high = middle - 1;
+    }
+    return [low, index - this.#offsets[low]];
+  }
+
+  ptsAt(index) { const [part, at] = this.#locate(index); return this.#parts[part].ptsAt(at); }
+  durationAt(index) { const [part, at] = this.#locate(index); return this.#parts[part].durationAt(at); }
+  keyframeAt(index) { const [part, at] = this.#locate(index); return this.#parts[part].keyframeAt(at); }
+  derivedDurationAt(index) { const [part, at] = this.#locate(index); return this.#parts[part].derivedDurationAt(at); }
+
+  setDuration(index, seconds, derived = false) {
+    const [part, at] = this.#locate(index);
+    this.#parts[part].setDuration(at, seconds, derived);
+  }
+
+  setDecodeFromIndex(index, first) {
+    if (!Number.isSafeInteger(first) || first < 0 || first > index) throw new TypeError("Invalid decoder dependency address.");
+    const [part, at] = this.#locate(index);
+    const offset = this.#offsets[part];
+    if (first >= offset) {
+      this.#parts[part].setDecodeFromIndex(at, first - offset);
+      this.#dependencies.delete(index);
+    } else {
+      this.#dependencies.set(index, first);
+    }
+  }
+
+  at(index) {
+    if (index < 0) index += this.length;
+    if (!Number.isSafeInteger(index) || index < 0 || index >= this.length) return undefined;
+    const [part, at] = this.#locate(index);
+    const packet = this.#parts[part].at(at);
+    if (this.#dependencies.has(index)) {
+      packet.decodeFromIndex = this.#dependencies.get(index);
+      delete packet.decodeDependencyUnknown;
+    } else if (packet.decodeFromIndex !== undefined) {
+      packet.decodeFromIndex += this.#offsets[part];
+    }
+    return packet;
+  }
+
+  firstAtOrAfter(seconds) {
+    if (!this.ptsAscending) throw new Error("Records whose times go back cannot be searched by time.");
+    let low = 0, high = this.length;
+    while (low < high) {
+      const middle = (low + high) >>> 1;
+      if (this.ptsAt(middle) < seconds) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
+
+  /** The tables are the container's: a request's index lets go of nothing. */
+  dispose() {}
+  releaseUnusedCapacity() {}
+
+  *[Symbol.iterator]() { for (let index = 0; index < this.length; index++) yield this.at(index); }
+  map(callback) { return Array.from(this, callback); }
+  filter(callback) { return Array.from(this).filter(callback); }
+  some(callback) { for (const packet of this) if (callback(packet)) return true; return false; }
+  every(callback) { for (const packet of this) if (!callback(packet)) return false; return true; }
+  reduce(callback, initial) {
+    let value = initial;
+    for (const packet of this) value = callback(value, packet);
+    return value;
+  }
+  slice(start = 0, end = this.length) {
+    const length = this.length;
+    if (start < 0) start = Math.max(0, length + start);
+    if (end < 0) end = Math.max(0, length + end);
+    const records = [];
+    for (let index = start; index < Math.min(end, length); index++) records.push(this.at(index));
+    return records;
+  }
+}

package/services/media/container/matroska-packets.js

@@ -10,17 +10,51 @@ const CLUSTER = 0x1f43b675;
 const LEVEL_ONE = new Set([CLUSTER, 0x1c53bb6b, 0x114d9b74, 0x1549a966, 0x1654ae6b, 0x1043a770, 0x1941a469, 0x1254c367]);
 
 /** Index frame addresses by reading structural headers, never frame bodies. */
-export async function readMatroskaPackets({ readRange, fileSize, portionBytes, layout, tracks, endSeconds, interval, packetMemory, state = {} }) {
+export async function readMatroskaPackets(params) {
+  const { tracks, endSeconds, interval, packetMemory, state = {} } = params;
+  const outcome = await walkMatroskaPackets({ ...params, state });
+  const declared = recordedTracks(tracks, params.trackIds);
+  const complete = outcome === "end";
+  return buildMatroskaIndex({ declared, streams: state.packets, endSeconds,
+    complete: complete ? new Set(declared.keys()) : new Set(), covered: complete ? null : interval, packetMemory });
+}
+
+/**
+ * The tracks a walk records: every media track, or only those named.
+ *
+ * @param {object[]} tracks
+ * @param {number[]} [trackIds]
+ * @returns {Map<number, object>}
+ */
+function recordedTracks(tracks, trackIds) {
+  return new Map(tracks.filter(track => ["video", "audio", "subtitle"].includes(track.type) &&
+    (!trackIds || trackIds.includes(track.trackNumber))).map(track => [track.trackNumber, track]));
+}
+
+/**
+ * Walk the clusters from where `state` stopped, recording the packets of the
+ * tracks it records, until the interval is covered (`isCovered`, by default
+ * over this state's own records), the walk reaches `until` — where another
+ * reading of the same tracks begins — or the segment ends.
+ *
+ * Only the named tracks are recorded: a soundtrack read on its own kept the
+ * picture's table as well, half of 37.9 MB on r47's order (torrent-tv/meta#178).
+ *
+ * @returns {Promise<"covered" | "until" | "end">}
+ */
+export async function walkMatroskaPackets({ readRange, fileSize, portionBytes, layout, tracks, trackIds, interval, packetMemory, state, until = null, isCovered = null }) {
   const reader = new ElementReader({ read: readRange, fileSize, portionBytes });
-  const declared = new Map(tracks.filter(track => ["video", "audio", "subtitle"].includes(track.type)).map(track => [track.trackNumber, track]));
+  const declared = recordedTracks(tracks, trackIds);
   if (!state.packets) {
     state.packets = new Map([...declared.keys()].map(id => [id, new PacketRecords(packetMemory)]));
     state.at ??= layout.firstClusterAt ?? layout.segmentEnd;
   }
   const packets = state.packets;
-  if (intervalCovered(declared, packets, interval)) return buildIndex(declared, packets, endSeconds, false, interval, packetMemory);
+  const covered = isCovered ?? (() => intervalCovered(declared, packets, interval));
+  const limit = Math.min(until ?? layout.segmentEnd, layout.segmentEnd);
+  if (covered()) return "covered";
   let at = state.at;
-  while (at < layout.segmentEnd) {
+  while (at < limit) {
     const element = state.cluster?.element ?? await validHeader(reader, at, layout.segmentEnd);
     if (element.id !== CLUSTER) {
       if (element.end === null) throw new Error("An unknown-sized Matroska element is not a Cluster.");
@@ -80,14 +114,16 @@ export async function readMatroskaPackets({ readRange, fileSize, portionBytes, l
       cursor = child.end;
       state.cluster.cursor = cursor;
       state.cluster.clusterTicks = clusterTicks;
-      if (intervalCovered(declared, packets, interval)) return buildIndex(declared, packets, endSeconds, false, interval, packetMemory);
+      if (covered()) return "covered";
     }
     if (cursor <= at) throw new Error("Matroska packet walk did not advance.");
     at = element.end ?? cursor;
     state.at = at;
     state.cluster = null;
   }
-  return buildIndex(declared, packets, endSeconds, true, undefined, packetMemory);
+  if (at < layout.segmentEnd) return "until";
+  state.reachedEnd = true;
+  return "end";
 }
 
 function paddingForFrame(seconds, duration, count, frame) {
@@ -110,17 +146,28 @@ function intervalCovered(declared, packets, interval) {
   });
 }
 
-function buildIndex(declared, packets, endSeconds, complete, interval, packetMemory) {
+/** Whether the interval's requested tracks reach past its end; exported for a container's own chains. */
+export { intervalCovered as matroskaIntervalCovered };
+
+/**
+ * A packet index over the given streams of the declared tracks.
+ *
+ * `complete` names the tracks no packet follows; `covered` is the interval
+ * the other requested tracks are known to reach past. A declared track with no
+ * stream is declared empty.
+ */
+export function buildMatroskaIndex({ declared, streams, endSeconds, complete, covered, packetMemory }) {
   const index = new PacketIndex();
   for (const [id, track] of declared) {
     index.declareTrack(id, { type: track.type, codecId: track.codecId, codecRanges: track.codecRanges ?? [], prerollSeconds: track.seekPrerollSeconds ?? 0, reorderDepth: track.reorderDepth ?? 0 });
-    const stream = packets.get(id);
+    const stream = streams.get(id);
+    if (!stream) continue;
     let needsFollowing = false;
     for (let position = 0; position < stream.length; position++) {
       if (stream.durationAt(position) === 0 || stream.derivedDurationAt(position)) { needsFollowing = true; break; }
     }
     if (needsFollowing) {
-      const endFor = followingPacketTime(stream, complete ? endSeconds : null, packetMemory);
+      const endFor = followingPacketTime(stream, complete.has(id) ? endSeconds : null, packetMemory);
       try { for (let position = 0; position < stream.length; position++) {
         if (stream.durationAt(position) !== 0 && !stream.derivedDurationAt(position)) continue;
         const pts = stream.ptsAt(position), end = endFor(pts);
@@ -128,9 +175,9 @@ function buildIndex(declared, packets, endSeconds, complete, interval, packetMem
       } } finally { endFor.dispose(); }
     }
     index.sharePackets(id, stream);
-    if (complete) index.complete(id);
-    else if (interval.trackIds ? interval.trackIds.includes(id) : ["video", "audio"].includes(track.type)) {
-      index.coverThrough(id, interval.to);
+    if (complete.has(id)) index.complete(id);
+    else if (covered && (covered.trackIds ? covered.trackIds.includes(id) : ["video", "audio"].includes(track.type))) {
+      index.coverThrough(id, covered.to);
     }
   }
   return index;

package/services/torrent/download/registry.js

@@ -20,6 +20,7 @@ import { SwarmSelection } from "./SwarmSelection.js";
 import { SelectionLimit } from "./SelectionLimit.js";
 import { futureDownload } from "./FutureDownload.js";
 import { urgencyName } from "../demand/index.js";
+import { pieceStoreOf } from "../piece-store-of.js";
 
 /** @type {WeakMap<object, { register: DemandRegister, selection: SwarmSelection }>} */
 const byTorrent = new WeakMap();
@@ -30,44 +31,117 @@ let publishingSelections = false;
 /** What the last reconciliation told the swarm to fetch first, for the pool to say. */
 let firstClassStated = "none";
 
-/** The last forecast that finished on the maps it was begun on, and how to tell it still describes them. */
+/**
+ * The last forecast that finished on the maps it was begun on: its result, the
+ * demand and withdrawal revisions of each torrent it was worked out on, and
+ * whether nothing has changed since; overdue once a missing piece outlived its
+ * forecast time.
+ *
+ * @type {{ result: Map<object, { measuredAt: number, arrivals: Map<number, number | null> }>,
+ *   versions: Map<object, { demand: number, withdrawal: number }>, describes: () => boolean, overdue?: boolean } | null}
+ */
 let latestForecast = null;
+/** Who is told that a forecast failed: this layer decides from demand and storage and writes no log of its own. */
+let reportForecastFailure = () => {};
 
 /**
- * The arrival forecast of every live download map, shared by concurrent file
- * reports.
- *
- * ANSWERED WITH THE LAST ONE FINISHED. Each forecast states when it was made
- * (`measuredAt`), and a newer one is worked out beside the answer whenever the
- * maps or the held pieces have changed since, one at a time. Awaited afresh on
- * every report, it held a waiting viewer's answer 3-11.6 s on Home Assistant
- * while it took about a quarter of a second of work, the machine 83-94 % busy:
- * the page asks again only once answered, so the hold after a seek to 100 s
- * lasted 52.5 s (r51, torrent-tv/meta#178). Only the first report waits, when
- * there is no forecast yet. A forecast whose maps change while it is worked out
- * is not kept.
- *
- * @returns {Promise<Map<object, { measuredAt: number, arrivals: Map<number, number | null> }>>}
+ * Say where a failed forecast is reported.
+ *
+ * @param {(error: unknown) => void} report
+ * @returns {void}
  */
-export function forecastDownloads() {
-  if (!futurePending && !latestForecast?.isCurrent()) {
-    const entries = [...live];
-    const revisions = entries.map(entry => entry.register.revision);
-    const withdrawals = entries.map(entry => entry.withdrawalRevision);
-    const verified = entries.map(entry => entry.verifiedRevision);
-    const isCurrent = () => entries.length === live.size && !entries.some((entry, index) =>
-        !live.has(entry) || entry.register.revision !== revisions[index] ||
-        entry.withdrawalRevision !== withdrawals[index]);
-    // Kept while no piece has arrived either: a piece forecast to come has
-    // either come, or its time is past and it must be forecast again.
-    const describes = () => isCurrent() && entries.every((entry, index) => entry.verifiedRevision === verified[index]);
-    futurePending = futureDownload(entries, { isCurrent }).then(result => {
-      if (!isCurrent()) return new Map();
-      latestForecast = { result, isCurrent: describes };
-      return result;
-    }).finally(() => { futurePending = null; });
+export function onForecastFailure(report) {
+  reportForecastFailure = report;
+}
+
+/**
+ * When this torrent's wanted pieces are forecast to be held, as far as the last
+ * finished forecast says so for the map it now has; null when that forecast
+ * does not describe it.
+ *
+ * NEVER AWAITED. Awaited afresh on every report, the forecast held a waiting
+ * viewer's answer 3-11.6 s on Home Assistant while it took about a quarter of a
+ * second of work, the machine 83-94 % busy: the page asks again only once
+ * answered, so the hold after a seek to 100 s lasted 52.5 s (r51,
+ * torrent-tv/meta#178). A newer forecast is worked out beside the answer,
+ * one at a time, whenever the maps, the withdrawals or the held pieces have
+ * changed since the last one.
+ *
+ * ONLY FOR THE MAP IT WAS WORKED OUT ON. The order pieces come in follows the
+ * map, so a forecast made before the map changed says nothing of the new one,
+ * and neither does one made before a piece was withdrawn: that piece goes back
+ * into the queue ahead of others. Such a torrent gets null, and the caller
+ * estimates its missing bytes from the measured rate, as before any forecast.
+ *
+ * AND WHAT IS HELD IS READ NOW, FROM THE STORE. A piece held now is available
+ * now whatever was forecast. A piece still missing whose forecast time has
+ * passed did not come when forecast — its peer stopped, or it failed
+ * verification — and is unknown, not available: read as `max(0, at - now)`,
+ * 2.99.11's stale time made a missing piece count as held.
+ *
+ * @param {object} torrent
+ * @param {{ now?: number, findStore?: (torrent: object) => object | null }} [options]
+ * @returns {{ measuredAt: number, arrivals: Map<number, number | null> } | null}
+ */
+export function downloadForecastOf(torrent, { now = Date.now(), findStore = pieceStoreOf } = {}) {
+  const entry = byTorrent.get(torrent);
+  if (!entry) return null;
+  refreshForecast();
+  const forecast = latestForecast?.result.get(torrent);
+  const version = latestForecast?.versions.get(entry);
+  if (!forecast || !version || version.demand !== entry.register.revision ||
+      version.withdrawal !== entry.withdrawalRevision) return null;
+  const store = findStore(torrent);
+  const arrivals = new Map();
+  let overdue = false;
+  for (const [piece, at] of forecast.arrivals) {
+    const held = store?.locationOf ? store.locationOf(piece) !== "missing" : torrent.bitfield?.get(piece) === true;
+    if (!held && Number.isFinite(at) && at <= now) overdue = true;
+    arrivals.set(piece, held ? now : Number.isFinite(at) && at > now ? at : null);
+  }
+  // A piece that did not come when forecast is a change no event announces:
+  // the next forecast is begun on it, as on one.
+  if (overdue) {
+    latestForecast.overdue = true;
+    refreshForecast();
   }
-  return latestForecast ? Promise.resolve(latestForecast.result) : futurePending;
+  return { measuredAt: forecast.measuredAt, arrivals };
+}
+
+/**
+ * Begin a forecast of every live map unless one is being worked out or the
+ * last one still describes them.
+ *
+ * A failed forecast is said and leaves the last one in place; the next report
+ * begins another. Left unhandled, its rejection would end nobody's wait and
+ * reach the thread's own handler as an unhandled rejection.
+ *
+ * @returns {void}
+ */
+function refreshForecast() {
+  if (futurePending || (latestForecast?.describes() && !latestForecast.overdue)) return;
+  const entries = [...live];
+  const revisions = entries.map(entry => entry.register.revision);
+  const withdrawals = entries.map(entry => entry.withdrawalRevision);
+  const verified = entries.map(entry => entry.verifiedRevision);
+  const isCurrent = () => entries.length === live.size && !entries.some((entry, index) =>
+      !live.has(entry) || entry.register.revision !== revisions[index] ||
+      entry.withdrawalRevision !== withdrawals[index]);
+  // Worked out again once a piece has arrived as well: the pieces queued
+  // behind it now come sooner.
+  const describes = () => isCurrent() && entries.every((entry, index) => entry.verifiedRevision === verified[index]);
+  const versions = new Map(entries.map((entry, index) =>
+    [entry, { demand: revisions[index], withdrawal: withdrawals[index] }]));
+  futurePending = futureDownload(entries, { isCurrent }).then(result => {
+    if (isCurrent()) latestForecast = { result, versions, describes };
+  }, error => {
+    try { reportForecastFailure(error); } catch { /* a failing report must not end the next forecast */ }
+  }).finally(() => { futurePending = null; });
+}
+
+/** The forecast being worked out, for checks that wait for it to finish. */
+export function forecastSettled() {
+  return futurePending ?? Promise.resolve();
 }
 
 /**

package/services/torrent/worker/worker.js

@@ -28,7 +28,7 @@ import { Readable } from "node:stream";
 import { heldFileBytes } from "./held-file-bytes.js";
 import { createSendStream } from "./channel.js";
 import { readFragments, supplyFiguresFor } from "./piece-reader.js";
-import { demandFor, forecastDownloads, selectionLimitOf } from "../download/registry.js";
+import { demandFor, downloadForecastOf, onForecastFailure, selectionLimitOf } from "../download/registry.js";
 import { heldRangesOf, ownsItsMemory, readHeldBytes, readHeldRanges } from "./held-bytes.js";
 import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
 import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
@@ -79,6 +79,8 @@ const { SharedPieceStore, collectStoreStats, findSharedStore, machineReserveByte
 // been asked for anything yet.
 const dhtBootstrap = await resolveDhtBootstrap();
 
+onForecastFailure(error => logger.warn(`download: the arrival forecast failed, the last one stands: ${error?.stack ?? error}`));
+
 const pool = new TorrentPool({
   memoryBytes: workerData?.memoryBytes,
   dhtBootstrap,
@@ -473,7 +475,7 @@ async function runCommand(command, params, id) {
       // interval invented from one point is exactly what this work removes.
       const file = Array.isArray(torrent?.files) ? torrent.files[params.fileIndex] : null;
       const forecastAt = performance.now();
-      const forecast = file && params.forecast === true ? (await forecastDownloads()).get(torrent) : null;
+      const forecast = file && params.forecast === true ? downloadForecastOf(torrent) : null;
       const forecastMs = params.forecast === true ? performance.now() - forecastAt : 0;
       const stats = pool.getFileStats(torrent, params.fileIndex);
       const ranges = [];

package/services/viewer/playback-readiness.js

@@ -350,7 +350,12 @@ export function predictPlaybackReadiness(input = {}) {
       for (const range of source.downloadForecast.ranges ?? []) {
         if (!Number.isFinite(range?.availableAt) || !Number.isSafeInteger(range.start) ||
           !Number.isSafeInteger(range.end) || range.start < 0 || range.end <= range.start) continue;
-        const arrival = Math.max(0, (range.availableAt - now) / 1000);
+        // A forecast time already past says nothing of when the bytes come:
+        // held bytes are in the residence, and missing ones are estimated
+        // below. Read as an arrival at zero, a piece forecast to come and
+        // still missing counted as held (torrent-tv/meta#178).
+        if (range.availableAt <= now) continue;
+        const arrival = (range.availableAt - now) / 1000;
         // The entire protocol piece becomes readable after verification;
         // bytes inside it never arrive proportionally to their file address.
         sourceByteService.get(id).push({ start: range.start, end: range.end, begin: arrival, finish: arrival });