@torrent-tv/proxy 2.100.1 → 2.100.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.100.1",
+  "version": "2.100.3",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -892,6 +892,7 @@ export async function startProxyServer({
         const params = await containerOver({ sourceKey: output.file.sourceKey, fileIndex });
         if (!params) return { kind: "needs-source" };
         if (timing) {
+          params.walkAccount = timing.walk ??= { lockMs: 0, reads: 0, bytes: 0 };
           params.onTimed = (_statement, waitedMs, readMs) => {
             timing.reads += 1;
             timing.queuedMs += waitedMs;
@@ -971,10 +972,16 @@ export async function startProxyServer({
           const answer = input.read(source.fileIndex, start, end);
           return answer && typeof answer.then !== "function" && answer.end === end ? answer.bytes : null;
         };
+        const holdsUntil = start => {
+          const range = source.input.ranges.find(([first, last]) => first <= start && start <= last);
+          return range ? range[1] + 1 : null;
+        };
         const startedAt = Date.now();
-        void container.learnPackets({ from: source.input.from, to: source.input.to, trackIds: [...sounds] }, read).then(learned =>
+        void container.learnPackets({ from: source.input.from, to: source.input.to, trackIds: [...sounds] }, read, holdsUntil).then(learned =>
           logger.info(`encode input #${from}..#${to} of ${output.outputKey}: soundtrack packets of track(s) ${[...sounds].join(",")} ` +
-            `over ${source.input.from.toFixed(3)}-${source.input.to.toFixed(3)}s ${learned ? "walked" : "not all walked"} from its copy in ${Date.now() - startedAt}ms`),
+            `over ${source.input.from.toFixed(3)}-${source.input.to.toFixed(3)}s ` +
+            (learned === "walked" ? "walked" : `not all walked (${learned === "memory" ? "memory refused" : "the copy does not hold the rest"})`) +
+            ` from its copy in ${Date.now() - startedAt}ms`),
         error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`));
       }
     },

package/services/encode/EncodeInputs.js

@@ -648,6 +648,9 @@ function describeTiming(timing, now) {
     `${timing.resolutions} resolution(s) ${timing.resolveMs}ms ` +
     `(${timing.reads} file read(s): queued ${timing.queuedMs}ms, reading ${timing.readMs}ms), ` +
     `held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms` +
+    // Where a packet input's walk spent its read: waiting for the same
+    // track's earlier walks, or reading the file (r62, torrent-tv/meta#178).
+    (timing.walk ? `; packet walk waited ${timing.walk.lockMs}ms for its track and read ${timing.walk.reads} time(s), ${timing.walk.bytes} bytes of the file` : "") +
     (timing.openApproximate ? "; sized by the host's startup open, this output has not opened yet" : "");
 }
 

package/services/media/ContainerOrchestrator.js

@@ -215,7 +215,7 @@ export class ContainerOrchestrator {
         value = await container.readSourceRanges(params.packetInterval);
         if (value.kind !== "result") return { ...value, requestId };
       } else if (statement === "packets") {
-        value = await container.readPacketIndex(params.packetInterval, { isCurrent: params.isCurrent });
+        value = await container.readPacketIndex(params.packetInterval, { isCurrent: params.isCurrent, account: params.walkAccount });
         if (value === null) return { kind: "terminal", reason: "request-obsolete", requestId };
         await value?.prepareAudioDependencies?.(params.packetInterval, container.readRange);
       } else {

package/services/media/container/MatroskaContainer.js

@@ -789,8 +789,10 @@ export class MatroskaContainer extends Container {
    *   A walk whose request is no longer current stops between clusters and
    *   answers null. `readRange`, when given, is where this one walk reads
    *   instead of the file's own reader; what it does not hold is unavailable.
+   *   `account`, when given, is told how long each track's walk waited for
+   *   that track's earlier walks and what it read (`lockMs`, `reads`, `bytes`).
    */
-  async readPacketIndex(interval, { isCurrent, readRange } = {}) {
+  async readPacketIndex(interval, { isCurrent, readRange, holdsUntil, account } = {}) {
     if (this.#packets) return this.#packets;
     const layout = await this.#segmentLayout();
     if (!layout) throw new Error("Matroska Segment is absent.");
@@ -810,7 +812,7 @@ export class MatroskaContainer extends Container {
     let whole = true;
     for (const track of requested) {
       const chain = await this.#readTrackPackets(layout, tracks, track, start, interval, isCurrent,
-        readRange ? strictReader(readRange, this.fileSize) : this.readRange);
+        readRange ? strictReader(readRange, this.fileSize) : this.readRange, account, readRange ? holdsUntil : null);
       if (!chain) return null;
       streams.set(track.trackNumber, chain.records);
       if (chain.reachesEnd) complete.add(track.trackNumber);
@@ -835,15 +837,18 @@ export class MatroskaContainer extends Container {
    *
    * @param {{ from: number, to: number, trackIds: number[] }} interval
    * @param {(start: number, end: number) => Buffer | null} readRange - Null where it does not hold the bytes.
-   * @returns {Promise<boolean>} Whether the interval's packets are recorded.
+   * @param {(start: number) => number | null} [holdsUntil] - One past the last byte held from `start` on, if known.
+   * @returns {Promise<"walked" | "missing" | "memory">} Whether the
+   *   interval's packets are recorded, or what stopped the walk: bytes the
+   *   copy does not hold, or memory refused.
    */
-  async learnPackets(interval, readRange) {
+  async learnPackets(interval, readRange, holdsUntil = null) {
     try {
-      return await this.readPacketIndex(interval, { readRange }) !== null;
+      return await this.readPacketIndex(interval, { readRange, holdsUntil }) !== null ? "walked" : "missing";
     } catch (error) {
-      // What the copy does not hold, or memory refused: the sound's own walk
-      // reads it later, from where this one stopped.
-      if (isUnavailable(error) || error instanceof IndexMemoryUnavailable) return false;
+      // The sound's own walk reads the rest later, from where this one stopped.
+      if (isUnavailable(error)) return "missing";
+      if (error instanceof IndexMemoryUnavailable) return "memory";
       throw error;
     }
   }
@@ -861,13 +866,21 @@ export class MatroskaContainer extends Container {
    *
    * @returns {Promise<{ records: PacketRecordsChain, startAt: number, reachesEnd: boolean }>}
    */
-  async #readTrackPackets(layout, tracks, track, start, interval, isCurrent, read) {
+  async #readTrackPackets(layout, tracks, track, start, interval, isCurrent, reader, account = null, holdsUntil = null) {
     const id = track.trackNumber;
     const previous = this.#packetWalks.get(id) ?? Promise.resolve();
     let finished;
     const walk = previous.then(() => new Promise(resolve => { finished = resolve; }));
     this.#packetWalks.set(id, walk);
+    const waitedFrom = Date.now();
     await previous;
+    const read = account ? async (start, end) => {
+      const bytes = await reader(start, end);
+      account.reads += 1;
+      account.bytes += bytes.length;
+      return bytes;
+    } : reader;
+    if (account) account.lockMs += Date.now() - waitedFrom;
     try {
       const readings = this.#packetReadings.get(id) ?? [];
       this.#packetReadings.set(id, readings);
@@ -918,7 +931,7 @@ export class MatroskaContainer extends Container {
         const following = readings.find(reading => reading.startAt > current.startAt);
         const outcome = await walkMatroskaPackets({ readRange: read, fileSize: this.fileSize,
           portionBytes: this.portionBytes, layout, tracks, trackIds: [id], interval, packetMemory: this.packetMemory,
-          state: current, until: following?.startAt ?? null, isCovered: covered, readsThrough, isCurrent });
+          state: current, until: following?.startAt ?? null, isCovered: covered, readsThrough, isCurrent, holdsUntil });
         if (outcome === "obsolete") return null;
         if (outcome !== "until") break;
         current.next = following;

package/services/media/container/matroska-packets.js

@@ -49,7 +49,7 @@ function recordedTracks(tracks, trackIds) {
  *
  * @returns {Promise<"covered" | "until" | "end" | "obsolete">}
  */
-export async function walkMatroskaPackets({ readRange, fileSize, portionBytes, layout, tracks, trackIds, interval, packetMemory, state, until = null, isCovered = null, readsThrough = null, isCurrent = null }) {
+export async function walkMatroskaPackets({ readRange, fileSize, portionBytes, layout, tracks, trackIds, interval, packetMemory, state, until = null, isCovered = null, readsThrough = null, isCurrent = null, holdsUntil = null }) {
   // INSIDE A CLUSTER EVERY BLOCK HEADER IS READ, so its bytes are read a
   // portion at a time up to the cluster's end, and never past it. Read a
   // header at a time, each a read of a few bytes from the torrent thread, the
@@ -61,8 +61,14 @@ export async function walkMatroskaPackets({ readRange, fileSize, portionBytes, l
   // (r54). The headers lie throughout the portion, so it is all needed.
   // Between clusters only their headers are read.
   let readableUntil = null;
+  // `holdsUntil` bounds a portion by what the reader holds, so a walk over a
+  // copy reads up to its end what it needs, not a portion past it (r63).
   const reader = new ElementReader({ read: readRange, fileSize, portionBytes, prefetch: true,
-    readableUntil: start => Math.max(readableUntil ?? start, readsThrough?.(start) ?? start) });
+    readableUntil: start => {
+      const reach = Math.max(readableUntil ?? start, readsThrough?.(start) ?? start);
+      const held = holdsUntil?.(start);
+      return Number.isFinite(held) ? Math.min(reach, held) : reach;
+    } });
   const declared = recordedTracks(tracks, trackIds);
   if (!state.packets) {
     state.packets = new Map([...declared.keys()].map(id => [id, new PacketRecords(packetMemory)]));