@torrent-tv/proxy 2.97.3 → 2.97.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.3",
+  "version": "2.97.4",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/services/encode/AdmittedInput.js

@@ -28,12 +28,12 @@ export async function admitInput({ sources, reserve, readRanges }) {
         !Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
         throw new Error("Available-only input reader returned incomplete ranges.");
       }
-      hash.update(JSON.stringify(source.input.ranges));
-      for (const buffer of buffers) {
-        hash.update(String(buffer.length));
-        hash.update(":");
-        hash.update(buffer);
-      }
+      // Named by the source and what is stated about its packets below, not by
+      // the bytes: verified pieces and strict reads make the same ranges of the
+      // same source the same bytes (torrent-tv/meta#178). The check that each
+      // packet's address holds its expected payload (`expectedHash`) stays: a
+      // verified piece does not prove an address inside it was worked out right.
+      hash.update(JSON.stringify([source.sourceKey, source.fileIndex, source.input.ranges]));
       for (const input of source.input.tracks) {
         const shifted = { ...input,
           ...(Number.isFinite(input.sourceEndSeconds) ? { sourceEndSeconds: input.sourceEndSeconds - (source.timeShiftSeconds ?? 0) } : {}),

package/services/encode/EncodeRuns.js

@@ -1039,7 +1039,6 @@ export class EncodeRuns {
     });
     if (admittedInput.original) {
       run.originalInput = admittedInput;
-      run.inputFingerprint = admittedInput.fingerprint;
       // Every number of the stretch answers to its one key, so a failure
       // anywhere inside it is recorded against the stretch that was commanded.
       run.admittedInputKeys = new Map(Array.from({ length: inputEnd - safeIndex + 1 }, (_, offset) => [safeIndex + offset, inputKey]));
@@ -1052,7 +1051,6 @@ export class EncodeRuns {
       // this run works and is ready when the plan places the next run there.
       if (inputEnd < mayReach) this.#host.encodeInputs?.take(session, inputEnd + 1, mayReach);
     } else if (admittedInput) {
-      run.inputFingerprint = admittedInput.fingerprint;
       run.admittedInputKeys = new Map([[safeIndex, inputKey]]);
       let nextIndex = startIndex + 1;
       const canAppend = index => this.isLive(session) && run.isAlive &&

package/services/encode/InputFailures.js

@@ -12,7 +12,7 @@ export function failedAdmittedInput(ended, hardwareEncode = false) {
   return ended.ending === ENCODE_EXIT.SHORT || ended.ending === ENCODE_EXIT.FAILED;
 }
 
-/** A failed input is retried only after its bytes or output parameters change. */
+/** A failed input is retried only after its identity — source, file, ranges and what is stated about them — or the output parameters change. */
 export class InputFailures {
   #failures = new Map();
 

package/services/encode/OriginalInput.js

@@ -25,8 +25,13 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
         !Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
         throw new Error("Original input reader returned incomplete ranges.");
       }
+      // THE INPUT IS NAMED BY WHAT IT IS, NOT BY ITS BYTES. Every source is a
+      // torrent whose pieces are verified before they are stored, and a read
+      // gives every byte or none, so the same source, file, ranges and
+      // selections are the same bytes. Hashing the bytes as well took 13.45 s
+      // of 180 s of the main thread on a Matroska film whose picture and sound
+      // copy the same ~30 MB a piece (torrent-tv/meta#178).
       hash.update(JSON.stringify([source.sourceKey, source.fileIndex, source.input.ranges, source.input.selections]));
-      for (const buffer of buffers) hash.update(buffer);
       held.set(source.fileIndex, { source, buffers });
     }
     let released = false;

package/services/media/container/AviContainer.js

@@ -13,6 +13,7 @@ import { VideoTrack } from "../tracks/VideoTrack.js";
 import { AudioTrack } from "../tracks/AudioTrack.js";
 import { ContainerTrack } from "../tracks/ContainerTrack.js";
 import { PacketIndex } from "./PacketIndex.js";
+import { IndexMemoryUnavailable } from "./memory-unavailable.js";
 import { readAviIndex } from "./avi-index.js";
 import { isUnavailable } from "./unavailable.js";
 import { Mpeg4PictureTiming } from "./mpeg4-picture-timing.js";
@@ -34,6 +35,8 @@ export class AviContainer extends Container {
   #packets = null;
   /** The sound packets of an indexed file while they are being stated, so two asks build them once. */
   #soundPacketsReading = null;
+  /** The bytes the sound packet table was refused, while memory still could not hold them. */
+  #soundPacketsRefused = null;
   #scan = null;
   #declarations = null;
   #workTags = null;
@@ -330,9 +333,25 @@ export class AviContainer extends Container {
     // copied 2.3 GB for 0.19 GB of its own (torrent-tv/meta#166).
     const indexed = await this.#readIndex();
     if (indexed) {
+      // A refusal of memory for the table is kept for as long as the memory's
+      // owner says the same reservation still would not fit. Not kept, the
+      // next reader walked the index, counted the packets and was refused
+      // again — once per soundtrack segment, 1943 times in 8.3 s for one
+      // two-hour film (torrent-tv/meta#178). Asked at each use, the refusal
+      // cannot outlive the state of memory it was given in.
+      if (this.#soundPacketsRefused !== null) {
+        if (this.packetMemory?.fits?.(this.#soundPacketsRefused) === false) {
+          throw new IndexMemoryUnavailable(this.#soundPacketsRefused);
+        }
+        this.#soundPacketsRefused = null;
+      }
       this.#soundPacketsReading ??= this.#soundPacketsOf(indexed).then(
         packets => (this.#packets = packets),
-        error => { this.#soundPacketsReading = null; throw error; });
+        error => {
+          this.#soundPacketsReading = null;
+          if (error instanceof IndexMemoryUnavailable) this.#soundPacketsRefused = error.bytes;
+          throw error;
+        });
       return this.#soundPacketsReading;
     }
     const tracks = await this.readTracks();

package/services/storage/IndexMemory.js

@@ -41,32 +41,39 @@ export class IndexMemory {
     let file = this.#files.get(key);
     if (file) return file.memory;
     file = { sourceKey, held: 0, needed: 0, records: new Set(), closed: false };
-    file.memory = { forRecord: (record, kind = "packets") => {
-      file.records.add(record);
-      let disposed = false;
-      return {
-        reserve: bytes => {
-          if (file.closed || disposed) return false;
-          if (this.#held + bytes > this.#allowed) {
-            const needed = file.held + bytes;
-            if (needed > file.needed) { file.needed = needed; this.#requestRevision(); }
-            return false;
-          }
-          file.held += bytes;
-          this.#held += bytes;
-          if (kind === "packets") this.#packetBytes += bytes;
-          if (file.held >= file.needed) file.needed = 0;
-          return true;
-        },
-        release: bytes => {
-          file.held -= bytes;
-          this.#held -= bytes;
-          if (kind === "packets") this.#packetBytes -= bytes;
-          this.#requestRevision();
-        },
-        dispose: () => { disposed = true; file.records.delete(record); }
-      };
-    } };
+    file.memory = {
+      // Whether a reservation of this many bytes would succeed now. Asked by a
+      // reader that keeps a refusal, so the refusal never outlives the state
+      // of memory it was given in — an event comes only when the allowance
+      // changes, and bytes released under the same allowance raise none.
+      fits: bytes => !file.closed && this.#held + bytes <= this.#allowed,
+      forRecord: (record, kind = "packets") => {
+        file.records.add(record);
+        let disposed = false;
+        return {
+          reserve: bytes => {
+            if (file.closed || disposed) return false;
+            if (this.#held + bytes > this.#allowed) {
+              const needed = file.held + bytes;
+              if (needed > file.needed) { file.needed = needed; this.#requestRevision(); }
+              return false;
+            }
+            file.held += bytes;
+            this.#held += bytes;
+            if (kind === "packets") this.#packetBytes += bytes;
+            if (file.held >= file.needed) file.needed = 0;
+            return true;
+          },
+          release: bytes => {
+            file.held -= bytes;
+            this.#held -= bytes;
+            if (kind === "packets") this.#packetBytes -= bytes;
+            this.#requestRevision();
+          },
+          dispose: () => { disposed = true; file.records.delete(record); }
+        };
+      }
+    };
     this.#files.set(key, file);
     return file.memory;
   }

package/services/viewer/DownloadMaps.js

@@ -136,15 +136,42 @@ export class DownloadMaps {
         urgent = current.urgent;
         deadlineAt = Number.isFinite(current.deadlineAt) ? current.deadlineAt - leadSeconds * 1000 : current.deadlineAt;
       }
-      file.metadata.set(statement, { ranges: result.ranges, requestId: result.requestId, priority, urgent, deadlineAt, interval, leadSeconds, scope, order });
-    } else if (result.kind !== "needs-memory") {
+      const entry = { ranges: result.ranges, requestId: result.requestId, priority, urgent, deadlineAt, interval, leadSeconds, scope, order };
+      const previous = file.metadata.get(statement);
+      if (previous && sameMetadata(previous, entry) && this.#standing(file)) return this.#unchanged(file);
+      file.metadata.set(statement, entry);
+    } else if (result.kind === "needs-memory") {
       // Allocation refusal says nothing about whether previously missing
-      // source bytes have arrived. Keep their demand until the read resolves.
-      file.metadata.delete(statement);
+      // source bytes have arrived. Keep their demand until the read resolves —
+      // and since it changes nothing in the map, nothing is built for it: the
+      // sound packet table of a two-hour AVI was refused once per soundtrack
+      // segment, 1943 times in 8.3 s, each a build and sort of every zone of
+      // the file (torrent-tv/meta#178).
+      if (this.#standing(file)) return this.#unchanged(file);
+    } else if (!file.metadata.delete(statement) && this.#standing(file)) {
+      return this.#unchanged(file);
     }
     return this.#emit(file);
   }
 
+  /**
+   * Whether the map last built for this file still stands, so a statement that
+   * changes nothing needs no new one: nothing `#emit` reads has changed since
+   * it ran (`dirty`), and the publication it made was not refused — `#emit`
+   * forgets the shape when the torrent thread did not apply the map or the
+   * publication failed, and that is what makes the next statement, changed or
+   * not, send it again.
+   */
+  #standing(file) {
+    return !file.dirty && Boolean(file.shape);
+  }
+
+  #unchanged(file) {
+    this.#statsOf(file).skipped += 1;
+    this.#report(file);
+    return Promise.resolve();
+  }
+
   withdrawMetadata(sourceKey, fileIndex, statement) {
     const file = this.#files.get(`${sourceKey}:${fileIndex}`);
     return file?.metadata.delete(statement) ? this.#emit(file) : Promise.resolve();
@@ -188,7 +215,7 @@ export class DownloadMaps {
     const key = `${sourceKey}:${fileIndex}`;
     let file = this.#files.get(key);
     if (!file) {
-      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve() };
+      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
       this.#files.set(key, file);
     }
     return file;
@@ -196,6 +223,8 @@ export class DownloadMaps {
 
   #setZones(file, zones) {
     file.zones = zones;
+    // What `#emit` reads has changed, whether or not it is published now.
+    file.dirty = true;
     file.inputs = new Map();
     for (const zone of zones) {
       if (typeof zone.outputKey !== "string" || !Number.isSafeInteger(zone.index)) continue;
@@ -208,6 +237,7 @@ export class DownloadMaps {
   }
 
   #emit(file) {
+    file.dirty = false;
     const builtFrom = this.#clock();
     const zones = [...file.zones, ...file.nativeZones];
     for (const { ranges, requestId, priority, urgent, deadlineAt, order } of file.metadata.values()) {
@@ -297,7 +327,8 @@ export class DownloadMaps {
       `(${total(stats.appliedMs)}ms in all); ` +
       `stages of a sent map — built ${stage(stats.sentBuildMs)}, waited behind the previous ${stage(stats.waitedMs)}, ` +
       `posted ${stage(stats.postMs)}, post to the torrent thread's handler ${stage(stats.toHandlerMs)}, ` +
-      `reply back ${stage(stats.replyMs)}; maps built and not sent took ${total([stats.unsentBuildMs])}ms here`);
+      `reply back ${stage(stats.replyMs)}; maps built and not sent took ${total([stats.unsentBuildMs])}ms here; ` +
+      `${stats.skipped} statement(s) changed nothing and built no map`);
     file.stats = emptyStats(now);
   }
 }
@@ -305,7 +336,7 @@ export class DownloadMaps {
 /** @param {number} since */
 function emptyStats(since) {
   return { since, sent: 0, unchanged: 0, zones: [], deliveredMs: [], appliedMs: [],
-    sentBuildMs: [], waitedMs: [], postMs: [], toHandlerMs: [], replyMs: [], unsentBuildMs: 0 };
+    sentBuildMs: [], waitedMs: [], postMs: [], toHandlerMs: [], replyMs: [], unsentBuildMs: 0, skipped: 0 };
 }
 
 /** One stage's median, largest and sum, in milliseconds with tenths. @param {number[]} values */
@@ -316,6 +347,22 @@ function stage(values) {
   return `median ${sorted[Math.floor(sorted.length / 2)].toFixed(1)}ms max ${sorted.at(-1).toFixed(1)}ms sum ${Math.round(sum)}ms`;
 }
 
+/**
+ * Whether two metadata demands are the same demand: every field, the byte
+ * ranges and the interval by value. The fields that do not reach the
+ * publication — the interval, the preparation time, the scope — decide how the
+ * demand is repriced and withdrawn later, so a change of them is a change.
+ */
+function sameMetadata(left, right) {
+  return left.requestId === right.requestId && Object.is(left.priority, right.priority) &&
+    left.urgent === right.urgent && Object.is(left.deadlineAt, right.deadlineAt) &&
+    Object.is(left.leadSeconds, right.leadSeconds) && left.scope === right.scope && left.order === right.order &&
+    (left.interval?.from ?? null) === (right.interval?.from ?? null) &&
+    (left.interval?.to ?? null) === (right.interval?.to ?? null) &&
+    left.ranges.length === right.ranges.length &&
+    left.ranges.every(([start, end], index) => start === right.ranges[index][0] && end === right.ranges[index][1]);
+}
+
 /** How many numbers describe one zone's part in what the torrent does. */
 const ZONE_FIELDS = 7;