@torrent-tv/proxy 2.92.5 → 2.93.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.92.5",
+  "version": "2.93.0",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/api/sources/fingerprint/get.js

@@ -0,0 +1,31 @@
+/**
+ * The OpenSubtitles hash of one file of a registered source.
+ *
+ * GET /api/sources/:sourceKey/files/:fileIndex/fingerprint
+ *
+ * Answers `200 { hash, size }` once both edges of the file are here,
+ * `202 { status: "pending" }` while they are not (the demand for them is already
+ * stated; the page asks again), and `404` for a file that has no hash because it
+ * is shorter than 128 KiB. Computing it reads 128 KiB, so it costs nothing worth
+ * a limit; it does not start a download of anything else.
+ *
+ * @param {import("fastify").FastifyRequest} req
+ * @param {import("fastify").FastifyReply} reply
+ * @param {{
+ *   sourceRegistry: { get: (key: string) => unknown },
+ *   inspectFingerprint: (address: { sourceKey: string, fileIndex: number, requestId?: string }) => Promise<{ kind: string, value?: { hash: string, size: number }, reason?: string }>
+ * }} deps
+ * @returns {Promise<void>}
+ */
+export async function handleApiSourceFingerprintGet(req, reply, { sourceRegistry, inspectFingerprint }) {
+  const sourceKey = typeof req.params?.sourceKey === "string" ? req.params.sourceKey.trim() : "";
+  const fileIndex = Number(req.params?.fileIndex);
+  if (!sourceKey || !Number.isInteger(fileIndex) || fileIndex < 0) {
+    return reply.code(400).send({ error: "sourceKey and a file index are required." });
+  }
+  if (!sourceRegistry.get(sourceKey)) return reply.code(404).send({ error: "Source key was not found." });
+  const result = await inspectFingerprint({ sourceKey, fileIndex, requestId: req.id });
+  if (result.kind === "result" && result.value) return reply.send({ hash: result.value.hash, size: result.value.size });
+  if (result.kind === "terminal") return reply.code(404).send({ error: `This file has no hash (${result.reason ?? "unavailable"}).` });
+  return reply.code(202).send({ status: "pending" });
+}

package/server.js

@@ -24,6 +24,7 @@ import { handleApiLinkProbeGet } from "./routes/api/link-probe/get.js";
 import { handleApiSourcesPost } from "./routes/api/sources/post.js";
 import { handleApiSourceStatsGet } from "./routes/api/sources/stats/get.js";
 import { handleApiSourceFilesGet } from "./routes/api/sources/files/get.js";
+import { handleApiSourceFingerprintGet } from "./routes/api/sources/fingerprint/get.js";
 import { handleApiSourceWarmPost } from "./routes/api/sources/warm/post.js";
 import { handleApiSourceViewerPost } from "./routes/api/sources/viewer/post.js";
 import { handleApiPlaybackPlanPost } from "./routes/api/playback-plan/post.js";
@@ -971,6 +972,15 @@ export async function startProxyServer({
   app.get("/api/sources/:sourceKey/files", async (req, reply) =>
     handleApiSourceFilesGet(req, reply, { sourceRegistry, torrentPool, viewers: outputParts.viewers })
   );
+  app.get("/api/sources/:sourceKey/files/:fileIndex/fingerprint", async (req, reply) =>
+    handleApiSourceFingerprintGet(req, reply, {
+      sourceRegistry,
+      inspectFingerprint: async (address) => {
+        const params = await containerOver(address);
+        return params ? containerOrchestrator.inspect(params, "fingerprint") : { kind: "pending" };
+      }
+    })
+  );
   app.post("/api/sources/:sourceKey/warm", async (req, reply) =>
     handleApiSourceWarmPost(req, reply, {
       sourceRegistry,

package/services/media/ContainerOrchestrator.js

@@ -16,7 +16,8 @@
  */
 
 import { ContainerFactory } from "./container/ContainerFactory.js";
-import { BytesUnavailable, isUnavailable } from "./container/unavailable.js";
+import { BytesUnavailable, isUnavailable, strictReader } from "./container/unavailable.js";
+import { OSHASH_EDGE_BYTES, hasOshash, oshash } from "./oshash.js";
 import { IndexMemoryUnavailable } from "./container/memory-unavailable.js";
 import { logger } from "../../utils/logger.js";
 
@@ -29,6 +30,21 @@ import { logger } from "../../utils/logger.js";
  */
 export const CONTAINER_HEAD_BYTES = 256 * 1024;
 
+/**
+ * The size of the file and its OpenSubtitles hash, from the two edges. A read
+ * that has not arrived throws `BytesUnavailable`, so nothing partial is kept.
+ *
+ * @param {{ readRange: (start: number, end: number) => Promise<Buffer | null>, fileSize: number }} params
+ * @returns {Promise<{ hash: string, size: number } | null>} `null` for a file too short to have one.
+ */
+async function readFingerprint({ readRange, fileSize }) {
+  if (!hasOshash(fileSize)) return null;
+  const read = strictReader(readRange, fileSize);
+  const head = await read(0, OSHASH_EDGE_BYTES - 1);
+  const tail = await read(fileSize - OSHASH_EDGE_BYTES, fileSize - 1);
+  return { hash: oshash(fileSize, head, tail), size: fileSize };
+}
+
 export class ContainerOrchestrator {
   #reads = new Map();
   #lifetimes = new Map();
@@ -73,7 +89,7 @@ export class ContainerOrchestrator {
 
   /** Read one statement without converting missing bytes into an empty answer. */
   async inspect(params, statement = "tracks") {
-    if (!["container", "tracks", "media-info", "keyframes", "packets"].includes(statement)) {
+    if (!["container", "tracks", "media-info", "keyframes", "packets", "fingerprint"].includes(statement)) {
       throw new TypeError(`Unknown media statement: ${statement}`);
     }
     const key = `${params.sourceKey}:${params.fileIndex}`;
@@ -105,6 +121,11 @@ export class ContainerOrchestrator {
     this.#activityEpoch++;
     this.#activeReads++;
     try {
+      if (statement === "fingerprint") {
+        // A fact about the bytes at the two ends of the file, whatever the format.
+        const value = await readFingerprint(params);
+        return value ? { kind: "result", value, requestId } : { kind: "terminal", reason: "file-too-short", requestId };
+      }
       const container = await this.containerFor(params);
       if (!container) {
         const result = { kind: "terminal", reason: "format-not-supported", requestId };

package/services/media/oshash.js

@@ -0,0 +1,41 @@
+/**
+ * @file The OpenSubtitles file hash ("moviehash", "oshash").
+ *
+ * The size of the file plus its first 64 KiB and its last 64 KiB, each read as
+ * little-endian 64-bit words and added modulo 2^64; written as 16 hexadecimal
+ * digits. Databases of releases (OpenSubtitles, StashDB, ThePornDB) keep it per
+ * file, so a match names this exact release; nothing else about the file is
+ * needed, which is why it can be computed from the two edges the proxy downloads
+ * first.
+ */
+
+/** Bytes read from each end of the file. */
+export const OSHASH_EDGE_BYTES = 65536;
+
+/**
+ * Whether a file is long enough to have the hash: both edges must be whole and
+ * distinct, as the format of the hash requires.
+ *
+ * @param {number} size
+ * @returns {boolean}
+ */
+export function hasOshash(size) {
+  return Number.isInteger(size) && size >= OSHASH_EDGE_BYTES * 2;
+}
+
+/**
+ * @param {number} size - The size of the file in bytes.
+ * @param {Uint8Array} head - Its first {@link OSHASH_EDGE_BYTES} bytes.
+ * @param {Uint8Array} tail - Its last {@link OSHASH_EDGE_BYTES} bytes.
+ * @returns {string} Sixteen lowercase hexadecimal digits.
+ */
+export function oshash(size, head, tail) {
+  if (!hasOshash(size)) throw new RangeError("a file shorter than 128 KiB has no oshash");
+  if (head.byteLength !== OSHASH_EDGE_BYTES || tail.byteLength !== OSHASH_EDGE_BYTES) throw new RangeError("each edge must be 64 KiB");
+  let sum = BigInt(size);
+  for (const edge of [head, tail]) {
+    const words = new DataView(edge.buffer, edge.byteOffset, edge.byteLength);
+    for (let at = 0; at < OSHASH_EDGE_BYTES; at += 8) sum += words.getBigUint64(at, true);
+  }
+  return BigInt.asUintN(64, sum).toString(16).padStart(16, "0");
+}

package/services/storage/piece-store/piece-lru.js

@@ -289,15 +289,16 @@ export class PieceLru {
    *
    * @returns {{ readers: number, names: string[], unionPieces: number, widestPieces: number, capacity: number }}
    */
-  demand() {
+  demand({ memoryRequiredOnly = false } = {}) {
     // WHO THEY ARE, not only how many. A "reader" here is whoever declared a
     // range, and on 2026-09-08 the field said `5 reader(s) want 24 piece(s) of
     // 25` on a session with two encoders — because the priority map declares one
     // range per zone and four of its zones were arriving as four readers. The
     // count alone could not say that, and choosing between "narrow the windows"
     // and "raise the allowance" was guesswork until the names were printed.
-    const names = [...this.#protected.keys()].map(String).sort();
-    const ranges = [...this.#protected.values()]
+    const entries = [...this.#protected.entries()].filter(([, range]) => !memoryRequiredOnly || range.memoryRequired);
+    const names = entries.map(([name]) => String(name)).sort();
+    const ranges = entries.map(([, range]) => range)
       .map((range) => ({ from: range.from, to: range.to }))
       .sort((left, right) => left.from - right.from);
     let unionPieces = 0;
@@ -432,9 +433,11 @@ export class PieceLru {
    *   urgent. A caller that states none is treated as wanting these pieces
    *   least of everyone who did state one, so an unstated range can never
    *   displace a stated one.
+   * @param {boolean} [memoryRequired] - Read windows require room; download
+   *   preferences can yield under pressure.
    * @returns {void}
    */
-  protect(readerId, from, to, urgency, deadlineAt, priority = 1) {
+  protect(readerId, from, to, urgency, deadlineAt, priority = 1, memoryRequired = true) {
     if (!Number.isInteger(from) || !Number.isInteger(to) || to < from) {
       return;
     }
@@ -443,7 +446,8 @@ export class PieceLru {
       to,
       urgency: Number.isFinite(urgency) ? Number(urgency) : Number.MAX_SAFE_INTEGER,
       deadlineAt,
-      priority
+      priority,
+      memoryRequired
     });
   }
 

package/services/storage/piece-store/shared-piece-store.js

@@ -658,10 +658,13 @@ export class SharedPieceStore {
    * range to declare.
    */
   get wantedBytes() {
+    // Download preferences may use spare RAM, but cannot become a required
+    // floor or a retained reading window after the map has been withdrawn.
     const demand = this.#lru.demand();
+    const required = this.#lru.demand({ memoryRequiredOnly: true });
     const pinned = this.#lru.pinnedCount;
     if (demand.readers > 0) {
-      this.#widestSeenPieces = Math.max(this.#widestSeenPieces, demand.widestPieces);
+      this.#widestSeenPieces = Math.max(this.#widestSeenPieces, required.widestPieces);
       const pieces = Math.max(MIN_RESIDENT_PIECES, demand.unionPieces, demand.widestPieces, pinned);
       // Plus room to absorb what arrives while a write is finishing. Asking for
       // exactly what the readers want leaves no free place ever, so every
@@ -710,7 +713,7 @@ export class SharedPieceStore {
     // also pin pieces without a priority-map range, so those measured pins are
     // part of the same minimum. Exceeding the share is the lesser failure, and
     // the line below says when it happens.
-    const demand = this.#lru.demand();
+    const demand = this.#lru.demand({ memoryRequiredOnly: true });
     const activeDemandPieces = Math.max(
       MIN_RESIDENT_PIECES,
       demand.readers > 0 ? demand.unionPieces : 0,
@@ -1848,8 +1851,8 @@ export class SharedPieceStore {
    * @param {number} [urgency]
    * @returns {void}
    */
-  protectRange(readerId, from, to, urgency, deadlineAt, priority) {
-    this.#lru.protect(readerId, from, to, urgency, deadlineAt, priority);
+  protectRange(readerId, from, to, urgency, deadlineAt, priority, memoryRequired = true) {
+    this.#lru.protect(readerId, from, to, urgency, deadlineAt, priority, memoryRequired);
     // A READER DECLARES ITSELF IN A MOMENT; the allowance was re-derived once a
     // minute. Between the two a read met whatever the store had shrunk to while
     // nobody was reading, and a claim gives up after five seconds — twelve

package/services/torrent/download/DeadlineScheduler.js

@@ -25,8 +25,7 @@ function requestArrival(wire, request, now) {
   return requestArrivals.get(request);
 }
 
-export function peerRequestCapacity(torrent, wire) {
-  const speed = Math.max(0, Number(wire.downloadSpeed?.()) || 0);
+export function peerRequestCapacity(torrent, wire, speed = Math.max(0, Number(wire.downloadSpeed?.()) || 0)) {
   if (wire.type === "webSeed") {
     return Math.min(1 + Math.ceil(speed / torrent.pieceLength), torrent.maxWebConns);
   }
@@ -36,8 +35,7 @@ export function peerRequestCapacity(torrent, wire) {
   return capacity;
 }
 
-export function requestCompletionAt(wire, bytes, now) {
-  const speed = Number(wire.downloadSpeed?.());
+export function requestCompletionAt(wire, bytes, now, speed = Number(wire.downloadSpeed?.())) {
   if (!(speed > 0)) return Number.POSITIVE_INFINITY;
   const queued = (wire.requests ?? []).reduce((total, request) => total + request.length, 0);
   return now + (queued + bytes) / speed * 1000;
@@ -116,26 +114,45 @@ export function downloadCandidates(entries, findStore = pieceStoreOf) {
 /** Run against protocol peers or an isolated future-state copy. */
 export function dispatchDownloadCandidates(candidates, now) {
   let requested = 0, reassigned = 0;
-  const hasCapacity = torrent => (torrent.wires ?? []).some(wire =>
-    !wire.destroyed && (wire.requests?.length ?? 0) < peerRequestCapacity(torrent, wire));
-  const writable = new Set(candidates.map(candidate => candidate.torrent).filter(hasCapacity));
+  // Rates and advertised limits cannot change during this synchronous pass.
+  // Read each peer once, and only inspect torrents once regardless of map size.
+  const readings = new Map();
+  const readingOf = (torrent, wire) => {
+    let reading = readings.get(wire);
+    if (!reading) {
+      const speed = Math.max(0, Number(wire.downloadSpeed?.()) || 0);
+      readings.set(wire, reading = { speed, capacity: peerRequestCapacity(torrent, wire, speed) });
+    }
+    return reading;
+  };
+  const usable = wire => !wire.destroyed &&
+    (!wire.peerChoking || (wire.hasFast && wire.peerAllowedFastSet?.length > 0));
+  const available = (torrent, wire) => usable(wire) &&
+    (wire.requests?.length ?? 0) < readingOf(torrent, wire).capacity;
+  const completionAt = (torrent, wire, bytes) =>
+    requestCompletionAt(wire, bytes, now, readingOf(torrent, wire).speed);
+  const writable = new Map();
+  for (const torrent of new Set(candidates.map(candidate => candidate.torrent))) {
+    const peers = (torrent.wires ?? []).filter(wire => available(torrent, wire));
+    if (peers.length) writable.set(torrent, peers);
+  }
   for (const candidate of candidates) {
     if (!writable.size) break;
     const { torrent, piece, deadlineAt, requestId } = candidate;
     if (!writable.has(torrent)) continue;
-    const wires = (torrent.wires ?? []).filter(wire => peerCanServe(wire, piece) &&
-      (wire.requests?.length ?? 0) < peerRequestCapacity(torrent, wire));
+    const peers = writable.get(torrent);
+    const wires = peers.filter(wire => peerCanServe(wire, piece) && available(torrent, wire));
     while (wires.length) {
-      wires.sort((a, b) => requestCompletionAt(a, BLOCK_BYTES, now) - requestCompletionAt(b, BLOCK_BYTES, now) ||
+      wires.sort((a, b) => completionAt(torrent, a, BLOCK_BYTES) - completionAt(torrent, b, BLOCK_BYTES) ||
         (a.requests?.length ?? 0) - (b.requests?.length ?? 0));
       const wire = wires[0];
       const before = wire.requests?.length ?? 0;
       let accepted = torrent._request(wire, piece, false);
       if (!accepted && Number.isFinite(deadlineAt)) {
-        const fasterAt = requestCompletionAt(wire, BLOCK_BYTES, now);
+        const fasterAt = completionAt(torrent, wire, BLOCK_BYTES);
         for (const slower of torrent.wires ?? []) {
           if (slower === wire || typeof slower.cancel !== "function") continue;
-          const lateAt = requestCompletionAt(slower, 0, now);
+          const lateAt = completionAt(torrent, slower, 0);
           if (!Number.isFinite(lateAt) || !(lateAt > deadlineAt && fasterAt < lateAt)) continue;
           const request = slower.requests?.find(one => one.piece === piece);
           if (!request) continue;
@@ -159,9 +176,9 @@ export function dispatchDownloadCandidates(candidates, now) {
         torrent.emit?.("download-requested", { piece, deadlineAt, requestId, peer: wire.remoteAddress, queued: wire.requests?.length ?? 0 });
       }
       if (!accepted || (wire.requests?.length ?? 0) <= before ||
-        (wire.requests?.length ?? 0) >= peerRequestCapacity(torrent, wire)) wires.shift();
+        (wire.requests?.length ?? 0) >= readingOf(torrent, wire).capacity) wires.shift();
     }
-    if (!hasCapacity(torrent)) writable.delete(torrent);
+    if (!peers.some(wire => available(torrent, wire))) writable.delete(torrent);
   }
   return { requested, reassigned };
 }

package/services/torrent/download/SwarmSelection.js

@@ -235,7 +235,9 @@ export class SwarmSelection {
         // The level goes with the range: it is what eviction compares when
         // everything resident is wanted by somebody, and dropping it here is
         // what left the store choosing by recency alone.
-        store.protectRange(window.claimant, range.from, range.to, window.urgency, window.deadlineAt, window.priority);
+        // The download map is an eviction preference. Actual reads own pins;
+        // declaring a whole-file tail must not set a whole-file memory floor.
+        store.protectRange(window.claimant, range.from, range.to, window.urgency, window.deadlineAt, window.priority, false);
         holding.add(window.claimant);
       }
     }