@torrent-tv/proxy 2.92.6 → 2.93.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.92.6",
+  "version": "2.93.1",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/api/sources/fingerprint/get.js

@@ -0,0 +1,31 @@
+/**
+ * The OpenSubtitles hash of one file of a registered source.
+ *
+ * GET /api/sources/:sourceKey/files/:fileIndex/fingerprint
+ *
+ * Answers `200 { hash, size }` once both edges of the file are here,
+ * `202 { status: "pending" }` while they are not (the demand for them is already
+ * stated; the page asks again), and `404` for a file that has no hash because it
+ * is shorter than 128 KiB. Computing it reads 128 KiB, so it costs nothing worth
+ * a limit; it does not start a download of anything else.
+ *
+ * @param {import("fastify").FastifyRequest} req
+ * @param {import("fastify").FastifyReply} reply
+ * @param {{
+ *   sourceRegistry: { get: (key: string) => unknown },
+ *   inspectFingerprint: (address: { sourceKey: string, fileIndex: number, requestId?: string }) => Promise<{ kind: string, value?: { hash: string, size: number }, reason?: string }>
+ * }} deps
+ * @returns {Promise<void>}
+ */
+export async function handleApiSourceFingerprintGet(req, reply, { sourceRegistry, inspectFingerprint }) {
+  const sourceKey = typeof req.params?.sourceKey === "string" ? req.params.sourceKey.trim() : "";
+  const fileIndex = Number(req.params?.fileIndex);
+  if (!sourceKey || !Number.isInteger(fileIndex) || fileIndex < 0) {
+    return reply.code(400).send({ error: "sourceKey and a file index are required." });
+  }
+  if (!sourceRegistry.get(sourceKey)) return reply.code(404).send({ error: "Source key was not found." });
+  const result = await inspectFingerprint({ sourceKey, fileIndex, requestId: req.id });
+  if (result.kind === "result" && result.value) return reply.send({ hash: result.value.hash, size: result.value.size });
+  if (result.kind === "terminal") return reply.code(404).send({ error: `This file has no hash (${result.reason ?? "unavailable"}).` });
+  return reply.code(202).send({ status: "pending" });
+}

package/server.js

@@ -24,6 +24,7 @@ import { handleApiLinkProbeGet } from "./routes/api/link-probe/get.js";
 import { handleApiSourcesPost } from "./routes/api/sources/post.js";
 import { handleApiSourceStatsGet } from "./routes/api/sources/stats/get.js";
 import { handleApiSourceFilesGet } from "./routes/api/sources/files/get.js";
+import { handleApiSourceFingerprintGet } from "./routes/api/sources/fingerprint/get.js";
 import { handleApiSourceWarmPost } from "./routes/api/sources/warm/post.js";
 import { handleApiSourceViewerPost } from "./routes/api/sources/viewer/post.js";
 import { handleApiPlaybackPlanPost } from "./routes/api/playback-plan/post.js";
@@ -971,6 +972,15 @@ export async function startProxyServer({
   app.get("/api/sources/:sourceKey/files", async (req, reply) =>
     handleApiSourceFilesGet(req, reply, { sourceRegistry, torrentPool, viewers: outputParts.viewers })
   );
+  app.get("/api/sources/:sourceKey/files/:fileIndex/fingerprint", async (req, reply) =>
+    handleApiSourceFingerprintGet(req, reply, {
+      sourceRegistry,
+      inspectFingerprint: async (address) => {
+        const params = await containerOver(address);
+        return params ? containerOrchestrator.inspect(params, "fingerprint") : { kind: "pending" };
+      }
+    })
+  );
   app.post("/api/sources/:sourceKey/warm", async (req, reply) =>
     handleApiSourceWarmPost(req, reply, {
       sourceRegistry,

package/services/media/ContainerOrchestrator.js

@@ -16,7 +16,8 @@
  */
 
 import { ContainerFactory } from "./container/ContainerFactory.js";
-import { BytesUnavailable, isUnavailable } from "./container/unavailable.js";
+import { BytesUnavailable, isUnavailable, strictReader } from "./container/unavailable.js";
+import { OSHASH_EDGE_BYTES, hasOshash, oshash } from "./oshash.js";
 import { IndexMemoryUnavailable } from "./container/memory-unavailable.js";
 import { logger } from "../../utils/logger.js";
 
@@ -29,6 +30,21 @@ import { logger } from "../../utils/logger.js";
  */
 export const CONTAINER_HEAD_BYTES = 256 * 1024;
 
+/**
+ * The size of the file and its OpenSubtitles hash, from the two edges. A read
+ * that has not arrived throws `BytesUnavailable`, so nothing partial is kept.
+ *
+ * @param {{ readRange: (start: number, end: number) => Promise<Buffer | null>, fileSize: number }} params
+ * @returns {Promise<{ hash: string, size: number } | null>} `null` for a file too short to have one.
+ */
+async function readFingerprint({ readRange, fileSize }) {
+  if (!hasOshash(fileSize)) return null;
+  const read = strictReader(readRange, fileSize);
+  const head = await read(0, OSHASH_EDGE_BYTES - 1);
+  const tail = await read(fileSize - OSHASH_EDGE_BYTES, fileSize - 1);
+  return { hash: oshash(fileSize, head, tail), size: fileSize };
+}
+
 export class ContainerOrchestrator {
   #reads = new Map();
   #lifetimes = new Map();
@@ -73,7 +89,7 @@ export class ContainerOrchestrator {
 
   /** Read one statement without converting missing bytes into an empty answer. */
   async inspect(params, statement = "tracks") {
-    if (!["container", "tracks", "media-info", "keyframes", "packets"].includes(statement)) {
+    if (!["container", "tracks", "media-info", "keyframes", "packets", "fingerprint"].includes(statement)) {
       throw new TypeError(`Unknown media statement: ${statement}`);
     }
     const key = `${params.sourceKey}:${params.fileIndex}`;
@@ -105,6 +121,11 @@ export class ContainerOrchestrator {
     this.#activityEpoch++;
     this.#activeReads++;
     try {
+      if (statement === "fingerprint") {
+        // A fact about the bytes at the two ends of the file, whatever the format.
+        const value = await readFingerprint(params);
+        return value ? { kind: "result", value, requestId } : { kind: "terminal", reason: "file-too-short", requestId };
+      }
       const container = await this.containerFor(params);
       if (!container) {
         const result = { kind: "terminal", reason: "format-not-supported", requestId };

package/services/media/oshash.js

@@ -0,0 +1,41 @@
+/**
+ * @file The OpenSubtitles file hash ("moviehash", "oshash").
+ *
+ * The size of the file plus its first 64 KiB and its last 64 KiB, each read as
+ * little-endian 64-bit words and added modulo 2^64; written as 16 hexadecimal
+ * digits. Databases of releases (OpenSubtitles, StashDB, ThePornDB) keep it per
+ * file, so a match names this exact release; nothing else about the file is
+ * needed, which is why it can be computed from the two edges the proxy downloads
+ * first.
+ */
+
+/** Bytes read from each end of the file. */
+export const OSHASH_EDGE_BYTES = 65536;
+
+/**
+ * Whether a file is long enough to have the hash: both edges must be whole and
+ * distinct, as the format of the hash requires.
+ *
+ * @param {number} size
+ * @returns {boolean}
+ */
+export function hasOshash(size) {
+  return Number.isInteger(size) && size >= OSHASH_EDGE_BYTES * 2;
+}
+
+/**
+ * @param {number} size - The size of the file in bytes.
+ * @param {Uint8Array} head - Its first {@link OSHASH_EDGE_BYTES} bytes.
+ * @param {Uint8Array} tail - Its last {@link OSHASH_EDGE_BYTES} bytes.
+ * @returns {string} Sixteen lowercase hexadecimal digits.
+ */
+export function oshash(size, head, tail) {
+  if (!hasOshash(size)) throw new RangeError("a file shorter than 128 KiB has no oshash");
+  if (head.byteLength !== OSHASH_EDGE_BYTES || tail.byteLength !== OSHASH_EDGE_BYTES) throw new RangeError("each edge must be 64 KiB");
+  let sum = BigInt(size);
+  for (const edge of [head, tail]) {
+    const words = new DataView(edge.buffer, edge.byteOffset, edge.byteLength);
+    for (let at = 0; at < OSHASH_EDGE_BYTES; at += 8) sum += words.getBigUint64(at, true);
+  }
+  return BigInt.asUintN(64, sum).toString(16).padStart(16, "0");
+}

package/services/torrent/download/DeadlineScheduler.js

@@ -46,6 +46,11 @@ export function peerCanServe(wire, piece) {
     wire.peerPieces?.get(piece);
 }
 
+function peerCanRequest(wire) {
+  return !wire.destroyed &&
+    (!wire.peerChoking || (wire.hasFast && wire.peerAllowedFastSet?.length > 0));
+}
+
 /** Identical map ordering for present requests and conditional future requests. */
 export function compareDownloadCandidates(a, b) {
   return a.deadlineAt - b.deadlineAt || b.priority - a.priority || a.order - b.order || a.piece - b.piece;
@@ -62,11 +67,18 @@ export class DeadlineScheduler {
     this.#store = findStore;
   }
 
-  reconcile(now = Date.now()) {
+  reconcile(now = Date.now(), { torrent = null, wire = null } = {}) {
     if (this.#running) return { requested: 0, reassigned: 0 };
+    // WebTorrent updates each peer in turn. A full or choked peer cannot issue
+    // a request, so its heartbeat must not scan every file's byte availability.
+    if (wire && (!peerCanRequest(wire) ||
+      (wire.requests?.length ?? 0) >= peerRequestCapacity(torrent, wire))) {
+      return { requested: 0, reassigned: 0 };
+    }
     this.#running = true;
     try {
-      return dispatchDownloadCandidates(downloadCandidates(this.#entries(), this.#store), now);
+      const entries = [...this.#entries()].filter(entry => !torrent || entry.torrent === torrent);
+      return dispatchDownloadCandidates(downloadCandidates(entries, this.#store), now, { wire });
     } finally {
       this.#running = false;
     }
@@ -112,7 +124,7 @@ export function downloadCandidates(entries, findStore = pieceStoreOf) {
 }
 
 /** Run against protocol peers or an isolated future-state copy. */
-export function dispatchDownloadCandidates(candidates, now) {
+export function dispatchDownloadCandidates(candidates, now, { wire: changedWire = null } = {}) {
   let requested = 0, reassigned = 0;
   // Rates and advertised limits cannot change during this synchronous pass.
   // Read each peer once, and only inspect torrents once regardless of map size.
@@ -125,15 +137,14 @@ export function dispatchDownloadCandidates(candidates, now) {
     }
     return reading;
   };
-  const usable = wire => !wire.destroyed &&
-    (!wire.peerChoking || (wire.hasFast && wire.peerAllowedFastSet?.length > 0));
-  const available = (torrent, wire) => usable(wire) &&
+  const available = (torrent, wire) => peerCanRequest(wire) &&
     (wire.requests?.length ?? 0) < readingOf(torrent, wire).capacity;
   const completionAt = (torrent, wire, bytes) =>
     requestCompletionAt(wire, bytes, now, readingOf(torrent, wire).speed);
   const writable = new Map();
   for (const torrent of new Set(candidates.map(candidate => candidate.torrent))) {
-    const peers = (torrent.wires ?? []).filter(wire => available(torrent, wire));
+    const peers = (changedWire ? [changedWire] : torrent.wires ?? [])
+      .filter(wire => available(torrent, wire));
     if (peers.length) writable.set(torrent, peers);
   }
   for (const candidate of candidates) {

package/services/torrent/download/registry.js

@@ -58,8 +58,8 @@ export function demandFor(torrent) {
   const entry = { torrent, register, selection: new SwarmSelection({ torrent, register }), withdrawalRevision: 0 };
   if (typeof torrent._request === "function" && typeof torrent._updateWire === "function") {
     entry.previousWireUpdate = torrent._updateWire;
-    torrent._updateWire = () => {
-      if (!publishingSelections) scheduler.reconcile();
+    torrent._updateWire = wire => {
+      if (!publishingSelections) scheduler.reconcile(Date.now(), { torrent, wire });
     };
   }
   byTorrent.set(torrent, entry);