@torrent-tv/proxy 2.101.20 → 2.101.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.101.20",
+  "version": "2.101.22",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/stream/get.js

@@ -7,6 +7,7 @@
  */
 
 import { open } from "node:fs/promises";
+import { Readable } from "node:stream";
 import { parseRange } from "../../utils/parse-range.js";
 import { logger } from "../../utils/logger.js";
 
@@ -109,49 +110,66 @@ function wholeFileFor(torrentPool, sourceKey, fileIndex) {
 }
 
 /**
- * Open the whole file a response under way can continue from, if there is one.
+ * Acquire the whole file a response can continue from, if there is one.
  *
- * Any failure to open it leaves the response on the torrent, which can still
+ * Any failure to acquire it leaves the response on the torrent, which can still
  * answer; a file the disk owner removed is also forgotten here, as the route
  * does before headers.
  *
- * @param {{ wholeFiles?: Map<string, { path: string, length: number, name: string }> }} torrentPool
+ * @param {{ wholeFiles?: Map<string, { path: string, length: number, name: string }>, acquireWholeFile: (source: { sourceKey: string, fileIndex: number }) => Promise<{ length: number, stream: (start: number, end: number, onRead: (length: number) => void) => import("node:stream").Readable | null, release: () => Promise<unknown> } | null> }} torrentPool
  * @param {string} sourceKey
  * @param {number} fileIndex
  * @param {number} length - The length the response was promised for.
- * @returns {Promise<import("node:fs/promises").FileHandle | null>}
+ * @returns {Promise<{ length: number, stream: (start: number, end: number, onRead: (length: number) => void) => import("node:stream").Readable | null, release: () => Promise<unknown> } | null>}
  */
-async function openWholeFileToContinue(torrentPool, sourceKey, fileIndex, length) {
+async function acquireWholeFileRead(torrentPool, sourceKey, fileIndex, length) {
   const whole = wholeFileFor(torrentPool, sourceKey, fileIndex);
   if (!whole || whole.length !== length) {
     return null;
   }
   try {
-    return await open(whole.path, "r");
+    const read = await torrentPool.acquireWholeFile({ sourceKey, fileIndex });
+    if (!read) {
+      forgetWholeFile(torrentPool, sourceKey, fileIndex, whole);
+      return null;
+    }
+    if (read.length !== length) {
+      await read.release();
+      forgetWholeFile(torrentPool, sourceKey, fileIndex, whole);
+      return null;
+    }
+    return read;
   } catch (error) {
     if (error.code === "ENOENT") {
-      const key = `${sourceKey.slice("torrent:".length).toLowerCase()}/${fileIndex}`;
-      if (torrentPool.wholeFiles.get(key) === whole) torrentPool.wholeFiles.delete(key);
+      forgetWholeFile(torrentPool, sourceKey, fileIndex, whole);
     }
     return null;
   }
 }
 
+function forgetWholeFile(torrentPool, sourceKey, fileIndex, whole) {
+  const key = `${sourceKey.slice("torrent:".length).toLowerCase()}/${fileIndex}`;
+  if (torrentPool.wholeFiles?.get(key) === whole) torrentPool.wholeFiles.delete(key);
+}
+
 /**
  * Write bytes `from`..`end` of an open file to a response, each write waited
  * for, and close the file.
  *
  * @param {import("node:http").ServerResponse} raw
- * @param {import("node:fs/promises").FileHandle} handle
+ * @param {{ stream: (start: number, end: number, onRead: (length: number) => void) => import("node:stream").Readable | null, release: () => Promise<unknown> }} read
  * @param {number} from - Inclusive.
  * @param {number} end - Inclusive.
  * @param {(length: number) => void} onWritten
  * @returns {Promise<number>} Bytes written.
  */
-async function writeFromWholeFile(raw, handle, from, end, onWritten) {
+async function writeFromWholeFile(raw, read, from, end, onWritten) {
   let written = 0;
+  let stream = null;
   try {
-    for await (const chunk of handle.createReadStream({ start: from, end, autoClose: false })) {
+    stream = read.stream(from, end, () => {});
+    if (!stream) throw new Error("Completed file did not provide the admitted range.");
+    for await (const chunk of stream) {
       if (raw.writableEnded || raw.destroyed) {
         break;
       }
@@ -162,7 +180,8 @@ async function writeFromWholeFile(raw, handle, from, end, onWritten) {
       onWritten(chunk.length);
     }
   } finally {
-    await handle.close().catch(() => undefined);
+    stream?.destroy();
+    await read.release();
   }
   return written;
 }
@@ -173,35 +192,41 @@ async function writeFromWholeFile(raw, handle, from, end, onWritten) {
  * @param {import("fastify").FastifyRequest} req
  * @param {import("fastify").FastifyReply} reply
  * @param {{ path: string, length: number, name: string }} file
- * @param {import("node:fs/promises").FileHandle} handle - Already open; closed for HEAD.
+ * @param {{ stream: (start: number, end: number, onRead: (length: number) => void) => import("node:stream").Readable | null, release: () => Promise<unknown> }} read
  * @returns {Promise<void> | void}
  */
-function serveWholeFile(req, reply, file, handle) {
+function serveWholeFile(req, reply, file, read) {
   const disposition = `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`;
-  if (req.method === "HEAD") {
-    reply.hijack();
-    reply.raw.writeHead(200, {
-      "Accept-Ranges": "bytes",
-      "Content-Type": "application/octet-stream",
-      "Content-Length": String(file.length),
-      "Content-Disposition": disposition
-    });
-    reply.raw.end();
-    return;
-  }
   reply.header("Accept-Ranges", "bytes");
   reply.header("Content-Type", "application/octet-stream");
   reply.header("Content-Disposition", disposition);
   const range = parseRange(req.headers.range, file.length);
+  if (file.length === 0) {
+    reply.header("Content-Length", "0");
+    void read.release().catch(() => undefined);
+    return reply.send(Readable.from([]));
+  }
+  const start = range?.start ?? 0;
+  const end = range?.end ?? file.length - 1;
+  const stream = read.stream(start, end, () => {});
+  if (!stream) {
+    void read.release().catch(() => undefined);
+    return null;
+  }
   if (!range) {
     reply.header("Content-Length", String(file.length));
-    return reply.send(handle.createReadStream());
+  } else {
+    reply.code(206);
+    reply.header("Content-Length", String(end - start + 1));
+    reply.header("Content-Range", `bytes ${start}-${end}/${file.length}`);
+  }
+  void read.release().catch(() => undefined);
+  try {
+    return reply.send(stream);
+  } catch (error) {
+    stream.destroy();
+    throw error;
   }
-  const { start, end } = range;
-  reply.code(206);
-  reply.header("Content-Length", String(end - start + 1));
-  reply.header("Content-Range", `bytes ${start}-${end}/${file.length}`);
-  return reply.send(handle.createReadStream({ start, end }));
 }
 
 export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null, noteInputWaiting = null }) {
@@ -224,23 +249,36 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
   // nothing the torrent would have told us.
   const whole = wholeFileFor(torrentPool, sourceKey, fileIndex);
   if (whole) {
-    let handle;
-    try {
-      // Open before committing headers: the disk owner may have removed an
-      // announced file. An open descriptor also preserves a live POSIX read.
-      handle = await open(whole.path, "r");
-    } catch (error) {
-      if (error.code !== "ENOENT") throw error;
-      const key = `${sourceKey.slice("torrent:".length).toLowerCase()}/${fileIndex}`;
-      if (torrentPool.wholeFiles.get(key) === whole) torrentPool.wholeFiles.delete(key);
-    }
-    if (handle) {
-      if (req.method === "HEAD") await handle.close();
+    if (req.method === "HEAD") {
+      let handle;
       try {
-        return serveWholeFile(req, reply, whole, handle);
+        handle = await open(whole.path, "r");
       } catch (error) {
-        await handle.close().catch(() => undefined);
-        throw error;
+        if (error.code !== "ENOENT") throw error;
+        forgetWholeFile(torrentPool, sourceKey, fileIndex, whole);
+      }
+      if (handle) {
+        await handle.close();
+        reply.hijack();
+        reply.raw.writeHead(200, {
+          "Accept-Ranges": "bytes",
+          "Content-Type": "application/octet-stream",
+          "Content-Length": String(whole.length),
+          "Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(whole.name)}`
+        });
+        reply.raw.end();
+        return;
+      }
+    } else {
+      const read = await acquireWholeFileRead(torrentPool, sourceKey, fileIndex, whole.length);
+      if (read) {
+        try {
+          const sent = serveWholeFile(req, reply, whole, read);
+          if (sent) return sent;
+        } catch (error) {
+          await read.release().catch(() => undefined);
+          throw error;
+        }
       }
     }
   }
@@ -338,7 +376,7 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
             fragment.release();
             break;
           }
-          continueFrom = await openWholeFileToContinue(torrentPool, sourceKey, fileIndex, file.length);
+          continueFrom = await acquireWholeFileRead(torrentPool, sourceKey, fileIndex, file.length);
           if (continueFrom) {
             fragment.release();
             break;
@@ -368,7 +406,7 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
         }
       } catch (error) {
         if (sent >= contentLength || req.raw.aborted || reply.raw.destroyed) throw error;
-        continueFrom = await openWholeFileToContinue(torrentPool, sourceKey, fileIndex, file.length);
+        continueFrom = await acquireWholeFileRead(torrentPool, sourceKey, fileIndex, file.length);
         if (!continueFrom) throw error;
       }
       if (continueFrom) {

package/services/storage/files/CompletedFiles.js

@@ -54,8 +54,13 @@ export class CompletedFiles {
 
   /** Assemblies in flight, so two passes cannot write one file at once. @type {Set<string>} */
   #writing = new Set();
+  /** Bytes promised to assemblies that have not yet become held files. @type {Map<string, number>} */
+  #reservations = new Map();
+  /** Serialize allowance changes and admission decisions. @type {Promise<unknown>} */
+  #admission = Promise.resolve();
   #readers = new Map();
   #removeAfterRead = new Set();
+  #removing = new Set();
 
   /**
    * What this store may hold, handed to it by the owner of the disk.
@@ -76,6 +81,8 @@ export class CompletedFiles {
 
   /** When each held file was last asked for. @type {Map<string, number>} */
   #lastRead = new Map();
+  /** Monotonic access order, so reads in the same clock tick remain ordered. */
+  #accessOrder = 0;
 
   /**
    * @param {object} params
@@ -142,17 +149,22 @@ export class CompletedFiles {
    */
   find(infoHash, fileIndex) {
     const key = this.#keyOf(infoHash, fileIndex);
-    if (this.#removeAfterRead.has(key)) return null;
+    if (this.#removeAfterRead.has(key) || this.#removing.has(key)) return null;
     const file = this.#held.get(key) ?? null;
     if (file) {
       // WHEN IT WAS LAST WANTED, which is the order it leaves in. A whole file
       // nobody has asked for in the longest is the one whose loss costs least:
       // it can be downloaded again, and a read of it falls back to the pieces.
-      this.#lastRead.set(key, Date.now());
+      this.#touch(key);
     }
     return file;
   }
 
+  /** Mark a file as more recently wanted than every earlier access. */
+  #touch(key) {
+    this.#lastRead.set(key, ++this.#accessOrder);
+  }
+
   /** Pin a completed file against eviction until all its offset reads finish. */
   acquire(infoHash, fileIndex) {
     const key = this.#keyOf(infoHash, fileIndex);
@@ -186,20 +198,38 @@ export class CompletedFiles {
    *   many files went.
    */
   async allow(bytes) {
-    this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
-    let removed = 0;
-    if (this.#allowanceBytes === null) {
+    return this.#withAdmission(async () => {
+      this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
+      const removed = await this.#trimToAllowance();
       return { bytes: this.bytes, removed };
-    }
-    while (this.bytes > this.#allowanceBytes) {
+    });
+  }
+
+  /** Run one storage decision at a time. @param {() => Promise<unknown> | unknown} action */
+  #withAdmission(action) {
+    const result = this.#admission.then(action, action);
+    this.#admission = result.catch(() => undefined);
+    return result;
+  }
+
+  /** Include bytes currently being assembled when enforcing the allowance. */
+  #reservedBytes() {
+    let total = 0;
+    for (const length of this.#reservations.values()) total += length;
+    return total;
+  }
+
+  /** Remove eligible unused files until held and reserved bytes fit. */
+  async #trimToAllowance() {
+    let removed = 0;
+    if (this.#allowanceBytes === null) return removed;
+    while (this.bytes + this.#reservedBytes() > this.#allowanceBytes) {
       const victim = this.#longestUnread();
-      if (!victim) {
-        break;
-      }
-      await this.#remove(victim);
+      if (!victim) break;
+      if (!await this.#remove(victim)) break;
       removed += 1;
     }
-    return { bytes: this.bytes, removed };
+    return removed;
   }
 
   /**
@@ -215,7 +245,7 @@ export class CompletedFiles {
     if (this.#allowanceBytes === null) {
       return true;
     }
-    return this.bytes + Math.max(0, Number(length) || 0) <= this.#allowanceBytes;
+    return this.bytes + this.#reservedBytes() + Math.max(0, Number(length) || 0) <= this.#allowanceBytes;
   }
 
   /**
@@ -227,7 +257,7 @@ export class CompletedFiles {
     let oldest = null;
     let oldestAt = Number.POSITIVE_INFINITY;
     for (const key of this.#held.keys()) {
-      if (this.#writing.has(key) || this.#readers.has(key)) {
+      if (this.#writing.has(key) || this.#readers.has(key) || this.#removing.has(key)) {
         continue;
       }
       const at = this.#lastRead.get(key) ?? 0;
@@ -243,17 +273,24 @@ export class CompletedFiles {
    * Take one whole file off the disk.
    *
    * @param {string} key
-   * @returns {Promise<void>}
+   * @returns {Promise<boolean>} Whether the file was removed.
    */
   async #remove(key) {
-    if (this.#readers.has(key)) { this.#removeAfterRead.add(key); return; }
+    if (this.#removing.has(key)) return false;
+    if (this.#readers.has(key)) { this.#removeAfterRead.add(key); return false; }
     const file = this.#held.get(key);
+    if (!file) return true;
+    this.#removing.add(key);
+    try {
+      await fs.rm(file.path, REMOVAL);
+    } catch {
+      this.#removing.delete(key);
+      return false;
+    }
     this.#held.delete(key);
     this.#lastRead.delete(key);
-    if (!file) {
-      return;
-    }
-    await fs.rm(file.path, REMOVAL).catch(() => undefined);
+    this.#removing.delete(key);
+    return true;
   }
 
   /**
@@ -344,18 +381,30 @@ export class CompletedFiles {
   async keep({ infoHash, fileIndex, length, name, open }) {
     const key = this.#keyOf(infoHash, fileIndex);
     const held = this.#held.get(key);
-    if (held) {
+    if (held && !this.#removing.has(key)) {
       return held;
     }
+    if (this.#removing.has(key)) return null;
     if (this.#writing.has(key)) {
       return null;
     }
-    if (!this.hasRoomFor(length)) {
-      // Assembling it and then removing it is the same bytes written for
-      // nothing. The pieces still serve the read; this file simply is not kept.
+    this.#writing.add(key);
+    const admitted = await this.#withAdmission(async () => {
+      if (this.#held.has(key)) return false;
+      if (this.#allowanceBytes !== null && length > this.#allowanceBytes) return false;
+      while (this.#allowanceBytes !== null &&
+        this.bytes + this.#reservedBytes() + length > this.#allowanceBytes) {
+        const victim = this.#longestUnread();
+        if (!victim) return false;
+        if (!await this.#remove(victim)) return false;
+      }
+      this.#reservations.set(key, length);
+      return true;
+    });
+    if (!admitted) {
+      await this.#withAdmission(() => { this.#writing.delete(key); });
       return null;
     }
-    this.#writing.add(key);
     const directory = path.join(this.#root, String(infoHash));
     const where = path.join(directory, String(fileIndex));
     const partial = `${where}.partial`;
@@ -367,6 +416,9 @@ export class CompletedFiles {
       let written = 0;
       try {
         for await (const chunk of source) {
+          if (written + chunk.length > length) {
+            throw new Error("Whole-file read exceeded its declared length");
+          }
           await handle.write(chunk);
           written += chunk.length;
         }
@@ -382,14 +434,24 @@ export class CompletedFiles {
       }
       await fs.rename(partial, where);
       const file = { path: where, length, name: String(name ?? fileIndex) };
-      this.#held.set(key, file);
+      await this.#withAdmission(async () => {
+        this.#reservations.delete(key);
+        this.#held.set(key, file);
+        this.#touch(key);
+        this.#writing.delete(key);
+        await this.#trimToAllowance();
+      });
       await this.#writeManifest(infoHash);
-      return file;
+      return this.#held.get(key) ?? null;
     } catch {
       await fs.rm(partial, REMOVAL).catch(() => undefined);
       return null;
     } finally {
-      this.#writing.delete(key);
+      await this.#withAdmission(async () => {
+        this.#reservations.delete(key);
+        this.#writing.delete(key);
+        await this.#trimToAllowance();
+      });
     }
   }