@torrent-tv/proxy 2.97.9 → 2.97.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.9",
+  "version": "2.97.11",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/services/encode/EncodeInputs.js

@@ -364,7 +364,7 @@ export class EncodeInputs {
    */
   async #admitStretch(request, firstSources) {
     const { output, from } = request;
-    const minimum = originalInputBytes(firstSources);
+    let minimum = originalInputBytes(firstSources);
     const stretches = new Map([[from, { sources: firstSources, bytes: minimum }]]);
     const held = new Map();
     const at = async end => {
@@ -383,8 +383,17 @@ export class EncodeInputs {
     };
     // Grows by doubling and then halves back: a handful of resolutions find
     // the end whether it is a piece away or the rest of the film.
+    // THE LEAST THAT MAKES THE PIECE IT IS FOR. An input read from the keyframe
+    // before starts one piece early; holding that piece alone would not make
+    // the one the viewer wants (torrent-tv/meta#178).
+    const serves = Math.max(from, Math.min(request.serves ?? from, request.to));
+    let floor = from;
+    if (serves > from) {
+      const covering = await at(serves);
+      if (covering) { floor = serves; minimum = covering.bytes; }
+    }
     const longest = async maxBytes => {
-      let good = from, bad = null, step = 1;
+      let good = floor, bad = null, step = 1;
       while (bad === null) {
         const end = good + step;
         if (end > request.to) bad = request.to + 1;
@@ -400,7 +409,7 @@ export class EncodeInputs {
     const sizing = this.stretchFor(output);
     const target = sizing?.bytes ?? null;
     if (sizing && !sizing.openMeasured) request.timing.openApproximate = true;
-    let end = target === null || this.#heldRanges === null ? from : await longest(target);
+    let end = target === null || this.#heldRanges === null ? floor : await longest(target);
     if (this.#requests.get(request.key) !== request) return undefined;
     this.#wanted.set(request.key, { wanted: stretches.get(end).bytes, required: minimum });
     await this.#spent(request.timing, "budgetMs", () => this.#revise());

package/services/encode/EncodeRun.js

@@ -232,6 +232,10 @@ export class EncodeRun {
    *   encoder itself names it on its own channel, and answers with the name that
    *   piece is served under — because making it servable is a rename, and only
    *   whoever owns the disk can perform one.
+   * @param {boolean} [params.publishesAlone] - Its pieces take nothing from the
+   *   file closed after them (a re-encoded picture, or sound served on its own),
+   *   so each is published on its own line once the completeness check proves it
+   *   whole, instead of when the next file closes (torrent-tv/meta#178).
    * @param {(name: string) => void} [params.onTail] - Called with the working
    *   name of a file this run closed but will not publish: past its last cut, or
    *   flushed when it was stopped. It begins with the next keyframe.
@@ -271,6 +275,7 @@ export class EncodeRun {
     onEnded,
     onSpeedMeasured,
     onClosed,
+    publishesAlone = false,
     onTail,
     onProgress,
     onProduced,
@@ -303,6 +308,7 @@ export class EncodeRun {
     // proof it is whole; nothing else here can prove that.
     this.onClosed = typeof onClosed === "function" ? onClosed : (name) => name;
     this.onTail = typeof onTail === "function" ? onTail : () => {};
+    this.publishesAlone = publishesAlone === true;
     this.lastSegmentIndex = typeof lastSegmentIndex === "function" ? lastSegmentIndex : () => null;
     this.inputUnavailable = typeof inputUnavailable === "function" ? inputUnavailable : () => false;
     this.argsDescribed = argsDescribed;
@@ -653,19 +659,29 @@ export class EncodeRun {
         this.#pendingClosed = null;
         continue;
       }
+      // A PIECE THAT NEEDS NOTHING AFTER IT does not wait for the next file:
+      // the completeness check reads its own bytes and decides whether it
+      // reached its cut. Waiting for the next file cost the viewer that file's
+      // whole encode — 5.6 s after a seek on the field's AVI, where the picture
+      // stood 13.2 s (torrent-tv/meta#178). One it cannot prove yet waits as
+      // before; the file after it publishes it or the stop judges it.
+      if (this.publishesAlone && this.#publishClosed(name, null, { tentative: true })) {
+        this.#pendingClosed = null;
+        continue;
+      }
       this.#pendingClosed = name;
     }
   }
 
-  #publishClosed(name, following = null, { quiet = false } = {}) {
-      this.#measureSpeed();
+  #publishClosed(name, following = null, { quiet = false, tentative = false } = {}) {
+      if (!tentative) this.#measureSpeed();
       // ITS SERVED NAME, which is what whoever owns the disk gives it in answer.
       // ffmpeg writes a piece under a working name and reports that; the piece
       // becomes servable by being renamed, and everything below works in the
       // name a request can actually ask for.
       let served;
       try {
-        served = this.onClosed(name, following, this.#closedTimings.get(name) ?? null);
+        served = this.onClosed(name, following, this.#closedTimings.get(name) ?? null, { tentative });
       } catch (error) {
         if (quiet) {
           // A piece closed before our stop that cannot be proved whole is not
@@ -684,10 +700,13 @@ export class EncodeRun {
         this.#signal("SIGTERM");
         return false;
       } finally {
-        this.#closedTimings.delete(name);
+        // A piece not proved whole on its own keeps its timing for the file
+        // after it.
+        if (!tentative || served) this.#closedTimings.delete(name);
       }
 
       if (!served) return false;
+      if (tentative) this.#measureSpeed();
       if (!this.#stopping) {
         this.#provenName = served;
       }

package/services/encode/EncodeRuns.js

@@ -386,7 +386,7 @@ export class EncodeRuns {
    * @param {string | null} [following] - The working name of the file the muxer closed after it.
    * @returns {object | null | false}
    */
-  #wholeClosedPiece(session, name, index, admitted = false, bytes = null, run = null, timing = null, following = null) {
+  #wholeClosedPiece(session, name, index, admitted = false, bytes = null, run = null, timing = null, following = null, { tentative = false } = {}) {
 
     const format = session.segmentFormat;
     if (!format?.readMediaRanges || !Number.isInteger(index) || index < 0) {
@@ -427,6 +427,9 @@ export class EncodeRuns {
         if (mediaRanges && run.cutShiftSeconds !== null) mediaRanges.cutShiftSeconds = run.cutShiftSeconds;
       }
       const { whole, throughSeconds, reason } = judgePiece(format, mediaRanges, cut, interval);
+      // A piece tried before the next file closed is judged again then; its
+      // refusal now is not a fault and is not said.
+      if (!whole && tentative) return false;
       if (!whole) {
         this.#host.logger.warn(
           `encode: not publishing piece ${index} of ${address.slice(0, 60)}: ` +
@@ -460,6 +463,7 @@ export class EncodeRuns {
             continue;
           }
           if (!judged.whole) {
+            if (tentative) return false;
             this.#host.logger.warn(`encode: not publishing piece ${index} of ${address}: ${judged.reason}`);
             return false;
           }
@@ -617,6 +621,18 @@ export class EncodeRuns {
     return byOutput;
   }
 
+  /**
+   * Whether a piece of this output takes nothing from the file closed after it:
+   * no sound moved into a piece of the picture, and no open-GOP leading
+   * pictures to read from it — a re-encoded picture, or sound served as its own
+   * output. Such a piece is published on its own line (torrent-tv/meta#178).
+   */
+  publishesAlone(session) {
+    return (session.spec.transcodesVideo || session.spec.carries === "audio-only") &&
+      !(session.segmentFormat?.id === "fmp4" && session.spec.audio && session.spec.carries !== "audio-only" &&
+        !this.#host.servesAudioSeparately(session));
+  }
+
   /** Whether a run copies the picture from the original file it reads. */
   #copiesOriginalPicture(session, admittedInput) {
     return Boolean(admittedInput?.original) && !session.spec.transcodesVideo && session.spec.carries !== "audio-only";
@@ -1082,7 +1098,12 @@ export class EncodeRuns {
       // Why this encoder exists, recorded with its argument list. It used to be
       // handed to a separate `start` call; there is no separate call now.
       because,
-      onClosed: (name, following, timing) => {
+      // ITS PIECES TAKE NOTHING FROM THE FILE CLOSED AFTER THEM: no sound moved
+      // into a piece of the picture, and no open-GOP leading pictures to read
+      // from it — a re-encoded picture, or sound served as its own output. Such
+      // a piece is published on its own line (torrent-tv/meta#178).
+      publishesAlone: this.publishesAlone(session),
+      onClosed: (name, following, timing, { tentative = false } = {}) => {
 
         const index = session.segmentFormat.segmentIndexFromName(
           session.segmentFormat.servedNameOf?.(name) ?? name);
@@ -1110,7 +1131,9 @@ export class EncodeRuns {
             throw error;
           }
         }
-        const mediaRanges = this.#wholeClosedPiece(session, name, index, admittedInput, bytes, run, timing, following);
+        const mediaRanges = this.#wholeClosedPiece(session, name, index, admittedInput, bytes, run, timing, following, { tentative });
+        // Tried on its own line and not proved whole: it waits for the next file.
+        if (mediaRanges === false && tentative) return null;
         if (mediaRanges === false) throw Object.assign(new Error(`Closed piece #${index} has incomplete media.`),
           { code: "ERR_INCOMPLETE_MEDIA" });
         const published = this.#host.segmentFiles.publish(address, name, session.segmentFormat, { mediaRanges, bytes });

package/services/encode/quality/HostLoad.js

@@ -183,6 +183,8 @@ export class HostLoad {
         takenAt: Date.now(),
         cpuSeconds: this.#host.readProxyCpuSeconds(),
         bytes: totals?.downloaded ?? null,
+        uploaded: totals?.uploaded ?? null,
+        served: totals?.served ?? null,
         threadCpuSeconds: totals?.threadCpuSeconds ?? null,
         metadata: this.#host.readMetadataActivity?.()
       };
@@ -213,7 +215,7 @@ export class HostLoad {
    * nothing, a price where it downloaded something, and what it moved against
    * the limit in force, said while anything moves or is limited.
    *
-   * @param {{ takenAt: number, bytes: number | null, threadCpuSeconds: number | null }} now
+   * @param {{ takenAt: number, bytes: number | null, uploaded: number | null, served: number | null, threadCpuSeconds: number | null }} now
    */
   #learnThreadCost(now) {
     const previous = this.#torrentSample;
@@ -227,9 +229,17 @@ export class HostLoad {
     if (!(elapsedSec > 0) || megabytes < 0 || now.threadCpuSeconds < previous.threadCpuSeconds) return;
     const cpuSeconds = (now.threadCpuSeconds - previous.threadCpuSeconds) / Math.max(1, os.cpus().length);
     const limit = this.downloadLimit;
-    if (megabytes > 0 || Number.isFinite(limit)) {
+    // What else the thread moved over the same interval, across every torrent:
+    // sent back to the swarm, and handed to readers (encoder inputs, container
+    // reads, `/stream`). Said beside the download so that the thread's
+    // processor can be set against each kind of work (torrent-tv/meta#178).
+    const movedOf = (key) => Number.isFinite(now[key]) && Number.isFinite(previous[key]) && now[key] >= previous[key]
+      ? (now[key] - previous[key]) / 1e6 : null;
+    const rateOf = (key) => movedOf(key) === null ? "n/a" : `${(movedOf(key) / elapsedSec).toFixed(2)} MB/s`;
+    if (megabytes > 0 || Number.isFinite(limit) || movedOf("served") > 0 || movedOf("uploaded") > 0) {
       logger.info(
-        `torrent: downloaded ${(megabytes / elapsedSec).toFixed(2)} MB/s over ${elapsedSec.toFixed(1)}s, ` +
+        `torrent: downloaded ${(megabytes / elapsedSec).toFixed(2)} MB/s, sent to peers ${rateOf("uploaded")}, ` +
+        `handed to readers ${rateOf("served")} over ${elapsedSec.toFixed(1)}s, ` +
         `limit ${Number.isFinite(limit) ? `${(limit / 1e6).toFixed(2)} MB/s` : "none"}; ` +
         `its thread drew ${Math.round(cpuSeconds / elapsedSec * 100)}% of the machine`
       );
@@ -431,12 +441,13 @@ export class HostLoad {
   }
 
   /**
-   * The bytes every torrent has downloaded and the torrent thread's processor
-   * seconds, read together by that thread, or null when it cannot be asked.
+   * The bytes every torrent has downloaded and sent to peers, the bytes the
+   * torrent thread handed to readers, and that thread's processor seconds, read
+   * together by that thread, or null when it cannot be asked.
    * Downloaded bytes only: every one is verified against the piece hash and
    * written to the store; a byte sent back to the swarm is neither.
    *
-   * @returns {Promise<{ downloaded: number | null, threadCpuSeconds: number | null } | null>}
+   * @returns {Promise<{ downloaded: number | null, uploaded: number | null, served: number | null, threadCpuSeconds: number | null } | null>}
    */
   async #torrentTotals() {
     if (typeof this.#host.getTorrentTotals !== "function") {
@@ -444,12 +455,8 @@ export class HostLoad {
     }
     try {
       const totals = await this.#host.getTorrentTotals();
-      const downloaded = Number(totals?.downloaded);
-      const threadCpuSeconds = Number(totals?.threadCpuSeconds);
-      return {
-        downloaded: Number.isFinite(downloaded) ? downloaded : null,
-        threadCpuSeconds: Number.isFinite(threadCpuSeconds) ? threadCpuSeconds : null
-      };
+      const read = (key) => { const value = Number(totals?.[key]); return Number.isFinite(value) ? value : null; };
+      return { downloaded: read("downloaded"), uploaded: read("uploaded"), served: read("served"), threadCpuSeconds: read("threadCpuSeconds") };
     } catch {
       return null; // the pool is busy or gone; a reading missed is not a fault
     }

package/services/torrent/download/throttle.js

@@ -0,0 +1,31 @@
+/**
+ * @file Setting WebTorrent's download limit so that lowering it cannot fail a
+ * peer's data (torrent-tv/meta#178).
+ *
+ * WebTorrent's limit is a `speed-limiter` ThrottleGroup over `limiter`'s
+ * TokenBucket. The group cuts what a peer sends into slices of `rate / 10`
+ * at the moment a slice is cut, and a slice that waits for tokens asks for
+ * its full length again when it retries. `setRate` makes the bucket as large
+ * as the new rate, and the bucket refuses any request larger than itself
+ * ("exceeds bucket size"), which ends that peer's stream. So a slice cut at
+ * 47 MB/s (4.7 MB) that is still waiting when the limit falls to 0.6 MB/s
+ * fails — checked in `test/no-torrent/download-throttle.test.js`.
+ *
+ * The slice is kept at one BitTorrent block, 16 KiB, the length a peer's
+ * request asks for, whatever the rate: no slice can then outgrow a bucket of
+ * at least one block. Below one block the slice is the limit itself, as the
+ * group requires; the processor share never sets a limit that low, since it
+ * never falls below what the watched films play.
+ */
+
+/** The length of one request in the BitTorrent wire protocol (BEP 3). */
+const BLOCK_BYTES = 16 * 1024;
+
+/**
+ * @param {{ throttleDownload: (rate: number) => unknown, throttleGroups?: { down?: { setChunksize: (bytes: number) => void } } }} client
+ * @param {number} rate - Bytes per second; -1 lifts the limit.
+ */
+export function applyDownloadRate(client, rate) {
+  client.throttleDownload(rate);
+  if (rate >= 1) client.throttleGroups?.down?.setChunksize(Math.min(BLOCK_BYTES, Math.floor(rate)));
+}

package/services/torrent/torrent-pool.js

@@ -19,6 +19,7 @@ import { pieceStoreOf } from "./piece-store-of.js";
 import { demandProgress } from "./demand-progress.js";
 import { Urgency, urgencyName } from "./demand/index.js";
 import { demandFor, firstClassStatement, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
+import { applyDownloadRate } from "./download/throttle.js";
 import { withdrawClaim } from "./download/withdraw-claim.js";
 import { deadlineOfMapZone, levelOfMapZone, priorityOfMapZone } from "./map-zones.js";
 import { deriveSourceKey } from "../../utils/torrent-source-key.js";
@@ -1273,7 +1274,8 @@ export class TorrentPool {
     if (!this.client || this.client.destroyed || typeof this.client.throttleDownload !== "function") return;
     const rate = Number.isFinite(bytesPerSecond) ? Math.max(1, Math.round(bytesPerSecond)) : -1;
     if (rate === this.#downloadLimit) return;
-    this.client.throttleDownload(rate);
+    // One block per slice, so a lower limit cannot fail a slice already waiting (download/throttle.js).
+    applyDownloadRate(this.client, rate);
     // Said when the limit is set or lifted, and when it moves by a factor of
     // two either way — the scale on which a limit changes what the swarm gives.
     const scale = rate < 0 ? null : Math.round(Math.log2(rate));

package/services/torrent/worker/worker.js

@@ -100,6 +100,14 @@ const readsById = new Map();
  */
 const openReads = new Map();
 
+/**
+ * Bytes this thread has handed to readers — encoder inputs, container reads and
+ * `/stream` — across every torrent. Said with the torrent totals so that the
+ * thread's processor can be set against each kind of work it does, not only
+ * against the bytes it downloads (torrent-tv/meta#178).
+ */
+let servedBytes = 0;
+
 /**
  * Shorthand for this file. The same path as `logger.info` anywhere else in the
  * thread — kept only because it reads better at the hundred call sites here.
@@ -261,6 +269,7 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
         fragment.release();
         break;
       }
+      servedBytes += fragment.length;
       // One fragment in flight at a time. Each one holds a piece pinned, and
       // the store guarantees only two resident pieces at its smallest budget —
       // holding two pins while asking for a third would deadlock it against
@@ -423,7 +432,7 @@ async function runCommand(command, params, id) {
       // bytes, so the price of a downloaded byte is taken over one interval
       // whether or not anything encodes (torrent-tv/meta#178).
       const usage = process.threadCpuUsage();
-      return { downloaded, uploaded, threadCpuSeconds: (usage.user + usage.system) / 1e6 };
+      return { downloaded, uploaded, served: servedBytes, threadCpuSeconds: (usage.user + usage.system) / 1e6 };
     }
 
 
@@ -521,6 +530,7 @@ async function runCommand(command, params, id) {
       const torrent = isUsableTorrentHandle(known) ? known : null;
       const bytes = await readWhileInUse(openReads, id, torrent, params.fileIndex,
         () => readHeldBytes(torrent, params.fileIndex, params.start, params.end, logger));
+      servedBytes += bytes?.length ?? 0;
       return { bytes };
     }
 
@@ -529,6 +539,7 @@ async function runCommand(command, params, id) {
       const torrent = isUsableTorrentHandle(known) ? known : null;
       const buffers = await readWhileInUse(openReads, id, torrent, params.fileIndex,
         () => readHeldRanges(torrent, params.fileIndex, params.ranges, params.maxBytes, logger));
+      for (const buffer of buffers ?? []) servedBytes += buffer?.length ?? 0;
       return { buffers };
     }