@torrent-tv/proxy 2.90.0 → 2.92.0

package/services/torrent/download/FutureDownload.js

@@ -0,0 +1,166 @@
+import { setImmediate as nextTurn } from "node:timers/promises";
+import { downloadCandidates, dispatchDownloadCandidates, peerCanServe, peerRequestCapacity, copyRequestArrival } from "./DeadlineScheduler.js";
+
+const BLOCK_BYTES = 16 * 1024;
+
+/** Replay the actual request scheduler against copied protocol state only. */
+export async function futureDownload(entries, { now = Date.now(), findStore, replayOnly = false, isCurrent = () => true } = {}) {
+  if (!isCurrent()) return new Map();
+  const candidates = downloadCandidates(entries, findStore);
+  const copies = new Map();
+  let clock = now;
+  for (const candidate of candidates) {
+    const source = candidate.torrent;
+    let copy = copies.get(source);
+    if (!copy) {
+      copy = { pieceLength: source.pieceLength, maxWebConns: source.maxWebConns, states: new Map(), wires: [], arrivals: new Map() };
+      copies.set(source, copy);
+      for (const peer of source.wires ?? []) {
+        if (peer.destroyed || (!peer.requests?.length && peer.peerChoking && !peer.hasFast)) continue;
+        const speed = Math.max(0, Number(peer.downloadSpeed?.()) || 0);
+        const available = new Set(candidates.filter(one => one.torrent === source && peer.peerPieces?.get(one.piece)).map(one => one.piece));
+        if (!available.size && !peer.requests?.length) continue;
+        const wire = { type: peer.type, destroyed: peer.destroyed, peerChoking: peer.peerChoking,
+          hasFast: peer.hasFast, peerAllowedFastSet: [...(peer.peerAllowedFastSet ?? [])], peerPieces: { get: piece => available.has(piece) },
+          peerExtendedHandshake: { reqq: peer.peerExtendedHandshake?.reqq }, downloadSpeed: () => speed,
+          requests: (peer.requests ?? []).map(request => copyRequestArrival(request,
+            { piece: request.piece, offset: request.offset, length: request.length, remaining: request.length })) };
+        wire.cancel = (piece, offset, length) => {
+          const index = wire.requests.findIndex(request => request.piece === piece && request.offset === offset && request.length === length);
+          if (index < 0) return;
+          wire.requests.splice(index, 1);
+          const state = copy.states.get(piece);
+          for (let block = offset / BLOCK_BYTES; state && block * BLOCK_BYTES < offset + length; block++) {
+            state.reserved.delete(block);
+            state.cursor = Math.min(state.cursor, block);
+          }
+        };
+        copy.wires.push(wire);
+      }
+      copy._request = (wire, piece) => {
+        const state = copy.states.get(piece);
+        if (!state || state.complete) return false;
+        let block = state.cursor;
+        while (block < state.count && (state.present.has(block) || state.reserved.has(block))) block++;
+        if (block === state.count) return false;
+        const offset = block * BLOCK_BYTES;
+        const length = wire.type === "webSeed" ? state.length - offset : Math.min(BLOCK_BYTES, state.length - offset);
+        for (let index = block; index * BLOCK_BYTES < offset + length; index++) state.reserved.add(index);
+        state.cursor = Math.ceil((offset + length) / BLOCK_BYTES);
+        wire.requests.push({ piece, offset, length, remaining: length });
+        return true;
+      };
+    }
+    const piece = source.pieces?.[candidate.piece];
+    if (!piece || !Number.isSafeInteger(piece.length) || piece.length <= 0 ||
+        (piece.missing !== piece.length && !Array.isArray(piece._buffer))) {
+      copy.arrivals.set(candidate.piece, null);
+      continue;
+    }
+    const count = Math.ceil(piece.length / BLOCK_BYTES);
+    const present = new Set();
+    for (let block = 0; block < count; block++) if (piece._buffer?.[block]) present.add(block);
+    const reserved = new Set();
+    for (const wire of copy.wires) for (const request of wire.requests) {
+      if (request.piece !== candidate.piece) continue;
+      for (let block = request.offset / BLOCK_BYTES; block * BLOCK_BYTES < request.offset + request.length; block++) reserved.add(block);
+    }
+    copy.states.set(candidate.piece, { length: piece.length, count, present, reserved, cursor: 0, complete: false });
+  }
+  const ordered = candidates.filter(candidate => copies.get(candidate.torrent).states.has(candidate.piece))
+    .map(candidate => ({ ...candidate, torrent: copies.get(candidate.torrent) }));
+  dispatchDownloadCandidates(ordered, clock);
+  const solved = new Set();
+  if (!replayOnly) for (const copy of copies.values()) {
+    if (copy.wires.length === 1 && !copy.wires[0].destroyed) {
+      finishSinglePeer(copy, ordered.filter(candidate => candidate.torrent === copy), now);
+      solved.add(copy);
+    }
+  }
+  const replay = ordered.filter(candidate => !solved.has(candidate.torrent));
+  const pendingCandidates = new Set(replay);
+  const candidateByCopy = new Map();
+  for (const candidate of replay) {
+    let pieces = candidateByCopy.get(candidate.torrent);
+    if (!pieces) candidateByCopy.set(candidate.torrent, pieces = new Map());
+    pieces.set(candidate.piece, candidate);
+  }
+  const wires = [...copies.values()].filter(copy => !solved.has(copy)).flatMap(copy => copy.wires.map(wire => ({ copy, wire })));
+  let steps = 0;
+  while (pendingCandidates.size) {
+    dispatchDownloadCandidates([...pendingCandidates], clock);
+    let next = null, delay = Infinity;
+    for (const entry of wires) {
+      const { wire } = entry;
+      if (!wire.requests.length || !(wire.downloadSpeed() > 0)) continue;
+      const arrivesIn = wire.requests[0].remaining / wire.downloadSpeed() * 1000;
+      if (arrivesIn < delay) { delay = arrivesIn; next = entry; }
+    }
+    if (!next || !Number.isFinite(delay)) break;
+    clock += delay;
+    for (const { copy, wire } of wires) {
+      const request = wire.requests[0];
+      if (!request) continue;
+      request.remaining = Math.max(0, request.remaining - delay / 1000 * wire.downloadSpeed());
+      if (wire === next.wire) request.remaining = 0;
+      if (request.remaining > 0) continue;
+      wire.requests.shift();
+      const state = copy.states.get(request.piece);
+      if (!state || state.complete) continue;
+      for (let block = request.offset / BLOCK_BYTES; block * BLOCK_BYTES < request.offset + request.length; block++) {
+        state.present.add(block);
+        state.reserved.delete(block);
+      }
+      if (state.present.size === state.count) {
+        state.complete = true;
+        state.present.clear();
+        state.reserved.clear();
+        copy.arrivals.set(request.piece, clock);
+        pendingCandidates.delete(candidateByCopy.get(copy)?.get(request.piece));
+      }
+    }
+    if (++steps % 256 === 0) {
+      await nextTurn();
+      if (!isCurrent()) return new Map();
+    }
+  }
+  return new Map([...copies].map(([torrent, copy]) => {
+    for (const piece of copy.states.keys()) if (!copy.arrivals.has(piece)) copy.arrivals.set(piece, null);
+    return [torrent, { measuredAt: now, arrivals: copy.arrivals }];
+  }));
+}
+
+/** With one peer there is no reassignment: its FIFO consumes the common map
+ * order. Sum that unchanged order instead of replaying every block arrival. */
+function finishSinglePeer(copy, candidates, now) {
+  const wire = copy.wires[0];
+  const speed = wire.downloadSpeed();
+  if (!(speed > 0) || !(peerRequestCapacity(copy, wire) > 0)) return;
+  let at = now;
+  for (const request of wire.requests) {
+    at += request.length / speed * 1000;
+    const state = copy.states.get(request.piece);
+    if (!state || state.complete) continue;
+    for (let block = request.offset / BLOCK_BYTES; block * BLOCK_BYTES < request.offset + request.length; block++) {
+      state.present.add(block);
+      state.reserved.delete(block);
+    }
+    if (state.present.size === state.count) {
+      state.complete = true;
+      state.present.clear();
+      state.reserved.clear();
+      copy.arrivals.set(request.piece, at);
+    }
+  }
+  for (const { piece } of candidates) {
+    const state = copy.states.get(piece);
+    if (state.complete || !peerCanServe(wire, piece)) continue;
+    let missing = 0;
+    for (let block = 0; block < state.count; block++) {
+      if (!state.present.has(block)) missing += Math.min(BLOCK_BYTES, state.length - block * BLOCK_BYTES);
+    }
+    at += missing / speed * 1000;
+    state.complete = true;
+    copy.arrivals.set(piece, at);
+  }
+}

package/services/torrent/download/SwarmSelection.js

@@ -23,7 +23,6 @@
  */
 
 import {
-  isConditional,
   piecesOf,
   selectionPriority,
   Urgency,
@@ -36,8 +35,6 @@ export class SwarmSelection {
   #register;
   /** What was last stated to the library, so a restatement can be a no-op. */
   #stated = new Map();
-  /** Pieces this instance marked for displacement, so it clears only its own. */
-  #displacing = null;
   /** Claimants whose windows are currently protected in memory. */
   #protectedInMemory = new Set();
   #findStore;
@@ -72,11 +69,8 @@ export class SwarmSelection {
    *   shared and the question is not a per-torrent one.
    * @returns {{ stated: number, withdrawn: number }}
    */
-  reconcile({ speculativeAllowed = true } = {}) {
-    const levels = this.#register.levelsToState(
-      (window) => this.#isSatisfied(window),
-      speculativeAllowed
-    );
+  reconcile() {
+    const levels = [...new Set(this.#register.windows().map(window => window.urgency))];
     /** @type {Map<string, { from: number, to: number, priority: number }>} */
     const wanted = new Map();
     for (const urgency of levels) {
@@ -102,6 +96,8 @@ export class SwarmSelection {
       withdrawn += 1;
     }
 
+    this.#cancelUnwanted(wanted.values());
+
     let stated = 0;
     for (const [key, range] of wanted) {
       // Re-stated when the library has dropped it, even though this instance
@@ -114,7 +110,6 @@ export class SwarmSelection {
       stated += 1;
     }
 
-    this.#markDisplacement();
     this.#projectIntoMemory();
     return { stated, withdrawn };
   }
@@ -125,7 +120,7 @@ export class SwarmSelection {
       this.#deselect(range);
     }
     this.#stated.clear();
-    this.#clearDisplacement();
+    this.#cancelUnwanted([]);
     const store = this.#findStore(this.#torrent);
     for (const claimant of this.#protectedInMemory) {
       store?.releaseProtection?.(claimant);
@@ -133,6 +128,27 @@ export class SwarmSelection {
     this.#protectedInMemory.clear();
   }
 
+  /** Cancel removed blocks in this pass; the wire callback releases reservations. */
+  #cancelUnwanted(ranges) {
+    const wanted = [...ranges];
+    for (const wire of this.#torrent.wires ?? []) {
+      for (const request of [...(wire.requests ?? [])]) {
+        if (wanted.some((range) => request.piece >= range.from && request.piece <= range.to)) continue;
+        if (typeof wire.cancel !== "function") continue;
+        try {
+          wire.cancel(request.piece, request.offset, request.length);
+          this.#torrent.emit?.("download-request-cancelled", {
+            piece: request.piece, offset: request.offset, length: request.length, reason: "demand-withdrawn"
+          });
+        } catch (error) {
+          this.#torrent.emit?.("download-request-cancel-failed", {
+            piece: request.piece, offset: request.offset, length: request.length, error: error?.message ?? String(error)
+          });
+        }
+      }
+    }
+  }
+
   /**
    * What is stated right now.
    *
@@ -171,30 +187,6 @@ export class SwarmSelection {
    *
    * @returns {void}
    */
-  #markDisplacement() {
-    const blocked = this.#register
-      .at(Urgency.BLOCKED)
-      .map((window) => this.#piecesFor(window))
-      .filter((range) => range !== null);
-    if (blocked.length === 0) {
-      this.#clearDisplacement();
-      return;
-    }
-    const nearest = Math.min(...blocked.map((range) => range.from));
-    const furthest = Math.max(...blocked.map((range) => range.to));
-    if (this.#displacing && this.#displacing.from === nearest && this.#displacing.to === furthest) {
-      return;
-    }
-    this.#clearDisplacement();
-    try {
-      this.#torrent.critical?.(nearest, furthest);
-      this.#displacing = { from: nearest, to: furthest };
-    } catch {
-      // silent-ok: displacement is an optimisation, and a torrent being torn
-      // down is not worth failing a read over.
-      this.#displacing = null;
-    }
-  }
 
   /**
    * Tell the piece store which bytes will be read soon, from the same stated
@@ -234,7 +226,7 @@ export class SwarmSelection {
       return;
     }
     const holding = new Set();
-    for (const urgency of [Urgency.BLOCKED, Urgency.NEAR]) {
+    for (const urgency of [...new Set(this.#register.windows().map(window => window.urgency))]) {
       for (const window of this.#register.at(urgency)) {
         const range = this.#piecesFor(window);
         if (!range) {
@@ -243,7 +235,7 @@ export class SwarmSelection {
         // The level goes with the range: it is what eviction compares when
         // everything resident is wanted by somebody, and dropping it here is
         // what left the store choosing by recency alone.
-        store.protectRange(window.claimant, range.from, range.to, window.urgency);
+        store.protectRange(window.claimant, range.from, range.to, window.urgency, window.deadlineAt, window.priority);
         holding.add(window.claimant);
       }
     }
@@ -255,17 +247,6 @@ export class SwarmSelection {
     this.#protectedInMemory = holding;
   }
 
-  /** @returns {void} */
-  #clearDisplacement() {
-    if (!this.#displacing || !Array.isArray(this.#torrent._critical)) {
-      this.#displacing = null;
-      return;
-    }
-    for (let index = this.#displacing.from; index <= this.#displacing.to; index += 1) {
-      this.#torrent._critical[index] = false;
-    }
-    this.#displacing = null;
-  }
 
   /**
    * The pieces a window covers, or null when the file or the torrent cannot
@@ -363,24 +344,14 @@ export class SwarmSelection {
    *
    * @returns {string}
    */
-  describe(speculativeAllowed = true) {
-    const stating = this.#register.levelsToState(
-      (window) => this.#isSatisfied(window),
-      speculativeAllowed
-    );
-    const speculative = stating.filter((urgency) => isConditional(urgency)).map(urgencyName);
+  describe() {
     const needs = this.#register
       .windows()
       .map((window) => `${urgencyName(window.urgency)}:${window.claimant}`);
     return (
       `download: ${this.#stated.size} instruction(s) to the swarm from ` +
       `${this.#register.size} stated need(s) [${needs.join(" ")}]` +
-      (speculative.length > 0
-        ? `; ${speculative.join(" and ")} also stated — nothing urgent is missing`
-        : "; nothing speculative is stated — something urgent is still missing") +
-      (this.#displacing
-        ? `; pieces ${this.#displacing.from}-${this.#displacing.to} may be taken from slow peers`
-        : "")
+      "; peer requests ordered by deadline and priority"
     );
   }
 }

package/services/torrent/download/registry.js

@@ -17,11 +17,31 @@
 
 import { DemandRegister } from "../demand/DemandRegister.js";
 import { SwarmSelection } from "./SwarmSelection.js";
+import { DeadlineScheduler } from "./DeadlineScheduler.js";
+import { futureDownload } from "./FutureDownload.js";
 
 /** @type {WeakMap<object, { register: DemandRegister, selection: SwarmSelection }>} */
 const byTorrent = new WeakMap();
 /** @type {Set<{ register: DemandRegister, selection: SwarmSelection }>} */
 const live = new Set();
+const scheduler = new DeadlineScheduler({ entries: () => live });
+let futurePending = null;
+let publishingSelections = false;
+
+/** Concurrent file reports share one snapshot of every live download map. */
+export function forecastDownloads() {
+  if (!futurePending) {
+    const entries = [...live];
+    const revisions = entries.map(entry => entry.register.revision);
+    const withdrawals = entries.map(entry => entry.withdrawalRevision);
+    const isCurrent = () => entries.length === live.size && !entries.some((entry, index) =>
+        !live.has(entry) || entry.register.revision !== revisions[index] ||
+        entry.withdrawalRevision !== withdrawals[index]);
+    futurePending = futureDownload(entries, { isCurrent }).then(result =>
+      isCurrent() ? result : new Map()).finally(() => { futurePending = null; });
+  }
+  return futurePending;
+}
 
 /**
  * The register and selection for a torrent, made on first use.
@@ -35,9 +55,19 @@ export function demandFor(torrent) {
     return held;
   }
   const register = new DemandRegister();
-  const entry = { register, selection: new SwarmSelection({ torrent, register }) };
+  const entry = { torrent, register, selection: new SwarmSelection({ torrent, register }), withdrawalRevision: 0 };
+  if (typeof torrent._request === "function" && typeof torrent._updateWire === "function") {
+    entry.previousWireUpdate = torrent._updateWire;
+    torrent._updateWire = () => {
+      if (!publishingSelections) scheduler.reconcile();
+    };
+  }
   byTorrent.set(torrent, entry);
   live.add(entry);
+  entry.onBytesChanged = () => reconcileAll();
+  entry.onWithdrawn = () => { entry.withdrawalRevision++; reconcileAll(); };
+  torrent.on?.("verified", entry.onBytesChanged);
+  torrent.on?.("piece-withdrawn", entry.onWithdrawn);
   return entry;
 }
 
@@ -74,6 +104,9 @@ export function forgetTorrent(torrent) {
   }
   held.selection.releaseAll();
   held.register.clear();
+  torrent.removeListener?.("verified", held.onBytesChanged);
+  torrent.removeListener?.("piece-withdrawn", held.onWithdrawn);
+  if (held.previousWireUpdate) torrent._updateWire = held.previousWireUpdate;
   byTorrent.delete(torrent);
   live.delete(held);
 }
@@ -89,14 +122,21 @@ export function forgetTorrent(torrent) {
  * @returns {{ torrents: number, speculativeAllowed: boolean, stated: number, withdrawn: number }}
  */
 export function reconcileAll() {
+  if (publishingSelections) return { torrents: live.size, speculativeAllowed: false, stated: 0, withdrawn: 0 };
   const entries = [...live];
   const speculativeAllowed = !entries.some((entry) => entry.selection.hasUrgentMissing());
   let stated = 0;
   let withdrawn = 0;
-  for (const entry of entries) {
-    const result = entry.selection.reconcile({ speculativeAllowed });
-    stated += result.stated;
-    withdrawn += result.withdrawn;
+  publishingSelections = true;
+  try {
+    for (const entry of entries) {
+      const result = entry.selection.reconcile({ speculativeAllowed, deadlineDriven: !!entry.previousWireUpdate });
+      stated += result.stated;
+      withdrawn += result.withdrawn;
+    }
+  } finally {
+    publishingSelections = false;
   }
+  scheduler.reconcile();
   return { torrents: entries.length, speculativeAllowed, stated, withdrawn };
 }

package/services/torrent/download/withdraw-claim.js

@@ -76,5 +76,7 @@ export function withdrawClaim({ index, files, torrent, warn = () => undefined })
     );
     return "refused";
   }
+  try { owner.emit?.("piece-withdrawn", index); }
+  catch (error) { warn(`piece withdrawal listener failed: ${error?.message ?? error}`); }
   return "withdrawn";
 }