@torrent-tv/proxy 2.87.14 → 2.87.16

package/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 2.87.16
+
+- **Fix**: Hold spilled source bytes across memory reservation so concurrent eviction cannot terminate ffmpeg input with a missing disk piece.
+- **Fix**: Publish a run's final closed segment only after complete input termination, retain earlier completed cuts, and drain output notifications before classifying an exit.
+- **Fix**: Preserve copied media presentation times when replacing per-segment headers with one shared init, including backward seeks after opening at a later position.
+- **Fix**: Feed measured segment intervals and separate browser track ranges into the service integral; retain media beyond holes, avoid duplicate transfer estimates, and price missing segments independently of another run's progress.
+
+## 2.87.15
+
+- **Fix**: Integrate a bounded, nonnegative forecast of recent measured service rates, so an old faster link sample cannot predict permanent delivery failure and prevent playback of fully prepared media.
+- **Fix**: Treat the already-held browser buffer as a measured lower bound on capacity when its loader target has been reduced by fragment retries.
+- **Chore**: Record forecast measurements when the playback-readiness reason changes.
+
 ## 2.87.14
 
 - **Fix**: Evaluate source-stall coverage against the continuous timeline already prepared on the proxy, while independently checking client-buffer continuity and delivery capacity.

package/docs/browser-proxy-contract.md

@@ -101,6 +101,15 @@ remaining-work estimates and share the measured download rate of their torrent.
 The browser reports buffer state and delivery measurements, but does not
 calculate or override readiness.
 
+Service work is the integral of a nonnegative measured-rate forecast. The latest
+rate relaxes over the observed measurement span toward the lower of its latest
+value and the recent mean; the measurement history covers the output look-ahead.
+A positive measured rate cannot become permanently zero from an old declining
+linear fit. A measured zero remains zero until a new measurement arrives.
+The reported loader limit is bounded below by the browser buffer already held.
+During startup waiting the browser refreshes stale delivery measurements through
+the existing link probe and reports their age to avoid clock-offset errors.
+
 ## Subtitles
 
 `GET /api/subtitles` requires `sourceKey` and `fileIndex`. Without `trackIndex`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.87.14",
+  "version": "2.87.16",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/routes/api/transcode-sessions/net-report/post.js

@@ -109,6 +109,7 @@ export async function handleApiTranscodeSessionNetReportPost(req, reply, { outpu
       linkSampleMbps: Number.isFinite(linkSampleMbps) && linkSampleMbps > 0 ? linkSampleMbps : undefined,
       linkSampleAt: Number.isFinite(linkSampleAt) && linkSampleAt > 0 ? linkSampleAt : undefined,
       bufferedAheadSec,
+      bufferedRanges: body.bufferedRanges,
       bufferLimitSeconds:
         Number.isFinite(bufferLimitSeconds) && bufferLimitSeconds > 0 ? bufferLimitSeconds : undefined,
       consumerId,

package/services/encode/EncodeRun.js

@@ -103,6 +103,10 @@ export class EncodeRun {
 
   /** Half a name left over from the last chunk of the encoder's own channel. */
   #closedTail = "";
+  #pendingClosed = null;
+  #inputTruncated = false;
+  #stderrReadTail = "";
+  #processExited = false;
 
   /**
    * The last piece named on the ready channel while the run was still running
@@ -242,7 +246,7 @@ export class EncodeRun {
     this.onProgress = typeof onProgress === "function" ? onProgress : () => {};
     // Told the NAME of every piece the encoder has closed. The name is the
     // proof it is whole; nothing else here can prove that.
-    this.onClosed = typeof onClosed === "function" ? onClosed : () => {};
+    this.onClosed = typeof onClosed === "function" ? onClosed : (name) => name;
     this.lastSegmentIndex = typeof lastSegmentIndex === "function" ? lastSegmentIndex : () => null;
     this.inputUnavailable = typeof inputUnavailable === "function" ? inputUnavailable : () => false;
     this.argsDescribed = argsDescribed;
@@ -420,6 +424,9 @@ export class EncodeRun {
       const line = String(chunk).trim();
       if (line.length > 0) {
         this.lastError = line;
+        const errorText = this.#stderrReadTail + String(chunk);
+        this.#inputTruncated ||= /Stream ends prematurely|Input\/output error|Error during demuxing/i.test(errorText);
+        this.#stderrReadTail = errorText.slice(-512);
         this.logger.warn(`ffmpeg #${this.from}..#${this.to} of ${this.address}: ${line}`);
       }
     });
@@ -428,7 +435,9 @@ export class EncodeRun {
       this.lastError = message;
       this.#finish(ENCODE_EXIT.FAILED, `the process could not be started: ${message}`, null, null);
     });
-    process.on("exit", (code, signal) => this.#onExit(code, signal));
+    // `close` follows drained stdio; `exit` can precede the final filename.
+    process.on("exit", () => { this.#processExited = true; });
+    process.on("close", (code, signal) => this.#onExit(code, signal));
   }
 
   /**
@@ -504,11 +513,23 @@ export class EncodeRun {
       if (name.length === 0) {
         continue;
       }
+      if (this.#stopping) continue;
+      // A subsequent closed file proves the previous one reached a cut.
+      // The last file can instead have been flushed by an input failure.
+      if (this.#pendingClosed !== null) {
+        this.#publishClosed(this.#pendingClosed);
+      }
+      this.#pendingClosed = name;
+    }
+  }
+
+  #publishClosed(name) {
       // ITS SERVED NAME, which is what whoever owns the disk gives it in answer.
       // ffmpeg writes a piece under a working name and reports that; the piece
       // becomes servable by being renamed, and everything below works in the
       // name a request can actually ask for.
-      const served = this.onClosed(name) ?? name;
+      const served = this.onClosed(name);
+      if (!served) return false;
       if (!this.#stopping) {
         this.#provenName = served;
       }
@@ -524,7 +545,7 @@ export class EncodeRun {
       if (Number.isInteger(index)) {
         this.noteProduced(index);
       }
-    }
+      return true;
   }
 
   noteSpeed(speedX) {
@@ -545,6 +566,12 @@ export class EncodeRun {
     if (this.#process === null || this.#ended) {
       return;
     }
+    // A cut closed before our stop stays complete; the file flushed by the
+    // subsequent signal must never be published as that same proof.
+    if (this.#pendingClosed !== null && !this.#processExited && !this.#inputTruncated) {
+      this.#publishClosed(this.#pendingClosed);
+      this.#pendingClosed = null;
+    }
     this.#stopping = true;
     this.#stopReason = because;
     this.#stopOrderedAt = this.now();
@@ -646,14 +673,24 @@ export class EncodeRun {
     // that exits cleanly at #11 has not finished, whatever the film's length;
     // and a run with no end has nothing but the film to be measured against.
     const endOfWork = Number.isFinite(endOfRun(this)) ? this.to : this.lastSegmentIndex();
+    const pendingIndex = this.#pendingClosed === null ? null :
+      this.indexOfName(this.#pendingClosed);
     const outcome = classifyEncodeExit({
       code,
-      producedThrough: this.#produced.size > 0 ? this.reached : null,
-      producedCount: this.#produced.size,
+      producedThrough: this.#inputTruncated ? this.reached :
+        (pendingIndex === this.head ? pendingIndex : this.reached),
+      producedCount: this.#produced.size + Number(pendingIndex === this.head && !this.#inputTruncated),
       lastSegmentIndex: endOfWork,
       inputUnavailable: this.inputUnavailable(this.lastError)
     });
     if (outcome === ENCODE_EXIT.COMPLETE) {
+      if (this.#pendingClosed !== null && !this.#inputTruncated) {
+        if (!this.#publishClosed(this.#pendingClosed)) {
+          this.#finish(ENCODE_EXIT.SHORT, "its final file could not be published", code, signal);
+          return;
+        }
+        this.#pendingClosed = null;
+      }
       this.#finish(ENCODE_EXIT.COMPLETE, "it reached the end of what it was given", code, signal);
       return;
     }

package/services/encode/EncodeRuns.js

@@ -827,7 +827,8 @@ export class EncodeRuns {
         session.timeline?.segmentCount > 0 ? session.timeline.segmentCount - 1 : null,
       inputUnavailable: (message) => isInputUnavailable(message),
       onProgress: (report) => this.#noteRunProgress(session, run, report),
-      indexOfName: (name) => session.segmentFormat.segmentIndexFromName(name),
+      indexOfName: (name) => session.segmentFormat.segmentIndexFromName(
+        session.segmentFormat.servedNameOf?.(name) ?? name),
       // Why this encoder exists, recorded with its argument list. It used to be
       // handed to a separate `start` call; there is no separate call now.
       because,

package/services/encode/segment-formats/fmp4.js

@@ -11,6 +11,9 @@
  */
 
 import {
+  rebaseSegmentDecodeTimes,
+  neutralizeEmptyEdits,
+  readPresentationRanges,
   readSelfContainedStartSeconds,
   readTrackTimescales,
   readVideoSampleSize,
@@ -210,6 +213,7 @@ export const fmp4Format = {
     const { firstFragment } = findFragmentBounds(bytes);
     return firstFragment > 0 ? bytes.subarray(0, firstFragment) : null;
   },
+  prepareSharedInit: neutralizeEmptyEdits,
 
   /**
    * A piece with its init header removed, and the trailing random-access index
@@ -275,6 +279,7 @@ export const fmp4Format = {
    * transcode itself; the box walk never descends into `mdat`.
    */
   needsSegmentRewrite: true,
+  readMediaRanges: readPresentationRanges,
 
   /**
    * Whether a segment carries every track the init promises.
@@ -425,13 +430,16 @@ export const fmp4Format = {
     return readVideoSampleSize(initBytes);
   },
 
-  prepareSegmentBytes(bytes, { startSeconds, initBytes }) {
+  prepareSegmentBytes(bytes, { startSeconds, initBytes, rawBytes }) {
     if (!initBytes || initBytes.length === 0) {
       // No init cached yet — nothing to read timescales from. The player always
       // fetches `#EXT-X-MAP` before any segment, so this is not reachable in
       // practice; serve unmodified rather than guess a timescale.
       return bytes;
     }
-    return stampSegmentStartTime(bytes, startSeconds, readTrackTimescales(initBytes));
+    const ownInit = rawBytes ? this.extractInit(rawBytes) : null;
+    return ownInit
+      ? rebaseSegmentDecodeTimes(bytes, ownInit, initBytes)
+      : stampSegmentStartTime(bytes, startSeconds, readTrackTimescales(initBytes));
   }
 };

package/services/encode/segment-formats/index.js

@@ -20,6 +20,7 @@ import { mpegtsFormat } from "./mpegts.js";
  *   output timeline, from the session's segment-boundary table.
  * @property {Buffer | null} initBytes - The init segment being served for this
  *   session, when the format has one (fMP4 needs it to read track timescales).
+ * @property {Buffer} [rawBytes] - Self-contained bytes before removing their init.
  */
 
 /**

package/services/encode/segment-formats/mp4-boxes.js

@@ -15,7 +15,7 @@
  *
  * @type {ReadonlySet<string>}
  */
-const CONTAINER_BOXES = new Set(["moov", "trak", "mdia", "minf", "stbl", "edts", "moof", "traf"]);
+const CONTAINER_BOXES = new Set(["moov", "mvex", "trak", "mdia", "minf", "stbl", "edts", "moof", "traf"]);
 
 /**
  * Walk the box tree, invoking `visit` for every box encountered.
@@ -95,6 +95,142 @@ export function readTrackTimescales(initSegment) {
   return timescales;
 }
 
+/** Movie presentation time minus track media time, in seconds per track. */
+export function readTrackEditOffsets(initSegment) {
+  const scales = readTrackTimescales(initSegment);
+  const offsets = new Map([...scales.keys()].map((id) => [id, 0]));
+  let movieScale = 0;
+  let trackId = null;
+  walkBoxes(initSegment, (type, start, end) => {
+    const version = initSegment[start];
+    if (type === "mvhd") {
+      const at = start + (version === 1 ? 20 : 12);
+      if (at + 4 <= end) movieScale = initSegment.readUInt32BE(at);
+    } else if (type === "tkhd") {
+      const at = start + (version === 1 ? 20 : 12);
+      if (at + 4 <= end) trackId = initSegment.readUInt32BE(at);
+    } else if (type === "elst" && scales.has(trackId) && movieScale > 0 && start + 8 <= end) {
+      const count = initSegment.readUInt32BE(start + 4);
+      const width = version === 1 ? 20 : 12;
+      let movieStart = 0;
+      for (let entry = 0, at = start + 8; entry < count && at + width <= end; entry++, at += width) {
+        const duration = version === 1 ? Number(initSegment.readBigUInt64BE(at)) : initSegment.readUInt32BE(at);
+        const mediaTime = version === 1 ? Number(initSegment.readBigInt64BE(at + 8)) : initSegment.readInt32BE(at + 4);
+        if (mediaTime === -1) {
+          movieStart += duration / movieScale;
+        } else {
+          offsets.set(trackId, movieStart - mediaTime / scales.get(trackId));
+          break;
+        }
+      }
+    }
+  });
+  return offsets;
+}
+
+/** An init shared across seeks has no segment-specific empty edit. */
+export function neutralizeEmptyEdits(initSegment) {
+  const neutral = Buffer.from(initSegment);
+  walkBoxes(neutral, (type, start, end) => {
+    if (type !== "elst" || start + 8 > end) return;
+    const version = neutral[start];
+    const width = version === 1 ? 20 : 12;
+    const count = neutral.readUInt32BE(start + 4);
+    for (let entry = 0, at = start + 8; entry < count && at + width <= end; entry++, at += width) {
+      const mediaTime = version === 1 ? neutral.readBigInt64BE(at + 8) : BigInt(neutral.readInt32BE(at + 4));
+      if (mediaTime !== -1n) continue;
+      if (version === 1) neutral.writeBigUInt64BE(0n, at);
+      else neutral.writeUInt32BE(0, at);
+    }
+  });
+  return neutral;
+}
+
+/** Preserve sample presentation times when replacing a piece's own init. */
+export function rebaseSegmentDecodeTimes(segment, ownInit, sessionInit) {
+  const scales = readTrackTimescales(sessionInit);
+  const ownOffsets = readTrackEditOffsets(ownInit);
+  const sessionOffsets = readTrackEditOffsets(sessionInit);
+  const stamped = Buffer.from(segment);
+  let trackId = null;
+  walkBoxes(stamped, (type, start, end) => {
+    if (type === "tfhd" && start + 8 <= end) {
+      trackId = stamped.readUInt32BE(start + 4);
+    } else if (type === "tfdt" && scales.has(trackId) && ownOffsets.has(trackId)) {
+      const version = stamped[start];
+      if (start + (version === 1 ? 12 : 8) > end) return;
+      const existing = version === 1 ? Number(stamped.readBigUInt64BE(start + 4)) : stamped.readUInt32BE(start + 4);
+      const value = existing + Math.round((ownOffsets.get(trackId) - sessionOffsets.get(trackId)) * scales.get(trackId));
+      if (value < 0) throw new Error("A fragment cannot be placed before its shared init's decode origin.");
+      if (version === 1) stamped.writeBigUInt64BE(BigInt(value), start + 4);
+      else if (value <= 0xffffffff) stamped.writeUInt32BE(value, start + 4);
+      else throw new Error("Fragment decode time exceeds its 32-bit field.");
+    }
+  });
+  return stamped;
+}
+
+/** Presentation intervals from the durations and composition times of samples. */
+export function readPresentationRanges(raw) {
+  const scales = readTrackTimescales(raw);
+  const edits = readTrackEditOffsets(raw);
+  const byTrack = new Map();
+  const defaultDurations = new Map();
+  let trackId = null;
+  let decodeTime = 0;
+  let defaultDuration = 0;
+  walkBoxes(raw, (type, start, end) => {
+    if (type === "trex" && start + 20 <= end) {
+      defaultDurations.set(raw.readUInt32BE(start + 4), raw.readUInt32BE(start + 12));
+    } else if (type === "tfhd" && start + 8 <= end) {
+      const flags = raw.readUIntBE(start + 1, 3);
+      trackId = raw.readUInt32BE(start + 4);
+      const at = start + 8 + ((flags & 1) ? 8 : 0) + ((flags & 2) ? 4 : 0);
+      defaultDuration = (flags & 8) && at + 4 <= end ? raw.readUInt32BE(at) :
+        defaultDurations.get(trackId) ?? 0;
+    } else if (type === "tfdt" && start + (raw[start] === 1 ? 12 : 8) <= end) {
+      decodeTime = raw[start] === 1 ? Number(raw.readBigUInt64BE(start + 4)) : raw.readUInt32BE(start + 4);
+    } else if (type === "trun" && scales.has(trackId) && start + 8 <= end) {
+      const scale = scales.get(trackId);
+      const flags = raw.readUIntBE(start + 1, 3);
+      const count = raw.readUInt32BE(start + 4);
+      let at = start + 8 + ((flags & 1) ? 4 : 0) + ((flags & 4) ? 4 : 0);
+      const width = Number(Boolean(flags & 0x100)) * 4 + Number(Boolean(flags & 0x200)) * 4 +
+        Number(Boolean(flags & 0x400)) * 4 + Number(Boolean(flags & 0x800)) * 4;
+      const ranges = byTrack.get(trackId) ?? [];
+      for (let sample = 0; sample < count && at + width <= end; sample++) {
+        const duration = flags & 0x100 ? raw.readUInt32BE(at) : defaultDuration;
+        if (flags & 0x100) at += 4;
+        if (flags & 0x200) at += 4;
+        if (flags & 0x400) at += 4;
+        const composition = flags & 0x800 ?
+          (raw[start] === 1 ? raw.readInt32BE(at) : raw.readUInt32BE(at)) : 0;
+        if (flags & 0x800) at += 4;
+        if (!(duration > 0)) return;
+        const position = (decodeTime + composition) / scale + (edits.get(trackId) ?? 0);
+        ranges.push({ start: position, end: position + duration / scale });
+        decodeTime += duration;
+      }
+      byTrack.set(trackId, ranges);
+    }
+  });
+  // Merge in presentation order: B frames arrive in decode order.
+  const mergedTracks = [...byTrack.values()].map((samples) => {
+    const merged = [];
+    for (const sample of samples.sort((left, right) => left.start - right.start)) {
+      const previous = merged.at(-1);
+      if (previous && sample.start - previous.end <= Number.EPSILON * Math.max(1, sample.end) * 8) {
+        previous.end = Math.max(previous.end, sample.end);
+      } else merged.push({ ...sample });
+    }
+    return merged;
+  });
+  // A multiplexed segment is playable only where every declared track exists.
+  return mergedTracks.reduce((common, ranges) => common.flatMap((left) => ranges
+    .map((right) => ({ start: Math.max(left.start, right.start), end: Math.min(left.end, right.end) }))
+    .filter(({ start, end }) => end > start)), mergedTracks[0] ?? []);
+}
+
 /**
  * Rewrite every fragment's `tfdt` so the segment declares that it starts at
  * `startSeconds` on the media timeline.

package/services/server/SegmentServing.js

@@ -419,6 +419,7 @@ export class SegmentServing {
         }
         const prepared = session.segmentFormat.prepareSegmentBytes(bytes, {
           startSeconds: stampStart,
+          rawBytes: raw,
           initBytes: this.#host.segmentStore.initOf(session.outputKey ?? "")
         });
         this.#host.encodeRuns.noteRunProducedSegment(session, filePath);
@@ -541,6 +542,7 @@ export class SegmentServing {
     const trueStart = selfContained ? format.readSegmentStartSeconds?.(raw) ?? null : null;
     const prepared = format.prepareSegmentBytes(bytes, {
       startSeconds: trueStart ?? this.#host.outputTimes.publishedStartTime(like, index),
+      rawBytes: raw,
       initBytes: this.#host.segmentStore.initOf(goneKey)
     });
     logger.info(`[hold] ${fileName} served from the stored pieces of ${goneKey}, the output that first answered it`);
@@ -802,7 +804,9 @@ export class SegmentServing {
         if (!found) {
           continue;
         }
-        const init = session.segmentFormat.extractInit(cached ?? await readFile(found));
+        const ownInit = session.segmentFormat.extractInit(cached ?? await readFile(found));
+        const init = ownInit && session.segmentFormat.prepareSharedInit ?
+          session.segmentFormat.prepareSharedInit(ownInit) : ownInit;
         if (!init || init.length === 0) {
           continue;
         }

package/services/server/ViewerRequests.js

@@ -568,7 +568,8 @@ export class ViewerRequests {
       const segments = Array.from({ length: timeline?.segmentCount ?? 0 }, (_, index) => ({
         index,
         startSeconds: timeline.publishedStartOf(index),
-        endSeconds: timeline.publishedStartOf(index + 1)
+        endSeconds: timeline.publishedStartOf(index + 1),
+        mediaRanges: this.#host.segmentStore.mediaRangesOf(output.outputKey, index)
       }));
       const sourceIndexes = [output.spec.video?.fileIndex, output.spec.audio?.fileIndex]
         .filter((fileIndex) => Number.isInteger(fileIndex) && fileIndex >= 0);
@@ -618,6 +619,8 @@ export class ViewerRequests {
         : await this.#host.quality.observedStreamMbps(output);
       tracks.push({
         id: output.outputKey,
+        clientRanges: viewerReading?.bufferedRanges?.[output === session ? "video" : "audio"] ??
+          viewerReading?.bufferedRanges?.media,
         sourceIds,
         processedSeconds: trackProgress?.processedSeconds,
         bitsPerMediaSecond: Number.isFinite(observedMbps) && observedMbps > 0 ? observedMbps * 1_000_000 : 0,
@@ -663,7 +666,7 @@ export class ViewerRequests {
       return Number.isFinite(segmentSeconds) && segmentSeconds > 0 ? segmentSeconds : 0;
     }));
 
-    return predictPlaybackReadiness({
+    const input = {
       now,
       positionSeconds,
       durationSeconds: session.file.durationSeconds,
@@ -675,7 +678,24 @@ export class ViewerRequests {
       sources: [...sourceMeasurements.values()],
       tracks,
       linkReadings: measurement.link.snapshot()
-    });
+    };
+    const forecast = predictPlaybackReadiness(input);
+    if (measurement.forecastReason !== forecast.reason) {
+      measurement.forecastReason = forecast.reason;
+      logger.info(`playback readiness ${session.id} consumer=${consumerId} ${JSON.stringify({
+        ...forecast,
+        positionSeconds,
+        durationSeconds: input.durationSeconds,
+        bufferLimitSeconds: input.bufferLimitSeconds,
+        lookaheadSeconds: input.lookaheadSeconds,
+        link: input.linkReadings,
+        sources: input.sources,
+        tracks: tracks.map(({ id, processedSeconds, readings, segments, readySegmentIndices }) => ({
+          id, processedSeconds, readings, segments: segments.length, prepared: readySegmentIndices.length
+        }))
+      })}`);
+    }
+    return forecast;
   }
 
   #rateStateFor(session, consumerId) {
@@ -687,7 +707,7 @@ export class ViewerRequests {
     const id = typeof consumerId === "string" ? consumerId : "";
     let state = byViewer.get(id);
     if (!state) {
-      state = { link: new RateTrend(), downloads: new Map(), tracks: new Map() };
+      state = { link: new RateTrend(this.#host.lookaheadSeconds), downloads: new Map(), tracks: new Map() };
       byViewer.set(id, state);
     }
     return state;
@@ -696,7 +716,7 @@ export class ViewerRequests {
   #downloadRateReadings(state, serviceId, downloadSpeed, now) {
     let trend = state.downloads.get(serviceId);
     if (!trend) {
-      trend = new RateTrend();
+      trend = new RateTrend(this.#host.lookaheadSeconds);
       state.downloads.set(serviceId, trend);
     }
     const speed = Number(downloadSpeed);
@@ -710,7 +730,7 @@ export class ViewerRequests {
     const id = output.outputKey;
     let trend = state.tracks.get(id);
     if (!trend) {
-      trend = new RateTrend();
+      trend = new RateTrend(this.#host.lookaheadSeconds);
       state.tracks.set(id, trend);
     }
     const reading = this.#host.encodeSpeedReadingOf?.(output, now) ?? null;