@torrent-tv/proxy 2.9.65 → 2.9.66

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.9.66
+
+- **New**: The container keyframe index now covers **MP4/MOV and AVI** as well as Matroska. MP4 reads the sync-sample and time-to-sample tables from `moov` — found by stepping over top-level box headers, so it works whether `moov` sits at the file start or the end, without scanning the gigabytes of `mdat` between them (verified on a 2 GB field file: **1145 keyframes in 625 ms**). AVI reads the trailing `idx1` table, still worth having because older releases are largely XviD-in-AVI and are exactly the files served by copying rather than re-encoding. Formats left out are documented in the module with the reason: MPEG-TS/M2TS carry no index anywhere by design, fragmented MP4 spreads timing across fragments instead of one table, and FLV/ASF have tables but effectively never appear in these releases.
+
 ## 2.9.65
 
 - **Fix**: Segment boundaries on the video-COPY path are now the source's **real** keyframe positions, read from the container's own index (`services/container-index/`), instead of an invented 4 s grid. ffmpeg can only cut a copied stream at existing keyframes, so the declared grid was simply false — measured on a field file, the true keyframe spacing is **10.43 s**, meaning roughly two of every three declared boundaries could not exist. Players punish this in two ways, both seen in the field 2026-08-02: on a long file the player stops trusting the playlist and walks it from segment #1 to locate a seek (a 1:30 seek produced requests #1, #2, #45, #86 … #1187 and never arrived), and on a short one it presents **audio with no picture**, because a segment beginning without a keyframe has nothing to decode from.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.9.65",
+  "version": "2.9.66",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/services/container-index/avi.js

@@ -0,0 +1,156 @@
+/**
+ * @file Keyframe index for AVI, read without downloading the file.
+ *
+ * AVI ends with an `idx1` chunk: one fixed-size entry per stream chunk, each
+ * carrying a flags word whose keyframe bit says whether that chunk starts a
+ * keyframe. Frame number times the video stream's frame duration gives the
+ * time, so the index alone is enough — no media has to be read.
+ *
+ * `idx1` lives at the end of the file and the top-level chunk headers state
+ * their sizes, so it is reached by stepping over headers (typically two hops:
+ * `LIST hdrl`, `LIST movi`), not by scanning.
+ *
+ * Still relevant despite the format's age: older releases are largely XviD in
+ * AVI, and those are exactly the files that get copied rather than re-encoded.
+ */
+
+const HEADER_BYTES = 8;
+const PROBE_BYTES = 4096;
+// Keyframe flag in an idx1 entry's flags word (AVIIF_KEYFRAME).
+const KEYFRAME_FLAG = 0x10;
+const IDX1_ENTRY_BYTES = 16;
+// Cap on the idx1 read. One entry per chunk, 16 bytes each — a long film runs
+// to a few MB; beyond this is not a normal index.
+const MAX_IDX1_BYTES = 64 * 1024 * 1024;
+
+/**
+ * Whether this looks like AVI: a RIFF container whose form type is `AVI `.
+ *
+ * @param {Buffer} head
+ * @returns {boolean}
+ */
+export function isAvi(head) {
+  return (
+    head.length >= 12 &&
+    head.toString("latin1", 0, 4) === "RIFF" &&
+    head.toString("latin1", 8, 12) === "AVI "
+  );
+}
+
+/**
+ * Microseconds per frame and the video stream's chunk id prefix, from the main
+ * header. Both live in the `hdrl` list near the file start.
+ *
+ * @param {Buffer} head
+ * @returns {{ microsecondsPerFrame: number } | null}
+ */
+function readMainHeader(head) {
+  // Top-level: "RIFF" size "AVI " then chunks. `avih` sits inside `LIST hdrl`.
+  let offset = 12;
+  while (offset + HEADER_BYTES <= head.length) {
+    const id = head.toString("latin1", offset, offset + 4);
+    const size = head.readUInt32LE(offset + 4);
+    if (size <= 0) {
+      return null;
+    }
+    if (id === "LIST") {
+      // Descend: list type follows the header, then its own chunks.
+      const listType = head.toString("latin1", offset + 8, offset + 12);
+      if (listType === "hdrl") {
+        let inner = offset + 12;
+        while (inner + HEADER_BYTES <= Math.min(head.length, offset + 8 + size)) {
+          const innerId = head.toString("latin1", inner, inner + 4);
+          const innerSize = head.readUInt32LE(inner + 4);
+          if (innerSize <= 0) {
+            return null;
+          }
+          if (innerId === "avih" && inner + 8 + 4 <= head.length) {
+            return { microsecondsPerFrame: head.readUInt32LE(inner + 8) };
+          }
+          inner += HEADER_BYTES + innerSize + (innerSize % 2);
+        }
+      }
+      offset += HEADER_BYTES + 4 + (size - 4) + ((size - 4) % 2);
+      continue;
+    }
+    offset += HEADER_BYTES + size + (size % 2);
+  }
+  return null;
+}
+
+/**
+ * Step over top-level chunks to find `idx1`.
+ *
+ * @param {(start: number, end: number) => Promise<Buffer | null>} readRange
+ * @param {number} fileSize
+ * @returns {Promise<{ offset: number, size: number } | null>}
+ */
+async function findIdx1(readRange, fileSize) {
+  let offset = 12; // Past "RIFF" size "AVI ".
+  while (offset + HEADER_BYTES < fileSize) {
+    const probe = await readRange(offset, Math.min(fileSize - 1, offset + HEADER_BYTES - 1));
+    if (!probe || probe.length < HEADER_BYTES) {
+      return null;
+    }
+    const id = probe.toString("latin1", 0, 4);
+    const size = probe.readUInt32LE(4);
+    if (size <= 0) {
+      return null;
+    }
+    if (id === "idx1") {
+      return { offset: offset + HEADER_BYTES, size };
+    }
+    // Chunks are word-aligned; a LIST carries its type inside the payload, so
+    // the same size arithmetic covers both cases.
+    offset += HEADER_BYTES + size + (size % 2);
+  }
+  return null;
+}
+
+/**
+ * Read the keyframe times of an AVI file.
+ *
+ * @param {(start: number, end: number) => Promise<Buffer | null>} readRange
+ * @param {number} fileSize
+ * @returns {Promise<number[] | null>} Ascending seconds, or null when the file
+ *   has no `idx1` (OpenDML-only index, interrupted write, damaged upload).
+ */
+export async function readAviKeyframeTimes(readRange, fileSize) {
+  const head = await readRange(0, Math.min(PROBE_BYTES - 1, fileSize - 1));
+  if (!head || !isAvi(head)) {
+    return null;
+  }
+  const mainHeader = readMainHeader(head);
+  if (!mainHeader || !mainHeader.microsecondsPerFrame) {
+    return null;
+  }
+
+  const idx1 = await findIdx1(readRange, fileSize);
+  if (!idx1 || idx1.size > MAX_IDX1_BYTES) {
+    return null;
+  }
+
+  const table = await readRange(idx1.offset, Math.min(fileSize - 1, idx1.offset + idx1.size - 1));
+  if (!table || table.length < IDX1_ENTRY_BYTES) {
+    return null;
+  }
+
+  const secondsPerFrame = mainHeader.microsecondsPerFrame / 1e6;
+  const times = [];
+  let videoFrame = 0;
+  for (let at = 0; at + IDX1_ENTRY_BYTES <= table.length; at += IDX1_ENTRY_BYTES) {
+    const chunkId = table.toString("latin1", at, at + 4);
+    // Video chunks are "##db" (uncompressed) or "##dc" (compressed); audio is
+    // "##wb" and must not advance the frame counter.
+    const isVideo = chunkId.endsWith("db") || chunkId.endsWith("dc");
+    if (!isVideo) {
+      continue;
+    }
+    const flags = table.readUInt32LE(at + 4);
+    if ((flags & KEYFRAME_FLAG) !== 0) {
+      times.push(videoFrame * secondsPerFrame);
+    }
+    videoFrame += 1;
+  }
+  return times.length > 0 ? times : null;
+}

package/services/container-index/index.js

@@ -3,24 +3,26 @@
  * from the container's own tables rather than by scanning the media.
  *
  * The problem it solves: on the video-COPY path ffmpeg can only cut segments at
- * the source's existing keyframes. If our playlist declares an even grid
- * instead, the declared times do not match the media, and players respond by
- * either walking the whole file to rebuild the timeline or showing audio with
- * no picture (both seen in the field 2026-08-02). Getting those positions by
- * decoding is not an option here — the file is served from a torrent, and a
- * full packet scan of 5.5 GB found 77 keyframes in 45 s without finishing.
+ * the source's existing keyframes. A playlist declaring an even grid instead is
+ * then false, and players punish it — either walking the whole file to rebuild
+ * the timeline, or presenting audio with no picture because a segment begins
+ * with nothing decodable (both seen in the field 2026-08-02; the file measured
+ * had 10.43 s keyframe spacing against our declared 4 s).
  *
- * Containers already store this. The reader takes a byte-range function and
- * does a couple of point reads, so it works over any transport and knows
- * nothing about torrents, HTTP or our session model.
+ * Scanning for the answer is not an option here: the file is served from a
+ * torrent, and a full packet scan of 5.5 GB found 77 keyframes in 45 s without
+ * finishing. Containers already store the table — this reads it with a couple
+ * of point reads (16 KB, 0.8 s for 570 keyframes on that same file).
  *
- * Format support is deliberately partial: a container we cannot index returns
- * null, and the caller falls back (re-encode with forced keyframes, which is
- * what Jellyfin, hls-media-server and hls-vod-too all do unconditionally).
+ * Transport-agnostic by construction: it takes a byte-range function and knows
+ * nothing about torrents, HTTP or our session model, which is what let it be
+ * verified standalone before being wired in.
  */
 
 import { logger } from "../../utils/logger.js";
 import { isMatroska, readMatroskaKeyframeTimes } from "./matroska.js";
+import { isMp4, readMp4KeyframeTimes } from "./mp4.js";
+import { isAvi, readAviKeyframeTimes } from "./avi.js";
 
 /**
  * @callback ReadRange
@@ -29,9 +31,30 @@ import { isMatroska, readMatroskaKeyframeTimes } from "./matroska.js";
  * @returns {Promise<Buffer | null>} The bytes, or null when unavailable.
  */
 
-// Enough to identify any supported container from its magic bytes.
+// Enough to identify every supported container from its opening bytes.
 const SNIFF_BYTES = 16;
 
+/**
+ * Container readers, in detection order. Each pairs a cheap magic-byte test
+ * with the reader for that format.
+ *
+ * Formats deliberately absent, and why:
+ *  - **MPEG-TS / M2TS** carry no index at all — the format is a continuous
+ *    broadcast stream with no table of contents anywhere. Nothing to read.
+ *  - **FLV / ASF-WMV** do have keyframe tables, but effectively never appear in
+ *    the releases this serves; adding them is mechanical if that changes.
+ *  - **Fragmented MP4** spreads its timing across fragments instead of a single
+ *    `moov` table; `readMp4KeyframeTimes` returns null for it rather than
+ *    guessing.
+ *
+ * @type {{ name: string, matches: (head: Buffer) => boolean, read: ReadRange extends never ? never : (readRange: ReadRange, fileSize: number) => Promise<number[] | null> }[]}
+ */
+const READERS = [
+  { name: "matroska", matches: isMatroska, read: readMatroskaKeyframeTimes },
+  { name: "mp4", matches: isMp4, read: readMp4KeyframeTimes },
+  { name: "avi", matches: isAvi, read: readAviKeyframeTimes }
+];
+
 /**
  * Read a file's keyframe times from its container index.
  *
@@ -49,28 +72,31 @@ export async function readKeyframeIndex({ readRange, fileSize, label = "" }) {
 
   const startedAt = Date.now();
   let times = null;
+  let format = "unrecognised";
   try {
     const sniff = await readRange(0, Math.min(SNIFF_BYTES - 1, fileSize - 1));
     if (!sniff) {
       return null;
     }
-    if (isMatroska(sniff)) {
-      times = await readMatroskaKeyframeTimes(readRange, fileSize);
+    const reader = READERS.find((candidate) => candidate.matches(sniff));
+    if (reader) {
+      format = reader.name;
+      times = await reader.read(readRange, fileSize);
     }
-    // Other containers fall through as null until their readers land; MP4's
-    // sync-sample table and AVI's index are the next candidates.
   } catch (error) {
     // A malformed or partially-downloaded index must never take playback down —
-    // it only means we do not know the grid, which the caller handles.
+    // it only means the grid is unknown, which the caller already handles.
     logger.warn(`container-index: failed to read index for "${label}": ${error?.message ?? error}`);
     return null;
   }
 
   const elapsedMs = Date.now() - startedAt;
   if (times) {
-    logger.info(`container-index: ${times.length} keyframes from the container index in ${elapsedMs}ms for "${label}"`);
+    logger.info(
+      `container-index: ${times.length} keyframes from the ${format} index in ${elapsedMs}ms for "${label}"`
+    );
   } else {
-    logger.info(`container-index: no usable index for "${label}" (${elapsedMs}ms)`);
+    logger.info(`container-index: no usable index for "${label}" (${format}, ${elapsedMs}ms)`);
   }
   return times;
 }

package/services/container-index/mp4.js

@@ -0,0 +1,246 @@
+/**
+ * @file Keyframe index for MP4/MOV, read without downloading the file.
+ *
+ * MP4 keeps its tables in a `moov` box: `stss` lists which samples are sync
+ * samples (keyframes) by number, and `stts` gives each sample's duration, so
+ * the two together turn "sample #N" into "second T". `moov` sits either at the
+ * start (files written for streaming) or at the end (the common case for a
+ * plain mux); box headers state their own size, so it is found by stepping over
+ * top-level boxes rather than scanning bytes — a couple of 64-byte reads even
+ * when `mdat` is gigabytes.
+ *
+ * Same purpose as the Matroska reader: on the video-COPY path the segment
+ * boundaries ARE the source's keyframes, and inventing an even grid instead
+ * makes players walk the whole file or present audio with no picture.
+ */
+
+// A 64-bit box size is signalled by a 32-bit size of 1, the real size following
+// in the next 8 bytes.
+const LARGE_SIZE_MARKER = 1;
+const HEADER_BYTES = 8;
+const LARGE_HEADER_BYTES = 16;
+// Enough to read any box header while walking the top level.
+const PROBE_BYTES = 64;
+// Cap on the moov read. A feature-length file indexes to a few hundred KB;
+// beyond this is not a normal index and not worth pulling over a torrent.
+const MAX_MOOV_BYTES = 32 * 1024 * 1024;
+
+/**
+ * Whether this looks like MP4/MOV — every real file opens with an `ftyp` box.
+ *
+ * @param {Buffer} head
+ * @returns {boolean}
+ */
+export function isMp4(head) {
+  return head.length >= 12 && head.toString("latin1", 4, 8) === "ftyp";
+}
+
+/**
+ * Read a box header at `offset`.
+ *
+ * @param {Buffer} buffer
+ * @param {number} offset
+ * @returns {{ type: string, size: number, headerBytes: number } | null}
+ */
+function readBoxHeader(buffer, offset) {
+  if (offset + HEADER_BYTES > buffer.length) {
+    return null;
+  }
+  const size32 = buffer.readUInt32BE(offset);
+  const type = buffer.toString("latin1", offset + 4, offset + 8);
+  if (size32 === LARGE_SIZE_MARKER) {
+    if (offset + LARGE_HEADER_BYTES > buffer.length) {
+      return null;
+    }
+    // High word is zero for any file we can practically handle.
+    const high = buffer.readUInt32BE(offset + 8);
+    const low = buffer.readUInt32BE(offset + 12);
+    return { type, size: high * 4294967296 + low, headerBytes: LARGE_HEADER_BYTES };
+  }
+  return { type, size: size32, headerBytes: HEADER_BYTES };
+}
+
+/**
+ * Walk the top-level boxes to find `moov`, reading only each box header.
+ *
+ * @param {(start: number, end: number) => Promise<Buffer | null>} readRange
+ * @param {number} fileSize
+ * @returns {Promise<{ offset: number, size: number, headerBytes: number } | null>}
+ */
+async function findMoov(readRange, fileSize) {
+  let offset = 0;
+  while (offset < fileSize) {
+    const probe = await readRange(offset, Math.min(fileSize - 1, offset + PROBE_BYTES - 1));
+    if (!probe || probe.length < HEADER_BYTES) {
+      return null;
+    }
+    const header = readBoxHeader(probe, 0);
+    // Size 0 means "extends to end of file" — legal only for the last box, and
+    // never for one we would step over.
+    if (!header || header.size <= 0) {
+      return null;
+    }
+    if (header.type === "moov") {
+      return { offset, size: header.size, headerBytes: header.headerBytes };
+    }
+    offset += header.size;
+  }
+  return null;
+}
+
+/**
+ * Find the first box of `type` directly inside a range of an already-read buffer.
+ *
+ * @param {Buffer} buffer
+ * @param {number} start
+ * @param {number} end
+ * @param {string} type
+ * @returns {{ dataOffset: number, end: number } | null}
+ */
+function findBox(buffer, start, end, type) {
+  let offset = start;
+  while (offset + HEADER_BYTES <= end) {
+    const header = readBoxHeader(buffer, offset);
+    if (!header || header.size <= 0) {
+      return null;
+    }
+    if (header.type === type) {
+      return { dataOffset: offset + header.headerBytes, end: Math.min(end, offset + header.size) };
+    }
+    offset += header.size;
+  }
+  return null;
+}
+
+/**
+ * All boxes of `type` directly inside a range.
+ *
+ * @param {Buffer} buffer
+ * @param {number} start
+ * @param {number} end
+ * @param {string} type
+ * @returns {{ dataOffset: number, end: number }[]}
+ */
+function findAllBoxes(buffer, start, end, type) {
+  const found = [];
+  let offset = start;
+  while (offset + HEADER_BYTES <= end) {
+    const header = readBoxHeader(buffer, offset);
+    if (!header || header.size <= 0) {
+      break;
+    }
+    if (header.type === type) {
+      found.push({ dataOffset: offset + header.headerBytes, end: Math.min(end, offset + header.size) });
+    }
+    offset += header.size;
+  }
+  return found;
+}
+
+/**
+ * Turn sample numbers into seconds using the time-to-sample table.
+ *
+ * `stts` is run-length encoded — pairs of (sample count, per-sample duration) —
+ * so one walk yields every sample's start time without expanding the table.
+ *
+ * @param {Buffer} buffer
+ * @param {{ dataOffset: number, end: number }} stts
+ * @param {number} timescale - Ticks per second.
+ * @param {Set<number>} wanted - Sample numbers (1-based).
+ * @returns {number[]} Seconds, ascending.
+ */
+function resolveSampleTimes(buffer, stts, timescale, wanted) {
+  const entryCount = buffer.readUInt32BE(stts.dataOffset + 4);
+  const times = [];
+  let sampleNumber = 1;
+  let ticks = 0;
+  let cursor = stts.dataOffset + 8;
+  for (let entry = 0; entry < entryCount && cursor + 8 <= stts.end; entry += 1) {
+    const count = buffer.readUInt32BE(cursor);
+    const delta = buffer.readUInt32BE(cursor + 4);
+    for (let index = 0; index < count; index += 1) {
+      if (wanted.has(sampleNumber)) {
+        times.push(ticks / timescale);
+      }
+      ticks += delta;
+      sampleNumber += 1;
+    }
+    cursor += 8;
+  }
+  return times;
+}
+
+/**
+ * Read the keyframe times of an MP4/MOV file.
+ *
+ * @param {(start: number, end: number) => Promise<Buffer | null>} readRange
+ * @param {number} fileSize
+ * @returns {Promise<number[] | null>} Ascending seconds, or null when the file
+ *   carries no usable index (fragmented MP4, truncated or damaged `moov`).
+ */
+export async function readMp4KeyframeTimes(readRange, fileSize) {
+  const moovBox = await findMoov(readRange, fileSize);
+  if (!moovBox || moovBox.size > MAX_MOOV_BYTES) {
+    return null;
+  }
+
+  const moov = await readRange(moovBox.offset, Math.min(fileSize - 1, moovBox.offset + moovBox.size - 1));
+  if (!moov || moov.length < moovBox.headerBytes) {
+    return null;
+  }
+
+  // Examine every track; the video one is whichever carries sync samples. A
+  // track with no `stss` has every sample a keyframe, so it constrains nothing
+  // and is skipped.
+  for (const trak of findAllBoxes(moov, moovBox.headerBytes, moov.length, "trak")) {
+    const mdia = findBox(moov, trak.dataOffset, trak.end, "mdia");
+    if (!mdia) {
+      continue;
+    }
+    const mdhd = findBox(moov, mdia.dataOffset, mdia.end, "mdhd");
+    if (!mdhd) {
+      continue;
+    }
+    // mdhd layout: version(1) + flags(3), then creation/modification times —
+    // 32-bit each in version 0, 64-bit in version 1 — then the timescale.
+    const version = moov[mdhd.dataOffset];
+    const timescaleOffset = version === 1 ? mdhd.dataOffset + 20 : mdhd.dataOffset + 12;
+    if (timescaleOffset + 4 > mdhd.end) {
+      continue;
+    }
+    const timescale = moov.readUInt32BE(timescaleOffset);
+    if (!timescale) {
+      continue;
+    }
+
+    const minf = findBox(moov, mdia.dataOffset, mdia.end, "minf");
+    const stbl = minf && findBox(moov, minf.dataOffset, minf.end, "stbl");
+    if (!stbl) {
+      continue;
+    }
+    const stss = findBox(moov, stbl.dataOffset, stbl.end, "stss");
+    const stts = findBox(moov, stbl.dataOffset, stbl.end, "stts");
+    if (!stss || !stts) {
+      continue;
+    }
+
+    const syncCount = moov.readUInt32BE(stss.dataOffset + 4);
+    const wanted = new Set();
+    for (let index = 0; index < syncCount; index += 1) {
+      const at = stss.dataOffset + 8 + index * 4;
+      if (at + 4 > stss.end) {
+        break;
+      }
+      wanted.add(moov.readUInt32BE(at));
+    }
+    if (wanted.size === 0) {
+      continue;
+    }
+
+    const times = resolveSampleTimes(moov, stts, timescale, wanted);
+    if (times.length > 0) {
+      return times;
+    }
+  }
+  return null;
+}