@torrent-tv/proxy 2.9.132 → 2.9.134

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2.9.134
+
+- **Fix**: The line reporting how long a segment waited before anyone decided to restart for it now prints. A restart backs off a segment or two from what was asked for, so the request that prompted it is recorded under a higher index than the run starts at; looking it up by the start index alone found nothing, and the instrument added in 2.9.132 never said a word. It now takes the earliest request at or above the index the run begins from.
+
+## 2.9.133
+
+- **Fix**: A restarted encoder no longer waits for its predecessor to die — every seek is about a second shorter. Runs shared one output directory, so two of them writing `segment-00042.mp4` at once would produce a file that is neither; the only defence was to kill the old run and block until it was gone. Measured 2.9.132 across four seeks: 712, 852, 882 and 1297 ms, against 11-15 ms of everything else a restart does. So that wait WAS the restart. Each run now writes into a directory of its own, which makes the collision impossible, so the new run starts at once and the old one is left to die in the background. Serving a segment searches the run directories newest-first, because a later run's answer supersedes an earlier one's — the older file may be the truncated output of a run that was killed mid-write, which is precisely what sharing a directory used to hide. Covered by a test that lays out two runs and insists the newer one wins.
+- **New**: How long a segment waited before anyone decided to restart for it. The restart costs about a second; a seek costs five to eight, so most of the wait happens before the decision is even taken, and nothing measured that gap.
+
 ## 2.9.132
 
 - **New**: A restarted encoder run says what its restart cost, and how much of that was waiting for the previous run to die. A seek costs 5-8 s in the field, and the reason on record — waiting for the previous ffmpeg to exit, measured once at 0.54-1.47 s — does not account for it. The remedy under consideration is a separate output directory per run, which removes the wait entirely but makes serving a segment a search across runs: the hottest path in the proxy, rebuilt on a guess about where the seconds go. So each stage states its own cost first. Two lines: how long SIGTERM took to be obeyed, and the total from the restart being asked for to the new run being announced.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.9.132",
+  "version": "2.9.134",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/services/hls-session-manager.js

@@ -1530,7 +1530,9 @@ export class HlsSessionManager {
     }
     let names;
     try {
-      names = (await readdir(session.dirPath))
+      names = (this.#runDirs(session).flatMap((dir) => {
+        try { return readdirSync(dir, { withFileTypes: false }); } catch { return []; }
+      }))
         .filter((name) => session.segmentFormat.isSegmentFileName(name))
         .sort();
     } catch {
@@ -1538,7 +1540,11 @@ export class HlsSessionManager {
     }
     for (const name of names) {
       try {
-        const init = session.segmentFormat.extractInit(await readFile(path.join(session.dirPath, name)));
+        const found = await this.#findProducedFile(session, name);
+        if (!found) {
+          continue;
+        }
+        const init = session.segmentFormat.extractInit(await readFile(found));
         if (init && init.length > 0) {
           return init;
         }
@@ -1764,7 +1770,9 @@ export class HlsSessionManager {
   async #observedStreamMbps(session) {
     let names;
     try {
-      names = await readdir(session.dirPath);
+      names = this.#runDirs(session).flatMap((dir) => {
+        try { return readdirSync(dir, { withFileTypes: false }); } catch { return []; }
+      });
     } catch {
       return null;
     }
@@ -1783,7 +1791,11 @@ export class HlsSessionManager {
     let bytes = 0;
     try {
       for (const index of completed) {
-        const st = await stat(path.join(session.dirPath, session.segmentFormat.segmentFileName(index)));
+        const segmentPath = await this.#findProducedFile(session, session.segmentFormat.segmentFileName(index));
+        if (!segmentPath) {
+          break;
+        }
+        const st = await stat(segmentPath);
         bytes += st.size;
       }
     } catch {
@@ -1999,7 +2011,9 @@ export class HlsSessionManager {
     let present;
     try {
       present = new Set();
-      for (const name of readdirSync(session.dirPath, { withFileTypes: false })) {
+      for (const name of this.#runDirs(session).flatMap((dir) => {
+        try { return readdirSync(dir, { withFileTypes: false }); } catch { return []; }
+      })) {
         if (!this.segmentFormat.isSegmentFileName(name)) {
           continue;
         }
@@ -2253,6 +2267,28 @@ export class HlsSessionManager {
     // 0.54-1.47 s — does not account for it. Before rebuilding the hottest path
     // in the proxy on a guess, make each stage state its own cost.
     const restartEnteredAt = Date.now();
+    // One directory per run. Two runs writing the same segment name at once
+    // produce a file that is neither, which is the only reason a restart ever
+    // had to wait for its predecessor to die.
+    session.runSerial = (session.runSerial ?? 0) + 1;
+    session.runDirPath = path.join(session.dirPath, `run-${session.runSerial}`);
+    await mkdir(session.runDirPath, { recursive: true });
+    // The restart backs off a segment or two from what was asked for, so the
+    // request that prompted it is recorded under a HIGHER index than the run
+    // starts at. Looking it up by the start index alone found nothing and the
+    // line never printed once.
+    let wantedAt = null;
+    for (const [index, at] of session.firstWantedAt ?? []) {
+      if (index >= startIndex && (wantedAt === null || at < wantedAt)) {
+        wantedAt = at;
+      }
+    }
+    if (typeof wantedAt === "number") {
+      logger.info(
+        `transcode ${session.id} restart for #${startIndex} decided ` +
+        `${restartEnteredAt - wantedAt}ms after it was first asked for`
+      );
+    }
     const generation = ++session.encodeRunGeneration;
     const previousFfmpeg = session.ffmpeg;
     // A suspended process does not act on SIGTERM until it is continued, so the
@@ -2264,11 +2300,16 @@ export class HlsSessionManager {
       } catch {
         // Best effort.
       }
+      // NOT awaited. The previous run has its own directory, so it cannot
+      // corrupt this one's output by still writing; letting it die in the
+      // background removes 0.7-1.3 s from every seek (measured 2.9.132, where
+      // that wait was essentially the whole cost of a restart).
       const termSentAt = Date.now();
-      await waitForChildExit(previousFfmpeg, ENCODE_RUN_TERMINATE_GRACE_MS);
-      logger.info(
-        `transcode ${session.id} restart: previous run took ${Date.now() - termSentAt}ms to exit after SIGTERM`
-      );
+      void waitForChildExit(previousFfmpeg, ENCODE_RUN_TERMINATE_GRACE_MS).then(() => {
+        logger.info(
+          `transcode ${session.id} restart: previous run took ${Date.now() - termSentAt}ms to exit after SIGTERM`
+        );
+      });
       if (!hasChildExited(previousFfmpeg)) {
         try {
           previousFfmpeg.kill("SIGKILL");
@@ -2477,7 +2518,7 @@ export class HlsSessionManager {
     logger.info(`transcode ${session.id} ${runLabel} ffmpeg ${describeFfmpegArgs(args)}`);
 
     const ffmpeg = spawn(this.ffmpegBin, args, {
-      cwd: session.dirPath,
+      cwd: session.runDirPath ?? session.dirPath,
       stdio: ["ignore", "pipe", "pipe"]
     });
     session.ffmpeg = ffmpeg;
@@ -2768,6 +2809,14 @@ export class HlsSessionManager {
    * @returns {void}
    */
   #ensureEncodingFor(session, index, requestSeq = Number.MAX_SAFE_INTEGER) {
+    // When this segment was FIRST asked for and nobody was producing it. The
+    // restart itself costs 0.7-1.3 s (measured 2.9.132), while a seek costs
+    // 5-8 s end to end — so most of the wait happens before a restart is even
+    // decided on, and that is what this records.
+    session.firstWantedAt ??= new Map();
+    if (!session.firstWantedAt.has(index)) {
+      session.firstWantedAt.set(index, Date.now());
+    }
     if (!session || session.state === "disposed" || index < 0) {
       return;
     }
@@ -3140,7 +3189,9 @@ export class HlsSessionManager {
    */
   #latestProducedSegment(session) {
     let highest = null;
-    for (const name of readdirSync(session.dirPath, { withFileTypes: false })) {
+    for (const name of this.#runDirs(session).flatMap((dir) => {
+      try { return readdirSync(dir, { withFileTypes: false }); } catch { return []; }
+    })) {
       if (!this.segmentFormat.isSegmentFileName(name)) {
         continue;
       }
@@ -3343,7 +3394,7 @@ export class HlsSessionManager {
         // piece, so the first one to exist supplies it.
         const bytes = session.usesExplicitCuts
           ? await this.#initFromFirstSegment(session)
-          : await readFile(path.join(session.dirPath, initFileName));
+          : await readFile((await this.#findProducedFile(session, initFileName)) ?? path.join(session.dirPath, initFileName));
         if (!bytes || bytes.length === 0) {
           return { kind: "warming-up" };
         }
@@ -3371,7 +3422,7 @@ export class HlsSessionManager {
       }
     }
 
-    const filePath = path.join(session.dirPath, fileName);
+    const filePath = (await this.#findProducedFile(session, fileName)) ?? path.join(session.dirPath, fileName);
     const isPlaylist = fileName === PLAYLIST_FILE_NAME;
     if (!isPlaylist) {
       // Where the viewer actually is. Recorded for every segment request,
@@ -3413,7 +3464,8 @@ export class HlsSessionManager {
         const index = session.segmentFormat.segmentIndexFromName(fileName);
         const isLast = index >= Math.max(0, (session.segmentBoundaries?.length ?? 1) - 2);
         if (!isLast && session.ffmpeg) {
-          const nextPath = path.join(session.dirPath, session.segmentFormat.segmentFileName(index + 1));
+          const nextName = session.segmentFormat.segmentFileName(index + 1);
+          const nextPath = (await this.#findProducedFile(session, nextName)) ?? path.join(session.dirPath, nextName);
           // The FIRST segment of a run cannot be judged by "the next one
           // exists": nothing is producing a next one yet, because the run has
           // only just begun here. Waiting for it holds precisely the segment a
@@ -3624,6 +3676,55 @@ export class HlsSessionManager {
     );
   }
 
+  /**
+   * The directories runs have written into, newest first.
+   *
+   * Runs used to share one directory, which is why a restart had to wait for
+   * the previous ffmpeg to die: two processes writing `segment-00042.mp4` at
+   * once produce a file that is neither. Measured 2.9.132, that wait was
+   * 0.7-1.3 s of every seek and essentially the whole cost of a restart.
+   * Given a directory each they cannot collide, so the new run starts at once
+   * and the old one is left to die in the background.
+   *
+   * Newest first because a later run's answer for a segment supersedes an
+   * earlier one's: the older file may be the truncated output of a run that was
+   * killed mid-write, which is exactly what sharing a directory used to hide.
+   *
+   * @param {HlsSession} session
+   * @returns {string[]}
+   */
+  #runDirs(session) {
+    try {
+      return readdirSync(session.dirPath, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory() && entry.name.startsWith("run-"))
+        .map((entry) => entry.name)
+        .sort((a, b) => Number(b.slice(4)) - Number(a.slice(4)))
+        .map((name) => path.join(session.dirPath, name));
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Where a produced file actually is, or null when no run has written it.
+   *
+   * @param {HlsSession} session
+   * @param {string} fileName
+   * @returns {Promise<string | null>}
+   */
+  async #findProducedFile(session, fileName) {
+    for (const dir of this.#runDirs(session)) {
+      const candidate = path.join(dir, fileName);
+      try {
+        await access(candidate);
+        return candidate;
+      } catch {
+        // Not this run's; try an older one.
+      }
+    }
+    return null;
+  }
+
   #holdForProduction(session, fileName, isPlaylist, options) {
     if (!isPlaylist) {
       this.#explainHold(session, fileName, "the file is not on disk");

package/test/segment-serve-wiring.test.js

@@ -226,3 +226,37 @@ test("a run's FIRST segment is served once the encoder has passed it, without wa
     "the encoder is past this segment's end, so it is finished — the absence of a next one says nothing"
   );
 });
+
+test("a segment is found in the run directory that produced it, newest run first", async (t) => {
+  const { manager, session, dirPath } = await managerWithReadySegment();
+  t.after(async () => {
+    await manager.disposeAll();
+    await rm(dirPath, { recursive: true, force: true });
+  });
+  // Runs write into a directory each — that is what lets a restart begin
+  // without waiting for its predecessor to die, which measured 0.7-1.3 s of
+  // every seek. A later run's answer supersedes an earlier one's, because the
+  // older file may be the truncated output of a run that was killed mid-write.
+  const { mkdir } = await import("node:fs/promises");
+  const piece = selfContainedPiece(SEGMENT_START_SECONDS);
+  await mkdir(path.join(dirPath, "run-1"), { recursive: true });
+  await mkdir(path.join(dirPath, "run-2"), { recursive: true });
+  await writeFile(path.join(dirPath, "run-1", "segment-00000.mp4"), Buffer.alloc(8));
+  await writeFile(path.join(dirPath, "run-2", "segment-00000.mp4"), piece);
+  await writeFile(path.join(dirPath, "run-2", "segment-00001.mp4"), piece);
+  await rm(path.join(dirPath, "segment-00000.mp4"));
+  await rm(path.join(dirPath, "segment-00001.mp4"));
+  session.encodeStartIndex = 0;
+
+  const result = await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
+
+  assert.equal(result.kind, "file", "a segment produced by a run must be found in that run's directory");
+  const chunks = [];
+  for await (const chunk of result.stream) {
+    chunks.push(chunk);
+  }
+  assert.ok(
+    Buffer.concat(chunks).length > 8,
+    "the newest run's output must win — the older file here is the 8-byte stub a killed run leaves"
+  );
+});