@torrent-tv/proxy 2.80.18 → 2.81.0

package/test/segment-store.test.js

@@ -1,216 +1,238 @@
-/**
- * @file One store of segments, addressed by what they are, and what it makes of
- * what a killed process left behind.
- */
-
-import test from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { directoryNameFor, SegmentStore } from "../services/encode/SegmentStore.js";
-import { fmp4Format } from "../services/segment-formats/fmp4.js";
-
-/**
- * @returns {{ store: SegmentStore, root: string, lines: string[] }}
- */
-function storeInATempRoot() {
-  const root = mkdtempSync(path.join(os.tmpdir(), "segment-store-"));
-  const lines = [];
-  const store = new SegmentStore({
-    root,
-    logger: { info: (line) => lines.push(line), warn: (line) => lines.push(line) }
-  });
-  return { store, root, lines };
-}
-
-/**
- * @param {string} dir
- * @param {number} index
- * @param {number} bytes
- */
-function writeSegment(dir, index, bytes = 16) {
-  writeFileSync(path.join(dir, fmp4Format.segmentFileName(index)), Buffer.alloc(bytes));
-}
-
-const KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
-
-test("two viewers of one output are given the same directory", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  // Which viewer asked never enters it — the address is what the segments are.
-  assert.equal(store.directoryFor(KEY), store.directoryFor(KEY));
-  assert.notEqual(store.directoryFor(KEY), store.directoryFor(`${KEY}:other`));
-});
-
-test("the directory says what it holds, so a later process can tell", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = store.directoryFor(KEY);
-  const names = readdirSync(dir);
-  assert.deepEqual(names, ["key.txt"]);
-});
-
-test("a segment is proven closed by the existence of the next one", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = store.directoryFor(KEY);
-  store.useFormat(KEY, fmp4Format);
-  writeSegment(dir, 0);
-  writeSegment(dir, 1);
-  writeSegment(dir, 2);
-
-  // The `hls` muxer carries no channel to report on and renames into place when
-  // it closes a piece, so on that branch existence IS the proof; the same
-  // answers for the pieces a previous life of this process left behind.
-  assert.deepEqual(store.provenNumbers(KEY), [0, 1]);
-  assert.equal(store.unprovenNumber(KEY), 2);
-});
-
-test("a file of no bytes is not a segment, whatever it is called", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = store.directoryFor(KEY);
-  store.useFormat(KEY, fmp4Format);
-  writeSegment(dir, 0);
-  writeSegment(dir, 1);
-  writeSegment(dir, 2, 0);
-
-  // What a run killed the moment after opening its next piece leaves behind.
-  // Counted as a segment once, it closed the only hole in the numbering and
-  // convinced the look-ahead the encoder had produced it.
-  assert.equal(store.pathOf(KEY, 2), null);
-  assert.deepEqual(store.provenNumbers(KEY), [0]);
-});
-
-test("a run's own stretch bounds what it unproves", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  store.useFormat(KEY, fmp4Format);
-  store.directoryFor(KEY);
-  for (let index = 0; index <= 5; index += 1) {
-    store.markClosed(KEY, index);
-  }
-
-  // A run of #0..#0 rewrites #0 and nothing else. Unbounded, this forgot the
-  // whole film beyond it — and with readiness a projection of what is proven,
-  // that is an output declaring itself unmade whenever an encoder starts near
-  // the beginning, which is a fresh encoder for every segment of it.
-  //
-  // Asked with no files on the disk, so the successor rule cannot answer for
-  // the statements and only the statements are under test.
-  store.forgetClosed(KEY, 0, 0);
-  assert.equal(store.isClosed(KEY, 0), false, "the one number it will rewrite");
-  assert.equal(store.isClosed(KEY, 1), true, "and nothing beyond its stretch");
-  assert.equal(store.isClosed(KEY, 5), true);
-
-  store.forgetClosed(KEY, 3);
-  assert.equal(store.isClosed(KEY, 2), true);
-  assert.equal(store.isClosed(KEY, 3), false, "no end given means to the end of the film");
-  assert.equal(store.isClosed(KEY, 5), false);
-});
-
-test("a directory that has not moved is not read again", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = store.directoryFor(KEY);
-  store.useFormat(KEY, fmp4Format);
-  writeSegment(dir, 0);
-  writeSegment(dir, 1);
-
-  const first = store.refresh(KEY);
-  const second = store.refresh(KEY);
-  assert.equal(first, second, "the same reading is handed back, not a fresh listing");
-});
-
-test("what a killed process left is found, named and counted", (t) => {
-  const { store, root, lines } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = path.join(root, directoryNameFor(KEY));
-  mkdirSync(dir, { recursive: true });
-  writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
-  writeSegment(dir, 0);
-  writeSegment(dir, 1);
-  const orphan = path.join(root, "0123456789abcdef");
-  mkdirSync(orphan, { recursive: true });
-  writeSegment(orphan, 0);
-
-  const swept = store.sweep();
-  assert.equal(swept.directories, 2);
-  assert.equal(swept.unidentified, 1, "the one with no key file cannot be matched to a request");
-  assert.equal(swept.segments, 3);
-  assert.match(lines.join("\n"), /startup sweep/);
-  assert.match(lines.join("\n"), /ended\s+without anything recording why/);
-});
-
-test("what survived a kill is taken back, minus the piece nothing proves", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = path.join(root, directoryNameFor(KEY));
-  mkdirSync(dir, { recursive: true });
-  writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
-  for (let index = 0; index <= 5; index += 1) {
-    writeSegment(dir, index);
-  }
-
-  const taken = store.adoptWhatSurvived(() => fmp4Format);
-
-  assert.equal(taken.adopted, 1);
-  assert.equal(taken.unprovenRemoved, 1, "the highest number was being written when the run died");
-  // Five kept rather than six thrown away: a copied segment's bytes depend only
-  // on the source, and re-encoding them costs a machine already short of it.
-  assert.deepEqual(store.provenNumbers(KEY), [0, 1, 2, 3, 4]);
-  assert.equal(store.pathOf(KEY, 5), null);
-});
-
-test("a directory that cannot name itself is thrown away rather than served", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const orphan = path.join(root, "0123456789abcdef");
-  mkdirSync(orphan, { recursive: true });
-  writeSegment(orphan, 0);
-
-  const taken = store.adoptWhatSurvived(() => fmp4Format);
-
-  assert.equal(taken.adopted, 0);
-  assert.equal(taken.dropped, 1);
-  assert.equal(readdirSync(root).length, 0);
-});
-
-test("an output this proxy can no longer serve is thrown away too", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = path.join(root, directoryNameFor(KEY));
-  mkdirSync(dir, { recursive: true });
-  writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
-  writeSegment(dir, 0);
-
-  const taken = store.adoptWhatSurvived(() => null);
-
-  assert.equal(taken.dropped, 1);
-  assert.equal(readdirSync(root).length, 0);
-});
-
-test("what the store weighs is reported, and dropping one output frees it", (t) => {
-  const { store, root } = storeInATempRoot();
-  t.after(() => rmSync(root, { recursive: true, force: true }));
-
-  const dir = store.directoryFor(KEY);
-  store.useFormat(KEY, fmp4Format);
-  writeSegment(dir, 0, 1000);
-  writeSegment(dir, 1, 1000);
-
-  assert.equal(store.stats().bytes, 2000);
-  store.drop(KEY, "nobody is watching it");
-  assert.equal(store.stats().outputs, 0);
-});
+/**
+ * @file One store of segments, addressed by what they are, and what it makes of
+ * what a killed process left behind.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { directoryNameFor, SegmentStore } from "../services/encode/SegmentStore.js";
+import { fmp4Format } from "../services/segment-formats/fmp4.js";
+
+/**
+ * @returns {{ store: SegmentStore, root: string, lines: string[] }}
+ */
+function storeInATempRoot() {
+  const root = mkdtempSync(path.join(os.tmpdir(), "segment-store-"));
+  const lines = [];
+  const store = new SegmentStore({
+    root,
+    logger: { info: (line) => lines.push(line), warn: (line) => lines.push(line) }
+  });
+  return { store, root, lines };
+}
+
+/**
+ * @param {string} dir
+ * @param {number} index
+ * @param {number} bytes
+ */
+function writeSegment(dir, index, bytes = 16) {
+  writeFileSync(path.join(dir, fmp4Format.segmentFileName(index)), Buffer.alloc(bytes));
+}
+
+const KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
+
+test("two viewers of one output are given the same directory", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  // Which viewer asked never enters it — the address is what the segments are.
+  assert.equal(store.directoryFor(KEY), store.directoryFor(KEY));
+  assert.notEqual(store.directoryFor(KEY), store.directoryFor(`${KEY}:other`));
+});
+
+test("the directory says what it holds, so a later process can tell", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  const names = readdirSync(dir);
+  assert.deepEqual(names, ["key.txt"]);
+});
+
+test("a piece under its served name is finished, and the last one too", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  writeSegment(dir, 0);
+  writeSegment(dir, 1);
+  writeSegment(dir, 2);
+
+  // It was "the NEXT number exists", which left the last piece of every run
+  // unprovable for ever and — the moment two runs share an output — declared a
+  // half-written file finished because somebody else had written the one after
+  // it. The name is the proof now, so all three count.
+  assert.deepEqual(store.provenNumbers(KEY), [0, 1, 2]);
+});
+
+test("a piece still being written is not a piece, and its name says so", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  writeSegment(dir, 0);
+  // What ffmpeg is writing into right now, tagged with the run that opened it.
+  writeFileSync(path.join(dir, "making-0-00001.mp4"), Buffer.alloc(64));
+
+  assert.deepEqual(store.provenNumbers(KEY), [0]);
+  assert.equal(store.isClosed(KEY, 1), false);
+  assert.equal(store.pathOf(KEY, 1), null);
+
+  // And closing it is one rename, after which it is servable.
+  assert.equal(store.publish(KEY, "making-0-00001.mp4", fmp4Format), "segment-00001.mp4");
+  assert.equal(store.isClosed(KEY, 1), true);
+});
+
+test("a file of no bytes is not a segment, whatever it is called", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  writeSegment(dir, 0);
+  writeSegment(dir, 1);
+  writeSegment(dir, 2, 0);
+
+  // A file of no bytes under a served name cannot arise from this proxy any
+  // more — a piece takes that name only on being closed — but a killed process
+  // can leave one, and taking it for a segment once closed the only hole in the
+  // numbering and convinced the look-ahead the encoder had produced it.
+  assert.equal(store.pathOf(KEY, 2), null);
+  assert.deepEqual(store.provenNumbers(KEY), [0, 1]);
+});
+
+test("clearing up after one run leaves every other run's work alone", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  // Two live runs, each with a piece open, plus finished pieces of both.
+  writeSegment(dir, 0);
+  writeSegment(dir, 100);
+  writeFileSync(path.join(dir, "making-0-00001.mp4"), Buffer.alloc(64));
+  writeFileSync(path.join(dir, "making-100-00101.mp4"), Buffer.alloc(64));
+
+  // The run that began at #0 ends. Its own unfinished piece goes and nothing
+  // else does — which the old rule could not manage: it took the highest SERVED
+  // name inside the ended run's stretch and judged its bytes, so with the
+  // naming rule above it would have removed #0, a piece that run had closed.
+  assert.equal(store.clearUpAfter(KEY, 0), 1);
+  assert.deepEqual(store.provenNumbers(KEY), [0, 100]);
+  assert.equal(
+    readdirSync(dir).includes("making-100-00101.mp4"),
+    true,
+    "the other run is still writing its own"
+  );
+});
+
+test("a directory that has not moved is not read again", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  writeSegment(dir, 0);
+  writeSegment(dir, 1);
+
+  const first = store.refresh(KEY);
+  const second = store.refresh(KEY);
+  assert.equal(first, second, "the same reading is handed back, not a fresh listing");
+});
+
+test("what a killed process left is found, named and counted", (t) => {
+  const { store, root, lines } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = path.join(root, directoryNameFor(KEY));
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
+  writeSegment(dir, 0);
+  writeSegment(dir, 1);
+  const orphan = path.join(root, "0123456789abcdef");
+  mkdirSync(orphan, { recursive: true });
+  writeSegment(orphan, 0);
+
+  const swept = store.sweep();
+  assert.equal(swept.directories, 2);
+  assert.equal(swept.unidentified, 1, "the one with no key file cannot be matched to a request");
+  assert.equal(swept.segments, 3);
+  assert.match(lines.join("\n"), /startup sweep/);
+  assert.match(lines.join("\n"), /ended\s+without anything recording why/);
+});
+
+test("what survived a kill is taken back whole, minus what was still being written", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = path.join(root, directoryNameFor(KEY));
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(path.join(dir, "key.txt"), `${KEY}
+`);
+  for (let index = 0; index <= 5; index += 1) {
+    writeSegment(dir, index);
+  }
+  // The piece the kernel interrupted. It never had a served name, so it cannot
+  // be mistaken for one — which is what the old rule could only guess at, by
+  // always throwing the highest number away.
+  writeFileSync(path.join(dir, "making-0-00006.mp4"), Buffer.alloc(64));
+
+  const taken = store.adoptWhatSurvived(() => fmp4Format);
+
+  assert.equal(taken.adopted, 1);
+  assert.equal(taken.unprovenRemoved, 1, "the one under a working name");
+  // All six kept rather than five: the highest served name used to be thrown
+  // away because nothing could prove it, and on the copy branch that is a piece
+  // whose bytes depend only on the source and cost a short machine to remake.
+  assert.deepEqual(store.provenNumbers(KEY), [0, 1, 2, 3, 4, 5]);
+});
+
+test("a directory that cannot name itself is thrown away rather than served", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const orphan = path.join(root, "0123456789abcdef");
+  mkdirSync(orphan, { recursive: true });
+  writeSegment(orphan, 0);
+
+  const taken = store.adoptWhatSurvived(() => fmp4Format);
+
+  assert.equal(taken.adopted, 0);
+  assert.equal(taken.dropped, 1);
+  assert.equal(readdirSync(root).length, 0);
+});
+
+test("an output this proxy can no longer serve is thrown away too", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = path.join(root, directoryNameFor(KEY));
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
+  writeSegment(dir, 0);
+
+  const taken = store.adoptWhatSurvived(() => null);
+
+  assert.equal(taken.dropped, 1);
+  assert.equal(readdirSync(root).length, 0);
+});
+
+test("what the store weighs is reported, and dropping one output frees it", (t) => {
+  const { store, root } = storeInATempRoot();
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+
+  const dir = store.directoryFor(KEY);
+  store.useFormat(KEY, fmp4Format);
+  writeSegment(dir, 0, 1000);
+  writeSegment(dir, 1, 1000);
+
+  assert.equal(store.stats().bytes, 2000);
+  store.drop(KEY, "nobody is watching it");
+  assert.equal(store.stats().outputs, 0);
+});

package/test/segments-are-shared.test.js

@@ -140,7 +140,7 @@ test("what nobody has read for long enough goes, and time is the only judge", (t
   clock += 60_000;
   assert.deepEqual(
     store.enforce({ idleMs: 3_600_000, maxBytes: 1e9 }),
-    { droppedIdle: 0, droppedForRoom: 0, bytes: 32 }
+    { droppedIdle: 0, droppedForRoom: 0, segmentsRemoved: 0, bytes: 32 }
   );
 
   clock += 7_200_000;

package/test/shared-piece-store.test.js

@@ -59,7 +59,7 @@ test("a stored piece comes back byte for byte", async () => {
     assert.deepEqual(read, piece(0));
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -79,7 +79,7 @@ test("pieces past the memory budget spill to disk and read back intact", async (
     }
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -99,7 +99,7 @@ test("the short last piece keeps its own length", async () => {
     assert.deepEqual(read, tail);
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -118,7 +118,7 @@ test("a buffer handed out survives later writes to the same slot", async () => {
     assert.deepEqual(held, piece(0), "the buffer changed under its holder");
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -139,7 +139,7 @@ test("a pinned piece is not evicted to make room", async () => {
     store.unpin(0);
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -160,7 +160,7 @@ test("refuses to make room when every resident piece is being read", async () =>
     store.unpin(0);
     store.unpin(1);
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -179,7 +179,7 @@ test("a piece revived from disk is readable by offset again", async () => {
     assert.deepEqual(view, piece(0));
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -205,7 +205,7 @@ test("a range within a piece is served correctly from memory and from disk", asy
     );
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -230,7 +230,7 @@ test("takes memory as it needs it, not the whole budget up front", async () => {
     assert.deepEqual(await get(store, 0), piece(0), "an early piece was disturbed");
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -257,7 +257,7 @@ test("counts where reads were served from, so the budget can be judged", async (
     assert.equal(stats.capacity, 2);
   } finally {
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -275,7 +275,7 @@ test("counts a refusal caused by pinned pieces", async () => {
     store.unpin(0);
     store.unpin(1);
     await new Promise((resolve) => store.destroy(resolve));
-    await fs.rm(directory, { recursive: true, force: true });
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
   }
 });
 
@@ -292,5 +292,5 @@ test("destroy removes the spill file", async () => {
   const after = await fs.readdir(directory);
   assert.equal(after.length, 0, "the spill file outlived the store");
 
-  await fs.rm(directory, { recursive: true, force: true });
+  await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
 });