kikoeru 0.1.2 → 0.1.3

package/README.md

@@ -60,8 +60,13 @@ address and the time it was found:
 
 The rules are simple:
 
-- Entry exists and is less than 5 hours old → kikoeru relays the sound.
-- Entry is missing or older than 5 hours → kikoeru answers 404.
+- Entry exists and its sound address has not expired yet → kikoeru
+  relays the sound. Sound addresses carry their own expiry time
+  (an `expire` value in the address, about 6 hours after the address
+  was found), and kikoeru serves the address until just before that
+  time. An address with no readable expiry is used until 6 hours
+  after it was found.
+- Entry is missing or its address has expired → kikoeru answers 404.
   (The shell pipeline will find a fresh address on its next run.)
 
 Sound addresses stop working after some hours. If YouTube answers
@@ -176,6 +181,14 @@ environment variable.
 | `KIKOERU_TOKEN` | (contents of the token file) | The shared secret |
 | `KIKOERU_TOKEN_FILE` | `~/.config/kikoeru/token` | Where the token is kept |
 | `YT_AUDIO_CACHE` | `~/.yt-audio/cache.json` | The shell pipeline's cache file |
+| `KIKOERU_CHUNK_SIZE` | `4194304` | Size of one upstream piece, in bytes |
+| `KIKOERU_FALLBACK_FRESH_SEC` | `21600` | How long an address with no readable expiry is used, in seconds |
+| `KIKOERU_EXPIRY_MARGIN_SEC` | `60` | Stop using an address this many seconds before it expires |
+| `KIKOERU_META_TTL_MS` | `3000000` | How long a resolved size and type are remembered, in milliseconds |
+
+The four numeric settings must be positive numbers. A value that is
+not a number, or is zero or negative, is ignored and the default is
+used instead.
 
 ## Example
 

package/package.json

@@ -1 +1 @@
-{"bin":{"kikoeru":"bin/kikoeru.js"},"description":"Streaming relay for YouTube audio: fetches googlevideo streams with small ranged requests and forwards them to the client without storing any files.","engines":{"node":">=18"},"license":"MIT","main":"src/server.js","name":"kikoeru","scripts":{"start":"node bin/kikoeru.js","test":"node --test \"test/**/*.test.js\""},"version":"0.1.2"}
+{"bin":{"kikoeru":"bin/kikoeru.js"},"description":"Streaming relay for YouTube audio: fetches googlevideo streams with small ranged requests and forwards them to the client without storing any files.","engines":{"node":">=18"},"license":"MIT","main":"src/server.js","name":"kikoeru","scripts":{"start":"node bin/kikoeru.js","test":"node --test \"test/**/*.test.js\""},"version":"0.1.3"}

package/src/server.js

@@ -17,8 +17,10 @@
  * Stream URL resolution is NOT this server's job. Extracting stream URLs
  * with yt-dlp belongs to the shell pipeline (yt-audio / yt-audio-batch);
  * this server only reads its cache (~/.yt-audio/cache.json) and relays:
- *   - entry present and younger than 5 h -> relay that URL.
- *   - entry missing or stale -> 404 (the shell refreshes it on its cycle).
+ *   - entry present and its URL not yet expired -> relay that URL.
+ *     (The URL's own `expire` parameter decides; a URL without one is
+ *     used until 6 h after its extraction time.)
+ *   - entry missing or expired -> 404 (the shell refreshes it on its cycle).
  * If the upstream answers 403 (URL expired server-side), the batch cache
  * is re-read once (the shell may have refreshed it in the meantime) and
  * the request is retried; if that still fails, the answer is 404.
@@ -36,14 +38,43 @@ const os = require('node:os');
 const crypto = require('node:crypto');
 
 const CHUNK_SIZE = 4 * 1024 * 1024; // upstream sub-range size
-const FRESH_SEC = 5 * 3600; // cached stream URLs are reused for 5 hours
+const FALLBACK_FRESH_SEC = 6 * 3600; // no expire param: ts + 6 h decides
+const EXPIRY_MARGIN_SEC = 60; // stop serving this long before a URL expires
 const META_TTL_MS = 50 * 60 * 1000; // resolved size/type/final-URL cache
 const VIDEO_ID_RE = /^[A-Za-z0-9_-]{11}$/;
 const ROUTE_RE = /^\/([^/]+)\/([^/]+)$/;
 
+function positiveNumber(raw, fallback) {
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+function positiveInt(raw, fallback) {
+  return Math.max(1, Math.floor(positiveNumber(raw, fallback)));
+}
+
+/**
+ * Fill the numeric knobs of a config object with validated values.
+ * loadConfig feeds it the raw environment strings; createServer and
+ * resolveStreamUrl run it too, so a hand-built config object gets
+ * the same treatment. Anything that is not a positive number falls
+ * back to the built-in default, so a typo can never change the
+ * behaviour silently.
+ */
+function normalizeConfig(cfg) {
+  const c = cfg || {};
+  return {
+    ...c,
+    chunkSize: positiveInt(c.chunkSize, CHUNK_SIZE),
+    fallbackFreshSec: positiveNumber(c.fallbackFreshSec, FALLBACK_FRESH_SEC),
+    expiryMarginSec: positiveNumber(c.expiryMarginSec, EXPIRY_MARGIN_SEC),
+    metaTtlMs: positiveNumber(c.metaTtlMs, META_TTL_MS),
+  };
+}
+
 function loadConfig(env = process.env) {
   const home = os.homedir();
-  return {
+  return normalizeConfig({
     host: env.KIKOERU_HOST || '127.0.0.1',
     port: Number(env.KIKOERU_PORT || 8791),
     token: env.KIKOERU_TOKEN || null,
@@ -51,7 +82,11 @@ function loadConfig(env = process.env) {
       env.KIKOERU_TOKEN_FILE || path.join(home, '.config/kikoeru/token'),
     batchCacheFile:
       env.YT_AUDIO_CACHE || path.join(home, '.yt-audio/cache.json'),
-  };
+    chunkSize: env.KIKOERU_CHUNK_SIZE,
+    fallbackFreshSec: env.KIKOERU_FALLBACK_FRESH_SEC,
+    expiryMarginSec: env.KIKOERU_EXPIRY_MARGIN_SEC,
+    metaTtlMs: env.KIKOERU_META_TTL_MS,
+  });
 }
 
 function ensureToken(cfg) {
@@ -77,16 +112,45 @@ function readJsonFile(file) {
 }
 
 /**
- * Resolve a fresh-enough stream URL for the video from the shell
+ * The expiry (epoch seconds) a googlevideo stream URL carries in its
+ * `expire` query parameter, or null when the URL has none that parses.
+ */
+function urlExpirySec(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  const raw = parsed.searchParams.get('expire');
+  if (!raw) return null;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : null;
+}
+
+/**
+ * Resolve a still-usable stream URL for the video from the shell
  * pipeline's batch cache, or null. This server never extracts URLs
  * itself (no yt-dlp, no cache of its own).
+ *
+ * Usability follows the URL itself: googlevideo URLs carry an
+ * `expire` parameter (epoch seconds, roughly 6 h after extraction)
+ * and are served until just before that moment. URLs without a
+ * parseable `expire` fall back to the extraction timestamp plus
+ * FALLBACK_FRESH_SEC.
  */
 function resolveStreamUrl(cfg, videoId) {
-  const batch = readJsonFile(cfg.batchCacheFile);
+  const c = normalizeConfig(cfg);
+  const batch = readJsonFile(c.batchCacheFile);
   const entry = batch && batch[videoId];
   if (!entry || !entry.url) return null;
   const now = Math.floor(Date.now() / 1000);
-  if (now - (entry.ts || 0) >= FRESH_SEC) return null;
+  const exp = urlExpirySec(entry.url);
+  if (exp !== null) {
+    return now < exp - c.expiryMarginSec ? entry.url : null;
+  }
+  const ts = entry.ts || 0;
+  if (!ts || now - ts >= c.fallbackFreshSec) return null;
   return entry.url;
 }
 
@@ -144,7 +208,7 @@ async function probeMeta(startUrl) {
 async function getMeta(cfg, state, videoId, force = false) {
   if (!force) {
     const hit = state.meta.get(videoId);
-    if (hit && Date.now() - hit.at < META_TTL_MS) return hit;
+    if (hit && Date.now() - hit.at < cfg.metaTtlMs) return hit;
   }
   const url = resolveStreamUrl(cfg, videoId);
   if (!url) return null;
@@ -249,7 +313,7 @@ async function handle(cfg, state, token, req, res) {
     // re-read batch cache.
     const tracker = { committed: false, wrote: false };
     try {
-      const firstEnd = Math.min(start + CHUNK_SIZE - 1, end);
+      const firstEnd = Math.min(start + cfg.chunkSize - 1, end);
       const up = await openUpstream(meta.finalUrl, {
         Range: `bytes=${start}-${firstEnd}`,
       });
@@ -275,7 +339,14 @@ async function handle(cfg, state, token, req, res) {
         }
       }
       if (firstEnd < end) {
-        await streamRest(meta.finalUrl, firstEnd + 1, end, res, tracker);
+        await streamRest(
+          meta.finalUrl,
+          firstEnd + 1,
+          end,
+          res,
+          tracker,
+          cfg.chunkSize,
+        );
       }
       res.end();
       return;
@@ -305,11 +376,11 @@ async function handle(cfg, state, token, req, res) {
   }
 }
 
-async function streamRest(finalUrl, start, end, res, tracker) {
+async function streamRest(finalUrl, start, end, res, tracker, chunkSize) {
   let pos = start;
   while (pos <= end) {
     if (res.destroyed || res.writableEnded) return;
-    const chunkEnd = Math.min(pos + CHUNK_SIZE - 1, end);
+    const chunkEnd = Math.min(pos + chunkSize - 1, end);
     const up = await openUpstream(finalUrl, {
       Range: `bytes=${pos}-${chunkEnd}`,
     });
@@ -337,10 +408,11 @@ async function streamRest(finalUrl, start, end, res, tracker) {
 }
 
 function createServer(cfg) {
-  const token = ensureToken(cfg);
+  const full = normalizeConfig(cfg);
+  const token = ensureToken(full);
   const state = { meta: new Map() };
   const server = http.createServer((req, res) => {
-    handle(cfg, state, token, req, res).catch((e) => {
+    handle(full, state, token, req, res).catch((e) => {
       console.error(`[kikoeru] handler error: ${e.message}`);
       if (!res.headersSent) {
         res.writeHead(500, { 'Content-Type': 'text/plain' });
@@ -354,9 +426,13 @@ function createServer(cfg) {
 module.exports = {
   createServer,
   loadConfig,
+  normalizeConfig,
   ensureToken,
   parseRange,
   resolveStreamUrl,
+  urlExpirySec,
   CHUNK_SIZE,
-  FRESH_SEC,
+  FALLBACK_FRESH_SEC,
+  EXPIRY_MARGIN_SEC,
+  META_TTL_MS,
 };

package/test/cache.test.js

@@ -1,16 +1,23 @@
 'use strict';
 
 // The batch cache is the only source of stream URLs. These tests pin
-// down how the server judges it: fresh entries are relayed, missing /
-// stale / broken caches are a plain 404, and the server never falls
-// over on a bad file. resolveStreamUrl is also unit-tested directly.
+// down how the server judges it: an entry is relayed while its URL is
+// unexpired (the URL's own `expire` parameter decides; without one,
+// the extraction time plus 6 hours does), missing / expired / broken
+// caches are a plain 404, and the server never falls over on a bad
+// file. resolveStreamUrl is also unit-tested directly.
 
 const { test, before, after } = require('node:test');
 const assert = require('node:assert');
 const fs = require('node:fs');
 const path = require('node:path');
 
-const { createServer, resolveStreamUrl, FRESH_SEC } = require('../src/server');
+const {
+  createServer,
+  resolveStreamUrl,
+  FALLBACK_FRESH_SEC,
+  EXPIRY_MARGIN_SEC,
+} = require('../src/server');
 const {
   listen,
   close,
@@ -74,6 +81,13 @@ function goodEntry(ts) {
   return { url: `http://127.0.0.1:${upPort}/media`, ts };
 }
 
+// An entry whose URL carries an `expire` parameter, like a real
+// googlevideo URL. The fake upstream serves /media regardless of the
+// query string; only kikoeru's freshness judgment reads the value.
+function expiringEntry(ts, expire) {
+  return { url: `http://127.0.0.1:${upPort}/media?expire=${expire}`, ts };
+}
+
 test('fresh entry (just written) is relayed', async () => {
   await withServer(cacheWith(goodEntry(nowSec())), async (port) => {
     const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
@@ -84,8 +98,8 @@ test('fresh entry (just written) is relayed', async () => {
   });
 });
 
-test('entry just inside the 5 hour freshness window is relayed', async () => {
-  const ts = nowSec() - FRESH_SEC + 120; // 2 minutes of margin
+test('entry just inside the fallback freshness window is relayed', async () => {
+  const ts = nowSec() - FALLBACK_FRESH_SEC + 120; // 2 minutes of margin
   await withServer(cacheWith(goodEntry(ts)), async (port) => {
     const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
       Range: 'bytes=0-9',
@@ -94,8 +108,8 @@ test('entry just inside the 5 hour freshness window is relayed', async () => {
   });
 });
 
-test('entry just past the 5 hour freshness window gets 404', async () => {
-  const ts = nowSec() - FRESH_SEC - 120; // 2 minutes of margin
+test('entry just past the fallback freshness window gets 404', async () => {
+  const ts = nowSec() - FALLBACK_FRESH_SEC - 120; // 2 minutes of margin
   await withServer(cacheWith(goodEntry(ts)), async (port) => {
     const r = await get(port, `/${TOKEN}/${VIDEO_ID}`);
     assert.equal(r.status, 404);
@@ -103,7 +117,7 @@ test('entry just past the 5 hour freshness window gets 404', async () => {
 });
 
 test('stale batch cache entry gets 404', async () => {
-  const ts = nowSec() - 6 * 3600; // older than 5 h
+  const ts = nowSec() - FALLBACK_FRESH_SEC - 120; // older than 6 h
   await withServer(cacheWith(goodEntry(ts)), async (port) => {
     const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
       Range: 'bytes=0-99',
@@ -112,6 +126,71 @@ test('stale batch cache entry gets 404', async () => {
   });
 });
 
+// --- expiry-based judgment (the URL's own `expire` parameter) --------
+
+test('old entry is still relayed while its URL is unexpired', async () => {
+  // Extracted more than 5 hours ago (past the old fixed window), but
+  // the URL itself stays valid for another 30 minutes.
+  const entry = expiringEntry(nowSec() - 5 * 3600 - 600, nowSec() + 1800);
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
+      Range: 'bytes=0-9',
+    });
+    assert.equal(r.status, 206);
+    assert.deepEqual(r.body, data.subarray(0, 10));
+  });
+});
+
+test('fresh entry gets 404 once its URL has expired', async () => {
+  const entry = expiringEntry(nowSec(), nowSec() - 60);
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`);
+    assert.equal(r.status, 404);
+  });
+});
+
+test('URL expiring within the safety margin gets 404', async () => {
+  const entry = expiringEntry(nowSec(), nowSec() + EXPIRY_MARGIN_SEC - 30);
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`);
+    assert.equal(r.status, 404);
+  });
+});
+
+test('URL expiring just beyond the safety margin is relayed', async () => {
+  const entry = expiringEntry(nowSec(), nowSec() + EXPIRY_MARGIN_SEC + 120);
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
+      Range: 'bytes=0-9',
+    });
+    assert.equal(r.status, 206);
+  });
+});
+
+test('unparseable expire falls back to the timestamp window (fresh)', async () => {
+  const entry = {
+    url: `http://127.0.0.1:${upPort}/media?expire=abc`,
+    ts: nowSec(),
+  };
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`, {
+      Range: 'bytes=0-9',
+    });
+    assert.equal(r.status, 206);
+  });
+});
+
+test('unparseable expire falls back to the timestamp window (stale)', async () => {
+  const entry = {
+    url: `http://127.0.0.1:${upPort}/media?expire=abc`,
+    ts: nowSec() - FALLBACK_FRESH_SEC - 120,
+  };
+  await withServer(cacheWith(entry), async (port) => {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`);
+    assert.equal(r.status, 404);
+  });
+});
+
 test('empty batch cache gets 404 for every video', async () => {
   const file = path.join(tmp, 'cache-empty.json');
   writeCache(file, {});
@@ -193,3 +272,39 @@ test('resolveStreamUrl returns null when the file cannot be read', () => {
   const file = path.join(tmp, 'no-such-cache.json');
   assert.equal(resolveStreamUrl({ batchCacheFile: file }, VIDEO_ID), null);
 });
+
+test('resolveStreamUrl serves an old entry whose URL is unexpired', () => {
+  const entry = expiringEntry(nowSec() - 5 * 3600 - 600, nowSec() + 1800);
+  const file = cacheWith(entry);
+  assert.equal(resolveStreamUrl({ batchCacheFile: file }, VIDEO_ID), entry.url);
+});
+
+test('resolveStreamUrl returns null for a fresh entry whose URL expired', () => {
+  const entry = expiringEntry(nowSec(), nowSec() - 60);
+  const file = cacheWith(entry);
+  assert.equal(resolveStreamUrl({ batchCacheFile: file }, VIDEO_ID), null);
+});
+
+test('resolveStreamUrl honours an expiryMarginSec override', () => {
+  // The URL stays valid for another 30 minutes: fine with the
+  // default 60-second margin, unusable with a 1-hour margin.
+  const entry = expiringEntry(nowSec(), nowSec() + 1800);
+  const file = cacheWith(entry);
+  assert.equal(resolveStreamUrl({ batchCacheFile: file }, VIDEO_ID), entry.url);
+  assert.equal(
+    resolveStreamUrl({ batchCacheFile: file, expiryMarginSec: 3600 }, VIDEO_ID),
+    null,
+  );
+});
+
+test('resolveStreamUrl honours a fallbackFreshSec override', () => {
+  // Extracted 1 hour ago, no expire parameter: fine with the
+  // default 6-hour window, stale with a 30-minute window.
+  const entry = goodEntry(nowSec() - 3600);
+  const file = cacheWith(entry);
+  assert.equal(resolveStreamUrl({ batchCacheFile: file }, VIDEO_ID), entry.url);
+  assert.equal(
+    resolveStreamUrl({ batchCacheFile: file, fallbackFreshSec: 1800 }, VIDEO_ID),
+    null,
+  );
+});

package/test/chunking.test.js

@@ -140,3 +140,43 @@ test('a client range inside one chunk causes exactly one data request', async ()
   const dataRequests = splitProbe(recordedRanges());
   assert.deepEqual(dataRequests, [[1000, 1999]]);
 });
+
+test('a chunkSize override shrinks the upstream sub-ranges', async () => {
+  const small = makeData(5000);
+  const up = makeMediaUpstream(small);
+  const upPort2 = await listen(up.server);
+  const dir = tmpDir();
+  const cache = path.join(dir, 'cache.json');
+  writeCache(cache, {
+    [VIDEO_ID]: { url: `http://127.0.0.1:${upPort2}/media`, ts: nowSec() },
+  });
+  const p = createServer({
+    host: '127.0.0.1',
+    port: 0,
+    token: TOKEN,
+    tokenFile: path.join(dir, 'token'),
+    batchCacheFile: cache,
+    chunkSize: 1024,
+  });
+  const port = await listen(p);
+  try {
+    const r = await get(port, `/${TOKEN}/${VIDEO_ID}`);
+    assert.equal(r.status, 200);
+    assert.deepEqual(r.body, small);
+    const chunks = up.state.requests
+      .filter((q) => q.url === '/media')
+      .map((q) => parseRangeHeader(q.range))
+      .filter(([s, e]) => !(s === 0 && e === 0));
+    assert.deepEqual(chunks, [
+      [0, 1023],
+      [1024, 2047],
+      [2048, 3071],
+      [3072, 4095],
+      [4096, 4999],
+    ]);
+  } finally {
+    await close(p);
+    await close(up.server);
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});

package/test/config.test.js

@@ -74,6 +74,43 @@ test('loadConfig honours every environment override', () => {
   assert.equal(cfg.batchCacheFile, '/tmp/somewhere/cache.json');
 });
 
+test('loadConfig defaults the numeric knobs to the built-in values', () => {
+  const cfg = loadConfig({});
+  assert.equal(cfg.chunkSize, 4 * 1024 * 1024);
+  assert.equal(cfg.fallbackFreshSec, 6 * 3600);
+  assert.equal(cfg.expiryMarginSec, 60);
+  assert.equal(cfg.metaTtlMs, 50 * 60 * 1000);
+});
+
+test('loadConfig honours the numeric knob overrides', () => {
+  const cfg = loadConfig({
+    KIKOERU_CHUNK_SIZE: '1048576',
+    KIKOERU_FALLBACK_FRESH_SEC: '7200',
+    KIKOERU_EXPIRY_MARGIN_SEC: '300',
+    KIKOERU_META_TTL_MS: '60000',
+  });
+  assert.equal(cfg.chunkSize, 1048576);
+  assert.equal(cfg.fallbackFreshSec, 7200);
+  assert.equal(cfg.expiryMarginSec, 300);
+  assert.equal(cfg.metaTtlMs, 60000);
+});
+
+test('loadConfig falls back to the defaults for invalid knob values', () => {
+  for (const bad of ['abc', '0', '-5', '']) {
+    const cfg = loadConfig({
+      KIKOERU_CHUNK_SIZE: bad,
+      KIKOERU_FALLBACK_FRESH_SEC: bad,
+      KIKOERU_EXPIRY_MARGIN_SEC: bad,
+      KIKOERU_META_TTL_MS: bad,
+    });
+    const label = JSON.stringify(bad);
+    assert.equal(cfg.chunkSize, 4 * 1024 * 1024, `chunkSize for ${label}`);
+    assert.equal(cfg.fallbackFreshSec, 6 * 3600, `fallbackFreshSec for ${label}`);
+    assert.equal(cfg.expiryMarginSec, 60, `expiryMarginSec for ${label}`);
+    assert.equal(cfg.metaTtlMs, 50 * 60 * 1000, `metaTtlMs for ${label}`);
+  }
+});
+
 // --- ensureToken ----------------------------------------------------------------
 
 test('ensureToken returns the configured token without touching the file', () => {

package/test/helpers.js

@@ -104,12 +104,15 @@ function makeMediaUpstream(data, contentType = 'audio/mp4') {
     const range = req.headers.range || null;
     state.requests.push({ url: req.url, range });
     if (!range) state.plainGets++;
-    if (req.url === '/redir') {
+    // Route on the pathname only: stream URLs may carry a query
+    // string (e.g. ?expire=...) and must still reach /media.
+    const pathname = (req.url || '').split('?')[0];
+    if (pathname === '/redir') {
       res.writeHead(302, { Location: '/media' });
       res.end();
       return;
     }
-    if (req.url !== '/media') {
+    if (pathname !== '/media') {
       res.writeHead(404);
       res.end();
       return;