@muhkoo/theater-transcoder 0.1.0

package/README.md

@@ -0,0 +1,39 @@
+# @muhkoo/theater-transcoder
+
+A tiny **native-ffmpeg worker** for [Muhkoo Theater](https://muhkoo-theater.apps.muhkoo.dev). Sign in as yourself and it drains your library's transcode queue using all your cores — so uploads finish far faster than in-browser WebAssembly (which is ~20–40× slower and capped at a few threads).
+
+Run it on a spare machine (a many-core server, say) **alongside or instead of** the browser. It coordinates with your browser tabs as a peer over your private Muhkoo Space: any device can claim a job, progress streams live everywhere, and pause/cancel from any device stops it here too.
+
+## Install & run
+
+```bash
+# one-off (no install)
+npx @muhkoo/theater-transcoder --username you
+
+# or install globally
+npm i -g @muhkoo/theater-transcoder
+MUHKOO_USERNAME=you MUHKOO_PASSWORD=… theater-transcoder
+```
+
+You'll be prompted for your password if it isn't in `$MUHKOO_PASSWORD`. Credentials never leave the machine — it runs the same **zero-knowledge login** the browser uses. `ffmpeg` is bundled (`ffmpeg-static`); nothing else to install. Requires **Node ≥ 20** (≥ 22 recommended).
+
+Leave it running. Upload videos in the app and this box starts chewing through the queue. Ctrl-C to stop.
+
+## How it works
+
+1. **ZK login** as you → obtains the group key for your per-user library Space via the always-online keeper (no other device need be online).
+2. **Claims** queued jobs from the shared `jobs` table (best-effort claim + heartbeat lease, identical to the browser worker).
+3. **Fetches the raw** upload by manifest (`client.storage.readByManifest` — over origin; the browser peers it over WebRTC on a LAN).
+4. **Transcodes** with native ffmpeg to HLS (720p, 6s segments, MSE-safe: `yuv420p` / `main` / stereo / keyframe-per-segment) — byte-compatible with what the browser writes.
+5. **Stores** each segment as an encrypted shard and writes the `media` row + marks the job done, then nudges the `@scanner` agent to fetch poster/metadata.
+
+Offline + P2P are auto-disabled in Node (no IndexedDB / WebRTC), so its shard and DB writes are identical to the browser's.
+
+## Options
+
+| flag | env | default |
+|---|---|---|
+| `--username` | `MUHKOO_USERNAME` | — (required) |
+| `--password` | `MUHKOO_PASSWORD` | prompt |
+| `--base` | `MUHKOO_BASE_URL` | `https://api.muhkoo.dev` |
+| `--app-key` | `MUHKOO_THEATER_KEY` | public prod key |

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@muhkoo/theater-transcoder",
+  "version": "0.1.0",
+  "description": "Drain a Muhkoo Theater transcode queue with NATIVE ffmpeg. Run it on any machine signed in as you to add transcoding power to your library.",
+  "type": "module",
+  "bin": {
+    "theater-transcoder": "src/index.js"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "license": "MIT",
+  "keywords": ["muhkoo", "transcode", "ffmpeg", "hls", "worker"],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@muhkoo/connect": "0.10.1-alpha.0",
+    "ffmpeg-static": "^5.2.0",
+    "snarkjs": "^0.7.5",
+    "ws": "^8.18.0"
+  }
+}

package/src/client.js

@@ -0,0 +1,76 @@
+/**
+ * Connect to Muhkoo as a user (headless), and open the per-user media Space.
+ *
+ * Mirrors the browser app: the media Space name is derived from the user's
+ * commitment, and its group key is obtained via the always-online keeper — so a
+ * worker can join with nobody else present and read/write the same encrypted
+ * shards the browser does. Offline + P2P are auto-disabled in Node (no
+ * IndexedDB / RTCPeerConnection), so shard + DB writes are byte-identical.
+ */
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { WebSocket } from "ws";
+import { Client } from "@muhkoo/connect";
+
+// Node 20's global WebSocket is flag-gated; ensure the Space socket has one.
+if (!globalThis.WebSocket) globalThis.WebSocket = WebSocket;
+
+/** snarkjs (Node) reads circuit assets off disk — download them to a temp dir. */
+async function downloadCircuits(baseUrl) {
+  const dir = await mkdtemp(join(tmpdir(), "muhkoo-circuits-"));
+  const fetchTo = async (path, file) => {
+    const res = await fetch(baseUrl + path);
+    if (!res.ok) throw new Error(`circuit asset ${path} → ${res.status}`);
+    const out = join(dir, file);
+    await writeFile(out, Buffer.from(await res.arrayBuffer()));
+    return out;
+  };
+  const [wasmUrl, zkeyUrl] = await Promise.all([
+    fetchTo("/circuits/build/preimagePoK_js/preimagePoK.wasm", "preimagePoK.wasm"),
+    fetchTo("/circuits/build/preimagePoK_0001.zkey", "preimagePoK_0001.zkey"),
+  ]);
+  return { wasmUrl, zkeyUrl };
+}
+
+/** The per-user media Space name (must match the browser app's ids.ts). */
+export function mediaSpaceName(commitment) {
+  const suffix = commitment ? commitment.replace(/\D/g, "").slice(-24) : "anon";
+  return `theater-${suffix}`;
+}
+
+/** Log in as the user and return a ready Client. */
+export async function connect({ baseUrl, appKey, username, password }) {
+  const circuits = await downloadCircuits(baseUrl);
+  const client = new Client({
+    baseUrl,
+    apiKey: appKey,
+    circuits,
+    offline: { enabled: false },
+    p2p: { enabled: false },
+  });
+  const user = await client.auth.zk.login(username, password);
+  return { client, user };
+}
+
+/** Join (or create) the user's media Space and wait until we hold the group key. */
+export async function openMediaSpace(client) {
+  const commitment = client.auth.zk.user?.commitment;
+  const name = mediaSpaceName(commitment);
+  let space;
+  try { space = await client.space.joinChannel(name); }
+  catch { space = await client.space.createChannel(name); }
+  const kr = space.keyring;
+  if (kr) {
+    try { await kr.loadFromCache(); } catch { /* new */ }
+    const deadline = Date.now() + 20000;
+    while (!kr.hasAnyKey() && Date.now() < deadline) {
+      await kr.requestKey().catch(() => {});
+      await kr.pullKeys().catch(() => {});
+      if (kr.hasAnyKey()) break;
+      await new Promise((r) => setTimeout(r, 800));
+    }
+    if (!kr.hasAnyKey()) throw new Error("could not obtain the library group key");
+  }
+  return { space, name };
+}

package/src/index.js

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+/**
+ * theater-transcoder — a native-ffmpeg worker for Muhkoo Theater.
+ *
+ * Sign in as yourself and it drains your library's transcode queue using all
+ * your cores. Run it on a beefy machine (a 48-core box, say) alongside — or
+ * instead of — the browser, and uploads finish far faster. Coordinates with your
+ * browsers as a peer over your private Space; the raw file comes from whichever
+ * device has it (P2P in the browser, origin here).
+ */
+import { connect, openMediaSpace } from "./client.js";
+import { startWorker } from "./worker.js";
+
+const HELP = `theater-transcoder — native-ffmpeg worker for Muhkoo Theater
+
+Usage:
+  theater-transcoder --username <you> [--password <pw>]
+
+Auth (credentials never leave your machine — they run the same zero-knowledge
+login the browser uses):
+  --username <u>        or  $MUHKOO_USERNAME
+  --password <p>        or  $MUHKOO_PASSWORD   (omit to be prompted; avoid on the CLI)
+
+Options:
+  --base <url>          API base (default https://api.muhkoo.dev, or $MUHKOO_BASE_URL)
+  --app-key <mk_*_pk_*> Theater app key (defaults to the public prod key)
+  -h, --help
+
+Requires: Node >= 20. ffmpeg is bundled (ffmpeg-static) — nothing to install.`;
+
+function parseArgs(argv) {
+  const a = {};
+  for (let i = 0; i < argv.length; i++) {
+    const t = argv[i];
+    if (t === "-h" || t === "--help") a.help = true;
+    else if (t.startsWith("--")) { const k = t.slice(2); a[k] = argv[i + 1]?.startsWith("--") || argv[i + 1] === undefined ? true : argv[++i]; }
+  }
+  return a;
+}
+
+async function promptHidden(q) {
+  if (!process.stdin.isTTY) return "";
+  process.stdout.write(q);
+  return new Promise((resolve) => {
+    process.stdin.setRawMode(true); process.stdin.resume(); process.stdin.setEncoding("utf8");
+    let buf = "";
+    const on = (ch) => {
+      if (ch === "\n" || ch === "\r" || ch === "") { process.stdin.setRawMode(false); process.stdin.pause(); process.stdin.removeListener("data", on); process.stdout.write("\n"); resolve(buf); }
+      else if (ch === "") process.exit(130);
+      else if (ch === "") buf = buf.slice(0, -1);
+      else buf += ch;
+    };
+    process.stdin.on("data", on);
+  });
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) { console.log(HELP); return; }
+
+  const baseUrl = (args.base || process.env.MUHKOO_BASE_URL || "https://api.muhkoo.dev").replace(/\/+$/, "");
+  const appKey = args["app-key"] || process.env.MUHKOO_THEATER_KEY || "mk_live_pk_80c0758c9e2f7d695d3f5f7fd2ee8458";
+  const username = args.username || process.env.MUHKOO_USERNAME;
+  let password = args.password || process.env.MUHKOO_PASSWORD;
+  if (!username) { console.error("Missing --username (or $MUHKOO_USERNAME).\n\n" + HELP); process.exit(1); }
+  if (!password) password = await promptHidden(`Password for ${username}: `);
+  if (!password) { console.error("A password is required."); process.exit(1); }
+
+  console.log(`Signing in as ${username} → ${baseUrl}…`);
+  const { client, user } = await connect({ baseUrl, appKey, username, password });
+  console.log(`✓ signed in as ${user.username} (…${user.commitment.slice(-8)})`);
+  const { space, name } = await openMediaSpace(client);
+  console.log(`✓ joined your library space "${name}"`);
+  startWorker({ client, space, appKey, baseUrl });
+  console.log("Ready — waiting for jobs. Leave this running; Ctrl-C to stop.");
+}
+
+main().catch((e) => { console.error("ERROR:", e?.stack || e?.message || e); process.exit(1); });

package/src/transcode.js

@@ -0,0 +1,112 @@
+/**
+ * Native-ffmpeg transcode → HLS, producing the SAME structure the browser writes
+ * so playback is identical: each ~6s segment is stored as an encrypted shard via
+ * `space.putFile` (→ manifest), and an HLS index `{variants:[{height,bandwidth,
+ * segments:[{duration,manifest}]}],duration}` is stored as its own file. Native
+ * ffmpeg has no wasm limits, so we use all cores (`-threads 0`) and a nicer
+ * preset than the browser's `ultrafast`.
+ */
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import ffmpegPath from "ffmpeg-static";
+
+// Match the browser's rendition ladder (single 720p for now).
+const RENDITIONS = [{ height: 720, videoBitrate: "2500k", audioBitrate: "128k", bandwidth: 2_800_000 }];
+const SEGMENT_SECONDS = 6;
+
+function parseDurations(m3u8) {
+  const out = [];
+  for (const line of m3u8.split("\n")) {
+    const m = line.match(/^#EXTINF:([\d.]+)/);
+    if (m) out.push(parseFloat(m[1]));
+  }
+  return out;
+}
+
+function hhmmssToSec(s) {
+  const m = /(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(s);
+  return m ? Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]) : 0;
+}
+
+/** Run one ffmpeg rendition to HLS; report fraction via stderr `time=` vs total. */
+function runFfmpeg(inPath, dir, rend, playlist, onFrac, signal) {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "-i", inPath,
+      "-vf", `scale=-2:${rend.height}`,
+      "-threads", "0", // native: use all cores
+      "-c:v", "libx264", "-preset", "veryfast", "-b:v", rend.videoBitrate,
+      "-pix_fmt", "yuv420p", "-profile:v", "main", "-level", "4.0",
+      "-force_key_frames", `expr:gte(t,n_forced*${SEGMENT_SECONDS})`, "-sc_threshold", "0",
+      "-c:a", "aac", "-ac", "2", "-b:a", rend.audioBitrate,
+      "-hls_time", String(SEGMENT_SECONDS), "-hls_playlist_type", "vod",
+      "-hls_segment_filename", join(dir, `r${rend.height}_%03d.ts`),
+      "-y", playlist,
+    ];
+    const ff = spawn(ffmpegPath, args, { stdio: ["ignore", "ignore", "pipe"] });
+    let total = 0;
+    let stderr = "";
+    ff.stderr.on("data", (b) => {
+      const s = b.toString();
+      stderr = (stderr + s).slice(-4000);
+      if (!total) { const d = /Duration:\s*(\d+:\d+:[\d.]+)/.exec(s); if (d) total = hhmmssToSec(d[1]); }
+      const t = /time=(\d+:\d+:[\d.]+)/.exec(s);
+      if (t && total) onFrac(Math.max(0, Math.min(1, hhmmssToSec(t[1]) / total)));
+    });
+    if (signal) signal.addEventListener("abort", () => { try { ff.kill("SIGKILL"); } catch { /* dead */ } }, { once: true });
+    ff.on("error", reject);
+    ff.on("close", (code) => {
+      if (signal?.aborted) reject(new Error("aborted"));
+      else if (code === 0) resolve();
+      else reject(new Error(`ffmpeg exited ${code}: ${stderr.slice(-500)}`));
+    });
+  });
+}
+
+/**
+ * Transcode raw bytes into HLS shards stored in `space`.
+ * @returns {{ hls_index: unknown, duration: number, size: number }}
+ */
+export async function transcodeToHls({ rawBytes, filename, space, onProgress, signal }) {
+  const dir = await mkdtemp(join(tmpdir(), "tt-"));
+  try {
+    const ext = filename.match(/\.\w+$/)?.[0] || ".mp4";
+    const inPath = join(dir, `input${ext}`);
+    await writeFile(inPath, rawBytes);
+
+    const nR = RENDITIONS.length;
+    const variants = [];
+    let size = 0;
+    for (let r = 0; r < nR; r++) {
+      const rend = RENDITIONS[r];
+      const playlist = join(dir, `r${rend.height}.m3u8`);
+      onProgress?.({ fraction: r / nR, label: `Encoding ${rend.height}p…` });
+      await runFfmpeg(inPath, dir, rend, playlist, (f) => onProgress?.({ fraction: (r + f) / nR, label: `Encoding ${rend.height}p…` }), signal);
+
+      const durations = parseDurations(await readFile(playlist, "utf8"));
+      const segments = [];
+      for (let i = 0; i < durations.length; i++) {
+        if (signal?.aborted) throw new Error("aborted");
+        const name = `r${rend.height}_${String(i).padStart(3, "0")}.ts`;
+        const bytes = await readFile(join(dir, name));
+        size += bytes.length;
+        const { manifest } = await space.putFile(new Uint8Array(bytes), { name, type: "video/mp2t" });
+        segments.push({ duration: durations[i], manifest });
+        onProgress?.({ fraction: (r + 1) / nR, label: `Uploading ${i + 1}/${durations.length} segments…` });
+      }
+      variants.push({ height: rend.height, bandwidth: rend.bandwidth, segments });
+    }
+
+    const duration = variants[0]?.segments.reduce((s, x) => s + x.duration, 0) ?? 0;
+    const index = { variants, duration };
+    const { manifest: hls_index } = await space.putFile(
+      new TextEncoder().encode(JSON.stringify(index)),
+      { name: "hls-index.json", type: "application/json" },
+    );
+    return { hls_index, duration, size };
+  } finally {
+    await rm(dir, { recursive: true, force: true }).catch(() => {});
+  }
+}

package/src/worker.js

@@ -0,0 +1,121 @@
+/**
+ * The worker: drains the shared `jobs` queue with the same claim/lease + Space
+ * signaling protocol the browser uses, so browsers and transcoders coordinate as
+ * peers. Only jobs whose raw file has been persisted to shards (`input_manifest`
+ * set) are claimable here — a browser's freshly-created job is stored in the
+ * background, and the browser can also process it locally from memory.
+ */
+import { hostname } from "node:os";
+import { transcodeToHls } from "./transcode.js";
+
+const WORKER_ID = `node-${Math.random().toString(36).slice(2, 10)}`;
+const WORKER_LABEL = `the transcoder (${hostname()})`;
+const STALE_MS = 90_000;
+const POLL_MS = 4_000;
+
+const log = (...a) => console.log(new Date().toISOString().slice(11, 19), ...a);
+
+export function startWorker({ client, space, appKey, baseUrl }) {
+  const commitment = client.auth.zk.user.commitment;
+  const jobs = () => client.db.table("jobs");
+  const media = () => client.db.table("media");
+  let activeJobId = null;
+  let abort = null; // AbortController for the running ffmpeg
+
+  const postSignal = async (sig) => {
+    try { await space.sendMessage({ _t: "jobsig", src: WORKER_ID, ...sig }); } catch { /* best-effort */ }
+  };
+
+  // Cross-device control: another device asks to pause/cancel a job we're running.
+  space.onMessage((e) => {
+    const b = e?.message?.body;
+    if (!b || b._t !== "jobsig" || b.src === WORKER_ID) return;
+    if (b.kind === "control" && b.jobId === activeJobId && abort) { log("remote stop for job", b.jobId); abort.abort(); }
+  });
+
+  const patch = (id, values) => jobs().update(id, { ...values, updated_at: Date.now() });
+
+  async function claimNext() {
+    const { rows } = await jobs().query({
+      where: [{ column: "owner", op: "eq", value: commitment }],
+      orderBy: { column: "created_at", dir: "asc" },
+      limit: 100,
+    });
+    const now = Date.now();
+    const cand = rows.find((j) =>
+      j.input_manifest && (
+        j.status === "queued" ||
+        (j.status === "processing" && (!j.heartbeat_at || now - j.heartbeat_at > STALE_MS))
+      ));
+    if (!cand) return null;
+    await patch(cand._id, { status: "processing", worker_id: WORKER_ID, worker_label: WORKER_LABEL, claimed_at: now, heartbeat_at: now });
+    const fresh = await jobs().get(cand._id);
+    const row = fresh?.row ?? fresh;
+    if (!row || row.worker_id !== WORKER_ID) return claimNext(); // lost the race
+    return row;
+  }
+
+  async function process(job) {
+    activeJobId = job._id;
+    abort = new AbortController();
+    let lastBeat = 0;
+    const onProgress = ({ fraction, label }) => {
+      const now = Date.now();
+      if (now - lastBeat > 4000) {
+        lastBeat = now;
+        void patch(job._id, { progress: fraction, phase: label, heartbeat_at: now }).catch(() => {});
+        void postSignal({ kind: "progress", jobId: job._id, progress: fraction, phase: label, worker: WORKER_LABEL });
+      }
+    };
+    try {
+      log(`claim job ${job._id} "${job.title}" — downloading raw…`);
+      const { data } = await client.storage.readByManifest(job.input_manifest);
+      log(`transcoding ${(data.length / 1e6).toFixed(1)}MB…`);
+      const { hls_index, duration, size } = await transcodeToHls({
+        rawBytes: data, filename: job.filename || "input.mp4", space, onProgress, signal: abort.signal,
+      });
+      const { row } = await media().insert({
+        title: job.title, year: null, genre: null, description: null, kind: "video",
+        duration, poster: null, hls_index, size, owner: commitment, created_at: Date.now(),
+      });
+      await patch(job._id, { status: "done", progress: 1, phase: "Done", media_id: row._id });
+      await postSignal({ kind: "done", jobId: job._id });
+      void requestScan(row._id, job.filename || job.title);
+      log(`✓ done job ${job._id} → media ${row._id}`);
+    } catch (e) {
+      if (abort.signal.aborted) { log(`job ${job._id} stopped remotely`); }
+      else { await patch(job._id, { status: "error", error: String(e?.message || e) }).catch(() => {}); log(`✗ job ${job._id} failed:`, e?.message || e); }
+    } finally {
+      activeJobId = null; abort = null;
+    }
+  }
+
+  // Best-effort: ask the @scanner agent to enrich the new media (title/poster).
+  let scannerSpace;
+  async function requestScan(mediaId, filename) {
+    try {
+      if (!scannerSpace) scannerSpace = await client.space.joinChannel("scanner").catch(() => client.space.createChannel("scanner"));
+      await scannerSpace.sendMessage({ contents: `@scanner scan mediaId=${mediaId} file=${JSON.stringify(filename)}` });
+    } catch { /* enrichment is optional */ }
+  }
+
+  let stopped = false;
+  (async () => {
+    log(`worker ${WORKER_ID} draining queue as ${client.auth.zk.user.username} (${WORKER_LABEL})`);
+    // Real-time wake on new jobs; the poll below is the durable fallback.
+    space.onMessage((e) => { const b = e?.message?.body; if (b?._t === "jobsig" && b.kind === "new" && b.src !== WORKER_ID) tick(); });
+    let ticking = false;
+    async function tick() {
+      if (ticking || stopped) return;
+      ticking = true;
+      try { for (let job = await claimNext(); job; job = await claimNext()) await process(job); }
+      catch (e) { log("loop error:", e?.message || e); }
+      finally { ticking = false; }
+    }
+    await tick();
+    const iv = setInterval(tick, POLL_MS);
+    process.on?.("SIGINT", () => { stopped = true; clearInterval(iv); log("shutting down"); process.exit(0); });
+  })();
+
+  void baseUrl; void appKey; // reserved for future direct-API needs
+}