@muhkoo/theater-transcoder 0.3.2 → 0.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@muhkoo/theater-transcoder",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Drain a Muhkoo Theater transcode queue with NATIVE ffmpeg. Run it on any machine signed in as you to add transcoding power to your library.",
   "type": "module",
   "bin": {
@@ -25,10 +25,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@muhkoo/connect": "0.10.2-alpha.0",
+    "@muhkoo/connect": "0.10.4-alpha.0",
     "ffmpeg-static": "^5.2.0",
     "node-datachannel": "^0.12.0",
     "snarkjs": "^0.7.5",
     "ws": "^8.18.0"
-  }
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

package/src/worker.js

@@ -38,6 +38,33 @@ export function startWorker({ client, space, appKey, baseUrl }) {
 
   const patch = (id, values) => jobs().update(id, { ...values, updated_at: Date.now() });
 
+  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+  // Fetch the raw by manifest, retrying while the uploader's background upload
+  // finishes landing on origin (or a P2P peer appears). A missing-shard error is
+  // transient here — the bytes are still arriving — so we back off and retry
+  // instead of failing the job. Only a genuine, persistent gap eventually throws.
+  async function fetchRawWithRetry(space, manifest, job) {
+    const deadline = Date.now() + 8 * 60_000; // headroom for a slow big-file upload
+    let attempt = 0;
+    for (;;) {
+      try {
+        const { data } = await space.getFile(manifest);
+        return data;
+      } catch (e) {
+        const msg = String(e?.message || e);
+        const transient = /shard|unrecoverable|missing|404|network|fetch|econn|timeout/i.test(msg);
+        if (!transient || Date.now() > deadline) throw e;
+        attempt++;
+        const wait = Math.min(8000, 1500 * attempt);
+        log(`raw not fully available yet (${msg.slice(0, 70)}) — retry ${attempt} in ${wait}ms`);
+        await patch(job._id, { phase: "Fetching raw… (uploading)", heartbeat_at: Date.now() }).catch(() => {});
+        await postSignal({ kind: "progress", jobId: job._id, progress: 0, phase: "Fetching raw… (uploading)", worker: WORKER_LABEL });
+        await sleep(wait);
+      }
+    }
+  }
+
   async function claimNext() {
     const { rows } = await jobs().query({
       where: [{ column: "owner", op: "eq", value: commitment }],
@@ -86,7 +113,14 @@ export function startWorker({ client, space, appKey, baseUrl }) {
       // Read through the Space's getFile (NOT client.storage.readByManifest):
       // only the Space file surface has the P2P peer source wired, so this pulls
       // shards from the uploader's browser over WebRTC first, origin as fallback.
-      const { data } = await space.getFile(manifest);
+      //
+      // Because the uploader returns the manifest as soon as shards are cached +
+      // announced (background upload), origin may not hold every shard yet when we
+      // first ask — and if P2P can't reach the uploader, a shard 404s and getFile
+      // fails "N shards missing". That's transient: the upload is still landing, so
+      // retry (with backoff) until we can reassemble or we time out. The heartbeat
+      // ticker keeps our claim alive throughout.
+      const data = await fetchRawWithRetry(space, manifest, job);
       log(`transcoding ${(data.length / 1e6).toFixed(1)}MB…`);
       const { hls_index, duration, size } = await transcodeToHls({
         rawBytes: data, filename: job.filename || "input.mp4", space, onProgress, signal: abort.signal,