channel-worker 2.1.3 → 2.1.5

package/lib/command-poller.js

@@ -1158,14 +1158,14 @@ class CommandPoller {
       });
       const offlineRenderers = renderers.filter(r => !runningRenderers.includes(r));
 
-      // 3. Crash recovery — re-launch offline profiles that have assigned commands
+      // 3. Crash recovery
+      // 3a. Offline profiles with assigned commands → reset + relaunch
       for (const r of offlineRenderers) {
         try {
           const cmdCount = await this.api.rendererHasCommands(r.nst_profile_id);
           if (cmdCount > 0) {
             console.log(`[scene-dispatch] Crash recovery: ${r.name} has ${cmdCount} commands but not running — relaunching`);
             try {
-              // Reset stuck "running" commands back to "pending" so extension can pick them up
               await this.api.resetRendererCommands(r.nst_profile_id);
               await this._launchRendererProfile(r);
               runningRenderers.push(r);
@@ -1177,17 +1177,41 @@ class CommandPoller {
         } catch {}
       }
 
+      // 3b. Running profiles with stale heartbeat + PENDING commands → extension never started
+      // Only restart if commands are still "pending" (not "running" = extension picked up but busy)
+      const HEARTBEAT_STALE = 3 * 60 * 1000; // 3 minutes (extension can be busy rendering for 5+ min)
+      const now = Date.now();
+      for (const r of [...runningRenderers]) {
+        try {
+          // Only check profiles that have pending commands (not running = actively processing)
+          const cmdCount = await this.api.rendererHasCommands(r.nst_profile_id);
+          if (cmdCount === 0) continue;
+          const hb = r.last_heartbeat ? new Date(r.last_heartbeat).getTime() : 0;
+          if (hb && (now - hb) < HEARTBEAT_STALE) continue;
+          console.log(`[scene-dispatch] Stuck: ${r.name} running but heartbeat stale (${hb ? Math.round((now - hb) / 1000) + 's' : 'never'}) — restarting`);
+          try {
+            await this.nst.stopProfile(r.nst_profile_id).catch(() => {});
+            await this.api.resetRendererCommands(r.nst_profile_id);
+            await this._launchRendererProfile(r);
+            console.log(`[scene-dispatch] ${r.name} restarted`);
+          } catch (err) {
+            console.error(`[scene-dispatch] Failed to restart ${r.name}: ${err.message}`);
+          }
+        } catch {}
+      }
+
       // 4. Check queued commands
       const queueCount = await this.api.getSceneQueueCount();
       if (!queueCount) { this._dispatching = false; return; }
 
-      // 5. Launch new profiles up to parallel limit
+      // 5. Launch new profiles up to parallel limit (recompute offline after recovery)
       const parallelLimit = parseInt(await this.api.getSetting('veo3_parallel_limit')) || 1;
-      console.log(`[scene-dispatch] running=${runningRenderers.length}/${parallelLimit} offline=${offlineRenderers.length} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
+      const stillOffline = renderers.filter(r => !runningRenderers.includes(r));
+      console.log(`[scene-dispatch] running=${runningRenderers.length}/${parallelLimit} offline=${stillOffline.length} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
 
-      const neededLaunches = Math.min(parallelLimit - runningRenderers.length, offlineRenderers.length, queueCount);
+      const neededLaunches = Math.min(parallelLimit - runningRenderers.length, stillOffline.length, queueCount);
       for (let li = 0; li < neededLaunches; li++) {
-        const toLaunch = offlineRenderers[li];
+        const toLaunch = stillOffline[li];
         console.log(`[scene-dispatch] Launching ${toLaunch.name} (running: ${runningRenderers.length + li}/${parallelLimit})`);
         try {
           await this._launchRendererProfile(toLaunch);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.1.3",
+  "version": "2.1.5",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {