channel-worker 2.5.1 → 2.5.3

package/lib/command-poller.js

@@ -1336,52 +1336,28 @@ class CommandPoller {
       } catch (e) {
         if (this.config.verbose) console.warn('[scene-dispatch] getMyWorker failed, using default:', e.message);
       }
-      const stillOffline = renderers.filter(r => !runningRenderers.includes(r));
-      console.log(`[scene-dispatch] running=${runningRenderers.length}/${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
-
-      // 5b. Close excess profiles — when parallelLimit drops (e.g., after a DOM→FlowKit
-      //     switch), idle profiles over the cap should shut down instead of staying open
-      //     and getting throttled at /flowkit/claim-command.
-      if (runningRenderers.length > parallelLimit) {
-        const excess = runningRenderers.length - parallelLimit;
-        const idleRenderers = [];
-        for (const r of runningRenderers) {
-          // Only daemon-launched profiles are eligible — skip manual launches
-          if (!this._profileLastActivity[r.nst_profile_id]) continue;
-          try {
-            const c = await this.api.rendererHasCommands(r.nst_profile_id);
-            if (c === 0) idleRenderers.push(r);
-          } catch {}
-        }
-        const toClose = idleRenderers.slice(0, excess);
-        for (const r of toClose) {
-          console.log(`[scene-dispatch] Closing excess profile ${r.name} (over cap ${parallelLimit})`);
-          try {
-            await this.nst.stopProfile(r.nst_profile_id);
-            delete this._profileLastActivity[r.nst_profile_id];
-            if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
-            const idx = runningRenderers.indexOf(r);
-            if (idx >= 0) runningRenderers.splice(idx, 1);
-          } catch (e) {
-            console.warn(`[scene-dispatch] Failed to close excess ${r.name}: ${e.message}`);
-          }
-        }
-      }
-
-      // Count how many running profiles are actually free (no commands)
-      let freeRunning = 0;
-      for (const r of runningRenderers) {
-        try {
-          const c = await this.api.rendererHasCommands(r.nst_profile_id);
-          if (c === 0) freeRunning++;
-        } catch {}
-      }
-      // Only launch new profiles if queue has more work than free profiles
-      const needNew = Math.max(0, queueCount - freeRunning);
-      const neededLaunches = Math.min(parallelLimit - runningRenderers.length, stillOffline.length, needNew);
-      for (let li = 0; li < neededLaunches; li++) {
-        const toLaunch = stillOffline[li];
-        console.log(`[scene-dispatch] Launching ${toLaunch.name} (running: ${runningRenderers.length + li}/${parallelLimit})`);
+      // Treat 'paused' renderers as NOT eligible to launch — operator clicked
+      // Pause on the Renderers tab; profile should stay closed until Resume.
+      const isPaused = (r) => r && r.health_state === 'paused'
+        && (!r.pause_until || new Date(r.pause_until).getTime() > Date.now());
+      const stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
+      const pausedCount = renderers.filter(isPaused).length;
+      console.log(`[scene-dispatch] running=${runningRenderers.length} cap=${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} paused=${pausedCount} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
+
+      // Launch policy: keep ALL non-paused owned renderers ONLINE whenever
+      // there's any queue work — they take turns claiming via the API's
+      // per-worker concurrency cap (Worker.parallel_limit). The cap limits
+      // CONCURRENT CLAIMS, not the number of profiles kept open: e.g. with
+      // parallel_limit=1 + 2 renderers veo01/veo02, both browsers stay open
+      // and alternate scene claims. Operator pauses a specific renderer if
+      // they want it closed (e.g. while solving captcha).
+      //
+      // Old "Close excess profiles" + parallelLimit-bounded launch logic was
+      // removed — they conflated "concurrent claims cap" with "max launched"
+      // and forced one of the renderers to sit unlaunched when cap < count.
+      const eligibleToLaunch = queueCount > 0 ? stillOffline : [];
+      for (const toLaunch of eligibleToLaunch) {
+        console.log(`[scene-dispatch] Launching ${toLaunch.name} (queue=${queueCount} running=${runningRenderers.length})`);
         try {
           await this._launchRendererProfile(toLaunch);
           runningRenderers.push(toLaunch);
@@ -1392,6 +1368,26 @@ class CommandPoller {
         }
       }
 
+      // Close any currently-running renderers that just became paused —
+      // operator clicked Pause while a profile was open. Skip if it has an
+      // in-flight command (let it finish naturally); cleanup happens on the
+      // next poll cycle.
+      for (const r of [...runningRenderers]) {
+        if (!isPaused(r)) continue;
+        try {
+          const c = await this.api.rendererHasCommands(r.nst_profile_id);
+          if (c > 0) continue;
+          console.log(`[scene-dispatch] Closing paused renderer ${r.name}`);
+          await this.nst.stopProfile(r.nst_profile_id);
+          delete this._profileLastActivity[r.nst_profile_id];
+          if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
+          const idx = runningRenderers.indexOf(r);
+          if (idx >= 0) runningRenderers.splice(idx, 1);
+        } catch (e) {
+          console.warn(`[scene-dispatch] Failed to close paused ${r.name}: ${e.message}`);
+        }
+      }
+
       if (runningRenderers.length === 0) {
         this._dispatching = false;
         return;
@@ -1438,17 +1434,24 @@ class CommandPoller {
     // two daemons would happily double-launch and corrupt cookies / kill
     // sessions for the underlying Google/FB accounts.
     if (renderer.nst_profile_id && this.api.leaseRenderer) {
+      // api.leaseRenderer returns json.data on success (a {nst_profile_id,
+      // leased_at, lessee} envelope) and THROWS on success=false. So:
+      //   - throw → another daemon holds the lease OR API rejected → skip
+      //   - return → lease acquired → proceed with NST launch
+      // Previous bug: checked `leaseRes.success !== true` but `.success` is
+      // already stripped from json.data → always falsy → every launch was
+      // wrongfully denied with reason="unknown".
       try {
         const leaseRes = await this.api.leaseRenderer(renderer.nst_profile_id);
-        if (!leaseRes || leaseRes.success !== true) {
-          const reason = leaseRes && leaseRes.message ? leaseRes.message : 'unknown';
-          console.warn(`[scene-dispatch] LEASE DENIED for ${renderer.name || renderer.nst_profile_id}: ${reason} — skipping launch`);
+        if (!leaseRes || !leaseRes.nst_profile_id) {
+          console.warn(`[scene-dispatch] lease returned no data for ${renderer.name || renderer.nst_profile_id} — skipping launch (API may be older)`);
           return;
         }
       } catch (e) {
-        // Treat lease failure as a soft block — the API might be down OR
-        // running an older version. Better to skip than risk double-launch.
-        console.warn(`[scene-dispatch] lease request failed for ${renderer.name || renderer.nst_profile_id}: ${e.message} — skipping launch`);
+        // Either API said success=false (lease held by peer / unowned /
+        // stale rules failed) OR network failure. Either way, do NOT
+        // launch — better idle than collision.
+        console.warn(`[scene-dispatch] LEASE DENIED for ${renderer.name || renderer.nst_profile_id}: ${e.message} — skipping launch`);
         return;
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.1",
+  "version": "2.5.3",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {