channel-worker 2.5.2 → 2.5.4

package/lib/command-poller.js

@@ -1336,26 +1336,81 @@ class CommandPoller {
       } catch (e) {
         if (this.config.verbose) console.warn('[scene-dispatch] getMyWorker failed, using default:', e.message);
       }
-      const stillOffline = renderers.filter(r => !runningRenderers.includes(r));
-      console.log(`[scene-dispatch] running=${runningRenderers.length}/${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
+      // Treat 'paused' renderers as NOT eligible to launch — operator clicked
+      // Pause on the Renderers tab; profile should stay closed until Resume.
+      const isPaused = (r) => r && r.health_state === 'paused'
+        && (!r.pause_until || new Date(r.pause_until).getTime() > Date.now());
+      const stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
+      const pausedCount = renderers.filter(isPaused).length;
+      console.log(`[scene-dispatch] running=${runningRenderers.length} cap=${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} paused=${pausedCount} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
+
+      // Launch policy: STRICT parallel_limit cap on physical launches.
+      // With pc2.parallel_limit=1 + 2 renderers (veo03/veo04 share same Veo
+      // Google account), only ONE of them is launched at any time. The
+      // second stays closed until the first finishes / is paused / cap is
+      // bumped. This prevents Google flagging "2 concurrent sessions on
+      // same account" → captcha.
+      //
+      // Earlier iteration tried "launch all, cap only claims" — that left
+      // both browsers open, both pinging Flow (token refresh, telemetry),
+      // and Veo's per-account session monitor counted them as 2 active
+      // sessions → tripped captcha on the supposedly-idle one.
+      const needNew = Math.max(0, queueCount);
+      const neededLaunches = Math.min(
+        parallelLimit - runningRenderers.length,
+        stillOffline.length,
+        needNew,
+      );
+      for (let li = 0; li < Math.max(0, neededLaunches); li++) {
+        const toLaunch = stillOffline[li];
+        console.log(`[scene-dispatch] Launching ${toLaunch.name} (${runningRenderers.length + li + 1}/${parallelLimit})`);
+        try {
+          await this._launchRendererProfile(toLaunch);
+          runningRenderers.push(toLaunch);
+          this._profileLastActivity[toLaunch.nst_profile_id] = Date.now();
+          console.log(`[scene-dispatch] ${toLaunch.name} launched`);
+        } catch (err) {
+          console.error(`[scene-dispatch] Failed to launch ${toLaunch.name}: ${err.message}`);
+        }
+      }
+
+      // Close any currently-running renderers that just became paused —
+      // operator clicked Pause while a profile was open. Skip if it has an
+      // in-flight command (let it finish naturally); cleanup happens on the
+      // next poll cycle.
+      for (const r of [...runningRenderers]) {
+        if (!isPaused(r)) continue;
+        try {
+          const c = await this.api.rendererHasCommands(r.nst_profile_id);
+          if (c > 0) continue;
+          console.log(`[scene-dispatch] Closing paused renderer ${r.name}`);
+          await this.nst.stopProfile(r.nst_profile_id);
+          delete this._profileLastActivity[r.nst_profile_id];
+          if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
+          const idx = runningRenderers.indexOf(r);
+          if (idx >= 0) runningRenderers.splice(idx, 1);
+        } catch (e) {
+          console.warn(`[scene-dispatch] Failed to close paused ${r.name}: ${e.message}`);
+        }
+      }
 
-      // 5b. Close excess profiles — when parallelLimit drops (e.g., after a DOM→FlowKit
-      //     switch), idle profiles over the cap should shut down instead of staying open
-      //     and getting throttled at /flowkit/claim-command.
+      // Close EXCESS running profiles when count > parallel_limit. Happens
+      // when operator drops the cap (eg 2→1 to throttle captcha) — the
+      // already-launched 2nd profile should retire so it doesn't keep
+      // refreshing tokens against the shared Google account. Only close
+      // if it has no in-flight command (let work finish first).
       if (runningRenderers.length > parallelLimit) {
         const excess = runningRenderers.length - parallelLimit;
-        const idleRenderers = [];
+        const idleRunning = [];
         for (const r of runningRenderers) {
-          // Only daemon-launched profiles are eligible — skip manual launches
           if (!this._profileLastActivity[r.nst_profile_id]) continue;
           try {
             const c = await this.api.rendererHasCommands(r.nst_profile_id);
-            if (c === 0) idleRenderers.push(r);
+            if (c === 0) idleRunning.push(r);
           } catch {}
         }
-        const toClose = idleRenderers.slice(0, excess);
-        for (const r of toClose) {
-          console.log(`[scene-dispatch] Closing excess profile ${r.name} (over cap ${parallelLimit})`);
+        for (const r of idleRunning.slice(0, excess)) {
+          console.log(`[scene-dispatch] Closing excess ${r.name} (cap=${parallelLimit})`);
           try {
             await this.nst.stopProfile(r.nst_profile_id);
             delete this._profileLastActivity[r.nst_profile_id];
@@ -1368,30 +1423,6 @@ class CommandPoller {
         }
       }
 
-      // Count how many running profiles are actually free (no commands)
-      let freeRunning = 0;
-      for (const r of runningRenderers) {
-        try {
-          const c = await this.api.rendererHasCommands(r.nst_profile_id);
-          if (c === 0) freeRunning++;
-        } catch {}
-      }
-      // Only launch new profiles if queue has more work than free profiles
-      const needNew = Math.max(0, queueCount - freeRunning);
-      const neededLaunches = Math.min(parallelLimit - runningRenderers.length, stillOffline.length, needNew);
-      for (let li = 0; li < neededLaunches; li++) {
-        const toLaunch = stillOffline[li];
-        console.log(`[scene-dispatch] Launching ${toLaunch.name} (running: ${runningRenderers.length + li}/${parallelLimit})`);
-        try {
-          await this._launchRendererProfile(toLaunch);
-          runningRenderers.push(toLaunch);
-          this._profileLastActivity[toLaunch.nst_profile_id] = Date.now();
-          console.log(`[scene-dispatch] ${toLaunch.name} launched`);
-        } catch (err) {
-          console.error(`[scene-dispatch] Failed to launch ${toLaunch.name}: ${err.message}`);
-        }
-      }
-
       if (runningRenderers.length === 0) {
         this._dispatching = false;
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.2",
+  "version": "2.5.4",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {