channel-worker 2.5.3 → 2.5.5

package/lib/command-poller.js

@@ -1340,24 +1340,78 @@ class CommandPoller {
       // Pause on the Renderers tab; profile should stay closed until Resume.
       const isPaused = (r) => r && r.health_state === 'paused'
         && (!r.pause_until || new Date(r.pause_until).getTime() > Date.now());
-      const stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
+      let stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
       const pausedCount = renderers.filter(isPaused).length;
       console.log(`[scene-dispatch] running=${runningRenderers.length} cap=${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} paused=${pausedCount} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
 
-      // Launch policy: keep ALL non-paused owned renderers ONLINE whenever
-      // there's any queue work — they take turns claiming via the API's
-      // per-worker concurrency cap (Worker.parallel_limit). The cap limits
-      // CONCURRENT CLAIMS, not the number of profiles kept open: e.g. with
-      // parallel_limit=1 + 2 renderers veo01/veo02, both browsers stay open
-      // and alternate scene claims. Operator pauses a specific renderer if
-      // they want it closed (e.g. while solving captcha).
+      // ROUND-ROBIN ROTATION — when a running renderer just finished its
+      // scene (no in-flight cmd) AND there's at least one OFFLINE sibling
+      // AND queue has more work, close the just-finished one so the next
+      // cycle launches the sibling. Result: scenes alternate veo03 →
+      // veo04 → veo03 → veo04 … instead of one renderer claiming
+      // everything. Required for shared-Google-account safety: keeping a
+      // browser idle on Flow page keeps refreshing tokens / pinging
+      // telemetry → Veo flags "2 concurrent sessions per account" → captcha.
+      // Closing it = 0 idle sessions, only 1 active at a time.
+      if (queueCount > 0 && stillOffline.length > 0 && runningRenderers.length > 0) {
+        const stoppedNames = [];
+        for (const r of [...runningRenderers]) {
+          // Skip externally-launched profiles (user opened manually via NST UI)
+          // — we don't own their lifecycle.
+          if (!this._profileLastActivity[r.nst_profile_id]) continue;
+          try {
+            const c = await this.api.rendererHasCommands(r.nst_profile_id);
+            if (c > 0) continue; // busy → keep alive
+            console.log(`[scene-dispatch] Round-robin: closing idle ${r.name} to rotate to sibling`);
+            await this.nst.stopProfile(r.nst_profile_id);
+            delete this._profileLastActivity[r.nst_profile_id];
+            if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
+            const idx = runningRenderers.indexOf(r);
+            if (idx >= 0) runningRenderers.splice(idx, 1);
+            stoppedNames.push(r.name);
+            // The now-closed renderer becomes a fresh offline candidate for
+            // future cycles (won't relaunch immediately — sibling goes first
+            // because its last_command_assigned_at is older).
+            stillOffline.push(r);
+          } catch (e) {
+            console.warn(`[scene-dispatch] Round-robin close failed for ${r.name}: ${e.message}`);
+          }
+        }
+        if (stoppedNames.length) {
+          console.log(`[scene-dispatch] Round-robin closed: [${stoppedNames.join(',')}] — next launch picks sibling`);
+        }
+      }
+
+      // Sort offline by last_command_assigned_at ASC (most idle first) so
+      // the launcher picks the round-robin partner, not the one that just
+      // finished. API populates last_command_assigned_at on every claim,
+      // so this naturally implements turn-taking across siblings.
+      stillOffline = stillOffline.slice().sort((a, b) => {
+        const ta = a.last_command_assigned_at ? new Date(a.last_command_assigned_at).getTime() : 0;
+        const tb = b.last_command_assigned_at ? new Date(b.last_command_assigned_at).getTime() : 0;
+        return ta - tb;
+      });
+
+      // Launch policy: STRICT parallel_limit cap on physical launches.
+      // With pc2.parallel_limit=1 + 2 renderers (veo03/veo04 share same Veo
+      // Google account), only ONE of them is launched at any time. The
+      // second stays closed until the first finishes / is paused / cap is
+      // bumped. This prevents Google flagging "2 concurrent sessions on
+      // same account" → captcha.
       //
-      // Old "Close excess profiles" + parallelLimit-bounded launch logic was
-      // removed — they conflated "concurrent claims cap" with "max launched"
-      // and forced one of the renderers to sit unlaunched when cap < count.
-      const eligibleToLaunch = queueCount > 0 ? stillOffline : [];
-      for (const toLaunch of eligibleToLaunch) {
-        console.log(`[scene-dispatch] Launching ${toLaunch.name} (queue=${queueCount} running=${runningRenderers.length})`);
+      // Earlier iteration tried "launch all, cap only claims" — that left
+      // both browsers open, both pinging Flow (token refresh, telemetry),
+      // and Veo's per-account session monitor counted them as 2 active
+      // sessions → tripped captcha on the supposedly-idle one.
+      const needNew = Math.max(0, queueCount);
+      const neededLaunches = Math.min(
+        parallelLimit - runningRenderers.length,
+        stillOffline.length,
+        needNew,
+      );
+      for (let li = 0; li < Math.max(0, neededLaunches); li++) {
+        const toLaunch = stillOffline[li];
+        console.log(`[scene-dispatch] Launching ${toLaunch.name} (${runningRenderers.length + li + 1}/${parallelLimit})`);
         try {
           await this._launchRendererProfile(toLaunch);
           runningRenderers.push(toLaunch);
@@ -1388,6 +1442,35 @@ class CommandPoller {
         }
       }
 
+      // Close EXCESS running profiles when count > parallel_limit. Happens
+      // when operator drops the cap (eg 2→1 to throttle captcha) — the
+      // already-launched 2nd profile should retire so it doesn't keep
+      // refreshing tokens against the shared Google account. Only close
+      // if it has no in-flight command (let work finish first).
+      if (runningRenderers.length > parallelLimit) {
+        const excess = runningRenderers.length - parallelLimit;
+        const idleRunning = [];
+        for (const r of runningRenderers) {
+          if (!this._profileLastActivity[r.nst_profile_id]) continue;
+          try {
+            const c = await this.api.rendererHasCommands(r.nst_profile_id);
+            if (c === 0) idleRunning.push(r);
+          } catch {}
+        }
+        for (const r of idleRunning.slice(0, excess)) {
+          console.log(`[scene-dispatch] Closing excess ${r.name} (cap=${parallelLimit})`);
+          try {
+            await this.nst.stopProfile(r.nst_profile_id);
+            delete this._profileLastActivity[r.nst_profile_id];
+            if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
+            const idx = runningRenderers.indexOf(r);
+            if (idx >= 0) runningRenderers.splice(idx, 1);
+          } catch (e) {
+            console.warn(`[scene-dispatch] Failed to close excess ${r.name}: ${e.message}`);
+          }
+        }
+      }
+
       if (runningRenderers.length === 0) {
         this._dispatching = false;
         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.3",
+  "version": "2.5.5",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {