channel-worker 2.5.4 → 2.5.6

package/lib/command-poller.js

@@ -139,6 +139,26 @@ class CommandPoller {
     }
     // Strip the '_pw' suffix to derive the script name (upload_youtube_pw → upload_youtube).
     const scriptName = command.type.replace(/_pw$/, '');
+
+    // PER-PROFILE MUTEX — two _pw scripts can't share the same NST profile.
+    // The NST browser is launched once and shared via CDP; concurrent
+    // page.goto / navigation calls from sibling scripts abort each other
+    // (saw `net::ERR_ABORTED` on FB while YT was navigating during the same
+    // publish click). Different profiles still run in parallel — this lock
+    // only serializes within a profile. Wait up to 30 min with 5s polling
+    // (upload typically 1-3 min); stale-cleanup picks up zombies separately.
+    if (!this._pwInFlight) this._pwInFlight = new Map();
+    const MUTEX_MAX_WAIT_MS = 30 * 60 * 1000;
+    const waitStart = Date.now();
+    while (this._pwInFlight.has(profileId)) {
+      if (Date.now() - waitStart > MUTEX_MAX_WAIT_MS) {
+        await this.api.updateCommand(command._id, { status: 'failed', error: `pw mutex timeout — profile ${profileId} busy with ${this._pwInFlight.get(profileId)} for >30min` });
+        return;
+      }
+      console.log(`[commands/pw] ${command.type} waiting — profile ${profileId} busy with ${this._pwInFlight.get(profileId)}`);
+      await new Promise(r => setTimeout(r, 5000));
+    }
+    this._pwInFlight.set(profileId, command.type);
     console.log(`[commands/pw] ${command.type} → scripts/${scriptName}.js (profile=${profileId})`);
 
     // Lazy-init NST manager (same pattern as the other handlers).
@@ -177,6 +197,10 @@ class CommandPoller {
     } catch (err) {
       console.error(`[commands/pw] ${command.type} failed: ${err.message}`);
       await this.api.postCommandResult(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });
+    } finally {
+      // Always release the per-profile mutex — even on throw — or sibling
+      // pw cmds for the same profile would hang forever.
+      if (this._pwInFlight) this._pwInFlight.delete(profileId);
     }
   }
 
@@ -1340,10 +1364,58 @@ class CommandPoller {
       // Pause on the Renderers tab; profile should stay closed until Resume.
       const isPaused = (r) => r && r.health_state === 'paused'
         && (!r.pause_until || new Date(r.pause_until).getTime() > Date.now());
-      const stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
+      let stillOffline = renderers.filter(r => !runningRenderers.includes(r) && !isPaused(r));
       const pausedCount = renderers.filter(isPaused).length;
       console.log(`[scene-dispatch] running=${runningRenderers.length} cap=${parallelLimit} (flowkit=${flowkitQ} dom=${domQ}) offline=${stillOffline.length} paused=${pausedCount} queue=${queueCount} names=[${runningRenderers.map(r=>r.name)}]`);
 
+      // ROUND-ROBIN ROTATION — when a running renderer just finished its
+      // scene (no in-flight cmd) AND there's at least one OFFLINE sibling
+      // AND queue has more work, close the just-finished one so the next
+      // cycle launches the sibling. Result: scenes alternate veo03 →
+      // veo04 → veo03 → veo04 … instead of one renderer claiming
+      // everything. Required for shared-Google-account safety: keeping a
+      // browser idle on Flow page keeps refreshing tokens / pinging
+      // telemetry → Veo flags "2 concurrent sessions per account" → captcha.
+      // Closing it = 0 idle sessions, only 1 active at a time.
+      if (queueCount > 0 && stillOffline.length > 0 && runningRenderers.length > 0) {
+        const stoppedNames = [];
+        for (const r of [...runningRenderers]) {
+          // Skip externally-launched profiles (user opened manually via NST UI)
+          // — we don't own their lifecycle.
+          if (!this._profileLastActivity[r.nst_profile_id]) continue;
+          try {
+            const c = await this.api.rendererHasCommands(r.nst_profile_id);
+            if (c > 0) continue; // busy → keep alive
+            console.log(`[scene-dispatch] Round-robin: closing idle ${r.name} to rotate to sibling`);
+            await this.nst.stopProfile(r.nst_profile_id);
+            delete this._profileLastActivity[r.nst_profile_id];
+            if (r.name) delete this._profileLastActivity[r.name.toLowerCase()];
+            const idx = runningRenderers.indexOf(r);
+            if (idx >= 0) runningRenderers.splice(idx, 1);
+            stoppedNames.push(r.name);
+            // The now-closed renderer becomes a fresh offline candidate for
+            // future cycles (won't relaunch immediately — sibling goes first
+            // because its last_command_assigned_at is older).
+            stillOffline.push(r);
+          } catch (e) {
+            console.warn(`[scene-dispatch] Round-robin close failed for ${r.name}: ${e.message}`);
+          }
+        }
+        if (stoppedNames.length) {
+          console.log(`[scene-dispatch] Round-robin closed: [${stoppedNames.join(',')}] — next launch picks sibling`);
+        }
+      }
+
+      // Sort offline by last_command_assigned_at ASC (most idle first) so
+      // the launcher picks the round-robin partner, not the one that just
+      // finished. API populates last_command_assigned_at on every claim,
+      // so this naturally implements turn-taking across siblings.
+      stillOffline = stillOffline.slice().sort((a, b) => {
+        const ta = a.last_command_assigned_at ? new Date(a.last_command_assigned_at).getTime() : 0;
+        const tb = b.last_command_assigned_at ? new Date(b.last_command_assigned_at).getTime() : 0;
+        return ta - tb;
+      });
+
       // Launch policy: STRICT parallel_limit cap on physical launches.
       // With pc2.parallel_limit=1 + 2 renderers (veo03/veo04 share same Veo
       // Google account), only ONE of them is launched at any time. The

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.4",
+  "version": "2.5.6",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {