@parall/daemon 1.36.1 → 1.37.0

package/dist/supervisor.js

@@ -5,11 +5,12 @@ import { effectiveLLMSourceExplicit } from '@parall/agent-core';
 import { ParallWs, } from '@parall/sdk';
 import { installClip, parseSource } from './clip-runtime/clip-installer.js';
 import { parseIpcCommands } from './clip-runtime/manifest.js';
-import { BrowserProfileManager, ClipProcessManager, ClipProvider, HubClient, } from './clip-runtime/index.js';
+import { BrowserProfilePool, ClipProcessManager, ClipProvider, HubClient, } from './clip-runtime/index.js';
 import { agentClaudeCredentialsFileFor, agentClaudeHomeFor, agentStateDirFor, agentWorkspaceDirFor, sharedClaudeCredentialsFileFor, } from './config.js';
 import { listDirectory } from './filesystem.js';
 import { assertAgentKey, getRuntimeAdapter } from './runtimes.js';
 import { applyRuntimeBinaryEnv } from './runtime-bin-resolver.js';
+import { detectedRuntimesEqual, detectRuntimes, summarizeDetectedRuntimes, } from './runtime-detector.js';
 import { prepareWorkspace } from './workspace.js';
 const RUNTIME_PACKAGES = {
     'claude-code': '@parall/claude-agent',
@@ -17,7 +18,12 @@ const RUNTIME_PACKAGES = {
     openclaw: '@parall/openclaw-agent',
 };
 const WORKSPACE_SETUP_RETRY_DELAY_MS = 5_000;
+const AGENT_CONFIG_REFRESH_RETRY_DELAY_MS = 5_000;
 const CLIP_RECONCILE_INTERVAL_MS = 5 * 60_000;
+// Runtime-CLI re-detection cadence. Cheap (three --version probes at most) and
+// only a CHANGED result posts a heartbeat, so this mainly bounds how long a
+// freshly installed CLI takes to show up in machine pickers.
+const RUNTIME_DETECT_INTERVAL_MS = 5 * 60_000;
 /**
  * Sleep that wakes early on abort. Returns true if the full delay elapsed,
  * false if aborted. Used by bootstrap retry and the outer keepalive in
@@ -63,8 +69,28 @@ export class DaemonSupervisor {
     // stream.start (the kill fires before the streamer registers and misses it,
     // leaving a live streamer on a profile the server just marked stopped).
     browserProfileOpQueues = new Map();
+    // Profiles with a revive (open/ensureRuntime) queued or running but not yet a live
+    // pool manager (count = in-flight revives). Reconnect reconcile snapshots this
+    // alongside the pool's active set: a stale queued open is invisible to
+    // pool.activeProfileIds() until it spawns, so without this it could drain after a
+    // missed offline stop/delete and start a runtime the server no longer wants.
+    browserProfilePendingRevives = new Map();
     workspaceSetupRetryTimers = new Map();
+    agentConfigRefreshRetryTimers = new Map();
     cancelledSpawns = new Set();
+    // Agents whose provider config changed while no child state existed yet
+    // (spawn in flight, or an attach fetch still in flight): spawnAgent replays
+    // handleAgentConfigUpdated after the spawn settles so an in-flight spawn
+    // cannot commit a stale provider_config snapshot and lose the event.
+    pendingConfigRefresh = new Set();
+    // Child states with a deliberate restart awaiting its credential mint. The
+    // state.child guard in restartChildNow cannot see a restart that has not
+    // spawned yet, so without this set the reconcile and live config-update
+    // paths could double-spawn the same agent. Keyed by ChildState identity,
+    // NOT agent ID: a stale in-flight restart from a detached state must not
+    // suppress the reattached agent's (new state's) restart — startChild's
+    // registration check already keeps the stale state from ever spawning.
+    restartingStates = new Set();
     ws = null;
     running = false;
     machineId = null;
@@ -82,12 +108,23 @@ export class DaemonSupervisor {
     connectedClipServiceUrl = null;
     stopResolve = null;
     updater = null;
-    healthConfirmed = false;
-    browserProfileManager = null;
+    // main #1671 (self-update health gate) and this PR (per-profile browser pool) added
+    // adjacent fields — keep both. healthGate replaces the old healthConfirmed flag;
+    // browserProfilePool replaces the old single browserProfileManager.
+    healthGate = null;
+    browserProfilePool = null;
     clipManager = null;
     clipProvider = null;
     clipReconcileTimer = null;
     clipReconcileInFlight = null;
+    // Last runtime-CLI detection result (null = not run yet). Re-detected
+    // periodically; only a CHANGED result re-posts the heartbeat so steady state
+    // stays write-free server-side.
+    detectedRuntimes = null;
+    runtimeDetectTimer = null;
+    runtimeDetectInFlight = null;
+    // Injectable for tests (real detection spawns --version subprocesses).
+    detectRuntimesFn = detectRuntimes;
     // Execution-side hub client for cross-machine dependency resolution
     // (GetBindings + Invoke). Cached by resolved endpoint so a clip_provider_url
     // rollout transparently rebuilds it against the new grey-cloud host.
@@ -101,6 +138,11 @@ export class DaemonSupervisor {
     setUpdater(updater) {
         this.updater = updater;
     }
+    /** Inject the process-level update health gate. The supervisor only feeds it
+     *  fact signals (machine.hello, supervisor-ended); the gate owns confirm policy. */
+    setHealthGate(gate) {
+        this.healthGate = gate;
+    }
     /** Start the supervisor. Returns a promise that resolves on `stop()`. */
     async run(signal) {
         if (this.running)
@@ -125,12 +167,12 @@ export class DaemonSupervisor {
         this.migrateFlatLayout();
         // Initialize Clip runtime manager (optional, for Pinix Clip subprocess management)
         if (process.env.PRLL_CLIP_RUNTIME_ENABLED === 'true') {
-            this.browserProfileManager = new BrowserProfileManager({
-                homeDir: path.join(this.config.rootStateDir, 'bb-browser'),
+            this.browserProfilePool = new BrowserProfilePool({
+                baseHomeDir: path.join(this.config.rootStateDir, 'bb-browser'),
                 log: this.log,
-                reportStatus: (profileId, status, errorMsg) => {
+                reportStatus: (profileId, status, errorMsg, generation) => {
                     this.client
-                        .reportBrowserProfileStatus(profileId, status, errorMsg)
+                        .reportBrowserProfileStatus(profileId, status, errorMsg, generation)
                         .catch((err) => this.log.warn(`browser profile status report failed: ${String(err)}`));
                 },
                 resolveProxy: (profileId) => this.resolveBrowserProfileProxy(profileId),
@@ -138,29 +180,49 @@ export class DaemonSupervisor {
             this.clipManager = new ClipProcessManager({
                 clipsDir: path.join(this.config.rootStateDir, 'clips'),
                 dataDir: path.join(this.config.rootStateDir, 'clip-data'),
-                browserProfileManager: this.browserProfileManager,
+                browserProfileManager: this.browserProfilePool,
                 // Execution side: nested browser dependency invokes resolve their
                 // binding and route through the hub (no local shortcut).
                 hubClient: () => this.getHubClient(),
                 ensureInstalled: (config) => this.ensureClipInstalled(config),
             });
             await this.reconcileMachineClips();
+            // Pre-fence every locally-known browser profile BEFORE registering as a hub
+            // Provider (which exposes hub browser invokes). Until the reconnect reconcile
+            // (fullReconcile below) has applied any reset/stop/delete missed while offline, a
+            // hub invoke must not run on a profile — it could revive the stale cookies a
+            // missed reset meant to clear (reset-before-open must hold across a reconnect).
+            // Fencing is synchronous and cheap (no bb-browser opens, so provider registration
+            // is never blocked on a slow/stuck Chromium); the reconcile un-fences the desired
+            // profiles after wiping. An invoke in the gap is rejected and the hub retries — no
+            // stale-disk execution.
+            const poolToFence = this.browserProfilePool;
+            if (poolToFence) {
+                for (const profileId of new Set([
+                    ...poolToFence.activeProfileIds(),
+                    ...poolToFence.localStateProfileIds(),
+                ])) {
+                    poolToFence.fence(profileId);
+                }
+            }
             // Register as a hub Provider unless the machine opted out
             // (provider_enabled=false). See applyClipProviderState.
             await this.applyClipProviderState();
             this.startClipReconcileTimer();
         }
-        // Report daemon version + self-update capability via heartbeat (best-effort).
-        // Capability tracks whether a service manager supervises us: the updater is
-        // created only in that case (see isSelfUpdateManaged in index.ts). A bare
-        // `npx` foreground run reports false so the server/UI won't offer a manual
-        // update it would silently ignore.
-        this.client
-            .postMachineHeartbeat({
-            daemonVersion: this.updater?.getLocalVersion(),
-            selfUpdateCapable: this.updater != null,
-        })
-            .catch((err) => this.log.warn(`daemon heartbeat report failed: ${String(err)}`));
+        // Report daemon version + self-update capability + detected runtime CLIs
+        // via heartbeat (best-effort, off the critical path — detection spawns
+        // `--version` probes). Capability tracks whether a service manager
+        // supervises us: the updater is created only in that case (see
+        // isSelfUpdateManaged in index.ts). A bare `npx` foreground run reports
+        // false so the server/UI won't offer a manual update it would silently
+        // ignore. Detection then re-runs periodically so installing a CLI shows
+        // up in machine pickers without a daemon restart.
+        void this.detectAndReportRuntimes(true);
+        this.runtimeDetectTimer = setInterval(() => {
+            void this.detectAndReportRuntimes(false);
+        }, RUNTIME_DETECT_INTERVAL_MS);
+        this.runtimeDetectTimer.unref?.();
         await this.fullReconcile();
         this.ws = new ParallWs({
             getTicket: () => this.client.getMachineWsTicket(),
@@ -169,15 +231,9 @@ export class DaemonSupervisor {
         });
         this.ws.on('machine.hello', (_data) => {
             this.log.info('machine WS connected (machine.hello)');
-            if (!this.healthConfirmed && this.updater) {
-                try {
-                    this.updater.confirmVersion();
-                    this.healthConfirmed = true;
-                }
-                catch (err) {
-                    this.log.warn(`confirmVersion failed: ${String(err)}`);
-                }
-            }
+            // Control-plane handshake fact — the health gate (process-level) owns the
+            // confirm / stability-window policy; the supervisor just reports the event.
+            this.healthGate?.onMachineHello();
             void (async () => {
                 await this.refreshMachineConfig();
                 await this.fullReconcile();
@@ -222,6 +278,10 @@ export class DaemonSupervisor {
                 void this.applyClipProviderState();
             }
         });
+        this.ws.on('machine.agent_config.updated', (data) => {
+            this.log.info(`WS: agent ${data.agent_id} provider config updated (effective llm_source=${data.llm_source})`);
+            void this.handleAgentConfigUpdated(data.agent_id);
+        });
         this.ws.on('machine.workspace.setup.requested', (data) => {
             this.log.info(`WS: workspace setup requested for agent ${data.agent_id}`);
             void this.handleWorkspaceSetupRequested(data.agent_id);
@@ -262,6 +322,13 @@ export class DaemonSupervisor {
         if (!this.running)
             return;
         this.running = false;
+        // This supervisor instance is ending (clean shutdown OR a runForever-caught
+        // soft crash). Reset the update health window before any cleanup that could
+        // throw, so a stale confirm timer can't fire across supervisor instances.
+        // Soft crash resets the timer but does NOT increment boot_count — rollback
+        // failures come only from process-level unclean exits (the daemon-running
+        // marker in index.ts).
+        this.healthGate?.onSupervisorEnded();
         if (this.ws) {
             this.ws.disconnect();
             this.ws = null;
@@ -271,7 +338,15 @@ export class DaemonSupervisor {
             clearTimeout(timer);
         }
         this.workspaceSetupRetryTimers.clear();
+        for (const timer of this.agentConfigRefreshRetryTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.agentConfigRefreshRetryTimers.clear();
         this.stopClipReconcileTimer();
+        if (this.runtimeDetectTimer) {
+            clearInterval(this.runtimeDetectTimer);
+            this.runtimeDetectTimer = null;
+        }
         for (const state of this.children.values()) {
             state.shuttingDown = true;
             if (state.restartTimer) {
@@ -288,9 +363,9 @@ export class DaemonSupervisor {
             exits.push(this.clipManager.stopAll());
             this.clipManager = null;
         }
-        if (this.browserProfileManager) {
-            exits.push(this.browserProfileManager.stop());
-            this.browserProfileManager = null;
+        if (this.browserProfilePool) {
+            exits.push(this.browserProfilePool.stop());
+            this.browserProfilePool = null;
         }
         await Promise.allSettled(exits);
         this.children.clear();
@@ -366,8 +441,21 @@ export class DaemonSupervisor {
                 }
                 await this.spawnAgent(userId, orgId, a);
             }
-            else if (!existing.child && !existing.restartTimer && !existing.shuttingDown) {
-                await this.restartChildNow(existing, 'reconcile found no live child');
+            else {
+                // Catch up on per-agent provider_config changes missed while the WS
+                // was down (machine.agent_config.updated is delivered live only):
+                // refresh the cache, and respawn a live child when the stored config
+                // actually changed. The serialized comparison is stable — both sides
+                // come from the same endpoint's JSON.
+                const configChanged = JSON.stringify(existing.providerConfig ?? null) !==
+                    JSON.stringify(a.provider_config ?? null);
+                existing.providerConfig = a.provider_config;
+                if (!existing.child && !existing.restartTimer && !existing.shuttingDown) {
+                    await this.restartChildNow(existing, 'reconcile found no live child');
+                }
+                else if (configChanged && !existing.shuttingDown) {
+                    await this.respawnChildForConfig(existing, 'provider config changed while disconnected');
+                }
             }
         }
         for (const [userId, state] of this.children) {
@@ -387,9 +475,46 @@ export class DaemonSupervisor {
         await this.reconcileMachineClips();
         await this.reconcileBrowserProfiles();
     }
+    // Wipe a missed reset before a revive, reporting `error` if the wipe fails. A failed
+    // wipe must NOT silently leave the row at `pending`: report `error` under the lifecycle
+    // generation (visible), keep the profile fenced (applyResetWipe fenced it) and the
+    // sidecar unwritten, then re-throw so the caller skips the open. The next reconcile's
+    // negative path retries the wipe because the SSOT reset_generation is still unapplied.
+    // Shared by the reconnect reconcile and the live `open` lifecycle path.
+    async wipeBeforeRevive(pool, profileId, resetGen, generation) {
+        try {
+            await pool.applyResetWipe(profileId, resetGen);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            this.client
+                .reportBrowserProfileStatus(profileId, 'error', message, generation)
+                .catch((e) => this.log.warn(`browser profile wipe-error report failed: ${String(e)}`));
+            throw err;
+        }
+    }
     async reconcileBrowserProfiles() {
-        if (!this.browserProfileManager)
+        const pool = this.browserProfilePool;
+        if (!pool)
             return;
+        // Snapshot the locally-live set BEFORE the server fetch. A profile opened via a
+        // WS lifecycle event DURING the fetch is NOT in this snapshot, so the negative
+        // convergence below (which is stale w.r.t. that open) can never release it.
+        // There is no periodic browser reconcile to re-open a wrongly-released profile —
+        // it only runs on startup / WS-hello — so a stale release would strand it until
+        // the next reconnect.
+        const activeBeforeList = new Set(pool.activeProfileIds());
+        // Also snapshot revives (open/ensure) queued but not yet a live manager: such a
+        // stale open is invisible to activeProfileIds() and would otherwise drain after a
+        // missed offline stop/delete and start a runtime the server no longer wants.
+        const pendingReviveBeforeList = new Set(this.browserProfilePendingRevives.keys());
+        // And ALL local profile state — per-profile homes AND legacy flat account files
+        // (the pool owns that storage layout). State with NO server row is a
+        // deleted/reassigned profile whose cookies/storage must be wiped from this host;
+        // a pre-upgrade profile never reopened after migration exists only as a legacy flat
+        // file, so home-only enumeration would miss it. Pre-fetch so state created during
+        // the fetch is left alone.
+        const localStateBeforeList = new Set(pool.localStateProfileIds());
         let profiles;
         try {
             profiles = await this.client.listMachineBrowserProfiles();
@@ -398,29 +523,163 @@ export class DaemonSupervisor {
             this.log.warn(`browser profile reconcile: list failed: ${String(err)}`);
             return;
         }
+        // The pool must still be the same instance after the await — stop() may have
+        // nulled/replaced it during a concurrent shutdown; bail rather than act on a
+        // torn-down pool (also avoids NPE-ing through Promise.allSettled into warnings).
+        if (this.browserProfilePool !== pool)
+            return;
+        // Converge local runtimes to the server's desired-live set (the server is the
+        // SSOT). The supervisor is the desired-state coordinator; the pool is the
+        // per-profile runtime registry. Every op here goes through
+        // enqueueBrowserProfileOp so all supervisor-originated profile work shares one
+        // ordering layer (the pool's own queue is the final per-profile mutex).
+        const desiredLive = new Set();
+        const serverIds = new Set(); // every byoc profile the server still lists (any status)
+        const serverResetGen = new Map();
+        const ops = [];
+        // reset_generation (server) is the SSOT for reset intent; the daemon's per-profile
+        // sidecar (read via pool.appliedResetGeneration) is only a local ack of "already
+        // wiped up to this generation". A missed reset (server newer than the local ack) must
+        // wipe local state BEFORE any open/ensure, so a reopen starts clean and never
+        // resurrects the cookies the reset cleared. Returns the server reset_generation to
+        // apply, or null if nothing is missed. Pure (no side effects).
+        const missedResetGen = (profileId) => {
+            const gen = serverResetGen.get(profileId) ?? 0;
+            return gen > pool.appliedResetGeneration(profileId) ? gen : null;
+        };
+        // Negative-path helper (a stopped profile with a missed reset): fence synchronously
+        // then enqueue a STANDALONE wipe. No revive follows, so a wipe failure simply leaves
+        // the profile fenced and the sidecar unwritten → the next reconcile retries (the
+        // SSOT reset_generation is still unapplied locally). Fail-closed by construction.
+        const enqueueResetWipeIfMissed = (profileId) => {
+            const gen = missedResetGen(profileId);
+            if (gen === null)
+                return false;
+            // Fence SYNCHRONOUSLY before the async wipe — same contract as the WS stop/reset
+            // path. On a machine-WS reconnect whose clip-provider stream stays live, a hub
+            // invoke could otherwise run on the stale runtime in the gap before the wipe op
+            // starts. (In the negative loop this re-asserts an already-set fence within the same
+            // synchronous iteration — atomic, no revive can observe an intermediate generation.)
+            pool.fence(profileId);
+            ops.push(this.enqueueBrowserProfileOp(profileId, () => pool.applyResetWipe(profileId, gen)));
+            return true;
+        };
         for (const profile of profiles) {
+            // The daemon supervises only machine-bound (byoc) profiles; hosted profiles
+            // (null machine_id) run on the platform pool and are never owned by a daemon.
+            if (!profile.machine_id)
+                continue;
+            serverIds.add(profile.id);
+            serverResetGen.set(profile.id, profile.reset_generation ?? 0);
             if (profile.status !== 'running' && profile.status !== 'pending')
                 continue;
-            // The daemon supervises only machine-bound (byoc) profiles; hosted
-            // profiles (null machine_id) run on the platform pool and are never owned
-            // by a daemon machine.
-            if (!profile.machine_id)
+            desiredLive.add(profile.id);
+            // A missed reset must wipe SUCCESSFULLY before this profile is revived. Fence
+            // synchronously now (reject hub invokes in the gap before the queued wipe runs).
+            const wipeGen = missedResetGen(profile.id);
+            if (wipeGen !== null)
+                pool.fence(profile.id);
+            // Capture the fence generation AFTER any fence bump so a stop arriving while the op
+            // waits makes it skip as stale; carry the lifecycle generation so the daemon's
+            // `running` report is fenced server-side. Drive desired state through the CAPTURED
+            // pool (not enqueueBrowserProfileLifecycle, which re-reads this.browserProfilePool
+            // and would negative-ACK `error` for a pending profile if stop() nulled the field).
+            const sinceSeq = pool.stopSeqOf(profile.id);
+            const generation = profile.lifecycle_generation;
+            // forceStatusReport ONLY for a `pending` row: the daemon's last report may already be
+            // `running` (Chromium survived the WS drop), so a deduped repeat would leave the
+            // server stuck at `pending`. A `running` row keeps the normal dedup.
+            const forceStatusReport = profile.status === 'pending';
+            ops.push(this.enqueueBrowserProfileRevive(profile.id, async () => {
+                // Reset-before-open, FAIL-CLOSED: a missed reset must wipe successfully BEFORE
+                // the revive un-fences and reopens. If applyResetWipe throws, the await
+                // propagates and ensureRuntime is SKIPPED — the profile stays fenced (the wipe
+                // fenced it and the applied-reset sidecar was NOT written), so the next reconcile
+                // retries because the SSOT reset_generation is still unapplied locally. This
+                // ordering is expressed locally; the enqueueBrowserProfileOp queue contract (a
+                // rejected op does not block later ops) is intentionally unchanged. A failed
+                // wipe reports `error` (visible) and re-throws here, so ensureRuntime is skipped.
+                if (wipeGen !== null)
+                    await this.wipeBeforeRevive(pool, profile.id, wipeGen, generation);
+                // Reconnect recovery is liveness-only: ensure the runtime is live (account +
+                // tab) for both pending and running. The original open's `start_url` is NOT
+                // replayed (it rides only the live lifecycle event, not persisted); a `pending`
+                // profile whose open event the daemon missed comes up at the default page rather
+                // than a targetless `about:blank`. Persisting start_url is a separate follow-up.
+                await pool.ensureRuntime(profile.id, sinceSeq, generation, { forceStatusReport });
+            }));
+        }
+        // Negative convergence over everything locally known but NOT desired — active
+        // runtimes, queued revives (open/ensure not yet a manager), and local state (homes +
+        // legacy flat files) — all from the PRE-FETCH snapshots, so a profile that appeared
+        // DURING the fetch is left alone (no periodic reconcile would undo a wrong teardown).
+        const candidates = new Set([
+            ...activeBeforeList,
+            ...pendingReviveBeforeList,
+            ...localStateBeforeList,
+        ]);
+        for (const profileId of candidates) {
+            if (desiredLive.has(profileId))
                 continue;
-            const machineId = profile.machine_id;
-            try {
-                if (profile.status === 'pending') {
-                    await this.enqueueBrowserProfileLifecycle({
-                        machine_id: machineId,
-                        profile_id: profile.id,
-                        action: 'open',
-                    });
+            const isActive = activeBeforeList.has(profileId);
+            const hasLocalState = localStateBeforeList.has(profileId);
+            // Pending-only revive (queued open/ensure, no runtime, no on-disk state at the
+            // pre-fetch snapshot): the synchronous fence alone neutralizes the stale open (it
+            // bumps stopSeq, so the queued revive skips as stale). Do NOT AWAIT a teardown — it
+            // would block behind the possibly-stuck queue and stall the whole reconcile (the
+            // queued-stale-open deadlock).
+            if (!isActive && !hasLocalState) {
+                pool.fence(profileId);
+                // Race: a pending-only revive can COMPLETE during the list fetch (create a manager
+                // + on-disk state) BEFORE this fence — the snapshot is then stale, so the fence
+                // alone leaves an orphaned runtime/disk the negative loop never converged. Converge
+                // it FIRE-AND-FORGET (NOT in `ops`: a still-stuck revive queue must not hang the
+                // reconcile; it runs once the queue drains — after the fenced open skips or
+                // completes — and is a no-op if nothing was created):
+                //   - deleted/reassigned    → wipeAbsent: the old host must keep no session data.
+                //   - present + missed reset → applyResetWipe: the completed open may have applied
+                //     only its event's (older) reset_generation, not the current one.
+                //   - present + stopped      → releaseRuntime: free the orphaned Chromium; a plain
+                //     stop keeps its cookies on disk.
+                let cleanup;
+                if (!serverIds.has(profileId)) {
+                    cleanup = pool.wipeAbsent(profileId);
                 }
                 else {
-                    await this.browserProfileManager.ensureRuntime(profile.id);
+                    const wipeGen = missedResetGen(profileId);
+                    cleanup =
+                        wipeGen !== null
+                            ? pool.applyResetWipe(profileId, wipeGen)
+                            : pool.releaseRuntime(profileId);
                 }
+                void cleanup.catch((err) => this.log.warn(`browser profile pending-revive cleanup failed for ${profileId}: ${String(err)}`));
+                continue;
             }
-            catch (err) {
-                this.log.warn(`browser profile reconcile failed for ${profile.id}: ${String(err)}`);
+            if (!serverIds.has(profileId)) {
+                // Absent from the server list → deleted/reassigned → fence + wipe local state (the
+                // old host must not keep cookies/storage). No status report — server is the SSOT.
+                pool.fence(profileId);
+                ops.push(this.enqueueBrowserProfileOp(profileId, () => pool.wipeAbsent(profileId)));
+            }
+            else if (enqueueResetWipeIfMissed(profileId)) {
+                // Present (stopped) with a missed reset → fenced synchronously + wipe enqueued.
+            }
+            else if (isActive) {
+                // Present (stopped) but the runtime is still live → fence + release the runtime
+                // only; a plain stop keeps its cookies on disk.
+                pool.fence(profileId);
+                ops.push(this.enqueueBrowserProfileOp(profileId, () => pool.releaseRuntime(profileId)));
+            }
+            // else: present + stopped + on-disk state + no runtime + no missed reset → LEAVE
+            // ALONE (do NOT fence). Its stopped-fence is already held — the start-sequence
+            // pre-fence on a fresh daemon, or the persisted pool state on a WS reconnect — so a
+            // hub invoke is still rejected. Re-fencing here would bump stopSeq and drop an `open`
+            // that arrived during the list fetch (it captured an older sinceSeq, and the stale
+            // `stopped` snapshot doesn't reflect it yet), stranding the server at `pending`.
+        }
+        for (const result of await Promise.allSettled(ops)) {
+            if (result.status === 'rejected') {
+                this.log.warn(`browser profile reconcile op failed: ${String(result.reason)}`);
             }
         }
     }
@@ -533,7 +792,9 @@ export class DaemonSupervisor {
     }
     async handleAgentDetached(agentId) {
         this.pendingWorkspaceSetup.delete(agentId);
+        this.pendingConfigRefresh.delete(agentId);
         this.clearWorkspaceSetupRetry(agentId);
+        this.clearAgentConfigRefreshRetry(agentId);
         if (this.spawningAgents.has(agentId)) {
             this.cancelledSpawns.add(agentId);
         }
@@ -574,26 +835,53 @@ export class DaemonSupervisor {
             }
         }
     }
-    async handleBrowserProfileLifecycle(data) {
-        if (!this.browserProfileManager) {
+    async handleBrowserProfileLifecycle(data, sinceSeq) {
+        if (!this.browserProfilePool) {
+            // Shutting down: stop() disconnects WS + nulls the pool, but a lifecycle op
+            // queued before that still runs. Bail silently rather than negative-ACK
+            // `error` for a profile that was merely mid-open during shutdown — a spurious
+            // error would strand it (reconcile only auto-heals pending/running). A genuine
+            // clip-runtime-disabled daemon is still running, so it falls through and ACKs.
+            if (!this.running)
+                return;
             this.log.warn('browser profile lifecycle event received but clip runtime is disabled');
             const status = data.action === 'stop' ? 'stopped' : 'error';
             const error = data.action === 'stop' ? undefined : 'Browser profile runtime is disabled on this daemon';
-            await this.client.reportBrowserProfileStatus(data.profile_id, status, error).catch((err) => {
+            // Carry the lifecycle generation so the report applies under the generation fence —
+            // without it a nil-gen `error` on an already-stopped row (the state a `reset` leaves)
+            // is silently dropped by the safe-state rule, hiding the failure.
+            await this.client
+                .reportBrowserProfileStatus(data.profile_id, status, error, data.generation)
+                .catch((err) => {
                 this.log.warn(`browser profile lifecycle negative ACK failed for ${data.profile_id}: ${String(err)}`);
             });
             return;
         }
         try {
             switch (data.action) {
-                case 'open':
-                    await this.browserProfileManager.openProfile(data.profile_id, data.start_url);
+                case 'open': {
+                    // Reset-before-open, FAIL-CLOSED (symmetric with reconnect reconcile): if this
+                    // open carries a reset the daemon hasn't applied locally, wipe FIRST and open
+                    // ONLY on success. A failed wipe throws to the catch below — openProfile is
+                    // skipped, the profile stays fenced (the wipe fenced it + the receipt pre-fence),
+                    // the applied-reset sidecar is NOT written, and the next reconnect reconcile
+                    // retries because the SSOT reset_generation is still unapplied. server
+                    // reset_generation is the SSOT; the sidecar is only the local applied-ack.
+                    const resetGen = data.reset_generation ?? 0;
+                    if (resetGen > this.browserProfilePool.appliedResetGeneration(data.profile_id)) {
+                        // Reports `error` + re-throws on failure (to the catch below), skipping the open.
+                        await this.wipeBeforeRevive(this.browserProfilePool, data.profile_id, resetGen, data.generation);
+                    }
+                    await this.browserProfilePool.openProfile(data.profile_id, data.start_url, sinceSeq, data.generation);
                     return;
+                }
                 case 'stop':
-                    await this.browserProfileManager.stopProfile(data.profile_id);
+                    await this.browserProfilePool.stopProfile(data.profile_id, data.generation);
                     return;
                 case 'reset':
-                    await this.browserProfileManager.resetProfile(data.profile_id);
+                    // Carry BOTH generations: lifecycle for the status report, reset for the
+                    // daemon's applied-reset sidecar (so a later reconnect won't re-wipe).
+                    await this.browserProfilePool.resetProfile(data.profile_id, data.generation, data.reset_generation);
                     return;
                 default:
                     this.log.warn(`unknown browser profile lifecycle action: ${data.action}`);
@@ -604,7 +892,35 @@ export class DaemonSupervisor {
         }
     }
     enqueueBrowserProfileLifecycle(data) {
-        return this.enqueueBrowserProfileOp(data.profile_id, () => this.handleBrowserProfileLifecycle(data));
+        // Pre-fence at event receipt (synchronously, before the op queues): a hub
+        // invoke bypasses this queue and hits the pool directly, so one authorized
+        // just before the server flipped the row to stopped could arrive while this
+        // stop/reset waits behind another op and spawn a transient Chromium the op
+        // then tears down. Fencing now rejects it immediately; open un-fences when it
+        // runs. (Only the WS stop/reset path reaches here with these actions; the
+        // reconcile open path passes action='open'.)
+        if (data.action === 'stop' || data.action === 'reset') {
+            this.browserProfilePool?.fence(data.profile_id);
+        }
+        else if (data.action === 'open' &&
+            this.browserProfilePool &&
+            (data.reset_generation ?? 0) > this.browserProfilePool.appliedResetGeneration(data.profile_id)) {
+            // reset-before-open on the LIVE open path too: if this open carries a reset the
+            // daemon hasn't applied (a prior wipe failed, or the reset event was missed), fence
+            // SYNCHRONOUSLY at receipt — a hub invoke (which bypasses this queue) must not hit
+            // the un-wiped home while the open waits in the queue. The handler wipes first and
+            // only opens on success (see handleBrowserProfileLifecycle).
+            this.browserProfilePool.fence(data.profile_id);
+        }
+        // Capture the fence generation NOW (at receipt, AFTER any pre-fence above) for an open,
+        // so a stop arriving while this open waits in the queue makes it skip as a stale revive.
+        const sinceSeq = data.action === 'open' ? this.browserProfilePool?.stopSeqOf(data.profile_id) : undefined;
+        const op = () => this.handleBrowserProfileLifecycle(data, sinceSeq);
+        // `open` is a revive — track it so reconnect reconcile can fence it even before
+        // it spawns a manager. stop/reset are not revives.
+        return data.action === 'open'
+            ? this.enqueueBrowserProfileRevive(data.profile_id, op)
+            : this.enqueueBrowserProfileOp(data.profile_id, op);
     }
     enqueueBrowserProfileOp(profileId, op) {
         const previous = this.browserProfileOpQueues.get(profileId) ?? Promise.resolve();
@@ -614,13 +930,35 @@ export class DaemonSupervisor {
         })
             .then(op);
         this.browserProfileOpQueues.set(profileId, next);
-        void next.finally(() => {
+        // The caller's returned `next` still surfaces a rejected op (the reconcile's
+        // allSettled handles it). This separate cleanup chain only deletes the queue-map
+        // entry, so it must swallow the rejection first — otherwise `.finally` re-throws it
+        // into a voided promise and a failing op (e.g. a reconnect reset wipe that throws)
+        // leaks an unhandledRejection. Mirrors BrowserProfilePool.enqueue's cleanup.
+        void next
+            .catch(() => { })
+            .finally(() => {
             if (this.browserProfileOpQueues.get(profileId) === next) {
                 this.browserProfileOpQueues.delete(profileId);
             }
         });
         return next;
     }
+    // Enqueue a REVIVE op (open / ensureRuntime), tracking it in
+    // browserProfilePendingRevives for the lifetime of the queued op so reconnect
+    // reconcile can fence a stale revive that hasn't spawned a manager yet. Registered
+    // synchronously at enqueue (so a snapshot taken now sees it) and cleared when the
+    // op settles. Use enqueueBrowserProfileOp (no tracking) for stop/reset/release/viewer.
+    enqueueBrowserProfileRevive(profileId, op) {
+        this.browserProfilePendingRevives.set(profileId, (this.browserProfilePendingRevives.get(profileId) ?? 0) + 1);
+        return this.enqueueBrowserProfileOp(profileId, op).finally(() => {
+            const remaining = (this.browserProfilePendingRevives.get(profileId) ?? 1) - 1;
+            if (remaining <= 0)
+                this.browserProfilePendingRevives.delete(profileId);
+            else
+                this.browserProfilePendingRevives.set(profileId, remaining);
+        });
+    }
     /**
      * Handle one viewer control command and reply via REST. Always answers the
      * request/reply bridge exactly once ({result} on success, {error:{message}}
@@ -628,7 +966,7 @@ export class DaemonSupervisor {
      * handleFilesystemBrowse. Serialized per profile by enqueueBrowserProfileViewer.
      */
     async handleBrowserProfileViewer(data) {
-        if (!this.browserProfileManager) {
+        if (!this.browserProfilePool) {
             await this.client
                 .postBrowserProfileViewerResponse(data.request_id, {
                 error: { message: 'Browser profile runtime is disabled on this daemon' },
@@ -637,7 +975,7 @@ export class DaemonSupervisor {
             return;
         }
         try {
-            const result = await this.browserProfileManager.handleViewerCommand(data.profile_id, data.session_id, data.command, data.input, data.turn);
+            const result = await this.browserProfilePool.handleViewerCommand(data.profile_id, data.session_id, data.command, data.input, data.turn);
             await this.client.postBrowserProfileViewerResponse(data.request_id, { result });
         }
         catch (err) {
@@ -736,6 +1074,40 @@ export class DaemonSupervisor {
         clearInterval(this.clipReconcileTimer);
         this.clipReconcileTimer = null;
     }
+    /**
+     * Detect runtime CLIs on this host and heartbeat the result. `initial`
+     * always reports (it also carries daemon version + self-update capability,
+     * replacing the old version-only startup heartbeat); periodic runs report
+     * only when the detection result changed, keeping steady state write-free.
+     * Serialized via `runtimeDetectInFlight` so a slow probe can't overlap the
+     * next interval tick and race the change comparison.
+     */
+    detectAndReportRuntimes(initial) {
+        if (this.runtimeDetectInFlight)
+            return this.runtimeDetectInFlight;
+        const run = (async () => {
+            const detected = await this.detectRuntimesFn(process.env, this.log);
+            const changed = this.detectedRuntimes == null || !detectedRuntimesEqual(this.detectedRuntimes, detected);
+            if (!initial && !changed)
+                return;
+            this.log.info(`runtime detection: ${summarizeDetectedRuntimes(detected)}`);
+            await this.client.postMachineHeartbeat({
+                daemonVersion: this.updater?.getLocalVersion(),
+                selfUpdateCapable: this.updater != null,
+                detectedRuntimes: detected,
+            });
+            // Commit the comparison baseline only after a successful post — a failed
+            // report must stay "changed" so the next tick retries it instead of
+            // silently dropping the result until the CLI set happens to change again.
+            this.detectedRuntimes = detected;
+        })().catch((err) => {
+            this.log.warn(`runtime detection report failed: ${String(err)}`);
+        });
+        this.runtimeDetectInFlight = run.finally(() => {
+            this.runtimeDetectInFlight = null;
+        });
+        return this.runtimeDetectInFlight;
+    }
     machineClipToConfig(clip) {
         const clipPath = path.join(this.config.rootStateDir, 'clips', clip.alias);
         return {
@@ -1021,20 +1393,132 @@ export class DaemonSupervisor {
         }
     }
     async respawnAllChildren() {
-        const states = [...this.children.values()];
-        for (const state of states) {
-            await this.terminateChild(state);
-        }
-        for (const state of states) {
+        // Same deliberate-restart choreography as the per-agent path: a bare
+        // terminateChild leaves the settle('exit') callback thinking the child
+        // crashed, which schedules a backoff timer that then blocks the immediate
+        // restart — the machine-level switch would wait out the backoff.
+        // respawnChildForConfig clears the timer and single-flights the restart.
+        for (const state of [...this.children.values()]) {
             if (!state.shuttingDown && this.running) {
-                await this.restartChildNow(state, 'llm_source changed');
+                await this.respawnChildForConfig(state, 'llm_source changed');
             }
         }
     }
+    /**
+     * Per-agent counterpart of the machine.config.updated respawn: ONE agent's
+     * explicit llm_source changed, so refresh that agent's cached
+     * provider_config from the server and respawn only its child — every other
+     * child keeps running. The event payload carries only the EFFECTIVE source;
+     * the cache must hold the RAW per-agent config (see ChildState
+     * .providerConfig) so startChild keeps resolving inherit-vs-override
+     * against the CURRENT machine default — caching the effective value would
+     * pin this agent across future machine-level default changes.
+     */
+    async handleAgentConfigUpdated(agentId) {
+        if (this.spawningAgents.has(agentId)) {
+            // A spawn in flight captured its provider_config from a pre-event fetch
+            // and `children` isn't populated yet — mark the agent dirty; spawnAgent
+            // replays this handler once the spawn settles (same idiom as
+            // pendingWorkspaceSetup). The replay compares configs, so it no-ops
+            // when the spawn already saw the change.
+            this.log.info(`agent ${agentId}: provider config updated during spawn — queued re-check`);
+            this.pendingConfigRefresh.add(agentId);
+            return;
+        }
+        const state = this.children.get(agentId);
+        if (!state) {
+            this.clearAgentConfigRefreshRetry(agentId);
+            // No child state and no spawn in flight — but handleAgentAttached may
+            // still be fetching its (pre-event, possibly stale) snapshot, so mark
+            // the agent dirty for the post-spawn re-check instead of dropping the
+            // only live signal. handleAgentDetached clears the flag if the agent
+            // leaves instead of spawning.
+            this.pendingConfigRefresh.add(agentId);
+            this.log.info(`agent ${agentId}: provider config updated but no running child — queued re-check for next spawn`);
+            return;
+        }
+        const result = await this.fetchAttachedAgent(agentId);
+        if (result.kind === 'retryable') {
+            // Transient list failure: retry rather than dropping the only live
+            // update signal (fullReconcile additionally re-syncs the cache on
+            // every WS reconnect).
+            this.log.warn(`agent ${agentId}: provider config refresh failed — scheduling retry`);
+            this.scheduleAgentConfigRefreshRetry(agentId);
+            return;
+        }
+        this.clearAgentConfigRefreshRetry(agentId);
+        if (result.kind === 'skip') {
+            // Agent gone or inactive — the detach path owns teardown.
+            this.log.info(`agent ${agentId}: provider config update for a gone/inactive agent — ignoring`);
+            return;
+        }
+        // Update the cache BEFORE the respawn so any restart path (including a
+        // pending crash-backoff timer) starts the child with the fresh config.
+        // Equal configs mean the change was already applied (spawn/reconcile got
+        // there first, or a duplicate event) — skip the pointless respawn.
+        const fresh = result.entry.provider_config;
+        const changed = JSON.stringify(state.providerConfig ?? null) !== JSON.stringify(fresh ?? null);
+        state.providerConfig = fresh;
+        if (!changed) {
+            this.log.info(`agent ${agentId}: provider config already current — no respawn needed`);
+            return;
+        }
+        await this.respawnChildForConfig(state, 'per-agent llm_source changed');
+    }
+    /**
+     * Intentional single-child respawn for a config change. Marks the state
+     * shuttingDown around the terminate so settleChild doesn't schedule a
+     * competing crash-backoff restart (which would also bump restartAttempts
+     * and delay the new route), clears any pending crash timer so
+     * restartChildNow isn't blocked by it, and bails if the agent detached
+     * while the child was terminating.
+     */
+    async respawnChildForConfig(state, reason) {
+        state.shuttingDown = true;
+        if (state.restartTimer) {
+            clearTimeout(state.restartTimer);
+            state.restartTimer = null;
+        }
+        await this.terminateChild(state);
+        if (this.children.get(state.agentId) !== state) {
+            return; // detached mid-restart — the detach path owns the lifecycle
+        }
+        state.shuttingDown = false;
+        await this.restartChildNow(state, reason);
+    }
+    scheduleAgentConfigRefreshRetry(agentId) {
+        if (!this.running || this.agentConfigRefreshRetryTimers.has(agentId))
+            return;
+        const timer = setTimeout(() => {
+            this.agentConfigRefreshRetryTimers.delete(agentId);
+            if (this.running) {
+                void this.handleAgentConfigUpdated(agentId);
+            }
+        }, AGENT_CONFIG_REFRESH_RETRY_DELAY_MS);
+        timer.unref?.();
+        this.agentConfigRefreshRetryTimers.set(agentId, timer);
+    }
+    clearAgentConfigRefreshRetry(agentId) {
+        const timer = this.agentConfigRefreshRetryTimers.get(agentId);
+        if (timer) {
+            clearTimeout(timer);
+            this.agentConfigRefreshRetryTimers.delete(agentId);
+        }
+    }
     async restartChildNow(state, reason) {
         if (!this.running || state.shuttingDown || state.child || state.restartTimer) {
             return;
         }
+        // Single-flight per child state: fullReconcile and
+        // handleAgentConfigUpdated can both reach here for the same agent (a WS
+        // reconnect fires both), and the state.child guard above cannot see a
+        // restart still awaiting its mint — a second entrant would double-spawn
+        // and orphan the first child (settleChild ignores a superseded
+        // state.child).
+        if (this.restartingStates.has(state)) {
+            return;
+        }
+        this.restartingStates.add(state);
         try {
             state.credential = await this.client.mintLaunchCredential(state.agentId);
             state.restartAttempts = 0;
@@ -1043,6 +1527,26 @@ export class DaemonSupervisor {
         }
         catch (err) {
             this.log.warn(`agent ${state.agentId}: restart mint failed (${reason}): ${String(err)}`);
+            // The old child is already terminated and no crash-backoff timer exists
+            // here — without a retry the agent stays down until the next reconcile
+            // or event. Re-enter through restartChildNow (re-mints the credential)
+            // on the standard backoff curve; guards mirror the settle path plus the
+            // registration-identity backstop.
+            if (this.running &&
+                !state.shuttingDown &&
+                this.children.get(state.agentId) === state &&
+                !state.restartTimer) {
+                const delay = Math.min(this.config.restartBackoffMs * 2 ** state.restartAttempts, this.config.restartBackoffMaxMs);
+                state.restartAttempts += 1;
+                this.log.warn(`agent ${state.agentId}: will retry restart in ${delay}ms`);
+                state.restartTimer = setTimeout(() => {
+                    state.restartTimer = null;
+                    void this.restartChildNow(state, reason);
+                }, delay);
+            }
+        }
+        finally {
+            this.restartingStates.delete(state);
         }
     }
     async spawnAgent(agentId, orgId, attached) {
@@ -1051,6 +1555,7 @@ export class DaemonSupervisor {
         }
         this.spawningAgents.add(agentId);
         let shouldReplaySetupRequest = false;
+        let shouldRecheckConfig = false;
         try {
             await this.spawnAgentOnce(agentId, orgId, attached);
         }
@@ -1058,12 +1563,23 @@ export class DaemonSupervisor {
             this.spawningAgents.delete(agentId);
             this.cancelledSpawns.delete(agentId);
             shouldReplaySetupRequest = this.pendingWorkspaceSetup.delete(agentId);
+            shouldRecheckConfig = this.pendingConfigRefresh.delete(agentId);
         }
         if (shouldReplaySetupRequest && this.running) {
             queueMicrotask(() => {
                 void this.handleWorkspaceSetupRequested(agentId);
             });
         }
+        else if (shouldRecheckConfig && this.running) {
+            // A provider-config update landed while this spawn was in flight, so
+            // the spawn may have committed a pre-event snapshot. Re-check now that
+            // the child state exists; the handler's config comparison no-ops when
+            // the spawn already picked up the fresh config. The workspace replay
+            // above subsumes this (it re-fetches the attached entry and respawns).
+            queueMicrotask(() => {
+                void this.handleAgentConfigUpdated(agentId);
+            });
+        }
     }
     async spawnAgentOnce(agentId, orgId, attached) {
         let credential;
@@ -1131,6 +1647,14 @@ export class DaemonSupervisor {
     startChild(state) {
         if (state.shuttingDown || !this.running)
             return;
+        // Never start a child for a state that is no longer the registered entry:
+        // a detach (or detach + reattach) can land while a deliberate restart is
+        // awaiting its credential mint — respawnChildForConfig restores
+        // shuttingDown=false before the detach handler deletes the entry, so this
+        // identity check is the backstop that keeps a stale state from spawning
+        // an untracked orphan child.
+        if (this.children.get(state.agentId) !== state)
+            return;
         if (!state.credential) {
             this.log.error(`startChild ${state.agentId}: no credential — bug`);
             return;
@@ -1202,7 +1726,14 @@ export class DaemonSupervisor {
             }
             settleChild('error', null, null, err);
         });
-        child.once('close', (code, signal) => settleChild('close', code, signal));
+        // Settle on 'exit', not 'close': with stdio ignore/inherit/inherit the
+        // parent holds no pipes to flush, and Node does not guarantee 'close'
+        // lands in the same tick as 'exit'. Deliberate respawn paths await
+        // terminateChild — an 'exit' listener registered AFTER this one — so
+        // settling here guarantees state.child is already cleared when
+        // terminateChild resolves and restartChildNow can run immediately
+        // instead of silently deferring to the crash-backoff timer.
+        child.once('exit', (code, signal) => settleChild('exit', code, signal));
         const stableTimer = setTimeout(() => {
             if (state.child === child) {
                 state.restartAttempts = 0;
@@ -1210,6 +1741,12 @@ export class DaemonSupervisor {
         }, Math.max(this.config.restartBackoffMs, 30_000));
         stableTimer.unref?.();
     }
+    /**
+     * SIGTERM the child (SIGKILL after 10s) and resolve once it has exited.
+     * settleChild's 'exit' listener is registered before this one, so
+     * state.child is guaranteed cleared by the time this resolves — callers
+     * may start a replacement child immediately after awaiting.
+     */
     async terminateChild(state) {
         const child = state.child;
         if (!child)