@songsid/agend 2.1.8 → 2.1.9

package/dist/fleet-manager.js

@@ -1,10 +1,10 @@
-import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
+import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync, accessSync, constants as fsConstants } from "node:fs";
 import { randomBytes, randomUUID } from "node:crypto";
 import { spawnSync } from "node:child_process";
 import { freemem, totalmem, cpus } from "node:os";
 import { createServer } from "node:http";
 import { Worker } from "node:worker_threads";
-import { join, dirname, basename } from "node:path";
+import { join, dirname, basename, delimiter, isAbsolute } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
 import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
@@ -41,7 +41,7 @@ import { WebhookEmitter } from "./webhook-emitter.js";
 import { TmuxControlClient } from "./tmux-control.js";
 import { safeHandler } from "./safe-async.js";
 import { RoutingEngine } from "./routing-engine.js";
-import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
+import { InstanceLifecycle, SupersededStartError, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
 import { TopicArchiver } from "./topic-archiver.js";
 import { StatuslineWatcher } from "./statusline-watcher.js";
 import { outboundHandlers } from "./outbound-handlers.js";
@@ -66,7 +66,10 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
 import { assertExplicitInstanceRemoval } from "./instance-removal.js";
 import { validateFleetConfig } from "./config-validator.js";
 import { readLastInboundAt } from "./daemon.js";
-import { clearPausedMarker } from "./pause-marker.js";
+import { clearPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
+import { DEFAULT_WARM_OVERFLOW, WakeCoordinator } from "./wake-coordinator.js";
+import { TargetQueueWorker } from "./target-queue-worker.js";
+import { resolveDeliveryWorkerMode } from "./types.js";
 import { isFleetStartCommandLine, readProcessCommandLine, releaseProcessFleetLock } from "./fleet-lock.js";
 import { isSetupComplete, markSetupComplete } from "./setup-marker.js";
 import { manualCleanupMessage, reapStaleTunnel } from "./tunnel/lease.js";
@@ -286,6 +289,14 @@ export class FleetManager {
     managerBootId = randomUUID();
     deliveryOutbox = null;
     deliveryOutboxRecovered = false;
+    /** Phase 2b: wakes paused targets for queued durable work (delivery_worker ≠ off). */
+    wakeCoordinator = null;
+    /**
+     * Phase 2c: targets whose durable lane a TargetQueueWorker owns
+     * (delivery_worker: on). Granted and released only in the pump's
+     * synchronous section and only while the lane is empty; the pump skips them.
+     */
+    queueWorkers = new Map();
     deliveryPumpScheduled = false;
     deliveryPumpRunning = false;
     deliveryPumpTimer = null;
@@ -570,9 +581,15 @@ export class FleetManager {
             this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
             this.deliveryOutboxRecovered = true;
         }
-        outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
+        outbox.on("admitted", () => { this.scheduleDeliveryOutboxPump(); this.wakeCoordinator?.kick(); });
         outbox.on("state", (event) => {
             this.scheduleDeliveryOutboxPump();
+            this.wakeCoordinator?.kick();
+            if (event.deliveryId && event.state === "delivered") {
+                const target = outbox.get(event.deliveryId)?.targetInstance;
+                if (target)
+                    this.wakeCoordinator?.noteDelivered(target);
+            }
             if (event.deliveryId && (event.state === "failed" || event.state === "uncertain")) {
                 const row = outbox.get(event.deliveryId);
                 if (row && row.kind !== "delivery_outcome_notice" && row.kind !== "post_restart_outcome_notice") {
@@ -591,8 +608,11 @@ export class FleetManager {
         outbox.on("content_digest_mismatch", (event) => {
             this.logger.warn(event, "Delivered text differs from the admitted text (content digest mismatch)");
         });
+        this.wakeCoordinator = this.createWakeCoordinator();
+        this.wakeCoordinator.start();
         outbox.on("expired", (event) => {
             this.scheduleDeliveryOutboxPump();
+            this.wakeCoordinator?.kick();
             if (event.count)
                 this.notifyFleetError(`${event.count} durable delivery row(s) reached the 24-hour outbox bound; ${event.uncertain ?? 0} unresolved submissions are uncertain and sender notices were queued.`);
         });
@@ -619,6 +639,9 @@ export class FleetManager {
                 kind: input.kind,
             }, "Durable cross-instance delivery accepted");
         this.scheduleDeliveryOutboxPump();
+        // Phase 2b: a paused wake_only target is woken by the coordinator, never by
+        // the pump; admission must tell it, or the work waits for the watchdog.
+        this.wakeCoordinator?.kick();
         return {
             deliveryId: admitted.delivery.deliveryId,
             state: admitted.delivery.state,
@@ -681,7 +704,7 @@ export class FleetManager {
         }
     }
     scheduleDeliveryOutboxPump(delayMs = 0) {
-        if (!this.deliveryOutbox || this.shuttingDown)
+        if (!this.deliveryOutbox || this.shuttingDown || !this.deliveryOutbox.isOpen)
             return;
         if (this.deliveryPumpTimer) {
             if (delayMs > 0)
@@ -701,14 +724,27 @@ export class FleetManager {
     }
     async runDeliveryOutboxPump() {
         const outbox = this.deliveryOutbox;
-        if (!outbox || this.shuttingDown || this.deliveryPumpRunning)
+        // A closed database is never queried: this runs from timers, where a throw
+        // is an unhandled rejection. Checking here covers the finally block too:
+        // the body below never awaits, so the database cannot close in between.
+        if (!outbox || this.shuttingDown || this.deliveryPumpRunning || !outbox.isOpen)
             return;
         this.deliveryPumpRunning = true;
         this.deliveryPumpScheduled = false;
         try {
-            outbox.expireStale();
+            outbox.expireStale(undefined, undefined, target => {
+                const failure = this.wakeCoordinator?.wakeFailure(target);
+                return failure ? `target could not be woken (${failure})` : undefined;
+            });
+            this.reconcileQueueWorkers(outbox);
             while (this.activeDurableTargets.size < 8) {
-                const claimed = outbox.claimNext(this.managerBootId, target => this.isInstanceRestarting(target) ? null : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
+                const claimed = outbox.claimNext(this.managerBootId, 
+                // Phase 2b: a wake_only target is claimed only once it is awake; the
+                // wake coordinator, not this pump, wakes it (no claim, no attempt).
+                // Phase 2c: a lane a queue worker owns is never claimed here.
+                target => this.queueWorkers.has(target) || this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target)
+                    ? null
+                    : this.daemons.get(target)?.bootId ?? null, this.activeDurableTargets);
                 if (!claimed)
                     break;
                 this.activeDurableTargets.add(claimed.targetInstance);
@@ -736,7 +772,67 @@ export class FleetManager {
             }
         }
     }
-    async dispatchDurableDelivery(claimed) {
+    /**
+     * Phase 2c ownership (design §1.5-A), called synchronously from the pump
+     * before its claim loop, so a grant and the pump's own claims cannot
+     * interleave. A worker is granted a target's lane only when the target is
+     * `on` and its lane is empty: no row in flight in the outbox and no pump
+     * dispatch holding it. It gives the lane back only when it is idle and the
+     * target is no longer `on` (or has nothing pending). A flag change never
+     * takes a lane mid-drain.
+     */
+    reconcileQueueWorkers(outbox) {
+        const pendingTargets = new Set(outbox.listPending().map(row => row.targetInstance));
+        for (const [target, worker] of this.queueWorkers) {
+            const keep = this.deliveryWorkerMode(target) === "on" && pendingTargets.has(target);
+            if (!keep && !worker.busy && !worker.inFlight)
+                this.queueWorkers.delete(target);
+        }
+        for (const target of pendingTargets) {
+            if (this.queueWorkers.has(target) || this.deliveryWorkerMode(target) !== "on")
+                continue;
+            const laneEmpty = !this.activeDurableTargets.has(target)
+                && outbox.countForTarget(target, ["delivering", "submission_started"]) === 0;
+            if (laneEmpty)
+                this.queueWorkers.set(target, this.createQueueWorker(target));
+        }
+        for (const worker of this.queueWorkers.values())
+            void worker.drain();
+    }
+    createQueueWorker(target) {
+        const worker = new TargetQueueWorker(target, {
+            owns: () => this.queueWorkers.get(target) === worker,
+            wanted: () => this.deliveryWorkerMode(target) === "on",
+            available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
+            blocked: () => this.isInstanceRestarting(target) || this.wakeCoordinator?.blocksClaim(target) === true,
+            daemonBootId: () => this.daemons.get(target)?.bootId ?? null,
+            claim: bootId => this.deliveryOutbox?.claimNext(this.managerBootId, candidate => candidate === target ? bootId : null, new Set()),
+            tryAcquireBudget: () => {
+                if (this.activeDurableTargets.size >= 8)
+                    return false;
+                this.activeDurableTargets.add(target);
+                return true;
+            },
+            releaseBudget: () => { this.activeDurableTargets.delete(target); },
+            dispatch: async (claimed) => {
+                try {
+                    await this.dispatchDurableDelivery(claimed, { holdLaneOnTransportError: true });
+                }
+                finally {
+                    // The row's state event can fire before the hand-off settles; the
+                    // pump pass it starts then still sees this worker busy and keeps it
+                    // as owner. One kick after the dispatch has really settled lets the
+                    // pump release an owner that is no longer wanted (or claim the next
+                    // row). The no-claim path never kicks, so no retry hot loop.
+                    this.scheduleDeliveryOutboxPump();
+                }
+            },
+            kickCoordinator: () => this.wakeCoordinator?.kick(),
+            logger: this.logger,
+        });
+        return worker;
+    }
+    async dispatchDurableDelivery(claimed, opts = {}) {
         const outbox = this.deliveryOutbox;
         if (!outbox)
             return;
@@ -762,7 +858,7 @@ export class FleetManager {
                     content: payload.content,
                     delivery_id: claimed.deliveryId,
                     delivery_attempt: String(claimed.attemptNo),
-                }, { waitForIdle: false });
+                }, { waitForIdle: false, noInlineWake: this.deliveryWorkerMode(target) !== "off" });
             }
             else {
                 payload.meta = {
@@ -777,6 +873,7 @@ export class FleetManager {
                 sent = await this.deliverToInstance(target, payload, {
                     isCrossInstance: true,
                     waitForIdle: claimed.kind !== "steer",
+                    noInlineWake: this.deliveryWorkerMode(target) !== "off",
                 });
             }
             if (!sent) {
@@ -789,7 +886,10 @@ export class FleetManager {
         catch (err) {
             const latest = outbox.get(claimed.deliveryId);
             const reason = err instanceof Error ? err.message : String(err);
-            if (latest?.state === "submission_started") {
+            if (opts.holdLaneOnTransportError && (latest?.state === "submission_started" || latest?.state === "reconciliation_pending")) {
+                this.logger.warn({ deliveryId: claimed.deliveryId, target, attempt, err: reason }, "Transport failed after the submission began; holding the lane for the daemon's verdict or its generation's reconciliation");
+            }
+            else if (latest?.state === "submission_started") {
                 outbox.complete(claimed.deliveryId, claimed.targetDaemonBootId, claimed.attemptNo, "uncertain", reason);
             }
             else {
@@ -1688,13 +1788,15 @@ export class FleetManager {
             exclude,
             isEvicting: name => this.warmCapEvicting.has(name),
             isGeneral: name => isGeneralInstance(this.fleetConfig, name),
-            isIdle: name => this.getInstanceExecutionState(name) === "idle",
+            // A work lease (Phase 2a) keeps a just-woken or mid-submission instance
+            // out of the eviction candidates; it is not idle in the sense that matters.
+            isIdle: name => this.getInstanceExecutionState(name) === "idle" && !this.lifecycle.hasWorkLease(name),
             lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
         });
         for (const victim of victims) {
             this.warmCapEvicting.add(victim);
             this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
-            this.lifecycle.pause(victim)
+            this.lifecycle.pause(victim, "warm_cap")
                 .catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
                 .finally(() => this.warmCapEvicting.delete(victim));
         }
@@ -1787,16 +1889,22 @@ export class FleetManager {
                 finish(false);
         });
     }
-    async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
+    async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false) {
         if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
             return false;
         await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
         if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
             return false;
         let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
+        if (this.lifecycle.isPaused(instanceName) && noInlineWake) {
+            // Paused after its row was claimed (an operator or auth pause): hand the
+            // row back; the coordinator decides whether and when to wake it.
+            this.wakeCoordinator?.kick();
+            return false;
+        }
         if (this.lifecycle.isPaused(instanceName)) {
             const wakeStartedAt = Date.now();
-            await this.lifecycle.wake(instanceName, 30_000);
+            await this.explicitWake(instanceName, 30_000);
             if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
                 return false;
             // Waking added one to the warm count — make room by evicting a different
@@ -1919,7 +2027,11 @@ export class FleetManager {
             ?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
         if (!waitForIdle) {
             if (this.lifecycle.isPaused(instanceName)) {
-                await this.lifecycle.wake(instanceName, 30_000);
+                if (options.noInlineWake) {
+                    this.wakeCoordinator?.kick();
+                    return false;
+                }
+                await this.explicitWake(instanceName, 30_000);
                 if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
                     return false;
                 this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
@@ -1935,7 +2047,7 @@ export class FleetManager {
             return true;
         }
         const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
-        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
+        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true));
         this.idleGatedDeliveryTails.set(instanceName, delivery);
         try {
             return await delivery;
@@ -1951,9 +2063,117 @@ export class FleetManager {
         const allowed = this.getChannelConfig(adapterId)?.access?.allowed_users ?? [];
         return allowed.some(entry => String(entry) === String(userId));
     }
+    /** Phase 2: delivery_worker for a target (instance override → fleet default → off). */
+    deliveryWorkerMode(target) {
+        return resolveDeliveryWorkerMode(this.fleetConfig, target);
+    }
+    /** Active instances mid-restart: they keep their warm slot through the stop → replacement gap. */
+    restartsHoldingSlot = new Set();
+    /**
+     * Instances counted against the wake coordinator's hard cap: running
+     * daemons, plus active instances whose restart is between stopping the old
+     * daemon and publishing the replacement.
+     */
+    warmInstanceNames() {
+        const names = new Set([...this.daemons.keys()].filter(name => this.getInstanceStatus(name) === "running"));
+        for (const name of this.restartsHoldingSlot)
+            names.add(name);
+        return [...names];
+    }
+    createWakeCoordinator() {
+        return new WakeCoordinator({
+            mode: target => this.deliveryWorkerMode(target),
+            available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
+            listPending: () => this.deliveryOutbox?.listPending() ?? [],
+            isPaused: target => this.lifecycle.isPaused(target),
+            pauseReason: target => readPauseReason(this.getInstanceDir(target)),
+            isRestarting: target => this.isInstanceRestarting(target),
+            wake: async (target) => {
+                await this.lifecycle.wake(target, 30_000, undefined, { source: "coordinator" });
+                // The soft cap still applies: evict a different, unleased idle instance.
+                this.enforceWarmCap(target);
+            },
+            residentNames: () => this.warmInstanceNames(),
+            warmCap: () => this.fleetConfig?.defaults?.warm_cap ?? 0,
+            warmOverflow: () => this.fleetConfig?.defaults?.warm_overflow ?? DEFAULT_WARM_OVERFLOW,
+            wokeAt: target => this.lifecycle.wokeAtFor(target),
+            notAcceptingReason: target => this.daemons.get(target)?.notAcceptingReason?.() ?? null,
+            notifyTarget: (target, text) => { this.notifyInstanceTopic(target, text); },
+            notifySender: (source, text) => { this.notifyInstanceTopic(source, text); },
+            kickPump: () => this.scheduleDeliveryOutboxPump(),
+            now: () => Date.now(),
+            logger: this.logger,
+        });
+    }
+    /** LifecycleContext: an operator or user wake clears the coordinator's park/backoff for it. */
+    onExternalWake(name) {
+        this.wakeCoordinator?.noteExternalWake(name);
+    }
+    /**
+     * Phase 2b shared warm-slot admission (design §1.3, §1.6) for every wake
+     * that is not the coordinator's own: an operator's wake, a user's message,
+     * an explicit start or a restart of a paused instance. With delivery_worker
+     * off, or for a target that is not paused (an active restart is already
+     * counted), nothing is reserved. A target already being woken is joined —
+     * its flight holds the slot — rather than reserved twice. Otherwise a slot
+     * is taken under the hard cap, an unleased idle instance is paused to make
+     * one, and if none can be, the wake is refused instead of exceeding the cap.
+     */
+    async acquireWakeSlot(name) {
+        const coordinator = this.wakeCoordinator;
+        if (!coordinator || this.deliveryWorkerMode(name) === "off" || !this.lifecycle.isPaused(name))
+            return null;
+        if (coordinator.isWaking(name))
+            return null;
+        let token = coordinator.tryReserve(name);
+        if (!token) {
+            const victim = this.lruUnleasedIdle(name);
+            if (victim) {
+                await this.lifecycle.pause(victim, "warm_cap");
+                if (coordinator.isWaking(name))
+                    return null;
+                token = coordinator.tryReserve(name);
+            }
+        }
+        if (!token) {
+            throw new Error(`No warm slot is free for '${name}' (warm_cap + overflow reached and no idle instance can be paused); try again when one goes idle`);
+        }
+        return token;
+    }
+    releaseWakeSlot(name, token) {
+        if (!token || !this.wakeCoordinator)
+            return;
+        this.wakeCoordinator.release(name, token);
+        this.wakeCoordinator.kick();
+    }
+    /**
+     * An operator's or user's wake (/wake, wake_instance, start_instance on a
+     * paused instance, Settings, a channel message). With delivery_worker off it
+     * is exactly the old lifecycle wake; otherwise it goes through the shared
+     * slot admission and joins any wake already in flight for the target.
+     */
+    async explicitWake(name, timeoutMs = 30_000) {
+        const token = await this.acquireWakeSlot(name);
+        try {
+            await this.lifecycle.wake(name, timeoutMs);
+        }
+        finally {
+            this.releaseWakeSlot(name, token);
+        }
+    }
+    /** The least-recently-active idle, unleased, non-general running instance other than `exclude`. */
+    lruUnleasedIdle(exclude) {
+        return this.warmInstanceNames()
+            .filter(name => name !== exclude
+            && !this.warmCapEvicting.has(name)
+            && !isGeneralInstance(this.fleetConfig, name)
+            && this.getInstanceExecutionState(name) === "idle"
+            && !this.lifecycle.hasWorkLease(name))
+            .sort((a, b) => (readLastInboundAt(this.getInstanceDir(a)) ?? 0) - (readLastInboundAt(this.getInstanceDir(b)) ?? 0))[0];
+    }
     async changeInstancePauseState(name, action) {
         if (action === "wake") {
-            await this.lifecycle.wake(name, 30_000);
+            await this.explicitWake(name, 30_000);
             this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
             return "awake";
         }
@@ -2072,12 +2292,14 @@ export class FleetManager {
      * and reconcile calls leave this false so a persisted pause remains paused
      * across a fleet restart.
      */
-    resumePaused = false) {
+    resumePaused = false, 
+    /** Phase 2a: the caller's transition, for a start made from inside a restart or wake. */
+    transition) {
         // Any start supersedes a pending automatic retry (it would otherwise fire
         // into a running instance — harmless, but noisy — or race this start).
         this.cancelStartupRetry(name);
         if (resumePaused && this.lifecycle.isPaused(name)) {
-            await this.lifecycle.wake(name, 30_000);
+            await this.explicitWake(name, 30_000);
             // A successful wake clears the persisted pause marker and produces a
             // fresh instance_state snapshot.  Drop any stale process error left by a
             // pre-pause crash so /api/fleet and `agend ls` converge on running.
@@ -2104,6 +2326,10 @@ export class FleetManager {
             this.logger.info({ name }, "Instance already running, skipping");
             return;
         }
+        // An explicit start (CLI/API) begins a fresh idle window; boot and
+        // reconcile starts keep the persisted last-inbound seed (Phase 2a).
+        if (resumePaused)
+            this.lifecycle.markActivitySeedNow(name);
         const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, backend, name);
@@ -2127,7 +2353,7 @@ export class FleetManager {
             kind,
             backend,
             model: this.resolveInstanceModel(name).display,
-        });
+        }, transition);
         // Only clear a stale process status after a real start succeeded.  Clearing
         // it before lifecycle.start() can turn a crash-loop daemon's dead pane into
         // a falsely running instance when lifecycle.start() returns early.
@@ -2141,18 +2367,18 @@ export class FleetManager {
         this.requestDiscordUsagePresenceRefresh();
     }
     /** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
-    async startPersistedPausedInstance(name) {
+    async startPersistedPausedInstance(name, transition) {
         const topicMode = this.fleetConfig?.channel?.mode === "topic"
             || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
         const fleetConfig = this.fleetConfig?.instances[name];
         if (fleetConfig) {
-            await this.startInstance(name, fleetConfig, topicMode);
+            await this.startInstance(name, fleetConfig, topicMode, "fleet-topic", false, transition);
             return;
         }
         const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
         if (!channel || !this.classicChannels)
             throw new Error(`Paused instance '${name}' is no longer configured`);
-        await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+        await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after), transition);
     }
     /**
      * Start instances with configurable concurrency and stagger delay.
@@ -2201,6 +2427,10 @@ export class FleetManager {
             return this.daemons.has(name);
         }
         catch (err) {
+            // Superseded by a stop/restart (Phase 2a): that transition owns the
+            // outcome. Retrying would start an instance the operator just stopped.
+            if (err instanceof SupersededStartError)
+                return false;
             this.logger.error({ err, name }, `Failed to start ${what}`);
             this.scheduleStartupRetry(name, 0);
             return false;
@@ -2322,6 +2552,9 @@ export class FleetManager {
             }
         }
         catch (err) {
+            // Superseded by a stop/restart: that transition owns the outcome.
+            if (err instanceof SupersededStartError)
+                return;
             this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
             this.scheduleStartupRetry(name, attempt + 1);
         }
@@ -2362,6 +2595,8 @@ export class FleetManager {
             return this.daemons.has(ch.instanceName);
         }
         catch (err) {
+            if (err instanceof SupersededStartError)
+                return false;
             this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
             this.scheduleStartupRetry(ch.instanceName, 0);
             return false;
@@ -2518,7 +2753,7 @@ export class FleetManager {
         }
         return false;
     }
-    async stopInstance(name) {
+    async stopInstance(name, transition) {
         this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
         this.cancelStartupRetry(name);
         this.failoverActive.delete(name);
@@ -2537,7 +2772,7 @@ export class FleetManager {
         // the hand-off snapshotted it).
         const run = (async () => {
             try {
-                await this.lifecycle.stop(name);
+                await this.lifecycle.stop(name, transition);
             }
             finally {
                 this.clearNoncePromptsForInstance(name);
@@ -2559,11 +2794,16 @@ export class FleetManager {
             return inFlight;
         }
         const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
-        const run = this.spawnGate.run({
+        // Phase 2a: invalidate synchronously so a start still spawning does not
+        // publish, then queue behind it as this instance's next transition. The
+        // spawn gate is taken inside the transition, never the other way round, so
+        // the start being waited for can still get its own gate slot.
+        this.lifecycle.invalidate(name);
+        const run = this.lifecycle.runTransition(name, transition => this.spawnGate.run({
             instanceName: name,
             workingDirectory,
             reason: "restart",
-        }, () => this.doRestartSingleInstance(name, opts))
+        }, () => this.doRestartSingleInstance(name, opts, transition)))
             .finally(() => {
             this.restartsInFlight.delete(name);
             this.scheduleDeliveryOutboxPump();
@@ -2574,19 +2814,73 @@ export class FleetManager {
     isInstanceRestarting(name) {
         return this.restartsInFlight.has(name);
     }
-    async doRestartSingleInstance(name, opts) {
+    /** InstanceLifecycleContext: an explicit wake joins the restart in flight. */
+    restartInFlight(name) {
+        return this.restartsInFlight.get(name);
+    }
+    /**
+     * Phase 2a: restarting a paused instance is a wake. The stop leaves the
+     * pause marker in place, and the start used to skip on it ("Persisted
+     * paused instance"), turning a resident pause into a marker-only one that
+     * nothing would wake. Clear it before the start; if the start fails, put it
+     * back so the instance stays wakeable instead of becoming plain stopped.
+     */
+    async startAfterRestart(name, wasPaused, pausedAt, reason, start) {
+        const instanceDir = this.getInstanceDir(name);
+        if (wasPaused)
+            clearPausedMarker(instanceDir);
+        this.lifecycle.markActivitySeedNow(name);
+        try {
+            await start();
+        }
+        catch (err) {
+            if (wasPaused && !this.daemons.has(name))
+                writePausedMarker(instanceDir, pausedAt ?? Date.now(), reason);
+            throw err;
+        }
+        finally {
+            this.lifecycle.clearActivitySeedNow(name);
+        }
+    }
+    async doRestartSingleInstance(name, opts, transition) {
         if (this.configPath) {
             this.loadConfig(this.configPath);
             this.routing.rebuild(this.fleetConfig);
             this.reregisterClassicChannels();
         }
         const config = this.fleetConfig?.instances[name];
+        const wasPaused = this.lifecycle.isPaused(name);
+        // A paused instance's restart is a wake: it takes a warm slot like any other
+        // (refused before anything is stopped when none can be had). An active
+        // restart is already counted and reserves nothing.
+        const slot = await this.acquireWakeSlot(name);
+        // An active instance keeps the slot it already occupies across its own
+        // stop → replacement gap: counted as warm until the replacement is
+        // published or the restart settles, so no other target can take it.
+        if (!wasPaused)
+            this.restartsHoldingSlot.add(name);
+        try {
+            await this.doRestartWithSlot(name, opts, transition, config, wasPaused);
+            // An explicit restart of a paused instance is an operator wake (§1.6):
+            // its success lifts the coordinator's park/backoff (the level stays).
+            // Automatic restarts never do.
+            if (wasPaused && opts?.explicit && this.daemons.has(name))
+                this.wakeCoordinator?.noteExternalWake(name);
+        }
+        finally {
+            this.restartsHoldingSlot.delete(name);
+            this.releaseWakeSlot(name, slot);
+        }
+    }
+    async doRestartWithSlot(name, opts, transition, config, wasPaused) {
+        const pausedAt = wasPaused ? (this.lifecycle.getLastPausedAt(name) ?? readPausedAt(this.getInstanceDir(name))) : null;
+        const pauseReason = wasPaused ? readPauseReason(this.getInstanceDir(name)) : null;
         if (config) {
-            await this.stopInstance(name);
+            await this.stopInstance(name, transition);
             if (opts?.freshStart)
                 this.writeFreshStartMarker(name);
             const topicMode = this.fleetConfig?.channel?.mode === "topic";
-            await this.startInstance(name, config, topicMode ?? false);
+            await this.startAfterRestart(name, wasPaused, pausedAt, pauseReason, () => this.startInstance(name, config, topicMode ?? false, "fleet-topic", false, transition));
             this.stormWindow.markRecovered(name);
             return;
         }
@@ -2595,11 +2889,11 @@ export class FleetManager {
         if (channelId) {
             const fleetBackend = this.fleetConfig?.defaults?.backend;
             const adapterId = this.classicChannels.getAdapterIdByInstance(name);
-            await this.stopInstance(name);
+            await this.stopInstance(name, transition);
             await new Promise(r => setTimeout(r, 1000)); // let tmux clean up
             if (opts?.freshStart)
                 this.writeFreshStartMarker(name);
-            await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+            await this.startAfterRestart(name, wasPaused, pausedAt, pauseReason, () => this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after), transition));
             this.stormWindow.markRecovered(name);
             return;
         }
@@ -6002,10 +6296,15 @@ export class FleetManager {
             ?? this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName)?.displayName
             ?? instanceName;
     }
+    /** Inherited keys such as __proto__ must never resolve to a mutable instance. */
+    ownInstanceConfig(instanceName) {
+        const instances = this.fleetConfig?.instances;
+        return instances && Object.hasOwn(instances, instanceName) ? instances[instanceName] : undefined;
+    }
     /** Persist identity to the instance's actual config store. Classic instances
      * are registry rows in classicBot.yaml, not fleet.yaml instance entries. */
     setInstanceDisplayName(instanceName, displayName) {
-        const fleetInstance = this.fleetConfig?.instances[instanceName];
+        const fleetInstance = this.ownInstanceConfig(instanceName);
         if (fleetInstance) {
             fleetInstance.display_name = displayName;
             this.saveFleetConfig();
@@ -6014,7 +6313,7 @@ export class FleetManager {
         return this.classicChannels?.setDisplayNameByInstance(instanceName, displayName) ?? false;
     }
     setInstanceDescription(instanceName, description) {
-        const fleetInstance = this.fleetConfig?.instances[instanceName];
+        const fleetInstance = this.ownInstanceConfig(instanceName);
         if (fleetInstance) {
             fleetInstance.description = description;
             this.saveFleetConfig();
@@ -6970,19 +7269,26 @@ export class FleetManager {
      * Settings picker (#1005/#1021), so an agent and an operator see one truth.
      */
     async listEmojisFor(instanceName, refresh = false) {
-        const self = this.fleetConfig?.instances[instanceName];
-        if (!self)
+        // Seeing is not setting: a ClassicBot instance has no per-instance stamp,
+        // but it reacts in its own channel like any bot and must be able to see
+        // the emojis it can use (only set_persona_emoji refuses it).
+        // Own entries only: an inherited name (constructor, __proto__, toString…)
+        // is not a fleet instance, as set_persona_emoji already holds.
+        const self = this.ownInstanceConfig(instanceName);
+        const classic = !self && this.isClassicPersonaTarget(instanceName);
+        if (!self && !classic)
             return { error: this.personaEmojiMissing(instanceName) };
         const worldId = this.getInstanceAdapterId(instanceName);
         const { platform } = this.resolveStatusEmojisFor(instanceName);
         const channel = (worldId ? this.worlds.get(worldId)?.channelConfig : undefined) ?? this.getChannelConfig(worldId ?? undefined);
-        const current = previewStatusEmojis({ platform, platformConfig: channel?.options?.status_emojis, instanceConfig: self.status_emojis });
+        const current = previewStatusEmojis({ platform, platformConfig: channel?.options?.status_emojis, instanceConfig: self?.status_emojis });
         const out = {
             platform: platform ?? null,
             statuses: current.entries.map(e => ({ status: e.key, value: e.value, source: e.source })),
             standard: platform === "telegram"
                 ? { reactions: [...TELEGRAM_REACTION_EMOJIS], note: "Telegram reacts only with these; progress_prefix may be any single emoji" }
                 : { suggestions: STATUS_EMOJI_SUGGESTIONS, note: "any single emoji works" },
+            ...(classic ? { note: "ClassicBot instances have no per-instance status emojis: the statuses above are the connection's, which an operator sets in Settings. You can still use any emoji listed here in your own reactions." } : {}),
         };
         if (platform === "discord" && worldId) {
             const listed = await this.listGuildEmojis(worldId, refresh);
@@ -7013,9 +7319,10 @@ export class FleetManager {
      * from that listed emoji's numeric id. Static PNG, size-capped, cached by id.
      */
     async previewEmojis(instanceName, args) {
-        const self = this.fleetConfig?.instances[instanceName];
-        if (!self)
+        // Seeing, like list_emojis: ClassicBot instances may preview too.
+        if (!this.ownInstanceConfig(instanceName) && !this.isClassicPersonaTarget(instanceName)) {
             return { error: this.personaEmojiMissing(instanceName) };
+        }
         const wanted = args.emojis;
         if (!Array.isArray(wanted) || wanted.length === 0 || wanted.some(e => typeof e !== "string")) {
             return { error: "emojis is required: a list of <:name:id> values from list_emojis" };
@@ -7110,7 +7417,7 @@ export class FleetManager {
      * An empty emoji removes that status's override.
      */
     async setPersonaEmoji(instanceName, args) {
-        const self = this.fleetConfig?.instances[instanceName];
+        const self = this.ownInstanceConfig(instanceName);
         if (!self)
             return { error: this.personaEmojiMissing(instanceName) };
         const status = (args.status ?? "delivered");
@@ -7155,8 +7462,9 @@ export class FleetManager {
         // Re-read after the validation above, which may have waited on Discord:
         // another set_persona_emoji or a Settings save can have changed another
         // status, or replaced this instance's config object, meanwhile. Change
-        // only this one key on what is there now.
-        const current = this.fleetConfig?.instances[instanceName];
+        // only this one key on what is there now. Ownership must still hold:
+        // removal during the await must not fall through to an inherited entry.
+        const current = this.ownInstanceConfig(instanceName);
         if (!current)
             return { error: this.personaEmojiMissing(instanceName) };
         const map = { ...(current.status_emojis ?? {}) };
@@ -7173,6 +7481,10 @@ export class FleetManager {
         const resolved = this.resolveStatusEmojisFor(instanceName);
         return { status, value: value ?? null, now: resolved[status], status_emojis: current.status_emojis ?? null };
     }
+    /** A ClassicBot instance: it may see (list/preview) emojis, never set a per-instance stamp. */
+    isClassicPersonaTarget(instanceName) {
+        return this.classicChannels?.getAll().some(c => c.instanceName === instanceName) ?? false;
+    }
     personaEmojiMissing(instanceName) {
         return this.classicChannels?.getAll().some(c => c.instanceName === instanceName)
             ? "ClassicBot instances have no per-instance status emojis; an operator sets the connection's in Settings"
@@ -7234,6 +7546,16 @@ export class FleetManager {
         const resolved = this.resolveStatusEmojisFor(instanceName, adapterId, adapter);
         return reactionForm(resolved.platform, resolved.received);
     }
+    /**
+     * The stamp for an inbound photo / file a classic bot saved (#1080): the
+     * instance's `status_emojis.photo|attachment`, then its connection's, then
+     * the built-in (📸/📎, or 👌/👍 on Telegram — what these paths always stamped).
+     * Anything else a saved attachment can be counts as a file.
+     */
+    savedAttachmentReactionFor(instanceName, adapter, adapterId, kind) {
+        const resolved = this.resolveStatusEmojisFor(instanceName, adapterId, adapter);
+        return reactionForm(resolved.platform, kind === "photo" ? resolved.photo : resolved.attachment);
+    }
     reactMessageStatus(instanceName, chatId, messageId, status, threadId) {
         // React via the adapter BOUND to this instance — NOT the first discord world.
         // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -9683,10 +10005,16 @@ export class FleetManager {
                 }
                 // The installer may only have added the binary to a profile PATH; a
                 // fresh login shell sees that, the fleet process's PATH may not.
-                if (!this.verifyBinaryOnLoginShell(info.binary)) {
+                const installedAt = this.locateBinaryOnLoginShell(info.binary);
+                if (!installedAt) {
                     await chat.adapter.sendText(chat.chatId, t("install.verify_failed", backend, info.binary), { threadId: chat.threadId }).catch(() => { });
                     return;
                 }
+                // #1059: make it reachable from this process too. `/login` lists what
+                // `which` finds on the fleet's own PATH, and instances resolve their
+                // binary the same way, so a binary only a login shell could see was
+                // installed, reported as verified, and then offered nowhere.
+                this.adoptBinaryDirectory(installedAt, backend);
                 if (!LOGIN_FLOWS[backend]) {
                     await chat.adapter.sendText(chat.chatId, t("install.success_no_login", backend), { threadId: chat.threadId }).catch(() => { });
                     return;
@@ -9746,15 +10074,35 @@ export class FleetManager {
         return t("install.cancelled", backend);
     }
     /** `command -v` on a login shell, so PATH additions from rc files count. */
-    verifyBinaryOnLoginShell(binary) {
+    /**
+     * The absolute path a fresh login shell resolves `binary` to, or null. Only
+     * a real executable file counts: `command -v` also answers with an alias
+     * definition or a function name, neither of which a spawn could run.
+     */
+    locateBinaryOnLoginShell(binary) {
         try {
-            const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe" });
-            return result.status === 0;
+            const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe", encoding: "utf8" });
+            if (result.status !== 0)
+                return null;
+            const path = String(result.stdout ?? "").trim().split("\n").pop()?.trim() ?? "";
+            if (!isAbsolute(path))
+                return null;
+            accessSync(path, fsConstants.X_OK);
+            return statSync(path).isFile() ? path : null;
         }
         catch {
-            return false;
+            return null;
         }
     }
+    /** Put an installed binary's directory on the fleet's PATH (#1059), once. */
+    adoptBinaryDirectory(binaryPath, backend) {
+        const dir = dirname(binaryPath);
+        const entries = (process.env.PATH ?? "").split(delimiter).filter(Boolean);
+        if (entries.includes(dir))
+            return;
+        process.env.PATH = [dir, ...entries].join(delimiter);
+        this.logger.info({ backend, dir }, "Added the installed CLI's directory to the fleet PATH");
+    }
     /** "Sign in now?" button after a successful install. */
     async handleInstallLoginConfirm(data, callbackAdapterId, receivingAdapter) {
         const claimed = this.consumeNonceCallback(INSTALL_LOGIN_CALLBACK_PREFIX, /^install-login:([0-9a-f]+):(go|later)$/, data, callbackAdapterId, receivingAdapter);
@@ -10325,9 +10673,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                     const reactAdapter = this.worlds.get(msg.adapterId ?? "")?.adapter ?? this.adapter;
                     const noMentionReactChatId = msg.threadId ?? msg.chatId;
                     if (reactAdapter && noMentionReactChatId && msg.messageId) {
-                        const emoji = msg.source === "telegram"
-                            ? (saved.kind === "photo" ? "👌" : "👍")
-                            : (saved.kind === "photo" ? "📸" : "📎");
+                        const emoji = this.savedAttachmentReactionFor(instanceName, reactAdapter, msg.adapterId, saved.kind);
                         reactAdapter.react(noMentionReactChatId, msg.messageId, emoji)
                             .catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
                     }
@@ -10358,9 +10704,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 return;
             // Attachments already saved at the top of the collab block.
             if (saved && classicAdapter && collabReactChatId && msg.messageId) {
-                const emoji = msg.source === "telegram"
-                    ? (saved.kind === "photo" ? "👌" : "👍")
-                    : (saved.kind === "photo" ? "📸" : "📎");
+                const emoji = this.savedAttachmentReactionFor(instanceName, classicAdapter, msg.adapterId, saved.kind);
                 classicAdapter.react(collabReactChatId, msg.messageId, emoji)
                     .catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
             }
@@ -10408,10 +10752,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             const reactAdapter = this.worlds.get(msg.adapterId ?? "")?.adapter ?? this.adapter;
             const reactChatId = msg.threadId ?? msg.chatId;
             if (saved && reactAdapter && reactChatId && msg.messageId) {
-                // Telegram only supports limited emoji for reactions; use 👌 for photo, 👍 for file
-                const emoji = msg.source === "telegram"
-                    ? (saved.kind === "photo" ? "👌" : "👍")
-                    : (saved.kind === "photo" ? "📸" : "📎");
+                const emoji = this.savedAttachmentReactionFor(instanceName, reactAdapter, msg.adapterId, saved.kind);
                 reactAdapter.react(reactChatId, msg.messageId, emoji)
                     .catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
             }
@@ -10457,9 +10798,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             classicMsgAdapter.react(reactChatId, msg.messageId, this.receivedReactionFor(instanceName, classicMsgAdapter, msg.adapterId))
                 .catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
             if (saved) {
-                const savedEmoji = msg.source === "telegram"
-                    ? (saved.kind === "photo" ? "👌" : "👍")
-                    : (saved.kind === "photo" ? "📸" : "📎");
+                const savedEmoji = this.savedAttachmentReactionFor(instanceName, classicMsgAdapter, msg.adapterId, saved.kind);
                 classicMsgAdapter.react(reactChatId, msg.messageId, savedEmoji)
                     .catch(e => this.logger.debug({ err: e.message }, "Auto-react failed"));
             }
@@ -11801,7 +12140,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
     }
     /** Start a classic channel instance with lightweight config */
-    async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
+    async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter, transition) {
         if (this.daemons.has(instanceName))
             return;
         const workDir = join(getAgendHome(), "workspaces", instanceName);
@@ -11828,7 +12167,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             ...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
         };
         const topicMode = this.fleetConfig?.channel?.mode === "topic";
-        await this.startInstance(instanceName, config, topicMode, "classic");
+        await this.startInstance(instanceName, config, topicMode, "classic", false, transition);
     }
     /** Handle /start slash command — register classic channel */
     async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
@@ -11940,6 +12279,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             clearInterval(this.replyObligationTimer);
             this.replyObligationTimer = null;
         }
+        this.wakeCoordinator?.stop();
         if (this.logRotateTimer) {
             clearInterval(this.logRotateTimer);
             this.logRotateTimer = null;
@@ -14001,7 +14341,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 this.logger.info({ name }, "Instance restart requested via HTTP");
                 (async () => {
                     try {
-                        await this.restartSingleInstance(name);
+                        await this.restartSingleInstance(name, { explicit: true });
                         this.logger.info({ name }, "Instance restarted");
                         this.emitSseEvent("status", this.getUiStatus());
                         res.writeHead(200);