@songsid/agend 2.1.2-beta.43 → 2.1.2-beta.45

package/dist/fleet-manager.js

@@ -236,6 +236,8 @@ export class FleetManager {
     instanceActivity = new Map();
     /** instanceName → tail of deliveries waiting for its IPC to come back. */
     ipcWaitTails = new Map();
+    /** instanceName → restart currently executing; concurrent callers join it. */
+    restartsInFlight = new Map();
     // Last user message delivered to each instance — used to react ✅ on completion.
     lastInboundMsg = new Map();
     topicArchiver;
@@ -588,7 +590,25 @@ export class FleetManager {
     getInstanceStatus(name) {
         if (this.lifecycle.isPaused(name))
             return "paused";
+        const daemon = this.lifecycle.daemons.get(name);
         const processStatus = this.instanceProcessStatus.get(name);
+        // IPC can be disconnected during a respawn, so the event which announces
+        // the new live pane may be missed.  The in-process daemon is authoritative
+        // in that case; do not leave a stale `crashed` cache masking an instance
+        // that has already recovered and can answer messages.
+        const daemonStatus = daemon?.getProcessStatus?.();
+        if (daemonStatus === "running") {
+            if (processStatus)
+                this.instanceProcessStatus.delete(name);
+            // A recovered in-process daemon is also the authority for clearing a
+            // marker left by a crash-loop/reconnect race.  This keeps standalone
+            // `agend ls` from seeing the old marker after the next API outage.
+            try {
+                unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
+            }
+            catch { /* absent */ }
+            return "running";
+        }
         if (processStatus)
             return processStatus;
         const pidPath = join(this.getInstanceDir(name), "daemon.pid");
@@ -670,6 +690,13 @@ export class FleetManager {
     cacheInstanceProcessStatus(name, status) {
         if (status === "running") {
             this.instanceProcessStatus.delete(name);
+            // A prior crash-loop marker is one-shot.  Successful respawn is the
+            // authoritative recovery signal even when the marker outlived an IPC
+            // disconnect and was not consumed by a new Daemon constructor.
+            try {
+                unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
+            }
+            catch { /* absent */ }
             return;
         }
         if (status !== "crashed" && status !== "stopped")
@@ -901,11 +928,41 @@ export class FleetManager {
         await new Promise(resolve => setTimeout(resolve, 250));
         await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
     }
-    async startInstance(name, config, topicMode, kind = "fleet-topic") {
+    async startInstance(name, config, topicMode, kind = "fleet-topic", 
+    /**
+     * Explicit starts (CLI/API) may resume a paused or failed daemon.  Startup
+     * and reconcile calls leave this false so a persisted pause remains paused
+     * across a fleet restart.
+     */
+    resumePaused = false) {
+        if (resumePaused && this.lifecycle.isPaused(name)) {
+            await this.lifecycle.wake(name, 30_000);
+            // A successful wake clears the persisted pause marker and produces a
+            // fresh instance_state snapshot.  Drop any stale process error left by a
+            // pre-pause crash so /api/fleet and `agend ls` converge on running.
+            this.instanceProcessStatus.delete(name);
+            return;
+        }
         if (this.lifecycle.isPaused(name)) {
             this.logger.info({ name }, "Persisted paused instance — skipping startup");
             return;
         }
+        if (this.lifecycle.daemons.has(name)) {
+            // A crash-loop daemon remains in the lifecycle map so its health monitor
+            // can expose the failure.  The old start path treated that object as
+            // already running and merely deleted the process-status cache, leaving a
+            // dead pane (and crash marker) behind.  An explicit start is a recovery
+            // request: tear down the failed daemon and build a fresh one.
+            if (resumePaused) {
+                const status = this.getInstanceStatus(name);
+                if (status === "crashed" || status === "stopped") {
+                    await this.restartSingleInstance(name);
+                    return;
+                }
+            }
+            this.logger.info({ name }, "Instance already running, skipping");
+            return;
+        }
         if (config.general_topic) {
             // antigravity (agy) does not read MCP instructions — fleet context and
             // routing instructions are not injected, so it cannot act as a dispatcher.
@@ -916,12 +973,19 @@ export class FleetManager {
             }
             this.ensureGeneralInstructions(config.working_directory, config.backend);
         }
-        this.instanceProcessStatus.delete(name);
         await this.lifecycle.start(name, config, topicMode, {
             kind,
             backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
             model: this.resolveInstanceModel(name).display,
         });
+        // Only clear a stale process status after a real start succeeded.  Clearing
+        // it before lifecycle.start() can turn a crash-loop daemon's dead pane into
+        // a falsely running instance when lifecycle.start() returns early.
+        this.instanceProcessStatus.delete(name);
+        try {
+            unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
+        }
+        catch { /* consumed or absent */ }
         // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
         await this.connectIpcToInstance(name);
     }
@@ -1030,6 +1094,21 @@ export class FleetManager {
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
     async restartSingleInstance(name, opts) {
+        // One restart at a time per instance. Multiple sources can ask concurrently
+        // (MCP revival, /restart, pty_error, model failover); a second stop/start
+        // interleaved with the first tears down the window the first just created.
+        // Later callers join the in-flight restart instead — its opts win.
+        const inFlight = this.restartsInFlight.get(name);
+        if (inFlight) {
+            this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
+            return inFlight;
+        }
+        const run = this.doRestartSingleInstance(name, opts)
+            .finally(() => this.restartsInFlight.delete(name));
+        this.restartsInFlight.set(name, run);
+        return run;
+    }
+    async doRestartSingleInstance(name, opts) {
         if (this.configPath) {
             this.loadConfig(this.configPath);
             this.routing.rebuild(this.fleetConfig);
@@ -6862,7 +6941,7 @@ When users create specialized instances, suggest these configurations:
                 (async () => {
                     try {
                         const topicMode = this.fleetConfig?.channel?.mode === "topic";
-                        await this.startInstance(name, config, topicMode ?? false);
+                        await this.startInstance(name, config, topicMode ?? false, "fleet-topic", true);
                         this.emitSseEvent("status", this.getUiStatus());
                         res.writeHead(200);
                         res.end(JSON.stringify({ ok: true }));