@songsid/agend 2.1.8 → 2.1.10-beta.1

package/dist/instance-lifecycle.js

@@ -9,7 +9,7 @@ import { isModelCompatible } from "./backend/types.js";
 import { safeHandler } from "./safe-async.js";
 import { t } from "./locale.js";
 import { EMBEDDED_CREDENTIAL_REMOTE_REPAIR_FAILED, EMBEDDED_CREDENTIAL_REMOTE_WARNING, secureGitHubRemotesForWorktree, } from "./remote-credential-check.js";
-import { clearPausedMarker, hasPausedMarker, readPausedAt, writePausedMarker } from "./pause-marker.js";
+import { clearPausedMarker, hasPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
 import { reportProviderRateLimit } from "./usage/provider-alerts.js";
 import { isFleetStartCommandLine } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
@@ -18,18 +18,37 @@ import { fetchClaudeUsage, fetchCodexUsage } from "./usage/providers.js";
 import { assertExplicitInstanceRemoval, authorizeExplicitInstanceRemoval } from "./instance-removal.js";
 import { capturePendingTargetReconciliation, finishTargetReconciliation, reconcileTargetBeforeStart, RECONCILIATION_WRITE_DRAIN_TIMEOUT_MS, retireCapturedTargetWindows, } from "./delivery-reconciliation.js";
 export { isFleetStartCommandLine } from "./fleet-lock.js";
+const localBin = (home) => join(home, ".local", "bin");
 /** Shared CLI metadata used by startup validation and ClassicBot onboarding. */
 export const BACKEND_INSTALLATION_INFO = {
-    "claude-code": { binary: "claude", install: "curl -fsSL https://claude.ai/install.sh | bash" },
+    "claude-code": { binary: "claude", install: "curl -fsSL https://claude.ai/install.sh | bash", binDirs: (_env, home) => [localBin(home)] },
     "gemini-cli": { binary: "gemini", install: "npm i -g @google/gemini-cli" },
     // The curl installer serves Linux and macOS (verified: 200, text/x-shellscript);
     // the previous `brew install --cask` form only worked on macOS.
-    "kiro-cli": { binary: "kiro-cli", install: "curl -fsSL https://cli.kiro.dev/install | bash" },
-    codex: { binary: "codex", install: "curl -fsSL https://chatgpt.com/codex/install.sh | sh" },
-    opencode: { binary: "opencode", install: "curl -fsSL https://opencode.ai/install | bash" },
-    antigravity: { binary: "agy", install: "curl -fsSL https://antigravity.google/cli/install.sh | bash" },
-    grok: { binary: "grok", install: "curl -fsSL https://x.ai/cli/install.sh | bash" },
-    muse: { binary: "muse", install: "curl -fsSL https://api.meta.ai/muse-launcher.sh | bash" },
+    "kiro-cli": { binary: "kiro-cli", install: "curl -fsSL https://cli.kiro.dev/install | bash", binDirs: (_env, home) => [localBin(home)] },
+    // install.sh: BIN_DIR="${CODEX_INSTALL_DIR:-$HOME/.local/bin}"; codex there is
+    // a symlink into ~/.codex/packages/standalone/current/bin (#1092).
+    codex: {
+        binary: "codex", install: "curl -fsSL https://chatgpt.com/codex/install.sh | sh",
+        binDirs: (env, home) => [env.CODEX_INSTALL_DIR, localBin(home)].filter((d) => !!d),
+    },
+    opencode: { binary: "opencode", install: "curl -fsSL https://opencode.ai/install | bash", binDirs: (_env, home) => [join(home, ".opencode", "bin")] },
+    antigravity: { binary: "agy", install: "curl -fsSL https://antigravity.google/cli/install.sh | bash", binDirs: (_env, home) => [localBin(home)] },
+    grok: {
+        binary: "grok", install: "curl -fsSL https://x.ai/cli/install.sh | bash",
+        binDirs: (env, home) => [env.GROK_BIN_DIR, join(home, ".grok", "bin")].filter((d) => !!d),
+    },
+    // #1092: the muse launcher keeps its binary NEXT TO ITSELF. Piped into bash
+    // it has no file of its own, so it downloaded the binary into the current
+    // directory and then started muse there, leaving no `muse` command anywhere.
+    // Its install mode (MUSE_LAUNCHER_INSTALL=1, read from the launcher) is run
+    // on the launcher saved as ~/.local/bin/muse: it downloads beside it and
+    // exits without launching.
+    muse: {
+        binary: "muse",
+        install: 'mkdir -p "$HOME/.local/bin" && curl -fsSL https://api.meta.ai/muse-launcher.sh -o "$HOME/.local/bin/muse" && chmod +x "$HOME/.local/bin/muse" && MUSE_LAUNCHER_INSTALL=1 "$HOME/.local/bin/muse"',
+        binDirs: (_env, home) => [localBin(home)],
+    },
 };
 /** Check one executable using the same PATH visible to the fleet process. */
 export function checkBinaryInstalled(binary) {
@@ -83,6 +102,19 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
     }
     return null;
 }
+/** Thrown by a start whose generation was invalidated (stop/restart) before it could publish. */
+export class SupersededStartError extends Error {
+    constructor(name) {
+        super(`Start of '${name}' was superseded by a stop or restart before it registered`);
+        this.name = "SupersededStartError";
+    }
+}
+/**
+ * How long a target just woken for queued work is held from idle auto-pause
+ * before its first queued delivery is claimed (Phase 2 design §1.6): the
+ * cross-instance idle gate (60s) plus a 60s grace.
+ */
+export const PRECLAIM_LEASE_MS = 120_000;
 /**
  * Working directories of the fleet's other instances on `backendName` (#984):
  * Codex consults them only to decide whether `resume --last` could take a
@@ -194,6 +226,21 @@ export class InstanceLifecycle {
     ctx;
     /** Active daemon processes: instanceName → Daemon */
     daemons = new Map();
+    /**
+     * Phase 2a: one transition (start / stop / wake / restart) at a time per
+     * instance. A call that passes the current transition's handle runs inline;
+     * anything else queues.
+     */
+    transitionTails = new Map();
+    activeTransition = new Map();
+    /** Bumped synchronously by stop/restart: a start begun under an older epoch must not publish. */
+    lifecycleEpochs = new Map();
+    /** Single wake flight per instance: concurrent wakes share it. */
+    wakeFlights = new Map();
+    /** Starts that begin their idle window now instead of at the persisted last inbound. */
+    activitySeedNow = new Set();
+    /** When each instance was last woken or started for work — the pre-claim lease clock. */
+    wokeAt = new Map();
     /** backend → last auth-error alert time, so one expiry sends one alert. */
     lastAuthAlertAt = new Map();
     /**
@@ -610,6 +657,10 @@ export class InstanceLifecycle {
             // restore its tools. With mcp_auto_restart (default) the daemon requests an
             // idle-gated restart itself — an immediate one would interrupt whatever the
             // agent is doing. With it off, tell the operator what to run, as before.
+            if (data.neverConnected) {
+                incident.deathNoticeSent = this.notifyIncident(name, "mcp_died", t(data.autoRestart ? "inst.mcp_never_connected_auto" : "inst.mcp_never_connected_manual", name));
+                return;
+            }
             incident.deathNoticeSent = this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具（無法 reply / 跨 instance 通訊）。\n`
                 + (data.autoRestart
                     ? "CLI 本身還在執行；等它閒置後會自動重啟以恢復工具（進行中的工作不會被打斷，session 會保留）。"
@@ -628,7 +679,7 @@ export class InstanceLifecycle {
                 return;
             }
             incident.deathNoticeSent = false;
-            this.notifyIncident(name, "mcp_recovered", t("inst.mcp_recovered", name));
+            this.notifyIncident(name, "mcp_recovered", t(data.source === "late_connect" ? "inst.mcp_connected_late" : "inst.mcp_recovered", name));
         }, this.ctx.logger, `daemon.mcp_recovered[${name}]`));
         daemon.on("mcp_proxy_reply", safeHandler((data) => {
             // The message itself goes out through the daemon's fleet_outbound path;
@@ -792,8 +843,9 @@ export class InstanceLifecycle {
             // Pattern-matched auth errors get a second opinion from the real CLI
             // before any pause/alert: an agent quoting "401 Unauthorized" in prose
             // must not pause the fleet. A working credential ends the incident here.
+            let authVerdict;
             if (data.type === "auth_error") {
-                const verdict = await this.verifyAuthError(name);
+                const verdict = authVerdict = await this.verifyAuthError(name);
                 if (verdict === "valid") {
                     this.ctx.logger.info({ name, backend: this.backendOf(name) }, "auth-error pattern ignored — token-free auth check passed (likely conversation text)");
                     // Dropping the incident is not enough: the daemon armed its auth
@@ -887,10 +939,10 @@ export class InstanceLifecycle {
                     const notificationTarget = this.ptyErrorNotificationTarget(name);
                     if (notificationTarget)
                         this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_capacity_backoff_exhausted", name, String(attempts)));
-                    void this.pause(name)
+                    void this.pause(name, "error")
                         .catch(err => this.ctx.logger.warn({ err, name }, "capacity-exhausted pause failed"))
                         .finally(() => { if (!this.isPaused(name))
-                        this.daemons.get(name)?.requestPauseWhenIdle(); });
+                        this.daemons.get(name)?.requestPauseWhenIdle({ reason: "error" }); });
                 }
                 else {
                     const delays = InstanceLifecycle.CAPACITY_BACKOFF_DELAYS_MS;
@@ -933,14 +985,96 @@ export class InstanceLifecycle {
                 // credit and lost work. pause() alone no-ops while the pane is busy (the
                 // usual state when auth fails), so mark it to pause as soon as it idles.
                 // Queued messages survive: delivery wakes a paused instance.
-                void this.pause(name)
-                    .catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
-                    .finally(() => { if (!this.isPaused(name))
-                    daemon.requestPauseWhenIdle(); });
+                if (data.type === "auth_error" && authVerdict === "unknown") {
+                    // Nothing vouched for an auth hit the CLI could not be asked about
+                    // (#1044): pause only if it is still on screen when the turn ends.
+                    // Never the direct pause here — on an already-idle pane it would
+                    // /quit before anything looked, and the check would never run. The
+                    // daemon evaluates at once, so an idle pane is decided now.
+                    daemon.requestPauseWhenIdle({ reconfirmAuth: true });
+                }
+                else {
+                    // The marker must say auth (2b never auto-wakes a login failure); any
+                    // other pause-action error is an error pause, not an operator one.
+                    const reason = data.type === "auth_error" ? "auth" : "error";
+                    void this.pause(name, reason)
+                        .catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
+                        .finally(() => { if (!this.isPaused(name))
+                        daemon.requestPauseWhenIdle({ reason }); });
+                }
             }
         }, this.ctx.logger, `daemon.pty_error[${name}]`));
     }
-    async start(name, config, topicMode, runtimeIdentity) {
+    epochOf(name) {
+        return this.lifecycleEpochs.get(name) ?? 0;
+    }
+    /** Invalidate every start/wake already running for `name` (they will not publish). */
+    invalidate(name) {
+        this.lifecycleEpochs.set(name, this.epochOf(name) + 1);
+    }
+    /** Whether `transition` is `name`'s transition and is the one running now. */
+    holds(name, transition) {
+        return transition !== undefined && transition.name === name && this.activeTransition.get(name) === transition.token;
+    }
+    /** Run `fn` as `name`'s next transition, or inline when `parent` is the transition running now. */
+    async runTransition(name, fn, parent) {
+        if (this.holds(name, parent))
+            return fn(parent);
+        const previous = this.transitionTails.get(name) ?? Promise.resolve();
+        const handle = { name, token: Symbol(name) };
+        const run = previous.then(() => {
+            this.activeTransition.set(name, handle.token);
+            return fn(handle);
+        }).finally(() => {
+            if (this.activeTransition.get(name) === handle.token)
+                this.activeTransition.delete(name);
+        });
+        const tail = run.then(() => { }, () => { });
+        this.transitionTails.set(name, tail);
+        try {
+            return await run;
+        }
+        finally {
+            if (this.transitionTails.get(name) === tail)
+                this.transitionTails.delete(name);
+        }
+    }
+    /** The next start of `name` is a wake/restart/explicit start: begin its idle window now. */
+    markActivitySeedNow(name) {
+        this.activitySeedNow.add(name);
+    }
+    /** Drop a seed request that no start consumed (the start returned early or failed before it). */
+    clearActivitySeedNow(name) {
+        this.activitySeedNow.delete(name);
+    }
+    /** Record that `name` was just woken or (re)started, opening its pre-claim lease window. */
+    noteWokeForWork(name, now = Date.now()) {
+        this.wokeAt.set(name, now);
+    }
+    /**
+     * Phase 2a work lease (design §1.6). Held while a durable submission to the
+     * target is in flight (unbounded: no timer releases a writer), or for
+     * PRECLAIM_LEASE_MS after a wake/start while queued work is still waiting to
+     * be claimed. It holds only the idle-timeout pause and warm-cap eviction —
+     * never an auth-deferred or operator pause.
+     */
+    hasWorkLease(name, now = Date.now()) {
+        const outbox = this.ctx.deliveryOutbox;
+        if (!outbox)
+            return false;
+        if (outbox.countForTarget(name, ["delivering", "submission_started"]) > 0)
+            return true;
+        const woke = this.wokeAt.get(name);
+        if (woke === undefined || now - woke >= PRECLAIM_LEASE_MS)
+            return false;
+        return outbox.countForTarget(name, ["queued", "retry_wait"]) > 0;
+    }
+    async start(name, config, topicMode, runtimeIdentity, transition) {
+        return this.runTransition(name, () => this.startInTransition(name, config, topicMode, runtimeIdentity), transition);
+    }
+    async startInTransition(name, config, topicMode, runtimeIdentity) {
+        const epoch = this.epochOf(name);
+        const seedNow = this.activitySeedNow.delete(name);
         if (this.daemons.has(name)) {
             this.ctx.logger.info({ name }, "Instance already running, skipping");
             return;
@@ -979,6 +1113,9 @@ export class InstanceLifecycle {
             model: config.model ?? "default",
         }, this.ctx.spawnGate, this.ctx.stormWindow, this.ctx.backendOutage);
         daemon.setDeliveryOutboxPort(this.ctx.deliveryOutbox ?? undefined);
+        daemon.setWorkLeaseCheck(() => this.hasWorkLease(name));
+        if (seedNow)
+            daemon.seedActivityNow();
         // Read at each spawn so a fleet.yaml reload is honoured (#984).
         daemon.setPeerWorkingDirectories(() => peerWorkingDirectories(this.ctx.fleetConfig, name, backendName));
         daemon.setStatusEmojiAvoidList(() => this.ctx.statusEmojiAvoidList?.(name));
@@ -1019,19 +1156,37 @@ export class InstanceLifecycle {
                 this.ctx.logger.warn({ name, ...reconciled }, "Some durable submissions need operator reconciliation");
             }
             if (!reconciled.safeToStart) {
-                const message = `Instance '${name}' remains stopped because its previous CLI window could not be confirmed retired during durable delivery reconciliation`;
+                const message = t("delivery.restart_blocked", name);
                 this.ctx.logger.error({ name, ...reconciled }, message);
                 this.ctx.notifyFleetError?.(message);
                 await daemon.abortStartup().catch(err => this.ctx.logger.warn({ err, name }, "Failed to dispose daemon after reconciliation could not verify old CLI exit"));
                 throw new Error(message);
             }
         }
+        if (this.epochOf(name) !== epoch) {
+            // A stop/restart arrived while this start was reconciling: do not spawn.
+            await daemon.abortStartup().catch(() => { });
+            throw new SupersededStartError(name);
+        }
+        // The startup scan can raise an incident (sign-in screen, fatal modal)
+        // before the handlers below exist and before this daemon is registered;
+        // hold it and deliver it once both are true.
+        daemon.holdStartupIncidents?.();
         await InstanceLifecycle.startOrDispose(daemon, name, this.ctx.logger);
+        // Publication fence (Phase 2a): the spawn cannot be cancelled, so a stop or
+        // restart requested meanwhile has been waiting behind this transition. A
+        // superseded start must not register, signal ready, or keep its window.
+        if (this.epochOf(name) !== epoch) {
+            await daemon.abortStartup().catch(err => this.ctx.logger.warn({ err, name }, "Failed to dispose a superseded start"));
+            throw new SupersededStartError(name);
+        }
         this.capacityBackoffBaselines.delete(name);
         this.daemons.set(name, daemon);
+        if (seedNow)
+            this.noteWokeForWork(name);
         this.ctx.onDaemonReady?.(name, daemon.bootId);
-        daemon.on("auto_pause_requested", safeHandler(async () => {
-            await this.pause(name);
+        daemon.on("auto_pause_requested", safeHandler(async (data) => {
+            await this.pause(name, data?.reason ?? "idle");
         }, this.ctx.logger, `autoPause[${name}]`));
         daemon.on("auto_paused", (data) => {
             this.ctx.eventLog?.insert(name, "instance_paused", { reason: "idle", paused_at: data.pausedAt });
@@ -1057,6 +1212,9 @@ export class InstanceLifecycle {
         this.attachDeliveryStatusHandlers(name, daemon);
         this.ctx.setTopicIcon(name, "green");
         this.ctx.touchActivity(name);
+        // Last: handlers attached and daemon registered, so a held startup incident
+        // (pause, notify) finds both.
+        daemon.releaseStartupIncidents?.();
     }
     isPaused(name) {
         return this.daemons.get(name)?.isPaused ?? hasPausedMarker(this.ctx.getInstanceDir(name));
@@ -1064,7 +1222,7 @@ export class InstanceLifecycle {
     getLastPausedAt(name) {
         return this.daemons.get(name)?.lastPausedAt ?? readPausedAt(this.ctx.getInstanceDir(name));
     }
-    async pause(name) {
+    async pause(name, reason = "operator") {
         // Final backstop shared by slash, MCP, Settings, warm-cap, and any future
         // caller. General is the coordinator and must remain resident to route work.
         if (isGeneralInstance(this.ctx.fleetConfig, name)) {
@@ -1089,7 +1247,7 @@ export class InstanceLifecycle {
         }
         this.ctx.stopStatuslineWatcher(name);
         try {
-            await daemon.pause();
+            await daemon.pause(reason);
         }
         finally {
             // A rejected/no-op pause leaves the instance active and must not strand
@@ -1098,25 +1256,65 @@ export class InstanceLifecycle {
                 this.ctx.startStatuslineWatcher(name);
         }
     }
-    async wake(name, timeoutMs = 30_000) {
+    /** When `name` was last woken or (re)started for work, if recorded. */
+    wokeAtFor(name) {
+        return this.wokeAt.get(name);
+    }
+    async wake(name, timeoutMs = 30_000, transition, opts = {}) {
+        const woke = await this.wakeInner(name, timeoutMs, transition);
+        // Only a successful operator/user wake lifts the coordinator's park.
+        if (opts.source !== "coordinator")
+            this.ctx.onExternalWake?.(name);
+        return woke;
+    }
+    async wakeInner(name, timeoutMs, transition) {
+        if (this.holds(name, transition))
+            return this.wakeInTransition(name, timeoutMs, transition);
+        // An explicit wake that meets a restart joins it: the restart of a paused
+        // instance is itself a wake (Phase 2a), and a second flight would race it.
+        const restarting = this.ctx.restartInFlight?.(name);
+        if (restarting) {
+            await restarting;
+            return;
+        }
+        const inFlight = this.wakeFlights.get(name);
+        if (inFlight)
+            return inFlight;
+        const flight = this.runTransition(name, handle => this.wakeInTransition(name, timeoutMs, handle))
+            .finally(() => { if (this.wakeFlights.get(name) === flight)
+            this.wakeFlights.delete(name); });
+        this.wakeFlights.set(name, flight);
+        return flight;
+    }
+    async wakeInTransition(name, timeoutMs, transition) {
+        const epoch = this.epochOf(name);
         const daemon = this.daemons.get(name);
         if (daemon) {
             await daemon.wake(timeoutMs);
+            // A stop/restart requested while the resident CLI was respawning is
+            // queued behind this wake and will tear it down next: do not open a
+            // pre-claim window or restart its watchers on the way out.
+            if (this.epochOf(name) !== epoch)
+                return;
         }
         else {
             const instanceDir = this.ctx.getInstanceDir(name);
             if (!hasPausedMarker(instanceDir))
                 throw new Error(`Cannot wake stopped instance '${name}'`);
             const pausedAt = readPausedAt(instanceDir) ?? Date.now();
+            const reason = readPauseReason(instanceDir);
             clearPausedMarker(instanceDir);
+            this.markActivitySeedNow(name);
             try {
-                await this.ctx.startPersistedPausedInstance(name);
+                await this.ctx.startPersistedPausedInstance(name, transition);
             }
             catch (err) {
-                writePausedMarker(instanceDir, pausedAt);
+                this.clearActivitySeedNow(name);
+                writePausedMarker(instanceDir, pausedAt, reason);
                 throw err;
             }
         }
+        this.noteWokeForWork(name);
         this.ctx.startStatuslineWatcher(name);
     }
     /**
@@ -1141,7 +1339,16 @@ export class InstanceLifecycle {
         await this.stop(name);
         return true;
     }
-    async stop(name) {
+    async stop(name, transition) {
+        // Synchronously, before queueing: a start still spawning must see this and
+        // not publish when it finishes (Phase 2a publication fence).
+        if (!this.holds(name, transition))
+            this.invalidate(name);
+        return this.runTransition(name, () => this.stopInTransition(name), transition);
+    }
+    async stopInTransition(name) {
+        this.invalidate(name);
+        this.wokeAt.delete(name);
         this.ctx.setTopicIcon(name, "remove");
         // B1: cancel any pending capacity backoff timer so a stopped/removed
         // instance is not restarted behind the user's back.
@@ -1336,9 +1543,10 @@ export class InstanceLifecycle {
             await ipc.close();
             this.ctx.instanceIpcClients.delete(name);
         }
-        // Remove from routing table
+        // Remove from routing table (this instance only: #1085, another Telegram
+        // world may own the same topic number)
         if (config.topic_id != null) {
-            this.ctx.routing.unregister(config.topic_id);
+            this.ctx.routing.unregister(config.topic_id, name);
         }
         // Remove from fleet config and save
         delete this.ctx.fleetConfig.instances[name];
@@ -1566,7 +1774,7 @@ export class InstanceLifecycle {
             if (newInstanceName && this.ctx.fleetConfig?.instances[newInstanceName]) {
                 delete this.ctx.fleetConfig.instances[newInstanceName];
                 if (createdTopicId)
-                    this.ctx.routing.unregister(createdTopicId);
+                    this.ctx.routing.unregister(createdTopicId, newInstanceName);
                 this.ctx.saveFleetConfig();
             }
             if (createdTopicId) {
@@ -1648,7 +1856,7 @@ export class InstanceLifecycle {
         }
         // 4. Remove old config + routing (so new instance can reuse the name/topic)
         if (topicId != null)
-            this.ctx.routing.unregister(topicId);
+            this.ctx.routing.unregister(topicId, instanceName);
         delete this.ctx.fleetConfig.instances[instanceName];
         this.ctx.saveFleetConfig();
         // 5. Clean instanceDir to avoid stale rotation-state.json / crash-history