@songsid/agend 2.1.4-beta.47 → 2.1.4-beta.49

package/dist/fleet-context.d.ts

@@ -93,6 +93,13 @@ export interface FleetContext {
     promptTip?(generalName: string, adapter: import("./channel/types.js").ChannelAdapter, chatId: string, threadId?: string): Promise<"posted" | "empty" | "unavailable">;
     /** Persistently enable advanced tips without waiting for the dismissal threshold. */
     unlockAdvancedTips?(userId: string): boolean;
+    /** `/install-cli` backend chooser buttons. Caller enforces fleet-admin. */
+    promptInstallBackends?(chat: {
+        adapter: import("./channel/types.js").ChannelAdapter;
+        adapterId: string;
+        chatId: string;
+        threadId?: string;
+    }): Promise<void>;
     /** `/login` backend chooser buttons. Caller enforces fleet-admin. */
     promptLoginBackends?(chat: {
         adapter: import("./channel/types.js").ChannelAdapter;

package/dist/fleet-manager.d.ts

@@ -22,6 +22,7 @@ import { ClassicChannelManager } from "./classic-channel-manager.js";
 import type { InstanceState } from "./backend/types.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
+import { BackendOutageTracker } from "./backend-outage.js";
 export declare function resolveReplyThreadId(argsThreadId: unknown, instanceConfig?: InstanceConfig): string | undefined;
 /**
  * Pure warm-cap victim selection (extracted for testability). Given the current
@@ -73,6 +74,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     readonly lifecycle: InstanceLifecycle;
     readonly stormWindow: StormWindow;
     readonly spawnGate: SpawnGate;
+    /** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
+    readonly backendOutage: BackendOutageTracker;
     /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons(): Map<string, import("./daemon.js").Daemon>;
     fleetConfig: FleetConfig | null;
@@ -143,6 +146,28 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private ipcWaitTails;
     /** instanceName → restart currently executing; concurrent callers join it. */
     private restartsInFlight;
+    /**
+     * Delayed automatic retries for instances whose startup failed. Before this a
+     * failed start was logged once and the instance stayed `stopped` until an
+     * operator noticed (2026-09-03: 4 kiro instances, all victims of the same
+     * backend outage during a post-update herd). instanceName → pending retry.
+     */
+    private startupRetries;
+    /** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
+    private explicitStopGeneration;
+    /** instanceName → outage hand-off currently executing; a repeat joins it. */
+    private handOffsInFlight;
+    /** instanceName → explicit stop currently executing; the outage hand-off waits for it. */
+    private stopsInFlight;
+    /** Aggregation window for the "N instances failed to start" notice. */
+    private startupRetryNotices;
+    /** Backoff between automatic startup retries; the last step repeats while the backend is down. */
+    static readonly STARTUP_RETRY_BACKOFF_MS: number[];
+    /** Hard cap on automatic startup retries (3 backoff steps + up to 3 more during a backend outage). */
+    static readonly STARTUP_RETRY_MAX_ATTEMPTS = 6;
+    /** Re-check interval when a retry is due but the tmux storm window still blocks spawns. */
+    static readonly STARTUP_RETRY_STORM_DEFER_MS = 60000;
+    static readonly STARTUP_RETRY_NOTICE_AGGREGATE_MS = 1000;
     private lastInboundMsg;
     private topicArchiver;
     controlClient: TmuxControlClient | null;
@@ -368,6 +393,65 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
      * TODO: per-instance startup timeout (existing issue, not introduced here)
      */
     private startInstancesWithConcurrency;
+    /**
+     * Start from an UNATTENDED path (fleet startup, full restart, config
+     * reconcile): nobody is watching the result, so a failure is logged and
+     * handed to the delayed automatic retry instead of leaving the instance
+     * `stopped` forever. Explicit operator/API starts call startInstance directly
+     * and keep their synchronous error. Returns whether the instance is up.
+     */
+    private startInstanceUnattended;
+    /**
+     * Schedule attempt `attempt` (0-based) of the automatic startup retry for an
+     * instance whose start just failed. Backoff 1 → 5 → 15 min; while the
+     * instance's backend is known to be unreachable the 15-min step repeats up to
+     * STARTUP_RETRY_MAX_ATTEMPTS, after which we give up with one notice. Each
+     * retry re-checks the world (still configured, not running, not paused, no
+     * tmux storm) and runs through the SpawnGate as "recovery" — so a herd of
+     * failed instances comes back at the gate's concurrency, never all at once.
+     */
+    scheduleStartupRetry(name: string, attempt: number): void;
+    /** Drop a pending automatic retry (an operator start/stop/restart supersedes it). */
+    cancelStartupRetry(name: string): void;
+    /** Pending automatic retry, if any (status display / tests). */
+    pendingStartupRetry(name: string): {
+        attempt: number;
+    } | null;
+    /**
+     * A daemon's crash-respawn hit the backend outage (startup_backend_unreachable):
+     * stop THAT daemon and schedule the delayed retry. Serialized against the
+     * operator paths: an in-flight restart is awaited first (it replaces the
+     * daemon itself), the stop is identity-checked so a fresh daemon registered
+     * meanwhile is never deleted, and an explicit stop/restart that began during
+     * the hand-off (generation bump) owns the outcome — no retry is scheduled
+     * behind an operator's back.
+     */
+    handOffToStartupRetry(name: string, daemon: unknown): Promise<void>;
+    private runStartupRetry;
+    /** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
+    private isConfiguredInstance;
+    /**
+     * Kind-aware start for the automatic retry: fleet-topic instances come from
+     * fleet.yaml, ClassicBot instances exist only in the classic channel manager
+     * and must be rebuilt through startClassicInstance (a fleet.yaml lookup alone
+     * silently dropped them — the retry timer fired and nothing happened).
+     */
+    private startConfiguredInstance;
+    /**
+     * Unattended ClassicBot start (fleet startup batch, full-restart batch, the
+     * classicBot.yaml reconcile): same contract as startInstanceUnattended —
+     * failures are logged and handed to the delayed automatic retry, whose
+     * kind-aware startConfiguredInstance rebuilds the Classic instance. Returns
+     * whether the instance is up.
+     */
+    private startClassicInstanceUnattended;
+    private backendNameOf;
+    /**
+     * One fleet-level notice per burst, not one per instance: a post-update herd
+     * fails many instances within the same second. Two notices per incident at
+     * most — "N failed, retrying in X" and, if it comes to that, "gave up on N".
+     */
+    private queueStartupRetryNotice;
     private runnableStartupCount;
     private configuredStartupInstanceNames;
     private restartProgressTarget;
@@ -832,6 +916,28 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
         chatId: string;
         threadId?: string;
     }): Promise<void>;
+    /**
+     * Backend chooser for a bare `/install-cli`, mirroring promptLoginBackends so
+     * both commands feel the same. Built on postNonceButtonPrompt rather than the
+     * `/model` selection coordinator: that is the mechanism `/login` already uses,
+     * and the one whose canonical-address binding (#682) makes the buttons answer
+     * in a Telegram General topic.
+     *
+     * Unlike the login chooser this does NOT filter to backends the fleet already
+     * runs. Installing is how you get a backend you do not have yet, so filtering
+     * by configured backends would hide the only entry the admin came for.
+     *
+     * gemini-cli is omitted: it is deprecated (see backend/factory.ts). Typing
+     * `/install-cli gemini-cli` still works — this only stops recommending it.
+     */
+    promptInstallBackends(chat: {
+        adapter: ChannelAdapter;
+        adapterId: string;
+        chatId: string;
+        threadId?: string;
+    }): Promise<void>;
+    /** Backend chooser button → start that backend's install session. */
+    private handleInstallBackendSelect;
     /**
      * Start a login session for one backend. Caller enforces admin.
      * Returns a status line to post, or null when a confirmation prompt was