@songsid/agend 2.1.2-beta.4 → 2.1.2-beta.40

package/dist/fleet-manager.d.ts

@@ -1,3 +1,5 @@
+import { type FleetMemory } from "./process-memory.js";
+import { ReplyDeduper } from "./reply-dedup.js";
 import type { FleetConfig, RawFleetConfig, InstanceConfig } from "./types.js";
 import { type RouteTarget } from "./fleet-context.js";
 import { EventLog } from "./event-log.js";
@@ -48,7 +50,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private static sighupHandlerInstalled;
     private children;
     readonly lifecycle: InstanceLifecycle;
-    /** @deprecated Use lifecycle.daemons — kept for backward compat */
+    /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons(): Map<string, import("./daemon.js").Daemon>;
     fleetConfig: FleetConfig | null;
     private rawFleetConfig;
@@ -98,15 +100,29 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private instanceIdleWaiters;
     private lastInboundUser;
     private cancelButtons;
+    /** Pending idle-edge retirement, one timer per instance. */
+    private cancelButtonIdleRetireTimers;
+    /** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
+    readonly replyDeduper: ReplyDeduper;
+    /** instanceName → what it is doing right now, when the backend can tell us. */
+    private instanceActivity;
+    /** instanceName → tail of deliveries waiting for its IPC to come back. */
+    private ipcWaitTails;
     private lastInboundMsg;
     private topicArchiver;
     controlClient: TmuxControlClient | null;
     classicChannels: ClassicChannelManager | null;
     private pendingClassicStarts;
     /** In-flight /model selections, keyed by nonce (see handleModelSelection). */
+    /** In-flight /effort selections, same coordinator shape as pendingModelSelects. */
+    private pendingEffortSelects;
     private pendingModelSelects;
     private failoverActive;
+    /** instance → when a click with no live button entry last fired a cancel. */
+    private staleCancelClickAt;
     readonly ipcStoppingInstances: Set<string>;
+    /** Set the moment a graceful stop begins — see isPlannedRestart(). */
+    private shuttingDown;
     /** Coalesce concurrent connection attempts for the same daemon socket. */
     private ipcConnectInFlight;
     /** At most one reconnect/backoff loop may exist per instance. */
@@ -117,6 +133,10 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private healthServer;
     private healthPortRetried;
     private updateCheckTimer;
+    private eventLogPruneTimer;
+    private logRotateTimer;
+    /** Days of event/activity history to keep. */
+    private static readonly EVENT_LOG_RETENTION_DAYS;
     private watchdogTimer;
     private startedAt;
     private mirrorBuffer;
@@ -127,9 +147,38 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     constructor(dataDir: string);
     private handleSighup;
     private scheduleReconcile;
+    /**
+     * Is the fleet going down (or coming back up) on purpose?
+     *
+     * Instances dying during a planned restart is the restart working, not an
+     * incident — but the code that notices a dead pane or a dead MCP server
+     * cannot tell the difference on its own. Two sources, because the noise
+     * starts before this process is even told to stop: `agend update` replaces
+     * the package on disk while this daemon is still running and still watching.
+     */
+    isPlannedRestart(): boolean;
     private finishStartup;
     lastActivityMs(name: string): number;
+    /**
+     * Is the instance between turns?
+     *
+     * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
+     * the control client's raw 2-second output-silence heuristic. The raw heuristic
+     * reads every >2s output lull as idle — and long silent tools (a build, a test
+     * run) or an LLM pause produce those constantly mid-turn. That misreading is
+     * what retired cancel buttons in the middle of long work (the 5-minute backstop
+     * fired during a lull) and froze their progress text (ticker skipped "idle"
+     * ticks). The silence heuristic remains only as the fallback for instances
+     * whose daemon has not reported a state yet.
+     */
     private getInstanceIdle;
+    /**
+     * True when the instance claims working/stuck but nothing has refreshed that
+     * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
+     * Measures the CACHE's age, not the button's — a healthy multi-hour run
+     * answers every query and never trips this.
+     */
+    private stateReportDead;
     webhookEmit(event: string, name: string, data?: Record<string, unknown>): void;
     getSysInfo(): import("./fleet-context.js").SysInfo;
     /** Load fleet.yaml and build routing table */
@@ -184,6 +233,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     getInstanceExecutionState(name: string): InstanceState | null;
     isClassicInstance(name: string): boolean;
     private cacheInstanceExecutionState;
+    private cancelIdleButtonRetirement;
+    private scheduleIdleButtonRetirement;
     private cacheInstanceProcessStatus;
     /**
      * Fleet-wide warm cap: if more than `defaults.warm_cap` instances are running,
@@ -197,6 +248,25 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private enforceWarmCap;
     private waitForInstanceIdle;
     private deliverWithIdleGate;
+    /**
+     * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
+     *
+     * A daemon that is restarting — `/restart`, crash recovery, a model switch —
+     * drops its socket for a few seconds. Any message arriving in that window used
+     * to fail instantly: the caller logged a warning, put ❌ on the user's message,
+     * and the message was gone. The user had to notice the ❌ and retype it. That is
+     * the "instance 訊息不容易掉" goal failing on the most predictable event there is.
+     *
+     * The wait is bounded. If the instance is genuinely down, this still throws and
+     * the ❌ still appears — just for a real failure rather than a restart.
+     *
+     * Ordering is preserved by serialising behind any waiter already queued for this
+     * instance, *including* when the socket happens to be up: otherwise a message
+     * arriving after the reconnect could overtake one that has been waiting for it.
+     */
+    private sendWhenConnected;
+    /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
+    private sendAfterIpcReturns;
     /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
     deliverToInstance(instanceName: string, payload: Record<string, unknown>, options?: DeliveryOptions): Promise<void>;
     /** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
@@ -204,7 +274,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     changeInstancePauseState(name: string, action: "pause" | "wake"): Promise<"paused" | "awake" | "not_idle">;
     /** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
     restartClassicInstanceFromSettings(instanceName: string): Promise<void>;
-    startInstance(name: string, config: InstanceConfig, topicMode: boolean): Promise<void>;
+    startInstance(name: string, config: InstanceConfig, topicMode: boolean, kind?: "fleet-topic" | "classic"): Promise<void>;
     /** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
     startPersistedPausedInstance(name: string): Promise<void>;
     /**
@@ -251,6 +321,42 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
         retryCount: number;
         lastError?: string;
     }>;
+    /**
+     * Real, checkable fleet health for `/health` and the operator.
+     *
+     * `status` is:
+     *  - `ok`       — at least one adapter connected and every configured instance
+     *                 that should be running is running
+     *  - `degraded` — reachable, but something the operator should look at (an
+     *                 adapter retrying, an instance crashed or stopped)
+     *  - `down`     — the fleet cannot do its job: no adapter is connected, so no
+     *                 message can arrive or be answered
+     *
+     * Deliberately does NOT gate the systemd watchdog — see the comment at the
+     * WATCHDOG timer for why.
+     */
+    getFleetHealth(): {
+        status: "ok" | "degraded" | "down";
+        uptime: number;
+        instances: {
+            configured: number;
+            running: number;
+            crashed: number;
+            paused: number;
+            stopped: number;
+        };
+        adapters: {
+            total: number;
+            connected: number;
+            states: Record<string, string>;
+        };
+        startupComplete: boolean;
+        /** See process-memory.ts: the fleet process and the whole service cgroup are
+         *  reported separately because they differ by ~60x and only one of them can
+         *  show a fleet-manager leak. */
+        memory: FleetMemory;
+        problems: string[];
+    };
     /** Start the primary adapter (backward-compatible, sets this.adapter) */
     private startSingleAdapter;
     /** Start an additional (non-primary) adapter */
@@ -265,6 +371,23 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private restartAdapter;
     /** Handle inbound message — transcribe voice if present, then route */
     private findGeneralInstance;
+    /**
+     * A user reacted to one of the bot's messages (#408).
+     *
+     * A reaction is context, not a message (#432, reworking #413): it never triggers
+     * an agent turn and never wakes anything. It is queued in the event log and rides
+     * into the instance's NEXT real message as one compact leading line —
+     * `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
+     * No pending reactions → no line → zero context spent, which is the common case.
+     */
+    private handleInboundReaction;
+    /**
+     * The queued-reaction summary for an instance's next real message, or {} when
+     * nothing is pending (the common case must add zero context). The consume
+     * callback is separate from the fetch so reactions are only marked once the
+     * message actually went out — a failed delivery keeps them queued.
+     */
+    private pendingReactionsMeta;
     private handleInboundMessage;
     /** Handle outbound tool calls from a daemon instance */
     /** Warn (but don't block) when rate limits are high. 30-min debounce per instance. */
@@ -272,6 +395,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private warnIfRateLimited;
     /** Handle outbound tool calls from a daemon instance */
     private handleOutboundFromInstance;
+    /** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
+    private afterReplyRouted;
     /** Handle tool status update from a daemon instance */
     private handleToolStatusFromInstance;
     private handleScheduleTrigger;
@@ -313,6 +438,68 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private static FAILOVER_RECOVER_PCT;
     checkModelFailover(name: string, fiveHourPct: number): void;
     toggleFleetCollab(instanceName: string): boolean;
+    /**
+     * Open the event log, tolerating a corrupt file.
+     *
+     * `events.db` holds history only — event rows and the activity feed. Nothing the
+     * fleet needs to run depends on it, and every consumer already uses
+     * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
+     * unreadable history file (a truncated WAL after a hard kill, a full disk)
+     * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
+     * running agent for a file whose only job is reporting.
+     *
+     * So: try, move a bad file aside and retry once with a fresh one, and if even
+     * that fails carry on without an event log.
+     */
+    /**
+     * Run `agend backend doctor` for the fleet's default backend and return its
+     * cleaned output.
+     *
+     * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
+     * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
+     * was frozen — no IPC, no adapter, no message delivery, no health responses,
+     * and critically no WATCHDOG ping, so a slow doctor could push past
+     * WatchdogSec and have systemd SIGABRT the fleet.
+     */
+    private runBackendDoctor;
+    /** Drop event/activity rows older than the retention window. Best-effort. */
+    /**
+     * Cap every instance's pipe-pane log, walking the instances **directory** rather
+     * than the config.
+     *
+     * A running instance rotates its own log on each health tick, so the ones that
+     * need this are the ones nothing else looks at:
+     *
+     *   - deleted instances, whose directory outlives the config entry. Nothing ever
+     *     touched these again. On the machine this was found on, one held 122 MB and
+     *     another 74 MB, out of 622 MB of pipe-pane logs in total.
+     *   - classic instances, which live in classicChannels, not fleetConfig.instances,
+     *     and so were never in the old config-driven loop at all.
+     *   - stopped instances, which have no health tick running.
+     *
+     * pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
+     * at animation rate. Unbounded growth here fills the disk, which takes the whole
+     * fleet down rather than one instance.
+     */
+    private rotateAllInstanceLogs;
+    private pruneEventLog;
+    private openEventLog;
+    /**
+     * Report a fleet-level fault (not attributable to one instance) to the General
+     * topic, so the operator learns about it without reading daemon.log.
+     *
+     * Throttled per distinct message: an unhandled rejection typically comes from a
+     * loop (a poller, a repeating timer), and one channel message per occurrence
+     * would bury the topic — which is worse than silence. First occurrence goes out
+     * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
+     * count.
+     *
+     * The log line is written by the caller regardless: if every adapter is down,
+     * the only notification path is the one that is broken.
+     */
+    notifyFleetError(text: string): void;
+    private static readonly FLEET_ERROR_THROTTLE_MS;
+    private fleetErrorNotices;
     notifyInstanceTopic(instanceName: string, text: string, extraOpts?: import("./channel/types.js").SendOpts): void;
     /** Send a "🛑 Cancel" button to the instance's topic/channel after delivery. */
     /**
@@ -322,8 +509,66 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
      */
     private handleSlashSave;
     /** Whether the instance currently has at least one live cancel button. */
+    /**
+     * A click on a cancel button, whether or not the fleet still tracks it.
+     *
+     * The old rule was "act only while an entry is live", which made a click on a
+     * button the fleet had forgotten a silent no-op — no cancel, no message, not
+     * even a log line. That is indistinguishable from a broken button, and it is
+     * what the "按鈕點了沒反應" reports were: the entry is briefly absent while a
+     * button is being replaced, and a delete that fails leaves the message on
+     * screen with no entry at all.
+     *
+     * So: honour the click if the instance is actually running, and say so plainly
+     * if it is not. The stale-click path is rate-limited because the original
+     * concern was real — a second click must not fire a second interrupt key at an
+     * instance that has already started a new turn.
+     */
+    private handleCancelClick;
     private hasCancelButton;
     sendCancelButton(instanceName: string, correlationId?: string): Promise<void>;
+    /**
+     * The cancel button's text for a given elapsed time.
+     *
+     * Below the threshold it keeps the original wording, so a normal quick answer
+     * looks exactly as it did before. Past it, the button doubles as the live
+     * progress indicator (#409) — the channel showed nothing at all during long work,
+     * and once the agent had replied once there was no sign it was still going.
+     */
+    static progressText(elapsedMs: number, activity?: string | null, minElapsedMs?: number): string;
+    /**
+     * Make a tool summary safe to paste into a channel message.
+     *
+     * The text is agent-controlled (it is built from tool inputs — file paths,
+     * shell commands), so it gets flattened to one line, capped, and stripped of
+     * the two Discord mass-mention triggers. Neither channel renders it with a
+     * parse mode, so no markup escaping is needed beyond that.
+     */
+    private static sanitizeActivity;
+    /**
+     * Remember what an instance is currently doing, for the progress line.
+     *
+     * Best-effort by design: only backends that expose a live activity feed report
+     * anything, and the progress line simply omits the detail for the rest. It is
+     * never used to decide anything — purely what the user is shown.
+     */
+    private cacheInstanceActivity;
+    /**
+     * Refresh the button's text in place while the instance keeps working.
+     *
+     * Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
+     * and the Bot API treats that as "clear the keyboard" — so editing with it would
+     * delete the very cancel button this is trying to keep alive.
+     */
+    /** Configured threshold before elapsed time appears, in ms. */
+    progressMinElapsedMs(): number;
+    private startProgressTicker;
+    /**
+     * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
+     * it has not, retire its button. Re-arming replaces the previous timer, so a
+     * burst of replies ends with exactly one pending check.
+     */
+    private armReplyGrace;
     /** Retire (delete) every cancel button belonging to an instance. */
     private retireInstanceButtons;
     /** Begin retiring one button (delete + bounded retry on failure). Idempotent:
@@ -333,6 +578,19 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private attemptButtonDelete;
     /** Clear an entry's timers (retry + idle-check) and drop it from the map. */
     private discardButton;
+    /**
+     * Mirror the live buttons to disk. The map is memory-only, so before this a
+     * fleet restart orphaned every button on screen: frozen "處理中…" text and a
+     * click that did nothing, forever. The ledger is tiny (a handful of rows) and
+     * written on every add/remove — no debounce needed at that rate.
+     */
+    private persistCancelButtons;
+    /**
+     * Delete the previous process's buttons. Runs once adapters are up: nothing
+     * from a previous fleet process can still be mid-turn from this process's
+     * point of view, so every ledger row is an orphan by definition.
+     */
+    private sweepOrphanedCancelButtons;
     /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
     private scheduleButtonRetry;
     /** Delete one button's message via its own adapter. Resolves on success,
@@ -416,6 +674,15 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     /** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
     private getModelOptions;
     /** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
+    /** Label an effort choice, marking the one currently configured. */
+    private effortChoiceLabel;
+    private effortMenuHeader;
+    /** `/effort` — DC Select Menu, or apply directly when a level is given. */
+    private handleEffortSlash;
+    /** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
+    promptEffortMenu(instanceName: string, userId: string, channelId: string, adapter: ChannelAdapter, chatId: string, threadId?: string): Promise<string | null>;
+    /** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
+    private handleEffortSelection;
     private handleModelSlash;
     /**
      * Show a TG inline-keyboard model-selection menu. Reuses the same
@@ -426,7 +693,43 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     /** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
     private handleModelSelection;
     /** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
+    /** AgEnD's canonical effort ladder, low → max. Backends expose a subset. */
+    static readonly EFFORT_LEVELS: readonly ["low", "medium", "high", "xhigh", "max"];
+    /** How this instance's backend applies an effort change. */
+    effortStrategyFor(instanceName: string): "runtime" | "restart" | "unsupported";
+    /** Effort levels this instance's backend actually accepts (empty = unsupported). */
+    effortLevelsFor(instanceName: string): string[];
+    /** Configured effort for an instance: per-instance, else fleet default, else none. */
+    resolveInstanceEffort(instanceName: string): {
+        effort: string | null;
+        source: "instance" | "fleet-default" | "unset";
+    };
+    /**
+     * Clamp a canonical level to the nearest one this backend supports.
+     *
+     * Clamping DOWN the ladder, never up: asking for `max` on a CLI that stops at
+     * `high` should get high, not silently fall to low. The caller reports the
+     * clamp — a user who asks for max and quietly receives high has been told the
+     * request succeeded when it did not.
+     */
+    static clampEffort(level: string, supported: string[]): string | null;
+    /**
+     * Apply a reasoning-effort level, mirroring applyModel's shape.
+     *
+     * runtime backends take `/effort <level>` in the pane and keep working;
+     * restart backends only read it at launch, so it is persisted and the
+     * instance respawns.
+     */
+    applyEffort(instanceName: string, requested: string): Promise<string>;
     applyModel(instanceName: string, model: string): Promise<string>;
+    /**
+     * The trailing "Current effort: …" line for a /model reply.
+     *
+     * Model and effort interact (a cheaper model at max effort is a different
+     * trade than a bigger one at low), so showing the effort in force right after
+     * a switch saves the round trip of asking. Empty when the backend has none.
+     */
+    private effortSuffix;
     /** Read recent chat log for agent context */
     private getRecentChatLog;
     /** Return a user-facing blocker without mutating ClassicBot state. */
@@ -447,7 +750,17 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     handleClassicStart(channelId: string, channelName: string, userId: string, guildId?: string, adapterId?: string, backend?: string): Promise<string>;
     /** Handle /stop slash command — unregister classic channel */
     handleClassicStop(channelId: string, adapterId?: string): Promise<string>;
+    /**
+     * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
+     * uncaughtException path calls this too, so overlapping runs were possible —
+     * each snapshotting the daemon map and calling stop() on the same daemons
+     * concurrently. Deliberately NOT `async`, so callers receive the same promise
+     * object rather than a fresh wrapper around it. The latch clears when the run
+     * settles, so a later genuine stop (after a restart) still does the work.
+     */
     stopAll(): Promise<void>;
+    private stopAllInFlight;
+    private doStopAll;
     /**
      * Prune stale external sessions by re-querying each daemon for live sessions.
      * Sessions in the registry that are no longer reported by any daemon are removed.