@songsid/agend 2.1.2-beta.8 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +1 -0
- package/dist/agent-cli.js +14 -0
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +25 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +20 -0
- package/dist/backend/antigravity.js +51 -4
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +59 -0
- package/dist/backend/claude-code.js +67 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +26 -0
- package/dist/backend/codex.js +100 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +25 -0
- package/dist/backend/grok.js +51 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +23 -0
- package/dist/backend/kiro.js +120 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +58 -0
- package/dist/backend/types.js +13 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +120 -8
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +86 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +25 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +30 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +242 -91
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +21 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +2 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +299 -7
- package/dist/daemon.js +1152 -311
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +14 -0
- package/dist/fleet-lock.d.ts +28 -0
- package/dist/fleet-lock.js +130 -0
- package/dist/fleet-lock.js.map +1 -0
- package/dist/fleet-manager.d.ts +348 -6
- package/dist/fleet-manager.js +1860 -262
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-instance.d.ts +9 -0
- package/dist/general-instance.js +11 -0
- package/dist/general-instance.js.map +1 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +58 -2
- package/dist/instance-lifecycle.js +231 -46
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +7 -1
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +9 -0
- package/dist/outbound-handlers.js +194 -31
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +11 -3
- package/dist/outbound-schemas.js +16 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/restart-progress.d.ts +41 -0
- package/dist/restart-progress.js +97 -0
- package/dist/restart-progress.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/service-installer.d.ts +28 -0
- package/dist/service-installer.js +72 -1
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +5 -3
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +40 -11
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +49 -9
- package/dist/topic-commands.js +270 -109
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +5 -1
- package/dist/tui-glyphs.js +6 -2
- package/dist/tui-glyphs.js.map +1 -1
- package/dist/types.d.ts +19 -2
- package/dist/ui/settings.html +1 -1
- package/dist/ui/view.html +123 -2
- package/dist/update-marker.d.ts +15 -0
- package/dist/update-marker.js +69 -0
- package/dist/update-marker.js.map +1 -0
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +121 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/provider-alerts.d.ts +33 -0
- package/dist/usage/provider-alerts.js +57 -0
- package/dist/usage/provider-alerts.js.map +1 -0
- package/dist/usage/providers.d.ts +130 -0
- package/dist/usage/providers.js +1063 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/statusline-usage.d.ts +20 -0
- package/dist/usage/statusline-usage.js +120 -0
- package/dist/usage/statusline-usage.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +197 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.js +8 -3
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +7 -3
- package/dist/web-api.js.map +1 -1
- package/dist/web-auth.d.ts +6 -0
- package/dist/web-auth.js +54 -0
- package/dist/web-auth.js.map +1 -0
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -4,7 +4,10 @@ import { createServer } from "node:http";
|
|
|
4
4
|
import { join, dirname, basename } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
|
-
import {
|
|
7
|
+
import { clearUpdateMarker, isUpdateInProgress } from "./update-marker.js";
|
|
8
|
+
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
9
|
+
import { readFleetMemory } from "./process-memory.js";
|
|
10
|
+
import { ReplyDeduper } from "./reply-dedup.js";
|
|
8
11
|
import { isScalar, parseDocument } from "yaml";
|
|
9
12
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
13
|
const __dirname = dirname(__filename);
|
|
@@ -21,12 +24,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
|
|
|
21
24
|
import { createAdapter } from "./channel/factory.js";
|
|
22
25
|
import { createBackend } from "./backend/factory.js";
|
|
23
26
|
import { isModelCompatible } from "./backend/types.js";
|
|
24
|
-
import { createLogger } from "./logger.js";
|
|
27
|
+
import { createLogger, rotateLogIfNeeded } from "./logger.js";
|
|
25
28
|
import { processAttachments } from "./channel/attachment-handler.js";
|
|
26
29
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
27
30
|
import { Scheduler } from "./scheduler/index.js";
|
|
28
31
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
29
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
32
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
30
33
|
import { DailySummary } from "./daily-summary.js";
|
|
31
34
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
32
35
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -38,6 +41,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
38
41
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
39
42
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
40
43
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
44
|
+
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
41
45
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
42
46
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
43
47
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
@@ -45,6 +49,10 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
|
|
|
45
49
|
import { validateFleetConfig } from "./config-validator.js";
|
|
46
50
|
import { readLastInboundAt } from "./daemon.js";
|
|
47
51
|
import { clearPausedMarker } from "./pause-marker.js";
|
|
52
|
+
import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
53
|
+
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
54
|
+
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
55
|
+
import { RestartProgress } from "./restart-progress.js";
|
|
48
56
|
import { getTmuxSession } from "./config.js";
|
|
49
57
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
50
58
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -83,9 +91,80 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
83
91
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
84
92
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
85
93
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
94
|
+
/**
|
|
95
|
+
* A queued turn can produce a very short idle edge while the CLI hands off to
|
|
96
|
+
* the next message. Do not retire the cancel button until that edge remains
|
|
97
|
+
* idle for this long; a working/stuck report during the grace cancels it.
|
|
98
|
+
*/
|
|
99
|
+
const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
|
|
100
|
+
/**
|
|
101
|
+
* How long after a reply an instance gets to resume working before its cancel
|
|
102
|
+
* button is retired. A short turn ends with a reply and never works again → the
|
|
103
|
+
* button disappears ~2 minutes after the answer. A multi-step run replies
|
|
104
|
+
* mid-flight and keeps going → the grace check sees "working" and leaves the
|
|
105
|
+
* button alone (the idle edge retires it when the run really ends).
|
|
106
|
+
*/
|
|
107
|
+
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
108
|
+
/**
|
|
109
|
+
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
110
|
+
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
111
|
+
* query each tick; a live daemon answers within milliseconds and refreshes the
|
|
112
|
+
* cache. When nothing has refreshed it for this long despite those pokes, the
|
|
113
|
+
* reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
|
|
114
|
+
* from 30 minutes ago proves nothing — the button may be retired.
|
|
115
|
+
*/
|
|
116
|
+
const STATE_REPORT_STALE_MS = 30 * 60_000;
|
|
117
|
+
/**
|
|
118
|
+
* Unconditional ceiling on a cancel button's life. Deliberately far beyond any
|
|
119
|
+
* legitimate run (multi-hour tasks are normal on this fleet): everything below
|
|
120
|
+
* this is decided by real state; a button that somehow survives a full day is
|
|
121
|
+
* wreckage, stuck or not.
|
|
122
|
+
*/
|
|
123
|
+
const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
|
|
124
|
+
/** A click on a button the fleet no longer tracks may fire at most this often. */
|
|
125
|
+
const STALE_CANCEL_CLICK_COOLDOWN_MS = 10_000;
|
|
126
|
+
/** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
|
|
127
|
+
const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
|
|
128
|
+
/**
|
|
129
|
+
* How often the cancel button's text is refreshed with elapsed working time.
|
|
130
|
+
*
|
|
131
|
+
* One edit per working instance per interval — at 60s that is trivial for both
|
|
132
|
+
* platforms' rate limits, and it reads as a live counter rather than a stale
|
|
133
|
+
* snapshot. Nothing new is posted, so the channel is never spammed: there is
|
|
134
|
+
* exactly one progress message per turn, and it is the cancel button itself.
|
|
135
|
+
*/
|
|
136
|
+
const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
|
|
137
|
+
/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
|
|
138
|
+
/**
|
|
139
|
+
* Default delay before the button starts showing elapsed time. Configurable via
|
|
140
|
+
* `defaults.progress_min_elapsed` (seconds) in fleet.yaml. 30s is the balance
|
|
141
|
+
* point: most quick answers finish inside it (no churn for ordinary turns),
|
|
142
|
+
* while anything real shows signs of life well before the old two minutes.
|
|
143
|
+
*/
|
|
144
|
+
const PROGRESS_MIN_ELAPSED_MS = 30_000;
|
|
145
|
+
/** How much of a tool summary the progress line will show before eliding. */
|
|
146
|
+
const PROGRESS_ACTIVITY_MAX_CHARS = 48;
|
|
147
|
+
/**
|
|
148
|
+
* Emoji AgEnD itself stamps on messages as the delivery-status ladder
|
|
149
|
+
* (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed). These are machine
|
|
150
|
+
* indicators, not opinions, so they never enter the reactions queue — from
|
|
151
|
+
* anyone. This exact-emoji filter is the ONLY bot filtering left: bot-to-bot
|
|
152
|
+
* reactions are otherwise delivered on purpose (agents signal each other), and
|
|
153
|
+
* 🫡 passes too — it reads as a deliberate acknowledgement, not plumbing.
|
|
154
|
+
*/
|
|
155
|
+
const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
156
|
+
/**
|
|
157
|
+
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
158
|
+
*
|
|
159
|
+
* Sized for a daemon restart (socket close → respawn → CLI ready), which is the
|
|
160
|
+
* event this exists for. Past it the delivery fails loudly as it always did.
|
|
161
|
+
*/
|
|
162
|
+
const IPC_RECONNECT_GRACE_MS = 30_000;
|
|
163
|
+
const IPC_RECONNECT_POLL_MS = 250;
|
|
86
164
|
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
87
165
|
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
88
166
|
const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
|
|
167
|
+
const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
|
|
89
168
|
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
90
169
|
export class FleetManager {
|
|
91
170
|
dataDir;
|
|
@@ -93,7 +172,7 @@ export class FleetManager {
|
|
|
93
172
|
static sighupHandlerInstalled = false;
|
|
94
173
|
children = new Map();
|
|
95
174
|
lifecycle;
|
|
96
|
-
/**
|
|
175
|
+
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
97
176
|
get daemons() { return this.lifecycle.daemons; }
|
|
98
177
|
fleetConfig = null;
|
|
99
178
|
rawFleetConfig = {};
|
|
@@ -108,8 +187,11 @@ export class FleetManager {
|
|
|
108
187
|
// same guild both receive every message). Bounded FIFO of recent message keys.
|
|
109
188
|
recentMessageIds = new Set();
|
|
110
189
|
accessManager = null;
|
|
111
|
-
/** Primary world (
|
|
112
|
-
get primaryWorld() {
|
|
190
|
+
/** Primary world (channels[0]), independent of concurrent adapter startup order. */
|
|
191
|
+
get primaryWorld() {
|
|
192
|
+
const adapterId = this.getPrimaryAdapterId();
|
|
193
|
+
return adapterId ? this.worlds.get(adapterId) : undefined;
|
|
194
|
+
}
|
|
113
195
|
routing = new RoutingEngine();
|
|
114
196
|
get routingTable() { return this.routing.map; }
|
|
115
197
|
instanceIpcClients = new Map();
|
|
@@ -151,6 +233,16 @@ export class FleetManager {
|
|
|
151
233
|
// reply, on cancel, or when a newer button supersedes it for the same
|
|
152
234
|
// instance. Per-button tracking means a failed delete never strands a button.
|
|
153
235
|
cancelButtons = new Map();
|
|
236
|
+
/** Pending idle-edge retirement, one timer per instance. */
|
|
237
|
+
cancelButtonIdleRetireTimers = new Map();
|
|
238
|
+
/** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
|
|
239
|
+
replyDeduper = new ReplyDeduper();
|
|
240
|
+
/** instanceName → what it is doing right now, when the backend can tell us. */
|
|
241
|
+
instanceActivity = new Map();
|
|
242
|
+
/** instanceName → tail of deliveries waiting for its IPC to come back. */
|
|
243
|
+
ipcWaitTails = new Map();
|
|
244
|
+
/** instanceName → restart currently executing; concurrent callers join it. */
|
|
245
|
+
restartsInFlight = new Map();
|
|
154
246
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
155
247
|
lastInboundMsg = new Map();
|
|
156
248
|
topicArchiver;
|
|
@@ -158,11 +250,17 @@ export class FleetManager {
|
|
|
158
250
|
classicChannels = null;
|
|
159
251
|
pendingClassicStarts = new Map();
|
|
160
252
|
/** In-flight /model selections, keyed by nonce (see handleModelSelection). */
|
|
253
|
+
/** In-flight /effort selections, same coordinator shape as pendingModelSelects. */
|
|
254
|
+
pendingEffortSelects = new Map();
|
|
161
255
|
pendingModelSelects = new Map();
|
|
162
256
|
// Model failover state
|
|
163
257
|
failoverActive = new Map(); // instance → current failover model
|
|
164
258
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
259
|
+
/** instance → when a click with no live button entry last fired a cancel. */
|
|
260
|
+
staleCancelClickAt = new Map();
|
|
165
261
|
ipcStoppingInstances = new Set();
|
|
262
|
+
/** Set the moment a graceful stop begins — see isPlannedRestart(). */
|
|
263
|
+
shuttingDown = false;
|
|
166
264
|
/** Coalesce concurrent connection attempts for the same daemon socket. */
|
|
167
265
|
ipcConnectInFlight = new Map();
|
|
168
266
|
/** At most one reconnect/backoff loop may exist per instance. */
|
|
@@ -176,6 +274,10 @@ export class FleetManager {
|
|
|
176
274
|
healthServer = null;
|
|
177
275
|
healthPortRetried = false;
|
|
178
276
|
updateCheckTimer = null;
|
|
277
|
+
eventLogPruneTimer = null;
|
|
278
|
+
logRotateTimer = null;
|
|
279
|
+
/** Days of event/activity history to keep. */
|
|
280
|
+
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
179
281
|
watchdogTimer = null;
|
|
180
282
|
startedAt = 0;
|
|
181
283
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
@@ -185,6 +287,7 @@ export class FleetManager {
|
|
|
185
287
|
sseClients = new Set();
|
|
186
288
|
webToken = null;
|
|
187
289
|
viewToken = null;
|
|
290
|
+
healthServerListening = false;
|
|
188
291
|
constructor(dataDir) {
|
|
189
292
|
this.dataDir = dataDir;
|
|
190
293
|
FleetManager.signalTarget = this;
|
|
@@ -214,7 +317,13 @@ export class FleetManager {
|
|
|
214
317
|
}
|
|
215
318
|
this.reloadPending = false;
|
|
216
319
|
this.reconcileInFlight = this.reconcileInstances()
|
|
217
|
-
.catch(err =>
|
|
320
|
+
.catch(err => {
|
|
321
|
+
// Almost always a YAML parse error. Log-only meant the user edited
|
|
322
|
+
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
323
|
+
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
324
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
325
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
|
|
326
|
+
})
|
|
218
327
|
.finally(() => {
|
|
219
328
|
this.reconcileInFlight = null;
|
|
220
329
|
if (this.reloadPending && this.startupComplete) {
|
|
@@ -222,16 +331,55 @@ export class FleetManager {
|
|
|
222
331
|
}
|
|
223
332
|
});
|
|
224
333
|
}
|
|
334
|
+
/**
|
|
335
|
+
* Is the fleet going down (or coming back up) on purpose?
|
|
336
|
+
*
|
|
337
|
+
* Instances dying during a planned restart is the restart working, not an
|
|
338
|
+
* incident — but the code that notices a dead pane or a dead MCP server
|
|
339
|
+
* cannot tell the difference on its own. Two sources, because the noise
|
|
340
|
+
* starts before this process is even told to stop: `agend update` replaces
|
|
341
|
+
* the package on disk while this daemon is still running and still watching.
|
|
342
|
+
*/
|
|
343
|
+
isPlannedRestart() {
|
|
344
|
+
return this.shuttingDown || isUpdateInProgress(this.dataDir);
|
|
345
|
+
}
|
|
225
346
|
finishStartup() {
|
|
226
347
|
this.startupComplete = true;
|
|
348
|
+
// We are the post-update fleet: the update is over by definition. Clearing
|
|
349
|
+
// it here (rather than in the update command, which exits before the new
|
|
350
|
+
// fleet is up) is what keeps the quiet window from outliving the restart.
|
|
351
|
+
clearUpdateMarker(this.dataDir);
|
|
227
352
|
if (this.reloadPending)
|
|
228
353
|
this.scheduleReconcile();
|
|
354
|
+
void this.sweepOrphanedCancelButtons();
|
|
229
355
|
}
|
|
230
356
|
// ── ArchiverContext bridge ────────────────────────────────────────────
|
|
231
357
|
lastActivityMs(name) {
|
|
232
358
|
return this.lastActivity.get(name) ?? 0;
|
|
233
359
|
}
|
|
360
|
+
/**
|
|
361
|
+
* Is the instance between turns?
|
|
362
|
+
*
|
|
363
|
+
* Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
|
|
364
|
+
* the control client's raw 2-second output-silence heuristic. The raw heuristic
|
|
365
|
+
* reads every >2s output lull as idle — and long silent tools (a build, a test
|
|
366
|
+
* run) or an LLM pause produce those constantly mid-turn. That misreading is
|
|
367
|
+
* what retired cancel buttons in the middle of long work (the 5-minute backstop
|
|
368
|
+
* fired during a lull) and froze their progress text (ticker skipped "idle"
|
|
369
|
+
* ticks). The silence heuristic remains only as the fallback for instances
|
|
370
|
+
* whose daemon has not reported a state yet.
|
|
371
|
+
*/
|
|
234
372
|
getInstanceIdle(name) {
|
|
373
|
+
// A daemon that is not running cannot be mid-turn. This is what a stale
|
|
374
|
+
// "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
|
|
375
|
+
// report ever arrives) must not override.
|
|
376
|
+
if (this.getInstanceStatus(name) !== "running")
|
|
377
|
+
return true;
|
|
378
|
+
const state = this.getInstanceExecutionState(name);
|
|
379
|
+
if (state === "working" || state === "stuck")
|
|
380
|
+
return false;
|
|
381
|
+
if (state === "idle")
|
|
382
|
+
return true;
|
|
235
383
|
try {
|
|
236
384
|
const widFile = join(this.getInstanceDir(name), "window-id");
|
|
237
385
|
if (!existsSync(widFile))
|
|
@@ -243,6 +391,18 @@ export class FleetManager {
|
|
|
243
391
|
return true;
|
|
244
392
|
}
|
|
245
393
|
}
|
|
394
|
+
/**
|
|
395
|
+
* True when the instance claims working/stuck but nothing has refreshed that
|
|
396
|
+
* claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
|
|
397
|
+
* Measures the CACHE's age, not the button's — a healthy multi-hour run
|
|
398
|
+
* answers every query and never trips this.
|
|
399
|
+
*/
|
|
400
|
+
stateReportDead(name) {
|
|
401
|
+
const cached = this.instanceStateCache.get(name);
|
|
402
|
+
if (!cached)
|
|
403
|
+
return false; // no claim to distrust — getInstanceIdle owns this case
|
|
404
|
+
return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
|
|
405
|
+
}
|
|
246
406
|
// ── LifecycleContext bridge methods ──────────────────────────────────────
|
|
247
407
|
webhookEmit(event, name, data) {
|
|
248
408
|
this.webhookEmitter?.emit(event, name, data);
|
|
@@ -304,10 +464,20 @@ export class FleetManager {
|
|
|
304
464
|
if (!this.classicChannels)
|
|
305
465
|
return;
|
|
306
466
|
const channels = this.classicChannels.getAll();
|
|
467
|
+
// Classic's persisted adapter is authoritative. Legacy adapter-less rows
|
|
468
|
+
// deterministically belong to channels[0], never to whichever bot happens
|
|
469
|
+
// to deliver the first message after startup/reconnect.
|
|
470
|
+
for (const ch of channels) {
|
|
471
|
+
const adapterId = ch.adapterId ?? this.getPrimaryAdapterId();
|
|
472
|
+
if (adapterId)
|
|
473
|
+
this.instanceWorldBinding.set(ch.instanceName, adapterId);
|
|
474
|
+
}
|
|
307
475
|
// Always update adapter openChannels (including empty — clears stale entries on /stop)
|
|
308
476
|
for (const [adapterId, w] of this.worlds) {
|
|
309
477
|
if (typeof w.adapter?.setOpenChannels === "function") {
|
|
310
|
-
const owned = channels
|
|
478
|
+
const owned = channels
|
|
479
|
+
.filter(ch => (ch.adapterId ?? this.getPrimaryAdapterId()) === adapterId)
|
|
480
|
+
.map(ch => ch.channelId);
|
|
311
481
|
w.adapter.setOpenChannels(owned);
|
|
312
482
|
}
|
|
313
483
|
}
|
|
@@ -385,58 +555,113 @@ export class FleetManager {
|
|
|
385
555
|
}
|
|
386
556
|
/** Get the adapter bound to an instance, falling back to primary adapter */
|
|
387
557
|
getAdapterForInstance(name) {
|
|
388
|
-
const worldId = this.
|
|
558
|
+
const worldId = this.getInstanceAdapterId(name);
|
|
389
559
|
if (worldId)
|
|
390
560
|
return this.worlds.get(worldId)?.adapter ?? this.adapter;
|
|
391
561
|
return this.adapter;
|
|
392
562
|
}
|
|
393
563
|
/** Get the world for an instance */
|
|
394
564
|
getWorldForInstance(name) {
|
|
395
|
-
const worldId = this.
|
|
396
|
-
return worldId ? this.worlds.get(worldId) :
|
|
565
|
+
const worldId = this.getInstanceAdapterId(name);
|
|
566
|
+
return worldId ? this.worlds.get(worldId) : undefined;
|
|
397
567
|
}
|
|
398
568
|
/** Get channel config for a specific adapter (by id), falling back to primary */
|
|
399
569
|
getChannelConfig(adapterId) {
|
|
570
|
+
const channels = this.fleetConfig?.channels
|
|
571
|
+
?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
|
|
400
572
|
if (adapterId) {
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
573
|
+
return channels.find(ch => (ch.id ?? ch.type) === adapterId)
|
|
574
|
+
?? this.worlds.get(adapterId)?.channelConfig
|
|
575
|
+
?? channels[0];
|
|
404
576
|
}
|
|
405
|
-
return
|
|
577
|
+
return channels[0];
|
|
406
578
|
}
|
|
407
579
|
/** Get the group_id for an instance's bound adapter */
|
|
408
580
|
getGroupIdForInstance(name) {
|
|
409
|
-
const
|
|
410
|
-
|
|
581
|
+
const adapterId = this.getInstanceAdapterId(name);
|
|
582
|
+
const world = adapterId ? this.worlds.get(adapterId) : undefined;
|
|
583
|
+
return world?.groupId ?? String(this.getChannelConfig(adapterId)?.group_id ?? "");
|
|
584
|
+
}
|
|
585
|
+
/** Configured primary adapter id. Never infer this from Map insertion order. */
|
|
586
|
+
getPrimaryAdapterId() {
|
|
587
|
+
const primary = this.fleetConfig?.channels?.[0] ?? this.fleetConfig?.channel;
|
|
588
|
+
if (primary)
|
|
589
|
+
return primary.id ?? primary.type;
|
|
590
|
+
// Defensive compatibility for callers/tests that provide a live world but
|
|
591
|
+
// no channel config. Real multi-adapter fleets always have channels[].
|
|
592
|
+
return this.worlds.keys().next().value;
|
|
593
|
+
}
|
|
594
|
+
/** Warn when a coordinator's adapter identity is ambiguous to the operator. */
|
|
595
|
+
warnUnboundGeneralChannelIds(fleet) {
|
|
596
|
+
const channels = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
|
|
597
|
+
if (channels.length <= 1)
|
|
598
|
+
return;
|
|
599
|
+
const adapterIds = channels.map(ch => ch.id ?? ch.type);
|
|
600
|
+
for (const [name, config] of Object.entries(fleet.instances)) {
|
|
601
|
+
if (!config.general_topic || config.channel_id)
|
|
602
|
+
continue;
|
|
603
|
+
this.logger.warn({
|
|
604
|
+
instance: name,
|
|
605
|
+
defaultAdapter: adapterIds[0],
|
|
606
|
+
availableAdapters: adapterIds,
|
|
607
|
+
}, "General instance has no channel_id in a multi-channel fleet; defaulting to the first adapter. Set channel_id explicitly.");
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Resolve the authoritative adapter identity for an instance.
|
|
612
|
+
* Fleet instances without channel_id and legacy Classic entries both belong
|
|
613
|
+
* to channels[0]. Runtime bindings remain available for external sessions.
|
|
614
|
+
*/
|
|
615
|
+
getInstanceAdapterId(name) {
|
|
616
|
+
const cfg = this.fleetConfig?.instances[name];
|
|
617
|
+
if (cfg)
|
|
618
|
+
return cfg.channel_id ?? this.getPrimaryAdapterId();
|
|
619
|
+
if (this.classicChannels?.getChannelIdByInstance(name) !== undefined) {
|
|
620
|
+
return this.classicChannels.getAdapterIdByInstance(name) ?? this.getPrimaryAdapterId();
|
|
621
|
+
}
|
|
622
|
+
return this.instanceWorldBinding.get(name) ?? this.getPrimaryAdapterId();
|
|
411
623
|
}
|
|
412
624
|
/**
|
|
413
625
|
* Bind an instance to a specific world (the bot that answers for it).
|
|
414
626
|
* fromInbound=true (binding inferred from which adapter received a message)
|
|
415
|
-
* must not override a configured identity
|
|
416
|
-
*
|
|
417
|
-
*
|
|
627
|
+
* must not override a configured identity. Fleet instances use channel_id or
|
|
628
|
+
* channels[0]; Classic instances use their persisted adapter (or channels[0]
|
|
629
|
+
* for a legacy entry). Only external sessions may bind from inbound traffic.
|
|
418
630
|
*/
|
|
419
631
|
bindInstanceAdapter(name, adapterId, fromInbound = false) {
|
|
420
|
-
const cfg = this.fleetConfig?.instances[name];
|
|
421
632
|
if (fromInbound) {
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
return;
|
|
428
|
-
if (cfg && !cfg.channel_id)
|
|
429
|
-
return; // fleet instance without explicit binding → use primary
|
|
430
|
-
// Classic instance: don't override an existing binding (authoritative from /start)
|
|
431
|
-
if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
|
|
633
|
+
const configuredId = this.getInstanceAdapterId(name);
|
|
634
|
+
if (this.fleetConfig?.instances[name]
|
|
635
|
+
|| this.classicChannels?.getChannelIdByInstance(name) !== undefined) {
|
|
636
|
+
if (configuredId)
|
|
637
|
+
this.instanceWorldBinding.set(name, configuredId);
|
|
432
638
|
return;
|
|
639
|
+
}
|
|
433
640
|
}
|
|
434
641
|
this.instanceWorldBinding.set(name, adapterId);
|
|
435
642
|
}
|
|
436
643
|
getInstanceStatus(name) {
|
|
437
644
|
if (this.lifecycle.isPaused(name))
|
|
438
645
|
return "paused";
|
|
646
|
+
const daemon = this.lifecycle.daemons.get(name);
|
|
439
647
|
const processStatus = this.instanceProcessStatus.get(name);
|
|
648
|
+
// IPC can be disconnected during a respawn, so the event which announces
|
|
649
|
+
// the new live pane may be missed. The in-process daemon is authoritative
|
|
650
|
+
// in that case; do not leave a stale `crashed` cache masking an instance
|
|
651
|
+
// that has already recovered and can answer messages.
|
|
652
|
+
const daemonStatus = daemon?.getProcessStatus?.();
|
|
653
|
+
if (daemonStatus === "running") {
|
|
654
|
+
if (processStatus)
|
|
655
|
+
this.instanceProcessStatus.delete(name);
|
|
656
|
+
// A recovered in-process daemon is also the authority for clearing a
|
|
657
|
+
// marker left by a crash-loop/reconnect race. This keeps standalone
|
|
658
|
+
// `agend ls` from seeing the old marker after the next API outage.
|
|
659
|
+
try {
|
|
660
|
+
unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
|
|
661
|
+
}
|
|
662
|
+
catch { /* absent */ }
|
|
663
|
+
return "running";
|
|
664
|
+
}
|
|
440
665
|
if (processStatus)
|
|
441
666
|
return processStatus;
|
|
442
667
|
const pidPath = join(this.getInstanceDir(name), "daemon.pid");
|
|
@@ -454,6 +679,10 @@ export class FleetManager {
|
|
|
454
679
|
getInstanceExecutionState(name) {
|
|
455
680
|
if (this.lifecycle.isPaused(name))
|
|
456
681
|
return null;
|
|
682
|
+
// Process status wins over a stale pane snapshot. A dead remain-on-exit pane
|
|
683
|
+
// can still contain the old ready marker and must never surface as Idle.
|
|
684
|
+
if (this.instanceProcessStatus.has(name))
|
|
685
|
+
return null;
|
|
457
686
|
return this.instanceStateCache.get(name)?.state ?? null;
|
|
458
687
|
}
|
|
459
688
|
isClassicInstance(name) {
|
|
@@ -471,22 +700,61 @@ export class FleetManager {
|
|
|
471
700
|
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
472
701
|
observedAt: numberOr(msg.observedAt, now),
|
|
473
702
|
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
703
|
+
// Fleet-manager receipt time, NOT the daemon's observation time: staleness
|
|
704
|
+
// asks "is anyone still reporting", which only the receiver can date.
|
|
705
|
+
receivedAt: now,
|
|
474
706
|
});
|
|
475
707
|
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
476
708
|
check();
|
|
477
709
|
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
478
710
|
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
479
711
|
// edge into idle, not on every idle heartbeat.
|
|
480
|
-
if (state === "idle" && previous?.state !== "idle")
|
|
712
|
+
if (state === "idle" && previous?.state !== "idle") {
|
|
481
713
|
this.enforceWarmCap();
|
|
714
|
+
// A queued message may turn this edge back into working almost
|
|
715
|
+
// immediately. Give that handoff a short grace before retiring the button.
|
|
716
|
+
this.scheduleIdleButtonRetirement(name);
|
|
717
|
+
}
|
|
718
|
+
else if (state !== "idle") {
|
|
719
|
+
this.cancelIdleButtonRetirement(name);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
cancelIdleButtonRetirement(name) {
|
|
723
|
+
const timer = this.cancelButtonIdleRetireTimers.get(name);
|
|
724
|
+
if (!timer)
|
|
725
|
+
return;
|
|
726
|
+
clearTimeout(timer);
|
|
727
|
+
this.cancelButtonIdleRetireTimers.delete(name);
|
|
728
|
+
}
|
|
729
|
+
scheduleIdleButtonRetirement(name) {
|
|
730
|
+
this.cancelIdleButtonRetirement(name);
|
|
731
|
+
const timer = setTimeout(() => {
|
|
732
|
+
// Ignore a superseded timer even if it was already queued to run.
|
|
733
|
+
if (this.cancelButtonIdleRetireTimers.get(name) !== timer)
|
|
734
|
+
return;
|
|
735
|
+
this.cancelButtonIdleRetireTimers.delete(name);
|
|
736
|
+
if (this.getInstanceExecutionState(name) === "idle") {
|
|
737
|
+
this.retireInstanceButtons(name);
|
|
738
|
+
}
|
|
739
|
+
}, CANCEL_BTN_IDLE_RETIRE_GRACE_MS);
|
|
740
|
+
timer.unref?.();
|
|
741
|
+
this.cancelButtonIdleRetireTimers.set(name, timer);
|
|
482
742
|
}
|
|
483
743
|
cacheInstanceProcessStatus(name, status) {
|
|
484
744
|
if (status === "running") {
|
|
485
745
|
this.instanceProcessStatus.delete(name);
|
|
746
|
+
// A prior crash-loop marker is one-shot. Successful respawn is the
|
|
747
|
+
// authoritative recovery signal even when the marker outlived an IPC
|
|
748
|
+
// disconnect and was not consumed by a new Daemon constructor.
|
|
749
|
+
try {
|
|
750
|
+
unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
|
|
751
|
+
}
|
|
752
|
+
catch { /* absent */ }
|
|
486
753
|
return;
|
|
487
754
|
}
|
|
488
755
|
if (status !== "crashed" && status !== "stopped")
|
|
489
756
|
return;
|
|
757
|
+
this.cancelIdleButtonRetirement(name);
|
|
490
758
|
this.instanceProcessStatus.set(name, status);
|
|
491
759
|
// Never display the last ready prompt as current execution state after its
|
|
492
760
|
// owning CLI process has exited.
|
|
@@ -517,7 +785,7 @@ export class FleetManager {
|
|
|
517
785
|
const victims = selectLruEvictions(warm, cap, {
|
|
518
786
|
exclude,
|
|
519
787
|
isEvicting: name => this.warmCapEvicting.has(name),
|
|
520
|
-
isGeneral: name => this.fleetConfig
|
|
788
|
+
isGeneral: name => isGeneralInstance(this.fleetConfig, name),
|
|
521
789
|
isIdle: name => this.getInstanceExecutionState(name) === "idle",
|
|
522
790
|
lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
|
|
523
791
|
});
|
|
@@ -583,12 +851,70 @@ export class FleetManager {
|
|
|
583
851
|
if (!idle) {
|
|
584
852
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
585
853
|
}
|
|
586
|
-
|
|
587
|
-
if (!ipc?.connected)
|
|
588
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
589
|
-
ipc.send(payload);
|
|
854
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
590
855
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
591
856
|
}
|
|
857
|
+
/**
|
|
858
|
+
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
859
|
+
*
|
|
860
|
+
* A daemon that is restarting — `/restart`, crash recovery, a model switch —
|
|
861
|
+
* drops its socket for a few seconds. Any message arriving in that window used
|
|
862
|
+
* to fail instantly: the caller logged a warning, put ❌ on the user's message,
|
|
863
|
+
* and the message was gone. The user had to notice the ❌ and retype it. That is
|
|
864
|
+
* the "instance 訊息不容易掉" goal failing on the most predictable event there is.
|
|
865
|
+
*
|
|
866
|
+
* The wait is bounded. If the instance is genuinely down, this still throws and
|
|
867
|
+
* the ❌ still appears — just for a real failure rather than a restart.
|
|
868
|
+
*
|
|
869
|
+
* Ordering is preserved by serialising behind any waiter already queued for this
|
|
870
|
+
* instance, *including* when the socket happens to be up: otherwise a message
|
|
871
|
+
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
872
|
+
*/
|
|
873
|
+
async sendWhenConnected(instanceName, payload) {
|
|
874
|
+
const queued = this.ipcWaitTails.get(instanceName);
|
|
875
|
+
if (!queued) {
|
|
876
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
877
|
+
if (ipc?.connected && ipc.send(payload))
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
const attempt = (queued ?? Promise.resolve())
|
|
881
|
+
.catch(() => { })
|
|
882
|
+
.then(() => this.sendAfterIpcReturns(instanceName, payload));
|
|
883
|
+
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
884
|
+
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
885
|
+
const tail = attempt.catch(() => { });
|
|
886
|
+
this.ipcWaitTails.set(instanceName, tail);
|
|
887
|
+
try {
|
|
888
|
+
await attempt;
|
|
889
|
+
}
|
|
890
|
+
finally {
|
|
891
|
+
// Only the last waiter clears the chain; while a queue is still draining the
|
|
892
|
+
// map must keep pointing at it or ordering is lost.
|
|
893
|
+
if (this.ipcWaitTails.get(instanceName) === tail) {
|
|
894
|
+
this.ipcWaitTails.delete(instanceName);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
899
|
+
async sendAfterIpcReturns(instanceName, payload) {
|
|
900
|
+
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
901
|
+
let warned = false;
|
|
902
|
+
for (;;) {
|
|
903
|
+
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
904
|
+
// so a cached reference would stay dead forever.
|
|
905
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
906
|
+
if (ipc?.connected && ipc.send(payload))
|
|
907
|
+
return;
|
|
908
|
+
if (Date.now() >= deadline) {
|
|
909
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
910
|
+
}
|
|
911
|
+
if (!warned) {
|
|
912
|
+
warned = true;
|
|
913
|
+
this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
|
|
914
|
+
}
|
|
915
|
+
await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
|
|
916
|
+
}
|
|
917
|
+
}
|
|
592
918
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
593
919
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
594
920
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
@@ -604,10 +930,7 @@ export class FleetManager {
|
|
|
604
930
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
605
931
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
606
932
|
}
|
|
607
|
-
|
|
608
|
-
if (!ipc?.connected)
|
|
609
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
610
|
-
ipc.send(payload);
|
|
933
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
611
934
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
612
935
|
// working must not trust the stale idle snapshot from before the send.
|
|
613
936
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
@@ -636,6 +959,9 @@ export class FleetManager {
|
|
|
636
959
|
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
637
960
|
return "awake";
|
|
638
961
|
}
|
|
962
|
+
if (isGeneralInstance(this.fleetConfig, name)) {
|
|
963
|
+
throw new Error(GENERAL_PAUSE_ERROR);
|
|
964
|
+
}
|
|
639
965
|
await this.lifecycle.pause(name);
|
|
640
966
|
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
641
967
|
}
|
|
@@ -655,11 +981,41 @@ export class FleetManager {
|
|
|
655
981
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
656
982
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
657
983
|
}
|
|
658
|
-
async startInstance(name, config, topicMode, kind = "fleet-topic"
|
|
984
|
+
async startInstance(name, config, topicMode, kind = "fleet-topic",
|
|
985
|
+
/**
|
|
986
|
+
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
987
|
+
* and reconcile calls leave this false so a persisted pause remains paused
|
|
988
|
+
* across a fleet restart.
|
|
989
|
+
*/
|
|
990
|
+
resumePaused = false) {
|
|
991
|
+
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
992
|
+
await this.lifecycle.wake(name, 30_000);
|
|
993
|
+
// A successful wake clears the persisted pause marker and produces a
|
|
994
|
+
// fresh instance_state snapshot. Drop any stale process error left by a
|
|
995
|
+
// pre-pause crash so /api/fleet and `agend ls` converge on running.
|
|
996
|
+
this.instanceProcessStatus.delete(name);
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
659
999
|
if (this.lifecycle.isPaused(name)) {
|
|
660
1000
|
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
661
1001
|
return;
|
|
662
1002
|
}
|
|
1003
|
+
if (this.lifecycle.daemons.has(name)) {
|
|
1004
|
+
// A crash-loop daemon remains in the lifecycle map so its health monitor
|
|
1005
|
+
// can expose the failure. The old start path treated that object as
|
|
1006
|
+
// already running and merely deleted the process-status cache, leaving a
|
|
1007
|
+
// dead pane (and crash marker) behind. An explicit start is a recovery
|
|
1008
|
+
// request: tear down the failed daemon and build a fresh one.
|
|
1009
|
+
if (resumePaused) {
|
|
1010
|
+
const status = this.getInstanceStatus(name);
|
|
1011
|
+
if (status === "crashed" || status === "stopped") {
|
|
1012
|
+
await this.restartSingleInstance(name);
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
this.logger.info({ name }, "Instance already running, skipping");
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
663
1019
|
if (config.general_topic) {
|
|
664
1020
|
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
665
1021
|
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
@@ -670,12 +1026,19 @@ export class FleetManager {
|
|
|
670
1026
|
}
|
|
671
1027
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
672
1028
|
}
|
|
673
|
-
this.instanceProcessStatus.delete(name);
|
|
674
1029
|
await this.lifecycle.start(name, config, topicMode, {
|
|
675
1030
|
kind,
|
|
676
1031
|
backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
|
|
677
1032
|
model: this.resolveInstanceModel(name).display,
|
|
678
1033
|
});
|
|
1034
|
+
// Only clear a stale process status after a real start succeeded. Clearing
|
|
1035
|
+
// it before lifecycle.start() can turn a crash-loop daemon's dead pane into
|
|
1036
|
+
// a falsely running instance when lifecycle.start() returns early.
|
|
1037
|
+
this.instanceProcessStatus.delete(name);
|
|
1038
|
+
try {
|
|
1039
|
+
unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
|
|
1040
|
+
}
|
|
1041
|
+
catch { /* consumed or absent */ }
|
|
679
1042
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
680
1043
|
await this.connectIpcToInstance(name);
|
|
681
1044
|
}
|
|
@@ -699,7 +1062,18 @@ export class FleetManager {
|
|
|
699
1062
|
* to avoid config file races. Stagger delay is group-to-group, not instance-to-instance.
|
|
700
1063
|
* TODO: per-instance startup timeout (existing issue, not introduced here)
|
|
701
1064
|
*/
|
|
702
|
-
async startInstancesWithConcurrency(entries, topicMode) {
|
|
1065
|
+
async startInstancesWithConcurrency(entries, topicMode, onReady) {
|
|
1066
|
+
// Persisted pauses are intentionally preserved across fleet restarts. Filter
|
|
1067
|
+
// them before grouping/staggering: startInstance() retains its own guard as
|
|
1068
|
+
// a final backstop, but putting a no-op entry in this queue still consumes a
|
|
1069
|
+
// full stagger slot for every distinct working directory.
|
|
1070
|
+
const runnableEntries = entries.filter(([name]) => !this.lifecycle.isPaused(name));
|
|
1071
|
+
const pausedCount = entries.length - runnableEntries.length;
|
|
1072
|
+
if (pausedCount > 0) {
|
|
1073
|
+
this.logger.info({ pausedCount }, "Paused instances excluded from startup queue");
|
|
1074
|
+
}
|
|
1075
|
+
if (runnableEntries.length === 0)
|
|
1076
|
+
return;
|
|
703
1077
|
const raw = this.fleetConfig?.defaults?.startup;
|
|
704
1078
|
const explicitConcurrency = raw?.concurrency;
|
|
705
1079
|
const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
|
|
@@ -714,10 +1088,10 @@ export class FleetManager {
|
|
|
714
1088
|
else {
|
|
715
1089
|
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
716
1090
|
concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
|
|
717
|
-
this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances:
|
|
1091
|
+
this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: runnableEntries.length }, "Adaptive startup concurrency");
|
|
718
1092
|
}
|
|
719
1093
|
const byWorkDir = new Map();
|
|
720
|
-
for (const [name, config] of
|
|
1094
|
+
for (const [name, config] of runnableEntries) {
|
|
721
1095
|
const dir = config.working_directory;
|
|
722
1096
|
if (!byWorkDir.has(dir))
|
|
723
1097
|
byWorkDir.set(dir, []);
|
|
@@ -760,7 +1134,14 @@ export class FleetManager {
|
|
|
760
1134
|
lastStartAt = Date.now();
|
|
761
1135
|
(async () => {
|
|
762
1136
|
for (const [name, config] of group) {
|
|
763
|
-
|
|
1137
|
+
try {
|
|
1138
|
+
await this.startInstance(name, config, topicMode);
|
|
1139
|
+
if (this.daemons.has(name))
|
|
1140
|
+
onReady?.(name);
|
|
1141
|
+
}
|
|
1142
|
+
catch (err) {
|
|
1143
|
+
this.logger.error({ err, name }, "Failed to start instance");
|
|
1144
|
+
}
|
|
764
1145
|
}
|
|
765
1146
|
})().finally(() => {
|
|
766
1147
|
running--;
|
|
@@ -774,8 +1155,41 @@ export class FleetManager {
|
|
|
774
1155
|
startNext();
|
|
775
1156
|
});
|
|
776
1157
|
}
|
|
1158
|
+
runnableStartupCount(fleet, includeClassic) {
|
|
1159
|
+
const names = this.configuredStartupInstanceNames(fleet, includeClassic);
|
|
1160
|
+
let count = 0;
|
|
1161
|
+
for (const name of names) {
|
|
1162
|
+
if (!this.lifecycle.isPaused(name))
|
|
1163
|
+
count++;
|
|
1164
|
+
}
|
|
1165
|
+
return count;
|
|
1166
|
+
}
|
|
1167
|
+
configuredStartupInstanceNames(fleet, includeClassic) {
|
|
1168
|
+
const names = new Set(Object.keys(fleet.instances));
|
|
1169
|
+
if (includeClassic) {
|
|
1170
|
+
for (const channel of this.classicChannels?.getAll() ?? [])
|
|
1171
|
+
names.add(channel.instanceName);
|
|
1172
|
+
}
|
|
1173
|
+
return [...names];
|
|
1174
|
+
}
|
|
1175
|
+
restartProgressTarget() {
|
|
1176
|
+
const generalName = this.findGeneralInstance();
|
|
1177
|
+
if (!generalName)
|
|
1178
|
+
return null;
|
|
1179
|
+
const adapter = this.getAdapterForInstance(generalName);
|
|
1180
|
+
const chatId = this.getGroupIdForInstance(generalName);
|
|
1181
|
+
if (!adapter || !chatId)
|
|
1182
|
+
return null;
|
|
1183
|
+
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
1184
|
+
return {
|
|
1185
|
+
adapter,
|
|
1186
|
+
chatId,
|
|
1187
|
+
threadId: topicId != null ? String(topicId) : undefined,
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
777
1190
|
async stopInstance(name) {
|
|
778
1191
|
this.failoverActive.delete(name);
|
|
1192
|
+
this.cancelIdleButtonRetirement(name);
|
|
779
1193
|
this.instanceStateCache.delete(name);
|
|
780
1194
|
this.instanceProcessStatus.delete(name);
|
|
781
1195
|
this.lastDeliveryAt.delete(name);
|
|
@@ -783,6 +1197,21 @@ export class FleetManager {
|
|
|
783
1197
|
}
|
|
784
1198
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
785
1199
|
async restartSingleInstance(name, opts) {
|
|
1200
|
+
// One restart at a time per instance. Multiple sources can ask concurrently
|
|
1201
|
+
// (MCP revival, /restart, pty_error, model failover); a second stop/start
|
|
1202
|
+
// interleaved with the first tears down the window the first just created.
|
|
1203
|
+
// Later callers join the in-flight restart instead — its opts win.
|
|
1204
|
+
const inFlight = this.restartsInFlight.get(name);
|
|
1205
|
+
if (inFlight) {
|
|
1206
|
+
this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
|
|
1207
|
+
return inFlight;
|
|
1208
|
+
}
|
|
1209
|
+
const run = this.doRestartSingleInstance(name, opts)
|
|
1210
|
+
.finally(() => this.restartsInFlight.delete(name));
|
|
1211
|
+
this.restartsInFlight.set(name, run);
|
|
1212
|
+
return run;
|
|
1213
|
+
}
|
|
1214
|
+
async doRestartSingleInstance(name, opts) {
|
|
786
1215
|
if (this.configPath) {
|
|
787
1216
|
this.loadConfig(this.configPath);
|
|
788
1217
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -838,7 +1267,10 @@ export class FleetManager {
|
|
|
838
1267
|
const eqIdx = trimmed.indexOf("=");
|
|
839
1268
|
if (eqIdx < 0)
|
|
840
1269
|
continue;
|
|
841
|
-
|
|
1270
|
+
// Accept `export KEY=value` — the shell-style form people paste from their
|
|
1271
|
+
// .bashrc. Without this the variable landed in process.env under the key
|
|
1272
|
+
// "export KEY" and silently did nothing.
|
|
1273
|
+
const key = trimmed.slice(0, eqIdx).replace(/^export\s+/, "").trim();
|
|
842
1274
|
const raw = trimmed.slice(eqIdx + 1);
|
|
843
1275
|
const value = raw.replace(/^["'](.*)["']$/, '$1');
|
|
844
1276
|
// .env file always wins over inherited shell env vars, so that
|
|
@@ -846,17 +1278,36 @@ export class FleetManager {
|
|
|
846
1278
|
process.env[key] = value;
|
|
847
1279
|
}
|
|
848
1280
|
}
|
|
1281
|
+
/** Initialize auth before any adapter can answer /dashboard. */
|
|
1282
|
+
initializeWebAuthTokens() {
|
|
1283
|
+
this.webToken = loadOrCreateWebToken(this.dataDir);
|
|
1284
|
+
this.viewToken = randomBytes(24).toString("hex");
|
|
1285
|
+
const viewTokenPath = join(this.dataDir, "view.token");
|
|
1286
|
+
writeFileSync(viewTokenPath, this.viewToken, { encoding: "utf8", mode: 0o600 });
|
|
1287
|
+
try {
|
|
1288
|
+
chmodSync(viewTokenPath, 0o600);
|
|
1289
|
+
}
|
|
1290
|
+
catch { /* best effort */ }
|
|
1291
|
+
this.healthServerListening = false;
|
|
1292
|
+
}
|
|
1293
|
+
getDashboardAccess() {
|
|
1294
|
+
return { ready: this.healthServerListening, token: this.webToken };
|
|
1295
|
+
}
|
|
849
1296
|
/** Start all instances from fleet config */
|
|
850
1297
|
async startAll(configPath) {
|
|
1298
|
+
const startupStartedAt = Date.now();
|
|
851
1299
|
FleetManager.signalTarget = this;
|
|
852
1300
|
this.startupComplete = false;
|
|
1301
|
+
// Cleared here, not at the end of doStopAll: a stop has an async tail, and
|
|
1302
|
+
// anything arriving during it is still part of the stop.
|
|
1303
|
+
this.shuttingDown = false;
|
|
853
1304
|
this.configPath = configPath;
|
|
854
1305
|
this.loadEnvFile();
|
|
855
1306
|
// Rotate fleet.log if oversized (before any logging)
|
|
856
|
-
const { rotateLogIfNeeded } = await import("./logger.js");
|
|
857
1307
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
858
1308
|
const fleet = this.loadConfig(configPath);
|
|
859
1309
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
1310
|
+
this.initializeWebAuthTokens();
|
|
860
1311
|
const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
|
|
861
1312
|
// Set tmux socket isolation for custom AGEND_HOME
|
|
862
1313
|
const { getTmuxSocketName: getSocket } = await import("./paths.js");
|
|
@@ -897,7 +1348,7 @@ export class FleetManager {
|
|
|
897
1348
|
}
|
|
898
1349
|
const pidPath = join(this.dataDir, "fleet.pid");
|
|
899
1350
|
writeFileSync(pidPath, String(process.pid), "utf-8");
|
|
900
|
-
this.eventLog =
|
|
1351
|
+
this.eventLog = this.openEventLog();
|
|
901
1352
|
// Initialize classic channel manager. The primary adapter (channels[0])
|
|
902
1353
|
// migrates legacy single-bot entries and names without a suffix. Classic
|
|
903
1354
|
// routing does NOT go through the routing engine (single-key, can't hold two
|
|
@@ -986,6 +1437,10 @@ export class FleetManager {
|
|
|
986
1437
|
// Rotate fleet.log daily too (besides the startup size check above), so a
|
|
987
1438
|
// long-running fleet doesn't accumulate an unbounded log.
|
|
988
1439
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1440
|
+
// Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
|
|
1441
|
+
// running instance's own log; this sweep is the safety net for every other
|
|
1442
|
+
// kind. One implementation, so the two cannot cover different sets.
|
|
1443
|
+
this.rotateAllInstanceLogs();
|
|
989
1444
|
}, () => {
|
|
990
1445
|
const instances = Object.keys(this.fleetConfig?.instances ?? {});
|
|
991
1446
|
const costMap = new Map();
|
|
@@ -1000,6 +1455,7 @@ export class FleetManager {
|
|
|
1000
1455
|
this.rotateInboxes();
|
|
1001
1456
|
// Auto-create/adopt a general dispatcher — ONLY for the primary adapter.
|
|
1002
1457
|
const channelConfigs = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
|
|
1458
|
+
this.warnUnboundGeneralChannelIds(fleet);
|
|
1003
1459
|
const primaryAdapterId = channelConfigs[0] ? (channelConfigs[0].id ?? channelConfigs[0].type) : undefined;
|
|
1004
1460
|
const generalInstances = Object.entries(fleet.instances).filter(([, inst]) => inst.general_topic === true);
|
|
1005
1461
|
let generalsCreated = false;
|
|
@@ -1107,10 +1563,13 @@ export class FleetManager {
|
|
|
1107
1563
|
const allEntries = Object.entries(fleet.instances);
|
|
1108
1564
|
const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
1109
1565
|
const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
1566
|
+
const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), startupStartedAt, this.logger);
|
|
1110
1567
|
if (generals.length > 0) {
|
|
1111
1568
|
for (const [name, cfg] of generals) {
|
|
1112
1569
|
try {
|
|
1113
1570
|
await this.startInstance(name, cfg, topicMode);
|
|
1571
|
+
if (this.daemons.has(name))
|
|
1572
|
+
startupProgress.markReady();
|
|
1114
1573
|
}
|
|
1115
1574
|
catch (err) {
|
|
1116
1575
|
this.logger.error({ err, name }, "Failed to start general instance");
|
|
@@ -1125,39 +1584,63 @@ export class FleetManager {
|
|
|
1125
1584
|
}
|
|
1126
1585
|
}
|
|
1127
1586
|
}
|
|
1128
|
-
//
|
|
1129
|
-
|
|
1587
|
+
// The adapter must exist before General can receive the progress message.
|
|
1588
|
+
// Start it after General is ready, in parallel with the remaining CLIs, so
|
|
1589
|
+
// progress is visible without adding adapter startup time to the critical path.
|
|
1590
|
+
let adapterStartup = null;
|
|
1591
|
+
let progressStart = Promise.resolve(false);
|
|
1592
|
+
if (topicMode && (fleet.channel || fleet.channels?.length)) {
|
|
1593
|
+
// An adapter becoming reachable during startup can receive messages; make
|
|
1594
|
+
// all existing topic ids routable before opening that inbound path.
|
|
1595
|
+
this.routing.rebuild(fleet);
|
|
1596
|
+
this.reregisterClassicChannels();
|
|
1597
|
+
adapterStartup = (async () => {
|
|
1598
|
+
try {
|
|
1599
|
+
await this.startSharedAdapter(fleet);
|
|
1600
|
+
}
|
|
1601
|
+
catch (err) {
|
|
1602
|
+
this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
|
|
1603
|
+
}
|
|
1604
|
+
})();
|
|
1605
|
+
progressStart = adapterStartup.then(() => startupProgress.start(this.restartProgressTarget()));
|
|
1606
|
+
}
|
|
1607
|
+
// The systemd watchdog answers exactly one question: is this process still
|
|
1608
|
+
// turning its event loop? Pinging from a timer proves that, and after the
|
|
1609
|
+
// blocking child-process calls were made async it is a meaningful signal —
|
|
1610
|
+
// a deadlocked or frozen fleet stops pinging and systemd restarts it.
|
|
1611
|
+
//
|
|
1612
|
+
// It deliberately does NOT gate on fleet health. "No adapter connected" or
|
|
1613
|
+
// "an instance crashed" must not kill the process: the fleet would be restarted
|
|
1614
|
+
// into the same broken state, and a user who has legitimately stopped every
|
|
1615
|
+
// instance would get a restart loop. Those conditions surface through /health
|
|
1616
|
+
// (which now returns 503) and through the General-topic notifications instead.
|
|
1130
1617
|
this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
|
|
1618
|
+
// EventLog.prune() existed but was never called, so `events` and `activity`
|
|
1619
|
+
// grew without bound for the life of the install. Prune once at startup and
|
|
1620
|
+
// daily after that; the timer is unref'd so it never holds the loop open.
|
|
1621
|
+
this.pruneEventLog();
|
|
1622
|
+
this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
|
|
1623
|
+
this.eventLogPruneTimer.unref?.();
|
|
1624
|
+
// Same shape for pipe-pane logs, and for the same reason: the only sweep that
|
|
1625
|
+
// covered them lived inside the daily-summary callback, so it did not run at
|
|
1626
|
+
// all when summaries were off.
|
|
1627
|
+
this.rotateAllInstanceLogs();
|
|
1628
|
+
this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
|
|
1629
|
+
this.logRotateTimer.unref?.();
|
|
1131
1630
|
// Phase 2: Start remaining instances with staggered concurrency
|
|
1132
1631
|
if (others.length > 0) {
|
|
1133
|
-
await this.startInstancesWithConcurrency(others, topicMode);
|
|
1632
|
+
await this.startInstancesWithConcurrency(others, topicMode, () => startupProgress.markReady());
|
|
1134
1633
|
}
|
|
1135
1634
|
if (topicMode && (fleet.channel || fleet.channels?.length)) {
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
}
|
|
1142
|
-
// Bind instances to their adapter (which bot answers on their behalf).
|
|
1143
|
-
// An explicit channel_id is authoritative — this is how a persona instance
|
|
1144
|
-
// picks its bot when several share one guild. Generals without a channel_id
|
|
1145
|
-
// fall back to a name-contains-adapterId heuristic.
|
|
1146
|
-
const channelConfigsForBind = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
|
|
1635
|
+
await adapterStartup;
|
|
1636
|
+
// Bind every fleet instance deterministically. Explicit channel_id wins;
|
|
1637
|
+
// otherwise channels[0] is authoritative. Do not infer identity from
|
|
1638
|
+
// concurrent adapter startup or whichever bot receives a message first.
|
|
1639
|
+
const primaryAdapterId = this.getPrimaryAdapterId();
|
|
1147
1640
|
for (const [name, config] of Object.entries(fleet.instances)) {
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
}
|
|
1152
|
-
if (!config.general_topic)
|
|
1153
|
-
continue;
|
|
1154
|
-
for (const ch of channelConfigsForBind) {
|
|
1155
|
-
const id = ch.id ?? ch.type;
|
|
1156
|
-
if (name.includes(id)) {
|
|
1157
|
-
this.bindInstanceAdapter(name, id);
|
|
1158
|
-
break;
|
|
1159
|
-
}
|
|
1160
|
-
}
|
|
1641
|
+
const adapterId = config.channel_id ?? primaryAdapterId;
|
|
1642
|
+
if (adapterId)
|
|
1643
|
+
this.bindInstanceAdapter(name, adapterId);
|
|
1161
1644
|
}
|
|
1162
1645
|
// Guard against a stale/invalid general topic_id. An old auto-general
|
|
1163
1646
|
// could have written the TG-convention "1" for a Discord general; the DC
|
|
@@ -1167,7 +1650,7 @@ export class FleetManager {
|
|
|
1167
1650
|
for (const [name, cfg] of Object.entries(this.fleetConfig.instances)) {
|
|
1168
1651
|
if (!cfg.general_topic || cfg.topic_id == null)
|
|
1169
1652
|
continue;
|
|
1170
|
-
const adapterId = this.
|
|
1653
|
+
const adapterId = this.getInstanceAdapterId(name);
|
|
1171
1654
|
if (this.getChannelConfig(adapterId)?.type === "discord" && !/^\d{17,}$/.test(String(cfg.topic_id))) {
|
|
1172
1655
|
this.logger.warn({ name, topic_id: cfg.topic_id }, "Discord general topic_id is not a valid channel — unbinding to avoid a crash loop");
|
|
1173
1656
|
delete cfg.topic_id;
|
|
@@ -1190,12 +1673,22 @@ export class FleetManager {
|
|
|
1190
1673
|
// Start classic channel instances (parallel, concurrency 3)
|
|
1191
1674
|
if (this.classicChannels) {
|
|
1192
1675
|
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
1193
|
-
const channels = this.classicChannels.getAll()
|
|
1676
|
+
const channels = this.classicChannels.getAll()
|
|
1677
|
+
.filter(ch => !this.lifecycle.isPaused(ch.instanceName));
|
|
1194
1678
|
const concurrency = 3;
|
|
1195
1679
|
let idx = 0;
|
|
1196
1680
|
while (idx < channels.length) {
|
|
1197
1681
|
const batch = channels.slice(idx, idx + concurrency);
|
|
1198
|
-
await Promise.allSettled(batch.map(
|
|
1682
|
+
await Promise.allSettled(batch.map(async (ch) => {
|
|
1683
|
+
try {
|
|
1684
|
+
await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1685
|
+
if (this.daemons.has(ch.instanceName))
|
|
1686
|
+
startupProgress.markReady();
|
|
1687
|
+
}
|
|
1688
|
+
catch (err) {
|
|
1689
|
+
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
1690
|
+
}
|
|
1691
|
+
}));
|
|
1199
1692
|
idx += concurrency;
|
|
1200
1693
|
}
|
|
1201
1694
|
}
|
|
@@ -1203,10 +1696,10 @@ export class FleetManager {
|
|
|
1203
1696
|
this.startStatuslineWatcher(name);
|
|
1204
1697
|
}
|
|
1205
1698
|
// Notify General topic that fleet is up
|
|
1206
|
-
const
|
|
1207
|
-
const total =
|
|
1208
|
-
const started = this.daemons.
|
|
1209
|
-
const allNotRunning =
|
|
1699
|
+
const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
|
|
1700
|
+
const total = configuredNames.length;
|
|
1701
|
+
const started = configuredNames.filter(name => this.daemons.has(name)).length;
|
|
1702
|
+
const allNotRunning = configuredNames.filter(name => !this.daemons.has(name));
|
|
1210
1703
|
const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
|
|
1211
1704
|
const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
|
|
1212
1705
|
const generalName = this.findGeneralInstance();
|
|
@@ -1214,7 +1707,15 @@ export class FleetManager {
|
|
|
1214
1707
|
const { createRequire } = await import("node:module");
|
|
1215
1708
|
const _require = createRequire(import.meta.url);
|
|
1216
1709
|
const agendVersion = _require("../package.json").version ?? "unknown";
|
|
1217
|
-
|
|
1710
|
+
await progressStart;
|
|
1711
|
+
const progressCompleted = await startupProgress.finish({
|
|
1712
|
+
running: started,
|
|
1713
|
+
total,
|
|
1714
|
+
version: agendVersion,
|
|
1715
|
+
pausedNames,
|
|
1716
|
+
failedNames,
|
|
1717
|
+
});
|
|
1718
|
+
if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
|
|
1218
1719
|
let text;
|
|
1219
1720
|
if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
1220
1721
|
text = t("fleet.ready", started, total, agendVersion);
|
|
@@ -1263,6 +1764,15 @@ export class FleetManager {
|
|
|
1263
1764
|
// rest of startup finishes. Replay one coalesced reload only after all
|
|
1264
1765
|
// startup-owned lifecycle work and signal handlers are in place.
|
|
1265
1766
|
this.finishStartup();
|
|
1767
|
+
// Tell systemd we are ready only now. This used to fire right after the
|
|
1768
|
+
// generals started — before adapters, classic instances, topic creation and the
|
|
1769
|
+
// health server — so `systemctl start` returned success while the fleet was
|
|
1770
|
+
// still deaf: no path existed for a user message to arrive.
|
|
1771
|
+
sdNotify("READY=1");
|
|
1772
|
+
const health = this.getFleetHealth();
|
|
1773
|
+
if (health.status !== "ok") {
|
|
1774
|
+
this.logger.warn({ health }, "Fleet started with problems — see /health");
|
|
1775
|
+
}
|
|
1266
1776
|
}
|
|
1267
1777
|
/**
|
|
1268
1778
|
* Delete inbox files older than retentionDays (by mtime). Cleans the shared
|
|
@@ -1379,6 +1889,67 @@ export class FleetManager {
|
|
|
1379
1889
|
getAdapterStates() {
|
|
1380
1890
|
return this.adapterState;
|
|
1381
1891
|
}
|
|
1892
|
+
/**
|
|
1893
|
+
* Real, checkable fleet health for `/health` and the operator.
|
|
1894
|
+
*
|
|
1895
|
+
* `status` is:
|
|
1896
|
+
* - `ok` — at least one adapter connected and every configured instance
|
|
1897
|
+
* that should be running is running
|
|
1898
|
+
* - `degraded` — reachable, but something the operator should look at (an
|
|
1899
|
+
* adapter retrying, an instance crashed or stopped)
|
|
1900
|
+
* - `down` — the fleet cannot do its job: no adapter is connected, so no
|
|
1901
|
+
* message can arrive or be answered
|
|
1902
|
+
*
|
|
1903
|
+
* Deliberately does NOT gate the systemd watchdog — see the comment at the
|
|
1904
|
+
* WATCHDOG timer for why.
|
|
1905
|
+
*/
|
|
1906
|
+
getFleetHealth() {
|
|
1907
|
+
const names = Object.keys(this.fleetConfig?.instances ?? {});
|
|
1908
|
+
const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
|
|
1909
|
+
for (const name of names) {
|
|
1910
|
+
const state = this.getInstanceStatus(name);
|
|
1911
|
+
if (state === "running")
|
|
1912
|
+
counts.running++;
|
|
1913
|
+
else if (state === "crashed")
|
|
1914
|
+
counts.crashed++;
|
|
1915
|
+
else if (state === "paused")
|
|
1916
|
+
counts.paused++;
|
|
1917
|
+
else
|
|
1918
|
+
counts.stopped++;
|
|
1919
|
+
}
|
|
1920
|
+
const states = {};
|
|
1921
|
+
let connected = 0;
|
|
1922
|
+
for (const [id, state] of this.adapterState) {
|
|
1923
|
+
states[id] = state.status;
|
|
1924
|
+
if (state.status === "connected")
|
|
1925
|
+
connected++;
|
|
1926
|
+
}
|
|
1927
|
+
const problems = [];
|
|
1928
|
+
if (this.adapterState.size > 0 && connected === 0)
|
|
1929
|
+
problems.push("no channel adapter is connected");
|
|
1930
|
+
if (counts.crashed > 0)
|
|
1931
|
+
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
1932
|
+
for (const [id, state] of this.adapterState) {
|
|
1933
|
+
if (state.status !== "connected")
|
|
1934
|
+
problems.push(`adapter ${id} is ${state.status}`);
|
|
1935
|
+
}
|
|
1936
|
+
if (!this.startupComplete)
|
|
1937
|
+
problems.push("startup has not completed");
|
|
1938
|
+
// "down" is reserved for "cannot receive or answer a message at all". A fleet
|
|
1939
|
+
// with adapters configured but none connected is exactly that.
|
|
1940
|
+
const status = this.adapterState.size > 0 && connected === 0
|
|
1941
|
+
? "down"
|
|
1942
|
+
: problems.length > 0 ? "degraded" : "ok";
|
|
1943
|
+
return {
|
|
1944
|
+
status,
|
|
1945
|
+
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
1946
|
+
instances: counts,
|
|
1947
|
+
adapters: { total: this.adapterState.size, connected, states },
|
|
1948
|
+
startupComplete: this.startupComplete,
|
|
1949
|
+
memory: readFleetMemory(),
|
|
1950
|
+
problems,
|
|
1951
|
+
};
|
|
1952
|
+
}
|
|
1382
1953
|
/** Start the primary adapter (backward-compatible, sets this.adapter) */
|
|
1383
1954
|
async startSingleAdapter(fleet, channelConfig) {
|
|
1384
1955
|
const botToken = process.env[channelConfig.bot_token_env];
|
|
@@ -1405,11 +1976,16 @@ export class FleetManager {
|
|
|
1405
1976
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
1406
1977
|
await this.handleInboundMessage(msg);
|
|
1407
1978
|
}, this.logger, "adapter.message"));
|
|
1979
|
+
this.adapter.on("reaction", safeHandler(async (r) => {
|
|
1980
|
+
await this.handleInboundReaction(r);
|
|
1981
|
+
}, this.logger, "adapter.reaction"));
|
|
1408
1982
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1409
1983
|
if (await this.handleClassicBackendSelection(data))
|
|
1410
1984
|
return;
|
|
1411
1985
|
if (await this.handleModelSelection(data))
|
|
1412
1986
|
return;
|
|
1987
|
+
if (await this.handleEffortSelection(data))
|
|
1988
|
+
return;
|
|
1413
1989
|
if (data.callbackData.startsWith("hang:")) {
|
|
1414
1990
|
const parts = data.callbackData.split(":");
|
|
1415
1991
|
const action = parts[1];
|
|
@@ -1430,12 +2006,7 @@ export class FleetManager {
|
|
|
1430
2006
|
return;
|
|
1431
2007
|
}
|
|
1432
2008
|
if (data.callbackData.startsWith("cancel:")) {
|
|
1433
|
-
|
|
1434
|
-
// Idempotent: a button click only acts while the button is live. A
|
|
1435
|
-
// second click (entry already cleared) is a no-op — don't re-send the
|
|
1436
|
-
// interrupt key. (The /cancel command path calls cancelInstance directly.)
|
|
1437
|
-
if (this.hasCancelButton(instanceName))
|
|
1438
|
-
this.cancelInstance(instanceName);
|
|
2009
|
+
this.handleCancelClick(data.callbackData.slice("cancel:".length), this.adapter, data);
|
|
1439
2010
|
return;
|
|
1440
2011
|
}
|
|
1441
2012
|
}, this.logger, "adapter.callback_query"));
|
|
@@ -1515,6 +2086,9 @@ export class FleetManager {
|
|
|
1515
2086
|
else if (data.command === "model") {
|
|
1516
2087
|
await this.handleModelSlash(data, adapterId);
|
|
1517
2088
|
}
|
|
2089
|
+
else if (data.command === "effort") {
|
|
2090
|
+
await this.handleEffortSlash(data, adapterId);
|
|
2091
|
+
}
|
|
1518
2092
|
else if (data.command === "cancel") {
|
|
1519
2093
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1520
2094
|
if (!name) {
|
|
@@ -1579,19 +2153,29 @@ export class FleetManager {
|
|
|
1579
2153
|
await data.respond(t("not_authorized"));
|
|
1580
2154
|
return;
|
|
1581
2155
|
}
|
|
2156
|
+
await data.respond(await this.runBackendDoctor());
|
|
2157
|
+
}
|
|
2158
|
+
else if (data.command === "usage") {
|
|
2159
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
2160
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
2161
|
+
// channel either way.
|
|
1582
2162
|
try {
|
|
1583
|
-
const {
|
|
1584
|
-
const
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
await data.respond(clean || "No output");
|
|
2163
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2164
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2165
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2166
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1588
2167
|
}
|
|
1589
2168
|
catch (err) {
|
|
1590
|
-
|
|
1591
|
-
await data.respond(output);
|
|
2169
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1592
2170
|
}
|
|
1593
2171
|
}
|
|
1594
2172
|
else if (data.command === "status") {
|
|
2173
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
2174
|
+
// instance's cost and IPC health.
|
|
2175
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
2176
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
2177
|
+
return;
|
|
2178
|
+
}
|
|
1595
2179
|
const text = await this.topicCommands.getStatusText();
|
|
1596
2180
|
await data.respond(text);
|
|
1597
2181
|
}
|
|
@@ -1637,7 +2221,9 @@ export class FleetManager {
|
|
|
1637
2221
|
this.probeCliEnvs();
|
|
1638
2222
|
this.adapter.on("started", safeHandler((username, userId) => {
|
|
1639
2223
|
this.logger.info(`Bot @${username} polling started. Ensure no other service is polling this bot token.`);
|
|
1640
|
-
|
|
2224
|
+
// Concurrent startup can insert a secondary world first. Update the
|
|
2225
|
+
// configured primary world, not Map insertion order.
|
|
2226
|
+
const w = this.worlds.get(adapterId);
|
|
1641
2227
|
if (w) {
|
|
1642
2228
|
w.botUsername = username;
|
|
1643
2229
|
if (userId)
|
|
@@ -1700,11 +2286,16 @@ export class FleetManager {
|
|
|
1700
2286
|
adapter.on("message", safeHandler(async (msg) => {
|
|
1701
2287
|
await this.handleInboundMessage(msg);
|
|
1702
2288
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
2289
|
+
adapter.on("reaction", safeHandler(async (r) => {
|
|
2290
|
+
await this.handleInboundReaction(r);
|
|
2291
|
+
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
1703
2292
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1704
2293
|
if (await this.handleClassicBackendSelection(data))
|
|
1705
2294
|
return;
|
|
1706
2295
|
if (await this.handleModelSelection(data))
|
|
1707
2296
|
return;
|
|
2297
|
+
if (await this.handleEffortSelection(data))
|
|
2298
|
+
return;
|
|
1708
2299
|
if (data.callbackData.startsWith("hang:")) {
|
|
1709
2300
|
const parts = data.callbackData.split(":");
|
|
1710
2301
|
const action = parts[1];
|
|
@@ -1724,10 +2315,7 @@ export class FleetManager {
|
|
|
1724
2315
|
return;
|
|
1725
2316
|
}
|
|
1726
2317
|
if (data.callbackData.startsWith("cancel:")) {
|
|
1727
|
-
|
|
1728
|
-
// Idempotent: only the first click (while the button is live) acts.
|
|
1729
|
-
if (this.hasCancelButton(instanceName))
|
|
1730
|
-
this.cancelInstance(instanceName);
|
|
2318
|
+
this.handleCancelClick(data.callbackData.slice("cancel:".length), adapter, data);
|
|
1731
2319
|
return;
|
|
1732
2320
|
}
|
|
1733
2321
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
@@ -1797,6 +2385,9 @@ export class FleetManager {
|
|
|
1797
2385
|
else if (data.command === "model") {
|
|
1798
2386
|
await this.handleModelSlash(data, adapterId);
|
|
1799
2387
|
}
|
|
2388
|
+
else if (data.command === "effort") {
|
|
2389
|
+
await this.handleEffortSlash(data, adapterId);
|
|
2390
|
+
}
|
|
1800
2391
|
else if (data.command === "cancel") {
|
|
1801
2392
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1802
2393
|
if (!name) {
|
|
@@ -1861,19 +2452,29 @@ export class FleetManager {
|
|
|
1861
2452
|
await data.respond(t("not_authorized"));
|
|
1862
2453
|
return;
|
|
1863
2454
|
}
|
|
2455
|
+
await data.respond(await this.runBackendDoctor());
|
|
2456
|
+
}
|
|
2457
|
+
else if (data.command === "usage") {
|
|
2458
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
2459
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
2460
|
+
// channel either way.
|
|
1864
2461
|
try {
|
|
1865
|
-
const {
|
|
1866
|
-
const
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
await data.respond(clean || "No output");
|
|
2462
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2463
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2464
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2465
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1870
2466
|
}
|
|
1871
2467
|
catch (err) {
|
|
1872
|
-
|
|
1873
|
-
await data.respond(output);
|
|
2468
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1874
2469
|
}
|
|
1875
2470
|
}
|
|
1876
2471
|
else if (data.command === "status") {
|
|
2472
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
2473
|
+
// instance's cost and IPC health.
|
|
2474
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
2475
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
2476
|
+
return;
|
|
2477
|
+
}
|
|
1877
2478
|
const text = await this.topicCommands.getStatusText();
|
|
1878
2479
|
await data.respond(text);
|
|
1879
2480
|
}
|
|
@@ -2029,6 +2630,9 @@ export class FleetManager {
|
|
|
2029
2630
|
else if (msg.type === "instance_process_state") {
|
|
2030
2631
|
this.cacheInstanceProcessStatus(name, msg.status);
|
|
2031
2632
|
}
|
|
2633
|
+
else if (msg.type === "instance_activity") {
|
|
2634
|
+
this.cacheInstanceActivity(name, msg.activity);
|
|
2635
|
+
}
|
|
2032
2636
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2033
2637
|
this.cacheInstanceExecutionState(name, msg);
|
|
2034
2638
|
if (msg.type === "instance_state_response") {
|
|
@@ -2099,9 +2703,22 @@ export class FleetManager {
|
|
|
2099
2703
|
if (existsSync(windowIdPath)) {
|
|
2100
2704
|
const windowId = readFileSync(windowIdPath, "utf-8").trim();
|
|
2101
2705
|
if (windowId) {
|
|
2706
|
+
// Async with an explicit timeout: this was execSync with NO timeout at
|
|
2707
|
+
// all, so a wedged tmux server blocked the whole fleet event loop
|
|
2708
|
+
// indefinitely — while we were here to diagnose a lost connection.
|
|
2709
|
+
// A timeout is also the correct signal: an unresponsive tmux server
|
|
2710
|
+
// means we cannot verify the pane, which is treated as dead (the same
|
|
2711
|
+
// conclusion the old code reached only by throwing).
|
|
2102
2712
|
try {
|
|
2103
|
-
const {
|
|
2104
|
-
|
|
2713
|
+
const { execFile } = await import("node:child_process");
|
|
2714
|
+
const { promisify } = await import("node:util");
|
|
2715
|
+
const { getTmuxSocketName } = await import("./paths.js");
|
|
2716
|
+
// Honour socket isolation: without -L this queried the user's default
|
|
2717
|
+
// tmux server instead of the fleet's, so under a custom AGEND_HOME the
|
|
2718
|
+
// check was meaningless (it reported every pane dead).
|
|
2719
|
+
const socket = getTmuxSocketName();
|
|
2720
|
+
const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
|
|
2721
|
+
await promisify(execFile)("tmux", args, { timeout: 5_000 });
|
|
2105
2722
|
}
|
|
2106
2723
|
catch {
|
|
2107
2724
|
// Pane dead — respawn
|
|
@@ -2122,6 +2739,12 @@ export class FleetManager {
|
|
|
2122
2739
|
if (this.adapterRestarting.has(id))
|
|
2123
2740
|
return;
|
|
2124
2741
|
this.adapterRestarting.add(id);
|
|
2742
|
+
// Reflect reality in adapterState throughout. This loop used to leave the state
|
|
2743
|
+
// untouched, so getAdapterStates() — and therefore /health and the dashboard —
|
|
2744
|
+
// kept reporting "connected" for an adapter that had been down for hours. An
|
|
2745
|
+
// adapter's true status was simply not knowable from inside the process.
|
|
2746
|
+
const previous = this.adapterState.get(id);
|
|
2747
|
+
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2125
2748
|
try {
|
|
2126
2749
|
for (let attempt = 1;; attempt++) {
|
|
2127
2750
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
@@ -2134,9 +2757,16 @@ export class FleetManager {
|
|
|
2134
2757
|
await adapter.stop().catch(() => { });
|
|
2135
2758
|
await adapter.start();
|
|
2136
2759
|
this.logger.info({ id, attempt }, "Adapter restarted successfully");
|
|
2760
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
2137
2761
|
return;
|
|
2138
2762
|
}
|
|
2139
|
-
catch {
|
|
2763
|
+
catch (err) {
|
|
2764
|
+
this.adapterState.set(id, {
|
|
2765
|
+
status: "retrying",
|
|
2766
|
+
retryCount: attempt,
|
|
2767
|
+
lastError: err?.message ?? String(err),
|
|
2768
|
+
});
|
|
2769
|
+
}
|
|
2140
2770
|
if (attempt % 10 === 0) {
|
|
2141
2771
|
this.logger.warn({ id, attempt }, "Adapter restart still failing");
|
|
2142
2772
|
}
|
|
@@ -2172,6 +2802,49 @@ export class FleetManager {
|
|
|
2172
2802
|
}
|
|
2173
2803
|
return generals[0];
|
|
2174
2804
|
}
|
|
2805
|
+
/**
|
|
2806
|
+
* A user reacted to one of the bot's messages (#408).
|
|
2807
|
+
*
|
|
2808
|
+
* A reaction is context, not a message (#432, reworking #413): it never triggers
|
|
2809
|
+
* an agent turn and never wakes anything. It is queued in the event log and rides
|
|
2810
|
+
* into the instance's NEXT real message as one compact leading line —
|
|
2811
|
+
* `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
|
|
2812
|
+
* No pending reactions → no line → zero context spent, which is the common case.
|
|
2813
|
+
*/
|
|
2814
|
+
async handleInboundReaction(r) {
|
|
2815
|
+
const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
|
|
2816
|
+
if (!instanceName) {
|
|
2817
|
+
this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
|
|
2818
|
+
return;
|
|
2819
|
+
}
|
|
2820
|
+
if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
|
|
2821
|
+
this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
|
|
2822
|
+
return;
|
|
2823
|
+
}
|
|
2824
|
+
this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
|
|
2825
|
+
if (r.action === "add") {
|
|
2826
|
+
this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2827
|
+
}
|
|
2828
|
+
else {
|
|
2829
|
+
// Withdrawn before anyone saw it → it never happened. See removeReaction.
|
|
2830
|
+
this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
/**
|
|
2834
|
+
* The queued-reaction summary for an instance's next real message, or {} when
|
|
2835
|
+
* nothing is pending (the common case must add zero context). The consume
|
|
2836
|
+
* callback is separate from the fetch so reactions are only marked once the
|
|
2837
|
+
* message actually went out — a failed delivery keeps them queued.
|
|
2838
|
+
*/
|
|
2839
|
+
pendingReactionsMeta(instanceName) {
|
|
2840
|
+
const pending = this.eventLog?.pendingReactions(instanceName);
|
|
2841
|
+
if (!pending)
|
|
2842
|
+
return { meta: {}, consume: () => { } };
|
|
2843
|
+
return {
|
|
2844
|
+
meta: { pending_reactions: pending.summary },
|
|
2845
|
+
consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
|
|
2846
|
+
};
|
|
2847
|
+
}
|
|
2175
2848
|
async handleInboundMessage(msg) {
|
|
2176
2849
|
const threadId = msg.threadId || undefined;
|
|
2177
2850
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
@@ -2509,6 +3182,7 @@ export class FleetManager {
|
|
|
2509
3182
|
}
|
|
2510
3183
|
this.warnIfRateLimited(generalInstance, msg);
|
|
2511
3184
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
|
|
3185
|
+
const generalReactions = this.pendingReactionsMeta(generalInstance);
|
|
2512
3186
|
try {
|
|
2513
3187
|
await this.deliverToInstance(generalInstance, {
|
|
2514
3188
|
type: "fleet_inbound",
|
|
@@ -2521,12 +3195,18 @@ export class FleetManager {
|
|
|
2521
3195
|
user_id: msg.userId,
|
|
2522
3196
|
ts: msg.timestamp.toISOString(),
|
|
2523
3197
|
thread_id: "",
|
|
2524
|
-
|
|
3198
|
+
// Fleet instances have an authoritative adapter binding. Multiple
|
|
3199
|
+
// bots in one guild can observe the same inbound message, so the
|
|
3200
|
+
// adapter whose event wins dedup is not necessarily the bot that
|
|
3201
|
+
// owns this instance.
|
|
3202
|
+
adapter_id: this.getInstanceAdapterId(generalInstance) ?? msg.adapterId,
|
|
2525
3203
|
source: msg.source,
|
|
2526
3204
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
3205
|
+
...generalReactions.meta,
|
|
2527
3206
|
...extraMeta,
|
|
2528
3207
|
},
|
|
2529
3208
|
});
|
|
3209
|
+
generalReactions.consume();
|
|
2530
3210
|
this.lastInboundUser.set(generalInstance, msg.username);
|
|
2531
3211
|
this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
|
|
2532
3212
|
this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
|
|
@@ -2602,6 +3282,7 @@ export class FleetManager {
|
|
|
2602
3282
|
this.setTopicIcon(instanceName, "blue");
|
|
2603
3283
|
this.warnIfRateLimited(instanceName, msg);
|
|
2604
3284
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
|
|
3285
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
2605
3286
|
try {
|
|
2606
3287
|
await this.deliverToInstance(instanceName, {
|
|
2607
3288
|
type: "fleet_inbound",
|
|
@@ -2614,12 +3295,18 @@ export class FleetManager {
|
|
|
2614
3295
|
user_id: msg.userId,
|
|
2615
3296
|
ts: msg.timestamp.toISOString(),
|
|
2616
3297
|
thread_id: msg.threadId ?? "",
|
|
2617
|
-
|
|
3298
|
+
// Canonicalize the reply context to the configured world. Whichever
|
|
3299
|
+
// sibling bot wins inbound dedup must not decide which bot replies.
|
|
3300
|
+
adapter_id: this.getInstanceAdapterId(instanceName) ?? msg.adapterId,
|
|
2618
3301
|
source: msg.source,
|
|
2619
3302
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
3303
|
+
...reactions.meta,
|
|
2620
3304
|
...extraMeta,
|
|
2621
3305
|
},
|
|
2622
3306
|
});
|
|
3307
|
+
// Only after the message actually went out. A failed delivery keeps the
|
|
3308
|
+
// reactions queued for the retry / the next message.
|
|
3309
|
+
reactions.consume();
|
|
2623
3310
|
}
|
|
2624
3311
|
catch (err) {
|
|
2625
3312
|
this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
|
|
@@ -2694,8 +3381,21 @@ export class FleetManager {
|
|
|
2694
3381
|
: (senderSessionName ? undefined : this.fleetConfig?.instances[instanceName]);
|
|
2695
3382
|
let threadId = resolveReplyThreadId(args.thread_id, routingConfig)
|
|
2696
3383
|
?? this.classicChannels?.getChannelIdByInstance(senderInstanceName ?? instanceName);
|
|
2697
|
-
// Select adapter
|
|
2698
|
-
|
|
3384
|
+
// Select the adapter from the daemon's exact last-inbound context. Message
|
|
3385
|
+
// ids are scoped to that bot/world; routing a secondary-world id through the
|
|
3386
|
+
// primary adapter produces a 404. Instance binding remains the compatibility
|
|
3387
|
+
// fallback for older daemons and calls without a live/persisted context.
|
|
3388
|
+
const contextAdapterId = typeof msg.adapterId === "string" && msg.adapterId
|
|
3389
|
+
? msg.adapterId
|
|
3390
|
+
: undefined;
|
|
3391
|
+
const contextWorld = contextAdapterId ? this.worlds.get(contextAdapterId) : undefined;
|
|
3392
|
+
if (contextAdapterId && !contextWorld) {
|
|
3393
|
+
respond(null, `Adapter world unavailable: ${contextAdapterId}`);
|
|
3394
|
+
return;
|
|
3395
|
+
}
|
|
3396
|
+
const outAdapter = contextWorld?.adapter
|
|
3397
|
+
?? this.getAdapterForInstance(senderInstanceName ?? instanceName)
|
|
3398
|
+
?? this.adapter;
|
|
2699
3399
|
if (!outAdapter) {
|
|
2700
3400
|
respond(null, "No adapter available");
|
|
2701
3401
|
return;
|
|
@@ -2708,25 +3408,33 @@ export class FleetManager {
|
|
|
2708
3408
|
delete args.thread_id;
|
|
2709
3409
|
threadId = undefined;
|
|
2710
3410
|
}
|
|
3411
|
+
// Reply dedup: retries land here when the agent was told a send failed
|
|
3412
|
+
// (daemon budget elapsed, shell tool killed) while the adapter send was
|
|
3413
|
+
// still in flight and about to succeed. One real send, everyone gets its
|
|
3414
|
+
// outcome; a genuinely failed send clears the entry so a retry passes.
|
|
3415
|
+
if (tool === "reply") {
|
|
3416
|
+
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3417
|
+
if (ticket.duplicate) {
|
|
3418
|
+
this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
|
|
3419
|
+
ticket.subscribe(respond);
|
|
3420
|
+
return;
|
|
3421
|
+
}
|
|
3422
|
+
const original = respond;
|
|
3423
|
+
const respondAndRecord = (result, error) => {
|
|
3424
|
+
ticket.complete(result, error);
|
|
3425
|
+
original(result, error);
|
|
3426
|
+
};
|
|
3427
|
+
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3428
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3429
|
+
return;
|
|
3430
|
+
}
|
|
3431
|
+
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
3432
|
+
ticket.complete(null, "reply not handled");
|
|
3433
|
+
original(null, "reply not handled");
|
|
3434
|
+
return;
|
|
3435
|
+
}
|
|
2711
3436
|
// Route standard channel tools (reply, react, edit_message, download_attachment)
|
|
2712
3437
|
if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
|
|
2713
|
-
if (tool === "reply") {
|
|
2714
|
-
// Agent answered — retire its pending cancel button and mark ✅ done.
|
|
2715
|
-
this.clearCancelButton(instanceName);
|
|
2716
|
-
this.reactDone(instanceName);
|
|
2717
|
-
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
2718
|
-
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
2719
|
-
this.emitSseEvent("message", {
|
|
2720
|
-
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
2721
|
-
text: (args.text ?? "").slice(0, 2000),
|
|
2722
|
-
ts: new Date().toISOString(),
|
|
2723
|
-
});
|
|
2724
|
-
// Log bot reply to classic instance chat-log
|
|
2725
|
-
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
2726
|
-
if (isClassic) {
|
|
2727
|
-
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
2728
|
-
}
|
|
2729
|
-
}
|
|
2730
3438
|
return;
|
|
2731
3439
|
}
|
|
2732
3440
|
// Log tool calls for activity visualization
|
|
@@ -2741,6 +3449,35 @@ export class FleetManager {
|
|
|
2741
3449
|
respond(null, `Unknown tool: ${tool}`);
|
|
2742
3450
|
}
|
|
2743
3451
|
}
|
|
3452
|
+
/** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
|
|
3453
|
+
afterReplyRouted(instanceName, args, senderSessionName) {
|
|
3454
|
+
// A reply is NOT proof the turn is over (#410) — but it is not proof of
|
|
3455
|
+
// more work either. Split the difference: an instance that is clearly
|
|
3456
|
+
// idle loses the button now; one that looks busy keeps it (re-posted
|
|
3457
|
+
// below the reply so it stays last in the channel), with a 2-minute
|
|
3458
|
+
// grace check — if it has NOT resumed working by then, the reply was the
|
|
3459
|
+
// end of the turn and the button goes. A multi-step run that keeps
|
|
3460
|
+
// working sails through the check and keeps its button.
|
|
3461
|
+
if (this.getInstanceIdle(instanceName)) {
|
|
3462
|
+
this.clearCancelButton(instanceName);
|
|
3463
|
+
}
|
|
3464
|
+
else {
|
|
3465
|
+
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
3466
|
+
}
|
|
3467
|
+
this.reactDone(instanceName);
|
|
3468
|
+
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
3469
|
+
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
3470
|
+
this.emitSseEvent("message", {
|
|
3471
|
+
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
3472
|
+
text: (args.text ?? "").slice(0, 2000),
|
|
3473
|
+
ts: new Date().toISOString(),
|
|
3474
|
+
});
|
|
3475
|
+
// Log bot reply to classic instance chat-log
|
|
3476
|
+
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
3477
|
+
if (isClassic) {
|
|
3478
|
+
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
3479
|
+
}
|
|
3480
|
+
}
|
|
2744
3481
|
/** Handle tool status update from a daemon instance */
|
|
2745
3482
|
handleToolStatusFromInstance(instanceName, msg) {
|
|
2746
3483
|
const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
@@ -3278,7 +4015,12 @@ export class FleetManager {
|
|
|
3278
4015
|
// the before/after runtime diff cannot observe.
|
|
3279
4016
|
for (const patch of explicitPatches) {
|
|
3280
4017
|
if (patch.remove) {
|
|
3281
|
-
|
|
4018
|
+
// YAML's deleteIn throws when an inherited nested key has no raw parent
|
|
4019
|
+
// (or a legacy scalar occupies that parent). Removing an override which
|
|
4020
|
+
// is already absent is an idempotent no-op, not a failed Settings save.
|
|
4021
|
+
if (this.rawFleetDocument.hasIn(patch.path)) {
|
|
4022
|
+
this.rawFleetDocument.deleteIn(patch.path);
|
|
4023
|
+
}
|
|
3282
4024
|
}
|
|
3283
4025
|
else {
|
|
3284
4026
|
const before = this.rawFleetDocument.getIn(patch.path);
|
|
@@ -3304,8 +4046,11 @@ export class FleetManager {
|
|
|
3304
4046
|
this.patchFleetDocument(document, [...path, i], before[i], after[i]);
|
|
3305
4047
|
}
|
|
3306
4048
|
// Remove from the end so YAML sequence indexes do not shift underneath us.
|
|
3307
|
-
for (let i = before.length - 1; i >= after.length; i--)
|
|
3308
|
-
|
|
4049
|
+
for (let i = before.length - 1; i >= after.length; i--) {
|
|
4050
|
+
const itemPath = [...path, i];
|
|
4051
|
+
if (document.hasIn(itemPath))
|
|
4052
|
+
document.deleteIn(itemPath);
|
|
4053
|
+
}
|
|
3309
4054
|
for (let i = shared; i < after.length; i++)
|
|
3310
4055
|
document.setIn([...path, i], after[i]);
|
|
3311
4056
|
return;
|
|
@@ -3327,7 +4072,11 @@ export class FleetManager {
|
|
|
3327
4072
|
return;
|
|
3328
4073
|
}
|
|
3329
4074
|
if (after === undefined) {
|
|
3330
|
-
|
|
4075
|
+
// Effective config contains inherited objects that may not exist in the
|
|
4076
|
+
// raw YAML at all. yaml.deleteIn() is not idempotent for a missing nested
|
|
4077
|
+
// parent, so guard it explicitly.
|
|
4078
|
+
if (document.hasIn(path))
|
|
4079
|
+
document.deleteIn(path);
|
|
3331
4080
|
}
|
|
3332
4081
|
else if (path.length === 0) {
|
|
3333
4082
|
document.contents = document.createNode(after);
|
|
@@ -3343,6 +4092,9 @@ export class FleetManager {
|
|
|
3343
4092
|
}
|
|
3344
4093
|
}
|
|
3345
4094
|
async removeInstance(name) {
|
|
4095
|
+
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
4096
|
+
// else evicted deleted entries, so it grew for the life of the process.
|
|
4097
|
+
forgetInstanceContext(name);
|
|
3346
4098
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
3347
4099
|
const config = this.fleetConfig?.instances[name];
|
|
3348
4100
|
if (this.scheduler && config?.topic_id) {
|
|
@@ -3448,47 +4200,212 @@ export class FleetManager {
|
|
|
3448
4200
|
this.collabInstances.add(instanceName);
|
|
3449
4201
|
return true;
|
|
3450
4202
|
}
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
4203
|
+
/**
|
|
4204
|
+
* Open the event log, tolerating a corrupt file.
|
|
4205
|
+
*
|
|
4206
|
+
* `events.db` holds history only — event rows and the activity feed. Nothing the
|
|
4207
|
+
* fleet needs to run depends on it, and every consumer already uses
|
|
4208
|
+
* `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
|
|
4209
|
+
* unreadable history file (a truncated WAL after a hard kill, a full disk)
|
|
4210
|
+
* threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
|
|
4211
|
+
* running agent for a file whose only job is reporting.
|
|
4212
|
+
*
|
|
4213
|
+
* So: try, move a bad file aside and retry once with a fresh one, and if even
|
|
4214
|
+
* that fails carry on without an event log.
|
|
4215
|
+
*/
|
|
4216
|
+
/**
|
|
4217
|
+
* Run `agend backend doctor` for the fleet's default backend and return its
|
|
4218
|
+
* cleaned output.
|
|
4219
|
+
*
|
|
4220
|
+
* Async on purpose: this was `execSync` with a 30s timeout, reachable by any
|
|
4221
|
+
* allowlisted user through `/doctor`. While it ran, the entire fleet event loop
|
|
4222
|
+
* was frozen — no IPC, no adapter, no message delivery, no health responses,
|
|
4223
|
+
* and critically no WATCHDOG ping, so a slow doctor could push past
|
|
4224
|
+
* WatchdogSec and have systemd SIGABRT the fleet.
|
|
4225
|
+
*/
|
|
4226
|
+
async runBackendDoctor() {
|
|
4227
|
+
const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
4228
|
+
const backend = this.fleetConfig?.defaults?.backend || "claude-code";
|
|
4229
|
+
try {
|
|
4230
|
+
const { execFile } = await import("node:child_process");
|
|
4231
|
+
const { promisify } = await import("node:util");
|
|
4232
|
+
// execFile with an argv array — no shell, so the backend name cannot be
|
|
4233
|
+
// interpreted as a command even if config is malformed.
|
|
4234
|
+
const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
|
|
4235
|
+
timeout: 30_000,
|
|
4236
|
+
encoding: "utf-8",
|
|
4237
|
+
});
|
|
4238
|
+
return stripAnsi(stdout) || "No output";
|
|
3470
4239
|
}
|
|
3471
|
-
|
|
3472
|
-
|
|
3473
|
-
|
|
3474
|
-
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
4240
|
+
catch (err) {
|
|
4241
|
+
const e = err;
|
|
4242
|
+
return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
|
|
3475
4243
|
}
|
|
3476
4244
|
}
|
|
3477
|
-
|
|
3478
|
-
// Sent after delivering a user message to an instance; clicking it (or
|
|
3479
|
-
// /cancel) sends Escape to the instance's pane to interrupt generation.
|
|
3480
|
-
/** Send a "🛑 Cancel" button to the instance's topic/channel after delivery. */
|
|
4245
|
+
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
3481
4246
|
/**
|
|
3482
|
-
*
|
|
3483
|
-
*
|
|
3484
|
-
*
|
|
4247
|
+
* Cap every instance's pipe-pane log, walking the instances **directory** rather
|
|
4248
|
+
* than the config.
|
|
4249
|
+
*
|
|
4250
|
+
* A running instance rotates its own log on each health tick, so the ones that
|
|
4251
|
+
* need this are the ones nothing else looks at:
|
|
4252
|
+
*
|
|
4253
|
+
* - deleted instances, whose directory outlives the config entry. Nothing ever
|
|
4254
|
+
* touched these again. On the machine this was found on, one held 122 MB and
|
|
4255
|
+
* another 74 MB, out of 622 MB of pipe-pane logs in total.
|
|
4256
|
+
* - classic instances, which live in classicChannels, not fleetConfig.instances,
|
|
4257
|
+
* and so were never in the old config-driven loop at all.
|
|
4258
|
+
* - stopped instances, which have no health tick running.
|
|
4259
|
+
*
|
|
4260
|
+
* pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
|
|
4261
|
+
* at animation rate. Unbounded growth here fills the disk, which takes the whole
|
|
4262
|
+
* fleet down rather than one instance.
|
|
3485
4263
|
*/
|
|
3486
|
-
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
4264
|
+
rotateAllInstanceLogs() {
|
|
4265
|
+
const root = join(this.dataDir, "instances");
|
|
4266
|
+
let entries;
|
|
4267
|
+
try {
|
|
4268
|
+
entries = readdirSync(root, { withFileTypes: true });
|
|
3490
4269
|
}
|
|
3491
|
-
|
|
4270
|
+
catch {
|
|
4271
|
+
return; // no instances directory yet
|
|
4272
|
+
}
|
|
4273
|
+
for (const entry of entries) {
|
|
4274
|
+
if (!entry.isDirectory())
|
|
4275
|
+
continue;
|
|
4276
|
+
// rotateLogIfNeeded is already best-effort and returns early on a missing
|
|
4277
|
+
// file, so a directory without a pipe-pane log costs one stat.
|
|
4278
|
+
rotateLogIfNeeded(join(root, entry.name, "output.log"));
|
|
4279
|
+
}
|
|
4280
|
+
}
|
|
4281
|
+
pruneEventLog() {
|
|
4282
|
+
try {
|
|
4283
|
+
this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
|
|
4284
|
+
}
|
|
4285
|
+
catch (err) {
|
|
4286
|
+
this.logger.warn({ err }, "Event log prune failed");
|
|
4287
|
+
}
|
|
4288
|
+
}
|
|
4289
|
+
openEventLog() {
|
|
4290
|
+
const dbPath = join(this.dataDir, "events.db");
|
|
4291
|
+
try {
|
|
4292
|
+
return new EventLog(dbPath);
|
|
4293
|
+
}
|
|
4294
|
+
catch (err) {
|
|
4295
|
+
this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
|
|
4296
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
4297
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
4298
|
+
try {
|
|
4299
|
+
renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
|
|
4300
|
+
}
|
|
4301
|
+
catch { /* may not exist */ }
|
|
4302
|
+
}
|
|
4303
|
+
try {
|
|
4304
|
+
return new EventLog(dbPath);
|
|
4305
|
+
}
|
|
4306
|
+
catch (retryErr) {
|
|
4307
|
+
// History is worth losing; a fleet that won't start is not.
|
|
4308
|
+
this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
|
|
4309
|
+
return null;
|
|
4310
|
+
}
|
|
4311
|
+
}
|
|
4312
|
+
}
|
|
4313
|
+
/**
|
|
4314
|
+
* Report a fleet-level fault (not attributable to one instance) to the General
|
|
4315
|
+
* topic, so the operator learns about it without reading daemon.log.
|
|
4316
|
+
*
|
|
4317
|
+
* Throttled per distinct message: an unhandled rejection typically comes from a
|
|
4318
|
+
* loop (a poller, a repeating timer), and one channel message per occurrence
|
|
4319
|
+
* would bury the topic — which is worse than silence. First occurrence goes out
|
|
4320
|
+
* immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
|
|
4321
|
+
* count.
|
|
4322
|
+
*
|
|
4323
|
+
* The log line is written by the caller regardless: if every adapter is down,
|
|
4324
|
+
* the only notification path is the one that is broken.
|
|
4325
|
+
*/
|
|
4326
|
+
notifyFleetError(text) {
|
|
4327
|
+
const now = Date.now();
|
|
4328
|
+
const key = text.slice(0, 200);
|
|
4329
|
+
const seen = this.fleetErrorNotices.get(key);
|
|
4330
|
+
if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
|
|
4331
|
+
seen.suppressed++;
|
|
4332
|
+
return;
|
|
4333
|
+
}
|
|
4334
|
+
const suppressed = seen?.suppressed ?? 0;
|
|
4335
|
+
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
4336
|
+
// Bound the map: it is keyed by message text, and a message with a varying
|
|
4337
|
+
// suffix (a path, an id) would otherwise grow it without limit.
|
|
4338
|
+
if (this.fleetErrorNotices.size > 100) {
|
|
4339
|
+
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
4340
|
+
if (oldest !== undefined)
|
|
4341
|
+
this.fleetErrorNotices.delete(oldest);
|
|
4342
|
+
}
|
|
4343
|
+
const body = suppressed > 0
|
|
4344
|
+
? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
|
|
4345
|
+
: text;
|
|
4346
|
+
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4347
|
+
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
4348
|
+
// itself still exists, and notifyInstanceTopic only needs adapter + group +
|
|
4349
|
+
// topic_id to post into it.
|
|
4350
|
+
const general = Object.entries(this.fleetConfig?.instances ?? {})
|
|
4351
|
+
.find(([, config]) => config.general_topic === true)?.[0];
|
|
4352
|
+
if (general) {
|
|
4353
|
+
this.notifyInstanceTopic(general, body);
|
|
4354
|
+
return;
|
|
4355
|
+
}
|
|
4356
|
+
// No General instance — fall back to the primary channel's group.
|
|
4357
|
+
const channelCfg = this.getChannelConfig();
|
|
4358
|
+
const groupId = channelCfg?.group_id;
|
|
4359
|
+
if (this.adapter && groupId) {
|
|
4360
|
+
this.adapter.sendText(String(groupId), body)
|
|
4361
|
+
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
4362
|
+
return;
|
|
4363
|
+
}
|
|
4364
|
+
this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
|
|
4365
|
+
}
|
|
4366
|
+
static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
|
|
4367
|
+
fleetErrorNotices = new Map();
|
|
4368
|
+
notifyInstanceTopic(instanceName, text, extraOpts) {
|
|
4369
|
+
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
4370
|
+
if (!adapter)
|
|
4371
|
+
return;
|
|
4372
|
+
const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
|
|
4373
|
+
const groupId = channelCfg?.group_id;
|
|
4374
|
+
// Fleet topic instance
|
|
4375
|
+
const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
4376
|
+
if (threadId != null && groupId) {
|
|
4377
|
+
adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
|
|
4378
|
+
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
|
|
4379
|
+
return;
|
|
4380
|
+
}
|
|
4381
|
+
// Classic instance: find its channelId from the classic manager
|
|
4382
|
+
const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
|
|
4383
|
+
if (classicChatId) {
|
|
4384
|
+
adapter.sendText(classicChatId, text, extraOpts)
|
|
4385
|
+
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
4386
|
+
return;
|
|
4387
|
+
}
|
|
4388
|
+
// Fallback: send to group without threadId
|
|
4389
|
+
if (groupId) {
|
|
4390
|
+
adapter.sendText(String(groupId), text, extraOpts)
|
|
4391
|
+
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
// ── Cancel button ────────────────────────────────────────────────────
|
|
4395
|
+
// Sent after delivering a user message to an instance; clicking it (or
|
|
4396
|
+
// /cancel) sends Escape to the instance's pane to interrupt generation.
|
|
4397
|
+
/** Send a "🛑 Cancel" button to the instance's topic/channel after delivery. */
|
|
4398
|
+
/**
|
|
4399
|
+
* Handle the DC `/save` slash command for both classic AND fleet-topic targets.
|
|
4400
|
+
* Picks the backend-appropriate command (kiro → /chat save, claude → /export);
|
|
4401
|
+
* unsupported backends get a clear error. Routes via classic paste or fleet IPC.
|
|
4402
|
+
*/
|
|
4403
|
+
async handleSlashSave(data, adapterId) {
|
|
4404
|
+
if (!this.classicChannels?.isAdmin(data.userId)) {
|
|
4405
|
+
await data.respond(t("admin.required"));
|
|
4406
|
+
return;
|
|
4407
|
+
}
|
|
4408
|
+
// Classic resolves per-bot (same-channel multi-bot); otherwise a fleet topic.
|
|
3492
4409
|
const classicName = this.classicChannels.getInstanceByChannel(data.channelId, adapterId);
|
|
3493
4410
|
const target = classicName
|
|
3494
4411
|
? { kind: "classic", name: classicName }
|
|
@@ -3521,6 +4438,40 @@ export class FleetManager {
|
|
|
3521
4438
|
await data.respond(t("save.sent", cmd, target.name));
|
|
3522
4439
|
}
|
|
3523
4440
|
/** Whether the instance currently has at least one live cancel button. */
|
|
4441
|
+
/**
|
|
4442
|
+
* A click on a cancel button, whether or not the fleet still tracks it.
|
|
4443
|
+
*
|
|
4444
|
+
* The old rule was "act only while an entry is live", which made a click on a
|
|
4445
|
+
* button the fleet had forgotten a silent no-op — no cancel, no message, not
|
|
4446
|
+
* even a log line. That is indistinguishable from a broken button, and it is
|
|
4447
|
+
* what the "按鈕點了沒反應" reports were: the entry is briefly absent while a
|
|
4448
|
+
* button is being replaced, and a delete that fails leaves the message on
|
|
4449
|
+
* screen with no entry at all.
|
|
4450
|
+
*
|
|
4451
|
+
* So: honour the click if the instance is actually running, and say so plainly
|
|
4452
|
+
* if it is not. The stale-click path is rate-limited because the original
|
|
4453
|
+
* concern was real — a second click must not fire a second interrupt key at an
|
|
4454
|
+
* instance that has already started a new turn.
|
|
4455
|
+
*/
|
|
4456
|
+
handleCancelClick(instanceName, adapter, data) {
|
|
4457
|
+
if (this.hasCancelButton(instanceName)) {
|
|
4458
|
+
this.cancelInstance(instanceName);
|
|
4459
|
+
return;
|
|
4460
|
+
}
|
|
4461
|
+
const lastAt = this.staleCancelClickAt.get(instanceName) ?? 0;
|
|
4462
|
+
if (Date.now() - lastAt < STALE_CANCEL_CLICK_COOLDOWN_MS)
|
|
4463
|
+
return;
|
|
4464
|
+
this.staleCancelClickAt.set(instanceName, Date.now());
|
|
4465
|
+
// cancelInstance returns false when there is no daemon — i.e. nothing to
|
|
4466
|
+
// cancel, which is the one case where the button really is dead.
|
|
4467
|
+
if (this.cancelInstance(instanceName)) {
|
|
4468
|
+
this.logger.info({ instanceName }, "Cancel click honoured with no live button entry");
|
|
4469
|
+
return;
|
|
4470
|
+
}
|
|
4471
|
+
this.logger.info({ instanceName }, "Cancel click on an expired button — instance not running");
|
|
4472
|
+
adapter?.editMessage(data.chatId, data.messageId, t("cancel.button_stale", instanceName), data.threadId)
|
|
4473
|
+
.catch(() => { });
|
|
4474
|
+
}
|
|
3524
4475
|
hasCancelButton(instanceName) {
|
|
3525
4476
|
for (const e of this.cancelButtons.values()) {
|
|
3526
4477
|
if (e.instanceName === instanceName)
|
|
@@ -3529,15 +4480,22 @@ export class FleetManager {
|
|
|
3529
4480
|
return false;
|
|
3530
4481
|
}
|
|
3531
4482
|
async sendCancelButton(instanceName, correlationId) {
|
|
3532
|
-
//
|
|
3533
|
-
//
|
|
3534
|
-
//
|
|
3535
|
-
|
|
4483
|
+
// Post first, retire after (see the tail of this method). Retiring up front
|
|
4484
|
+
// meant that from the delete until the new message came back — a chat API
|
|
4485
|
+
// round trip, and every reply goes through here — the instance had NO live
|
|
4486
|
+
// entry, while the old button was still on screen. A click in that window
|
|
4487
|
+
// hit `hasCancelButton() === false` and was silently dropped: the reported
|
|
4488
|
+
// "按鈕失效". If notifyAlert then failed, the button was simply gone.
|
|
3536
4489
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3537
4490
|
if (!adapter)
|
|
3538
4491
|
return;
|
|
3539
|
-
|
|
3540
|
-
|
|
4492
|
+
// Resolve the group through the world fallback (first world when unbound),
|
|
4493
|
+
// NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
|
|
4494
|
+
// `channels:` worlds the primary `channel:` block is empty, so an instance
|
|
4495
|
+
// with no world binding yet (fresh restart, cross-instance delegation)
|
|
4496
|
+
// resolved group_id to undefined and the button silently never appeared.
|
|
4497
|
+
const adapterId = this.getInstanceAdapterId(instanceName);
|
|
4498
|
+
const groupId = this.getGroupIdForInstance(instanceName) || undefined;
|
|
3541
4499
|
const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3542
4500
|
let chatId;
|
|
3543
4501
|
let threadId;
|
|
@@ -3553,8 +4511,12 @@ export class FleetManager {
|
|
|
3553
4511
|
if (!chatId && groupId)
|
|
3554
4512
|
chatId = String(groupId);
|
|
3555
4513
|
}
|
|
3556
|
-
if (!chatId)
|
|
4514
|
+
if (!chatId) {
|
|
4515
|
+
// A button that cannot be addressed must say so — this exact silence is how
|
|
4516
|
+
// "the cancel button sometimes never appears" stayed unreported-in-logs.
|
|
4517
|
+
this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
|
|
3557
4518
|
return;
|
|
4519
|
+
}
|
|
3558
4520
|
try {
|
|
3559
4521
|
const sent = await adapter.notifyAlert(chatId, {
|
|
3560
4522
|
type: "cancel",
|
|
@@ -3562,13 +4524,6 @@ export class FleetManager {
|
|
|
3562
4524
|
message: "👀 處理中…",
|
|
3563
4525
|
choices: [{ id: `cancel:${instanceName}`, label: t("cancel.button") }],
|
|
3564
4526
|
}, threadId ? { threadId } : undefined);
|
|
3565
|
-
// A concurrent sendCancelButton for the same instance may have posted its
|
|
3566
|
-
// own button while we awaited notifyAlert. Retire any other buttons for
|
|
3567
|
-
// this instance (not the one we just posted) so only the newest shows.
|
|
3568
|
-
for (const other of this.cancelButtons.values()) {
|
|
3569
|
-
if (other.instanceName === instanceName)
|
|
3570
|
-
this.retireButton(other);
|
|
3571
|
-
}
|
|
3572
4527
|
const entry = {
|
|
3573
4528
|
instanceName,
|
|
3574
4529
|
adapterId,
|
|
@@ -3577,7 +4532,16 @@ export class FleetManager {
|
|
|
3577
4532
|
threadId: sent.threadId ?? threadId,
|
|
3578
4533
|
correlationId,
|
|
3579
4534
|
retryCount: 0,
|
|
4535
|
+
// Elapsed time is measured from when this button was posted — i.e. from
|
|
4536
|
+
// when the work was handed over — not from the pane's working transition,
|
|
4537
|
+
// which resets if the CLI blips idle mid-turn.
|
|
4538
|
+
startedAt: Date.now(),
|
|
4539
|
+
// Matches the text notifyAlert just posted, so the first 60s tick does
|
|
4540
|
+
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
4541
|
+
// with nothing visibly changed.
|
|
4542
|
+
lastProgressText: "👀 處理中…",
|
|
3580
4543
|
};
|
|
4544
|
+
this.startProgressTicker(entry);
|
|
3581
4545
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
3582
4546
|
// button. Covers turns that end without hitting a clear trigger (reply /
|
|
3583
4547
|
// cancel / correlation). Cleared in discardButton when the entry is removed.
|
|
@@ -3586,18 +4550,167 @@ export class FleetManager {
|
|
|
3586
4550
|
clearInterval(entry.idleCheckTimer);
|
|
3587
4551
|
return;
|
|
3588
4552
|
}
|
|
3589
|
-
|
|
3590
|
-
this.
|
|
4553
|
+
const reason = this.getInstanceIdle(instanceName) ? "idle"
|
|
4554
|
+
: this.stateReportDead(instanceName) ? "state reports stopped"
|
|
4555
|
+
: Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
|
|
4556
|
+
: null;
|
|
4557
|
+
if (reason) {
|
|
4558
|
+
this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
|
|
3591
4559
|
this.retireButton(entry);
|
|
4560
|
+
return;
|
|
3592
4561
|
}
|
|
4562
|
+
// Still looks busy. The daemon only broadcasts on transitions, so ask for
|
|
4563
|
+
// a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
|
|
4564
|
+
// the staleness check honest; a dead one's silence is the evidence.
|
|
4565
|
+
this.instanceIpcClients.get(instanceName)?.send({
|
|
4566
|
+
type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
|
|
4567
|
+
});
|
|
3593
4568
|
}, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
|
|
3594
4569
|
this.cancelButtons.set(sent.messageId, entry);
|
|
4570
|
+
// Only now: at most one button per instance, but never zero. Covers both
|
|
4571
|
+
// the previous turn's button and any button a concurrent
|
|
4572
|
+
// sendCancelButton posted while we were awaiting notifyAlert.
|
|
4573
|
+
for (const other of [...this.cancelButtons.values()]) {
|
|
4574
|
+
if (other.instanceName === instanceName && other.messageId !== sent.messageId) {
|
|
4575
|
+
this.retireButton(other);
|
|
4576
|
+
}
|
|
4577
|
+
}
|
|
4578
|
+
this.persistCancelButtons();
|
|
3595
4579
|
this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
|
|
3596
4580
|
}
|
|
3597
4581
|
catch (e) {
|
|
3598
4582
|
this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
|
|
3599
4583
|
}
|
|
3600
4584
|
}
|
|
4585
|
+
/**
|
|
4586
|
+
* The cancel button's text for a given elapsed time.
|
|
4587
|
+
*
|
|
4588
|
+
* Below the threshold it keeps the original wording, so a normal quick answer
|
|
4589
|
+
* looks exactly as it did before. Past it, the button doubles as the live
|
|
4590
|
+
* progress indicator (#409) — the channel showed nothing at all during long work,
|
|
4591
|
+
* and once the agent had replied once there was no sign it was still going.
|
|
4592
|
+
*/
|
|
4593
|
+
static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
|
|
4594
|
+
if (elapsedMs < minElapsedMs)
|
|
4595
|
+
return "👀 處理中…";
|
|
4596
|
+
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
4597
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
4598
|
+
const seconds = totalSeconds % 60;
|
|
4599
|
+
const elapsed = minutes >= 60
|
|
4600
|
+
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
4601
|
+
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
4602
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
4603
|
+
return detail
|
|
4604
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
4605
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
4606
|
+
}
|
|
4607
|
+
/**
|
|
4608
|
+
* Make a tool summary safe to paste into a channel message.
|
|
4609
|
+
*
|
|
4610
|
+
* The text is agent-controlled (it is built from tool inputs — file paths,
|
|
4611
|
+
* shell commands), so it gets flattened to one line, capped, and stripped of
|
|
4612
|
+
* the two Discord mass-mention triggers. Neither channel renders it with a
|
|
4613
|
+
* parse mode, so no markup escaping is needed beyond that.
|
|
4614
|
+
*/
|
|
4615
|
+
static sanitizeActivity(activity) {
|
|
4616
|
+
if (!activity)
|
|
4617
|
+
return null;
|
|
4618
|
+
const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@$1").trim();
|
|
4619
|
+
if (!flat)
|
|
4620
|
+
return null;
|
|
4621
|
+
return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
|
|
4622
|
+
? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
|
|
4623
|
+
: flat;
|
|
4624
|
+
}
|
|
4625
|
+
/**
|
|
4626
|
+
* Remember what an instance is currently doing, for the progress line.
|
|
4627
|
+
*
|
|
4628
|
+
* Best-effort by design: only backends that expose a live activity feed report
|
|
4629
|
+
* anything, and the progress line simply omits the detail for the rest. It is
|
|
4630
|
+
* never used to decide anything — purely what the user is shown.
|
|
4631
|
+
*/
|
|
4632
|
+
cacheInstanceActivity(name, activity) {
|
|
4633
|
+
if (activity)
|
|
4634
|
+
this.instanceActivity.set(name, activity);
|
|
4635
|
+
else
|
|
4636
|
+
this.instanceActivity.delete(name);
|
|
4637
|
+
}
|
|
4638
|
+
/**
|
|
4639
|
+
* Refresh the button's text in place while the instance keeps working.
|
|
4640
|
+
*
|
|
4641
|
+
* Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
|
|
4642
|
+
* and the Bot API treats that as "clear the keyboard" — so editing with it would
|
|
4643
|
+
* delete the very cancel button this is trying to keep alive.
|
|
4644
|
+
*/
|
|
4645
|
+
/** Configured threshold before elapsed time appears, in ms. */
|
|
4646
|
+
progressMinElapsedMs() {
|
|
4647
|
+
const seconds = this.fleetConfig?.defaults
|
|
4648
|
+
?.progress_min_elapsed;
|
|
4649
|
+
if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) {
|
|
4650
|
+
return seconds * 1000;
|
|
4651
|
+
}
|
|
4652
|
+
return PROGRESS_MIN_ELAPSED_MS;
|
|
4653
|
+
}
|
|
4654
|
+
startProgressTicker(entry) {
|
|
4655
|
+
const tick = () => {
|
|
4656
|
+
if (!this.cancelButtons.has(entry.messageId)) {
|
|
4657
|
+
clearInterval(entry.progressTimer);
|
|
4658
|
+
return;
|
|
4659
|
+
}
|
|
4660
|
+
const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
|
|
4661
|
+
if (text === entry.lastProgressText)
|
|
4662
|
+
return; // nothing changed — skip the API call
|
|
4663
|
+
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
4664
|
+
if (!adapter?.editAlert)
|
|
4665
|
+
return;
|
|
4666
|
+
entry.lastProgressText = text;
|
|
4667
|
+
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
4668
|
+
type: "cancel",
|
|
4669
|
+
instanceName: entry.instanceName,
|
|
4670
|
+
message: text,
|
|
4671
|
+
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
4672
|
+
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
4673
|
+
.catch(err => {
|
|
4674
|
+
// A failed progress edit must never escalate: the button still works and
|
|
4675
|
+
// the next tick retries. Common causes are a deleted message or a
|
|
4676
|
+
// rate limit.
|
|
4677
|
+
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
4678
|
+
});
|
|
4679
|
+
};
|
|
4680
|
+
entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
|
|
4681
|
+
entry.progressTimer.unref?.();
|
|
4682
|
+
// One extra tick right when the threshold passes, so a 30s threshold shows
|
|
4683
|
+
// time at ~30s instead of waiting for the first 60s interval. Costs at most
|
|
4684
|
+
// one additional edit per turn that lives past the threshold.
|
|
4685
|
+
const firstAt = this.progressMinElapsedMs() - (Date.now() - (entry.startedAt ?? Date.now()));
|
|
4686
|
+
if (firstAt > 0 && firstAt < PROGRESS_UPDATE_INTERVAL_MS) {
|
|
4687
|
+
const firstTick = setTimeout(tick, firstAt);
|
|
4688
|
+
firstTick.unref?.();
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
/**
|
|
4692
|
+
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
4693
|
+
* it has not, retire its button. Re-arming replaces the previous timer, so a
|
|
4694
|
+
* burst of replies ends with exactly one pending check.
|
|
4695
|
+
*/
|
|
4696
|
+
armReplyGrace(instanceName) {
|
|
4697
|
+
for (const entry of this.cancelButtons.values()) {
|
|
4698
|
+
if (entry.instanceName !== instanceName)
|
|
4699
|
+
continue;
|
|
4700
|
+
if (entry.replyGraceTimer)
|
|
4701
|
+
clearTimeout(entry.replyGraceTimer);
|
|
4702
|
+
entry.replyGraceTimer = setTimeout(() => {
|
|
4703
|
+
entry.replyGraceTimer = undefined;
|
|
4704
|
+
if (!this.cancelButtons.has(entry.messageId))
|
|
4705
|
+
return;
|
|
4706
|
+
if (!this.getInstanceIdle(instanceName))
|
|
4707
|
+
return; // resumed — a long run keeps its button
|
|
4708
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
4709
|
+
this.retireButton(entry);
|
|
4710
|
+
}, REPLY_RETIRE_GRACE_MS);
|
|
4711
|
+
entry.replyGraceTimer.unref?.();
|
|
4712
|
+
}
|
|
4713
|
+
}
|
|
3601
4714
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
3602
4715
|
retireInstanceButtons(instanceName) {
|
|
3603
4716
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -3629,7 +4742,70 @@ export class FleetManager {
|
|
|
3629
4742
|
clearTimeout(entry.retryTimer);
|
|
3630
4743
|
if (entry.idleCheckTimer)
|
|
3631
4744
|
clearInterval(entry.idleCheckTimer);
|
|
4745
|
+
if (entry.progressTimer)
|
|
4746
|
+
clearInterval(entry.progressTimer);
|
|
4747
|
+
if (entry.replyGraceTimer)
|
|
4748
|
+
clearTimeout(entry.replyGraceTimer);
|
|
3632
4749
|
this.cancelButtons.delete(entry.messageId);
|
|
4750
|
+
this.persistCancelButtons();
|
|
4751
|
+
}
|
|
4752
|
+
/**
|
|
4753
|
+
* Mirror the live buttons to disk. The map is memory-only, so before this a
|
|
4754
|
+
* fleet restart orphaned every button on screen: frozen "處理中…" text and a
|
|
4755
|
+
* click that did nothing, forever. The ledger is tiny (a handful of rows) and
|
|
4756
|
+
* written on every add/remove — no debounce needed at that rate.
|
|
4757
|
+
*/
|
|
4758
|
+
persistCancelButtons() {
|
|
4759
|
+
try {
|
|
4760
|
+
const rows = [...this.cancelButtons.values()].map(e => ({
|
|
4761
|
+
instanceName: e.instanceName,
|
|
4762
|
+
adapterId: e.adapterId,
|
|
4763
|
+
chatId: e.chatId,
|
|
4764
|
+
messageId: e.messageId,
|
|
4765
|
+
threadId: e.threadId,
|
|
4766
|
+
}));
|
|
4767
|
+
writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
|
|
4768
|
+
}
|
|
4769
|
+
catch (err) {
|
|
4770
|
+
this.logger.debug({ err }, "Cancel button ledger write failed");
|
|
4771
|
+
}
|
|
4772
|
+
}
|
|
4773
|
+
/**
|
|
4774
|
+
* Delete the previous process's buttons. Runs once adapters are up: nothing
|
|
4775
|
+
* from a previous fleet process can still be mid-turn from this process's
|
|
4776
|
+
* point of view, so every ledger row is an orphan by definition.
|
|
4777
|
+
*/
|
|
4778
|
+
async sweepOrphanedCancelButtons() {
|
|
4779
|
+
const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
|
|
4780
|
+
let rows;
|
|
4781
|
+
try {
|
|
4782
|
+
if (!existsSync(ledgerPath))
|
|
4783
|
+
return;
|
|
4784
|
+
rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
|
|
4785
|
+
}
|
|
4786
|
+
catch {
|
|
4787
|
+
try {
|
|
4788
|
+
unlinkSync(ledgerPath);
|
|
4789
|
+
}
|
|
4790
|
+
catch { /* corrupt ledger — drop it */ }
|
|
4791
|
+
return;
|
|
4792
|
+
}
|
|
4793
|
+
for (const row of rows) {
|
|
4794
|
+
const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
|
|
4795
|
+
?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
|
|
4796
|
+
if (!adapter?.deleteMessage)
|
|
4797
|
+
continue;
|
|
4798
|
+
try {
|
|
4799
|
+
await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
|
|
4800
|
+
this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
|
|
4801
|
+
}
|
|
4802
|
+
catch (err) {
|
|
4803
|
+
// Best effort: the message may already be gone, or too old to delete.
|
|
4804
|
+
this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
|
|
4805
|
+
}
|
|
4806
|
+
}
|
|
4807
|
+
// The current process owns the ledger from here on.
|
|
4808
|
+
this.persistCancelButtons();
|
|
3633
4809
|
}
|
|
3634
4810
|
/** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
|
|
3635
4811
|
scheduleButtonRetry(entry, err) {
|
|
@@ -3743,7 +4919,7 @@ export class FleetManager {
|
|
|
3743
4919
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3744
4920
|
if (!adapter)
|
|
3745
4921
|
return;
|
|
3746
|
-
const channelCfg = this.getChannelConfig(this.
|
|
4922
|
+
const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
|
|
3747
4923
|
const groupId = channelCfg?.group_id;
|
|
3748
4924
|
if (!groupId)
|
|
3749
4925
|
return;
|
|
@@ -4294,6 +5470,12 @@ When users create specialized instances, suggest these configurations:
|
|
|
4294
5470
|
meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
|
|
4295
5471
|
}
|
|
4296
5472
|
}
|
|
5473
|
+
// Classic channels queue reactions like everyone else (#432 stored them, but
|
|
5474
|
+
// this path never attached them — reactions in a ClassicBot channel went into
|
|
5475
|
+
// the DB and were never seen again). Same contract as the topic paths:
|
|
5476
|
+
// consumed only after the delivery succeeded.
|
|
5477
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
5478
|
+
Object.assign(meta, reactions.meta);
|
|
4297
5479
|
try {
|
|
4298
5480
|
await this.deliverToInstance(instanceName, {
|
|
4299
5481
|
type: "fleet_inbound",
|
|
@@ -4301,6 +5483,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4301
5483
|
targetSession: instanceName,
|
|
4302
5484
|
meta,
|
|
4303
5485
|
});
|
|
5486
|
+
reactions.consume();
|
|
4304
5487
|
}
|
|
4305
5488
|
catch (err) {
|
|
4306
5489
|
this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
|
|
@@ -4439,6 +5622,170 @@ When users create specialized instances, suggest these configurations:
|
|
|
4439
5622
|
return env?.models ?? [];
|
|
4440
5623
|
}
|
|
4441
5624
|
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
5625
|
+
/** Label an effort choice, marking the one currently configured. */
|
|
5626
|
+
effortChoiceLabel(level, current) {
|
|
5627
|
+
return level === current ? `✓ ${level}` : level;
|
|
5628
|
+
}
|
|
5629
|
+
effortMenuHeader(instanceName) {
|
|
5630
|
+
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
5631
|
+
if (!effort)
|
|
5632
|
+
return "Current effort: (CLI default)";
|
|
5633
|
+
return source === "fleet-default"
|
|
5634
|
+
? `Current effort: ${effort} (fleet default)`
|
|
5635
|
+
: `Current effort: ${effort}`;
|
|
5636
|
+
}
|
|
5637
|
+
/** `/effort` — DC Select Menu, or apply directly when a level is given. */
|
|
5638
|
+
async handleEffortSlash(data, adapterId) {
|
|
5639
|
+
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
5640
|
+
await data.respond(t("permission.denied"));
|
|
5641
|
+
return;
|
|
5642
|
+
}
|
|
5643
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
5644
|
+
if (!name) {
|
|
5645
|
+
await data.respond(t("classic.no_agent"));
|
|
5646
|
+
return;
|
|
5647
|
+
}
|
|
5648
|
+
const requested = (typeof data.options?.level === "string" ? data.options.level.trim() : "")
|
|
5649
|
+
|| (data.text?.trim() ?? "");
|
|
5650
|
+
if (requested) {
|
|
5651
|
+
await data.respond(await this.applyEffort(name, requested));
|
|
5652
|
+
return;
|
|
5653
|
+
}
|
|
5654
|
+
const levels = this.effortLevelsFor(name);
|
|
5655
|
+
if (levels.length === 0) {
|
|
5656
|
+
await data.respond(`❌ ${this.backendNameForInstance(name)} has no reasoning-effort setting.`);
|
|
5657
|
+
return;
|
|
5658
|
+
}
|
|
5659
|
+
if (!data.respondChoices) {
|
|
5660
|
+
await data.respond(`Usage: /effort <${levels.join("|")}>`);
|
|
5661
|
+
return;
|
|
5662
|
+
}
|
|
5663
|
+
const current = this.resolveInstanceEffort(name).effort;
|
|
5664
|
+
const nonce = randomBytes(6).toString("hex");
|
|
5665
|
+
const choices = levels.map(l => ({
|
|
5666
|
+
id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
|
|
5667
|
+
label: this.effortChoiceLabel(l, current),
|
|
5668
|
+
}));
|
|
5669
|
+
const timer = setTimeout(() => this.pendingEffortSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
5670
|
+
timer.unref?.();
|
|
5671
|
+
this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
5672
|
+
try {
|
|
5673
|
+
await data.respondChoices(`${this.effortMenuHeader(name)}\nSelect a new effort level:`, choices);
|
|
5674
|
+
}
|
|
5675
|
+
catch (err) {
|
|
5676
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5677
|
+
clearTimeout(timer);
|
|
5678
|
+
this.logger.warn({ err, instanceName: name }, "effort menu failed");
|
|
5679
|
+
await data.respond(`Usage: /effort <${levels.join("|")}>`);
|
|
5680
|
+
}
|
|
5681
|
+
}
|
|
5682
|
+
/** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
|
|
5683
|
+
async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
5684
|
+
const levels = this.effortLevelsFor(instanceName);
|
|
5685
|
+
if (levels.length === 0) {
|
|
5686
|
+
return `❌ ${this.backendNameForInstance(instanceName)} has no reasoning-effort setting.`;
|
|
5687
|
+
}
|
|
5688
|
+
const current = this.resolveInstanceEffort(instanceName).effort;
|
|
5689
|
+
const nonce = randomBytes(6).toString("hex");
|
|
5690
|
+
const choices = levels.map(l => ({
|
|
5691
|
+
id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
|
|
5692
|
+
label: this.effortChoiceLabel(l, current),
|
|
5693
|
+
}));
|
|
5694
|
+
const respond = async (text) => {
|
|
5695
|
+
await adapter.sendText(chatId, text, { threadId });
|
|
5696
|
+
return undefined;
|
|
5697
|
+
};
|
|
5698
|
+
const timer = setTimeout(() => {
|
|
5699
|
+
const p = this.pendingEffortSelects.get(nonce);
|
|
5700
|
+
if (p) {
|
|
5701
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5702
|
+
p.respond("⏰ Effort selection expired.").catch(() => { });
|
|
5703
|
+
}
|
|
5704
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
5705
|
+
timer.unref?.();
|
|
5706
|
+
this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
5707
|
+
try {
|
|
5708
|
+
const menuMessageId = await adapter.promptUser(chatId, `${this.effortMenuHeader(instanceName)}\nSelect a new effort level:`, choices, { threadId });
|
|
5709
|
+
const pending = this.pendingEffortSelects.get(nonce);
|
|
5710
|
+
if (pending)
|
|
5711
|
+
pending.menuMessageId = menuMessageId;
|
|
5712
|
+
return null;
|
|
5713
|
+
}
|
|
5714
|
+
catch (err) {
|
|
5715
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5716
|
+
clearTimeout(timer);
|
|
5717
|
+
this.logger.warn({ err, instanceName }, "TG effort menu failed");
|
|
5718
|
+
return `Usage: /effort <${levels.join("|")}>`;
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
/** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
|
|
5722
|
+
async handleEffortSelection(data) {
|
|
5723
|
+
if (!data.callbackData.startsWith(EFFORT_SELECT_CALLBACK_PREFIX))
|
|
5724
|
+
return false;
|
|
5725
|
+
const match = data.callbackData.match(/^effort-select:([0-9a-f]+):(.+)$/);
|
|
5726
|
+
if (!match)
|
|
5727
|
+
return true;
|
|
5728
|
+
const pending = this.pendingEffortSelects.get(match[1]);
|
|
5729
|
+
if (!pending)
|
|
5730
|
+
return true;
|
|
5731
|
+
if (data.userId && data.userId !== pending.userId)
|
|
5732
|
+
return true;
|
|
5733
|
+
const cbChannel = data.threadId ?? data.chatId;
|
|
5734
|
+
if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
|
|
5735
|
+
return true;
|
|
5736
|
+
this.pendingEffortSelects.delete(match[1]);
|
|
5737
|
+
clearTimeout(pending.timer);
|
|
5738
|
+
const level = match[2];
|
|
5739
|
+
const progressText = `⏳ Setting ${pending.instanceName} effort to \`${level}\`…`;
|
|
5740
|
+
let progressMsgId;
|
|
5741
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
5742
|
+
const menuMessageId = pending.menuMessageId ?? data.messageId;
|
|
5743
|
+
if (menuMessageId && pending.adapter.editMessageRemoveButtons) {
|
|
5744
|
+
try {
|
|
5745
|
+
await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, menuMessageId, progressText, pending.adapterThreadId);
|
|
5746
|
+
progressMsgId = menuMessageId;
|
|
5747
|
+
}
|
|
5748
|
+
catch { /* fall back to a new message */ }
|
|
5749
|
+
}
|
|
5750
|
+
if (!progressMsgId) {
|
|
5751
|
+
try {
|
|
5752
|
+
const sent = await pending.adapter.sendText(pending.adapterChatId, progressText, { threadId: pending.adapterThreadId });
|
|
5753
|
+
progressMsgId = sent.messageId;
|
|
5754
|
+
}
|
|
5755
|
+
catch { /* non-fatal */ }
|
|
5756
|
+
}
|
|
5757
|
+
}
|
|
5758
|
+
else {
|
|
5759
|
+
await pending.respond(progressText).catch(() => { });
|
|
5760
|
+
}
|
|
5761
|
+
// Background-applied and guarded for the same reason as the model path: a
|
|
5762
|
+
// restart backend respawns the instance here, and an unguarded rejection
|
|
5763
|
+
// from a menu click must not take the fleet down.
|
|
5764
|
+
void (async () => {
|
|
5765
|
+
let result;
|
|
5766
|
+
try {
|
|
5767
|
+
result = await this.applyEffort(pending.instanceName, level);
|
|
5768
|
+
}
|
|
5769
|
+
catch (err) {
|
|
5770
|
+
this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
|
|
5771
|
+
result = `Effort switch to \`${level}\` failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
5772
|
+
}
|
|
5773
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
5774
|
+
if (progressMsgId) {
|
|
5775
|
+
pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
|
|
5776
|
+
pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
|
|
5777
|
+
});
|
|
5778
|
+
}
|
|
5779
|
+
else {
|
|
5780
|
+
pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
|
|
5781
|
+
}
|
|
5782
|
+
}
|
|
5783
|
+
else {
|
|
5784
|
+
await pending.respond(result).catch(() => { });
|
|
5785
|
+
}
|
|
5786
|
+
})();
|
|
5787
|
+
return true;
|
|
5788
|
+
}
|
|
4442
5789
|
async handleModelSlash(data, adapterId) {
|
|
4443
5790
|
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
4444
5791
|
await data.respond(t("permission.denied"));
|
|
@@ -4575,9 +5922,19 @@ When users create specialized instances, suggest these configurations:
|
|
|
4575
5922
|
// DC path: respond immediately with progress text
|
|
4576
5923
|
await pending.respond(progressText).catch(() => { });
|
|
4577
5924
|
}
|
|
4578
|
-
// Apply model in background — don't await here (keeps callback handler fast)
|
|
5925
|
+
// Apply model in background — don't await here (keeps callback handler fast).
|
|
5926
|
+
// Guarded: applyModel() restarts the instance, and an unguarded rejection here
|
|
5927
|
+
// meant a user picking from the /model menu could take the whole fleet down.
|
|
5928
|
+
// On failure the user gets told, rather than the click silently doing nothing.
|
|
4579
5929
|
void (async () => {
|
|
4580
|
-
|
|
5930
|
+
let result;
|
|
5931
|
+
try {
|
|
5932
|
+
result = await this.applyModel(pending.instanceName, model);
|
|
5933
|
+
}
|
|
5934
|
+
catch (err) {
|
|
5935
|
+
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
5936
|
+
result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
5937
|
+
}
|
|
4581
5938
|
if (pending.adapter && pending.adapterChatId) {
|
|
4582
5939
|
if (progressMsgId) {
|
|
4583
5940
|
pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
|
|
@@ -4595,6 +5952,110 @@ When users create specialized instances, suggest these configurations:
|
|
|
4595
5952
|
return true;
|
|
4596
5953
|
}
|
|
4597
5954
|
/** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
|
|
5955
|
+
/** AgEnD's canonical effort ladder, low → max. Backends expose a subset. */
|
|
5956
|
+
static EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"];
|
|
5957
|
+
/** How this instance's backend applies an effort change. */
|
|
5958
|
+
effortStrategyFor(instanceName) {
|
|
5959
|
+
try {
|
|
5960
|
+
const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
|
|
5961
|
+
const strategy = backend.getEffortStrategy?.() ?? "unsupported";
|
|
5962
|
+
// A backend claiming support but listing no levels is unusable either way.
|
|
5963
|
+
return strategy !== "unsupported" && (backend.getEffortLevels?.() ?? []).length > 0
|
|
5964
|
+
? strategy
|
|
5965
|
+
: "unsupported";
|
|
5966
|
+
}
|
|
5967
|
+
catch {
|
|
5968
|
+
return "unsupported";
|
|
5969
|
+
}
|
|
5970
|
+
}
|
|
5971
|
+
/** Effort levels this instance's backend actually accepts (empty = unsupported). */
|
|
5972
|
+
effortLevelsFor(instanceName) {
|
|
5973
|
+
try {
|
|
5974
|
+
const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
|
|
5975
|
+
if ((backend.getEffortStrategy?.() ?? "unsupported") === "unsupported")
|
|
5976
|
+
return [];
|
|
5977
|
+
return backend.getEffortLevels?.() ?? [];
|
|
5978
|
+
}
|
|
5979
|
+
catch {
|
|
5980
|
+
return [];
|
|
5981
|
+
}
|
|
5982
|
+
}
|
|
5983
|
+
/** Configured effort for an instance: per-instance, else fleet default, else none. */
|
|
5984
|
+
resolveInstanceEffort(instanceName) {
|
|
5985
|
+
const own = this.fleetConfig?.instances[instanceName]?.effort;
|
|
5986
|
+
if (own)
|
|
5987
|
+
return { effort: own, source: "instance" };
|
|
5988
|
+
const fallback = this.fleetConfig?.defaults?.effort;
|
|
5989
|
+
if (fallback)
|
|
5990
|
+
return { effort: fallback, source: "fleet-default" };
|
|
5991
|
+
return { effort: null, source: "unset" };
|
|
5992
|
+
}
|
|
5993
|
+
/**
|
|
5994
|
+
* Clamp a canonical level to the nearest one this backend supports.
|
|
5995
|
+
*
|
|
5996
|
+
* Clamping DOWN the ladder, never up: asking for `max` on a CLI that stops at
|
|
5997
|
+
* `high` should get high, not silently fall to low. The caller reports the
|
|
5998
|
+
* clamp — a user who asks for max and quietly receives high has been told the
|
|
5999
|
+
* request succeeded when it did not.
|
|
6000
|
+
*/
|
|
6001
|
+
static clampEffort(level, supported) {
|
|
6002
|
+
if (supported.includes(level))
|
|
6003
|
+
return level;
|
|
6004
|
+
const ladder = FleetManager.EFFORT_LEVELS;
|
|
6005
|
+
const wanted = ladder.indexOf(level);
|
|
6006
|
+
if (wanted < 0)
|
|
6007
|
+
return null;
|
|
6008
|
+
for (let i = wanted - 1; i >= 0; i--) {
|
|
6009
|
+
if (supported.includes(ladder[i]))
|
|
6010
|
+
return ladder[i];
|
|
6011
|
+
}
|
|
6012
|
+
return supported[0] ?? null;
|
|
6013
|
+
}
|
|
6014
|
+
/**
|
|
6015
|
+
* Apply a reasoning-effort level, mirroring applyModel's shape.
|
|
6016
|
+
*
|
|
6017
|
+
* runtime backends take `/effort <level>` in the pane and keep working;
|
|
6018
|
+
* restart backends only read it at launch, so it is persisted and the
|
|
6019
|
+
* instance respawns.
|
|
6020
|
+
*/
|
|
6021
|
+
async applyEffort(instanceName, requested) {
|
|
6022
|
+
const level = requested.trim().toLowerCase();
|
|
6023
|
+
const backendName = this.backendNameForInstance(instanceName);
|
|
6024
|
+
let strategy = "unsupported";
|
|
6025
|
+
let supported = [];
|
|
6026
|
+
try {
|
|
6027
|
+
const backend = createBackend(backendName, this.getInstanceDir(instanceName));
|
|
6028
|
+
strategy = backend.getEffortStrategy?.() ?? "unsupported";
|
|
6029
|
+
supported = backend.getEffortLevels?.() ?? [];
|
|
6030
|
+
}
|
|
6031
|
+
catch { /* treated as unsupported below */ }
|
|
6032
|
+
if (strategy === "unsupported" || supported.length === 0) {
|
|
6033
|
+
return `❌ ${backendName} has no reasoning-effort setting.`;
|
|
6034
|
+
}
|
|
6035
|
+
if (!FleetManager.EFFORT_LEVELS.includes(level)) {
|
|
6036
|
+
return `❌ Unknown effort level \`${level}\`. Use: ${FleetManager.EFFORT_LEVELS.join(", ")}.`;
|
|
6037
|
+
}
|
|
6038
|
+
const applied = FleetManager.clampEffort(level, supported);
|
|
6039
|
+
if (!applied)
|
|
6040
|
+
return `❌ ${backendName} accepts none of the canonical effort levels.`;
|
|
6041
|
+
const warn = applied === level
|
|
6042
|
+
? ""
|
|
6043
|
+
: `⚠️ Clamped to \`${applied}\` (\`${level}\` not supported by ${backendName}).\n`;
|
|
6044
|
+
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6045
|
+
// or the instance silently reverts on restart.
|
|
6046
|
+
if (this.fleetConfig?.instances[instanceName]) {
|
|
6047
|
+
this.fleetConfig.instances[instanceName].effort = applied;
|
|
6048
|
+
this.saveFleetConfig();
|
|
6049
|
+
}
|
|
6050
|
+
if (strategy === "runtime") {
|
|
6051
|
+
if (!this.instanceIpcClients.get(instanceName))
|
|
6052
|
+
return `${warn}❌ ${instanceName} is not running.`;
|
|
6053
|
+
this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
|
|
6054
|
+
return `${warn}✅ Set ${instanceName} effort to \`${applied}\` (runtime).`;
|
|
6055
|
+
}
|
|
6056
|
+
await this.restartSingleInstance(instanceName);
|
|
6057
|
+
return `${warn}✅ Set ${instanceName} effort to \`${applied}\` and restarted.`;
|
|
6058
|
+
}
|
|
4598
6059
|
async applyModel(instanceName, model) {
|
|
4599
6060
|
const backendName = this.backendNameForInstance(instanceName);
|
|
4600
6061
|
let strategy = "restart";
|
|
@@ -4607,7 +6068,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4607
6068
|
if (!this.instanceIpcClients.get(instanceName))
|
|
4608
6069
|
return `${warn}❌ ${instanceName} is not running.`;
|
|
4609
6070
|
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
4610
|
-
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime)
|
|
6071
|
+
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).${this.effortSuffix(instanceName)}`;
|
|
4611
6072
|
}
|
|
4612
6073
|
// restart: persist the model so the respawned CLI launches with it.
|
|
4613
6074
|
let persisted = false;
|
|
@@ -4622,7 +6083,24 @@ When users create specialized instances, suggest these configurations:
|
|
|
4622
6083
|
if (!persisted)
|
|
4623
6084
|
return `${warn}❌ Could not set model for ${instanceName}.`;
|
|
4624
6085
|
await this.restartSingleInstance(instanceName);
|
|
4625
|
-
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted
|
|
6086
|
+
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.${this.effortSuffix(instanceName)}`;
|
|
6087
|
+
}
|
|
6088
|
+
/**
|
|
6089
|
+
* The trailing "Current effort: …" line for a /model reply.
|
|
6090
|
+
*
|
|
6091
|
+
* Model and effort interact (a cheaper model at max effort is a different
|
|
6092
|
+
* trade than a bigger one at low), so showing the effort in force right after
|
|
6093
|
+
* a switch saves the round trip of asking. Empty when the backend has none.
|
|
6094
|
+
*/
|
|
6095
|
+
effortSuffix(instanceName) {
|
|
6096
|
+
if (this.effortLevelsFor(instanceName).length === 0)
|
|
6097
|
+
return "";
|
|
6098
|
+
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6099
|
+
if (!effort)
|
|
6100
|
+
return "\nCurrent effort: (CLI default)";
|
|
6101
|
+
return source === "fleet-default"
|
|
6102
|
+
? `\nCurrent effort: ${effort} (fleet default)`
|
|
6103
|
+
: `\nCurrent effort: ${effort}`;
|
|
4626
6104
|
}
|
|
4627
6105
|
/** Read recent chat log for agent context */
|
|
4628
6106
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
@@ -4857,11 +6335,28 @@ When users create specialized instances, suggest these configurations:
|
|
|
4857
6335
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
4858
6336
|
return t("classic.stopped");
|
|
4859
6337
|
}
|
|
4860
|
-
|
|
6338
|
+
/**
|
|
6339
|
+
* Idempotent while in flight: SIGINT and SIGTERM share one handler and the
|
|
6340
|
+
* uncaughtException path calls this too, so overlapping runs were possible —
|
|
6341
|
+
* each snapshotting the daemon map and calling stop() on the same daemons
|
|
6342
|
+
* concurrently. Deliberately NOT `async`, so callers receive the same promise
|
|
6343
|
+
* object rather than a fresh wrapper around it. The latch clears when the run
|
|
6344
|
+
* settles, so a later genuine stop (after a restart) still does the work.
|
|
6345
|
+
*/
|
|
6346
|
+
stopAll() {
|
|
6347
|
+
this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
|
|
6348
|
+
return this.stopAllInFlight;
|
|
6349
|
+
}
|
|
6350
|
+
stopAllInFlight = null;
|
|
6351
|
+
async doStopAll() {
|
|
4861
6352
|
this.startupComplete = false;
|
|
4862
6353
|
this.reloadPending = false;
|
|
6354
|
+
// Before anything is stopped: everything that dies from here on dies
|
|
6355
|
+
// because we asked it to. Set synchronously — doStopAll runs to its first
|
|
6356
|
+
// await in the same tick as the signal handler, so no event can slip in.
|
|
6357
|
+
this.shuttingDown = true;
|
|
4863
6358
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
4864
|
-
|
|
6359
|
+
sdNotifyBlocking("STOPPING=1");
|
|
4865
6360
|
if (this.watchdogTimer) {
|
|
4866
6361
|
clearInterval(this.watchdogTimer);
|
|
4867
6362
|
this.watchdogTimer = null;
|
|
@@ -4881,6 +6376,29 @@ When users create specialized instances, suggest these configurations:
|
|
|
4881
6376
|
clearInterval(this.updateCheckTimer);
|
|
4882
6377
|
this.updateCheckTimer = null;
|
|
4883
6378
|
}
|
|
6379
|
+
if (this.eventLogPruneTimer) {
|
|
6380
|
+
clearInterval(this.eventLogPruneTimer);
|
|
6381
|
+
this.eventLogPruneTimer = null;
|
|
6382
|
+
}
|
|
6383
|
+
if (this.logRotateTimer) {
|
|
6384
|
+
clearInterval(this.logRotateTimer);
|
|
6385
|
+
this.logRotateTimer = null;
|
|
6386
|
+
}
|
|
6387
|
+
// Cancel-button timers were never cleared here. The idle-check interval is not
|
|
6388
|
+
// unref'd, so it held the event loop open past shutdown and kept retrying
|
|
6389
|
+
// deletes against an adapter that was already gone.
|
|
6390
|
+
for (const entry of [...this.cancelButtons.values()]) {
|
|
6391
|
+
if (entry.retryTimer)
|
|
6392
|
+
clearTimeout(entry.retryTimer);
|
|
6393
|
+
if (entry.idleCheckTimer)
|
|
6394
|
+
clearInterval(entry.idleCheckTimer);
|
|
6395
|
+
if (entry.progressTimer)
|
|
6396
|
+
clearInterval(entry.progressTimer);
|
|
6397
|
+
}
|
|
6398
|
+
this.cancelButtons.clear();
|
|
6399
|
+
for (const timer of this.cancelButtonIdleRetireTimers.values())
|
|
6400
|
+
clearTimeout(timer);
|
|
6401
|
+
this.cancelButtonIdleRetireTimers.clear();
|
|
4884
6402
|
if (this.topicCleanupTimer) {
|
|
4885
6403
|
clearInterval(this.topicCleanupTimer);
|
|
4886
6404
|
this.topicCleanupTimer = null;
|
|
@@ -4937,6 +6455,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4937
6455
|
this.controlClient?.stop();
|
|
4938
6456
|
this.controlClient = null;
|
|
4939
6457
|
if (this.healthServer) {
|
|
6458
|
+
this.healthServerListening = false;
|
|
4940
6459
|
this.healthServer.close();
|
|
4941
6460
|
this.healthServer = null;
|
|
4942
6461
|
}
|
|
@@ -4948,6 +6467,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
4948
6467
|
catch (e) {
|
|
4949
6468
|
this.logger.debug({ err: e }, "Failed to remove fleet PID file");
|
|
4950
6469
|
}
|
|
6470
|
+
// The lock contains a nonce, so an older/shutting-down process can never
|
|
6471
|
+
// remove a lock acquired by a newer fleet owner.
|
|
6472
|
+
releaseProcessFleetLock();
|
|
4951
6473
|
}
|
|
4952
6474
|
/**
|
|
4953
6475
|
* Prune stale external sessions by re-querying each daemon for live sessions.
|
|
@@ -5099,6 +6621,15 @@ When users create specialized instances, suggest these configurations:
|
|
|
5099
6621
|
removedRatio,
|
|
5100
6622
|
validationErrors: validation.errors,
|
|
5101
6623
|
}, "Refusing unsafe fleet config reload; running configuration was kept");
|
|
6624
|
+
// Tell the operator. A silently ignored config edit is the most confusing
|
|
6625
|
+
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
6626
|
+
// with no explanation anywhere they are looking.
|
|
6627
|
+
const why = !validation.valid
|
|
6628
|
+
? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
|
|
6629
|
+
: unsafeEmpty
|
|
6630
|
+
? `it removed every instance (${oldCount} → 0)`
|
|
6631
|
+
: `it removed more than half the instances (${oldCount} → ${newCount})`;
|
|
6632
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
|
|
5102
6633
|
return;
|
|
5103
6634
|
}
|
|
5104
6635
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -5182,6 +6713,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5182
6713
|
clearTimeout(timeoutHandle);
|
|
5183
6714
|
}
|
|
5184
6715
|
this.logger.info("All instances idle — restarting...");
|
|
6716
|
+
const restartStartedAt = Date.now();
|
|
6717
|
+
// Capture the live adapter/topic before General's daemon is stopped. The
|
|
6718
|
+
// channel adapter remains connected throughout an in-process restart.
|
|
6719
|
+
const progressTarget = this.restartProgressTarget();
|
|
5185
6720
|
this.clearStatuslineWatchers();
|
|
5186
6721
|
for (const [, ipc] of this.instanceIpcClients) {
|
|
5187
6722
|
await ipc.close();
|
|
@@ -5206,15 +6741,25 @@ When users create specialized instances, suggest these configurations:
|
|
|
5206
6741
|
const fleet = this.loadConfig(this.configPath);
|
|
5207
6742
|
this.fleetConfig = fleet;
|
|
5208
6743
|
const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
|
|
6744
|
+
const restartProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), restartStartedAt, this.logger);
|
|
5209
6745
|
// Phase 1: generals first
|
|
5210
6746
|
const restartEntries = Object.entries(fleet.instances);
|
|
5211
6747
|
const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
5212
6748
|
const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
5213
6749
|
for (const [name, cfg] of restartGenerals) {
|
|
5214
|
-
|
|
6750
|
+
try {
|
|
6751
|
+
await this.startInstance(name, cfg, topicMode);
|
|
6752
|
+
if (this.daemons.has(name))
|
|
6753
|
+
restartProgress.markReady();
|
|
6754
|
+
}
|
|
6755
|
+
catch (err) {
|
|
6756
|
+
this.logger.error({ err, name }, "Failed to start general instance");
|
|
6757
|
+
}
|
|
5215
6758
|
}
|
|
6759
|
+
// General is ready again; now its topic can own the live progress message.
|
|
6760
|
+
await restartProgress.start(progressTarget);
|
|
5216
6761
|
if (restartOthers.length > 0) {
|
|
5217
|
-
await this.startInstancesWithConcurrency(restartOthers, topicMode);
|
|
6762
|
+
await this.startInstancesWithConcurrency(restartOthers, topicMode, () => restartProgress.markReady());
|
|
5218
6763
|
}
|
|
5219
6764
|
if (topicMode) {
|
|
5220
6765
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -5223,12 +6768,22 @@ When users create specialized instances, suggest these configurations:
|
|
|
5223
6768
|
// Restart classic channel instances (killed during orphan cleanup)
|
|
5224
6769
|
if (this.classicChannels) {
|
|
5225
6770
|
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
5226
|
-
const channels = this.classicChannels.getAll()
|
|
6771
|
+
const channels = this.classicChannels.getAll()
|
|
6772
|
+
.filter(ch => !this.lifecycle.isPaused(ch.instanceName));
|
|
5227
6773
|
const concurrency = 3;
|
|
5228
6774
|
let idx = 0;
|
|
5229
6775
|
while (idx < channels.length) {
|
|
5230
6776
|
const batch = channels.slice(idx, idx + concurrency);
|
|
5231
|
-
await Promise.allSettled(batch.map(
|
|
6777
|
+
await Promise.allSettled(batch.map(async (ch) => {
|
|
6778
|
+
try {
|
|
6779
|
+
await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
6780
|
+
if (this.daemons.has(ch.instanceName))
|
|
6781
|
+
restartProgress.markReady();
|
|
6782
|
+
}
|
|
6783
|
+
catch (err) {
|
|
6784
|
+
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
6785
|
+
}
|
|
6786
|
+
}));
|
|
5232
6787
|
idx += concurrency;
|
|
5233
6788
|
}
|
|
5234
6789
|
}
|
|
@@ -5237,15 +6792,23 @@ When users create specialized instances, suggest these configurations:
|
|
|
5237
6792
|
}
|
|
5238
6793
|
}
|
|
5239
6794
|
this.logger.info("Graceful restart complete");
|
|
6795
|
+
const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
|
|
6796
|
+
const total = configuredNames.length;
|
|
6797
|
+
const started = configuredNames.filter(name => this.daemons.has(name)).length;
|
|
6798
|
+
const allNotRunning2 = configuredNames.filter(name => !this.daemons.has(name));
|
|
6799
|
+
const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
|
|
6800
|
+
const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
|
|
6801
|
+
const { createRequire } = await import("node:module");
|
|
6802
|
+
const _require2 = createRequire(import.meta.url);
|
|
6803
|
+
const agendVersion2 = _require2("../package.json").version ?? "unknown";
|
|
6804
|
+
const progressCompleted = await restartProgress.finish({
|
|
6805
|
+
running: started,
|
|
6806
|
+
total,
|
|
6807
|
+
version: agendVersion2,
|
|
6808
|
+
pausedNames: pausedNames2,
|
|
6809
|
+
failedNames,
|
|
6810
|
+
});
|
|
5240
6811
|
if (groupId && this.adapter) {
|
|
5241
|
-
const total = Object.keys(fleet.instances).length;
|
|
5242
|
-
const started = this.daemons.size;
|
|
5243
|
-
const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
5244
|
-
const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
|
|
5245
|
-
const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
|
|
5246
|
-
const { createRequire } = await import("node:module");
|
|
5247
|
-
const _require2 = createRequire(import.meta.url);
|
|
5248
|
-
const agendVersion2 = _require2("../package.json").version ?? "unknown";
|
|
5249
6812
|
let restartText;
|
|
5250
6813
|
if (failedNames.length === 0 && pausedNames2.length === 0) {
|
|
5251
6814
|
restartText = t("fleet.ready", started, total, agendVersion2);
|
|
@@ -5257,8 +6820,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5257
6820
|
restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
|
|
5258
6821
|
+ (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
|
|
5259
6822
|
}
|
|
5260
|
-
|
|
5261
|
-
|
|
6823
|
+
if (!progressCompleted) {
|
|
6824
|
+
await this.adapter.sendText(String(groupId), restartText, notifyOpts)
|
|
6825
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
|
|
6826
|
+
}
|
|
5262
6827
|
// Notify each instance's channel — staggered to avoid rate limit storm
|
|
5263
6828
|
const instances = Object.entries(this.fleetConfig?.instances ?? {});
|
|
5264
6829
|
this.logger.info({ count: instances.length }, "Sending restart notification to instances (staggered)");
|
|
@@ -5298,10 +6863,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
5298
6863
|
// ── Update check ────────────────────────────────────────────────────
|
|
5299
6864
|
async checkForUpdates() {
|
|
5300
6865
|
try {
|
|
5301
|
-
|
|
6866
|
+
// Both npm lookups are async: as execSync they froze the fleet event loop for
|
|
6867
|
+
// up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
|
|
6868
|
+
// past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
|
|
6869
|
+
// for a background version check.
|
|
6870
|
+
const { execFile } = await import("node:child_process");
|
|
6871
|
+
const { promisify } = await import("node:util");
|
|
6872
|
+
const execFileP = promisify(execFile);
|
|
6873
|
+
const npmVersion = async (spec) => {
|
|
6874
|
+
const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
|
|
6875
|
+
return stdout.toString().trim();
|
|
6876
|
+
};
|
|
5302
6877
|
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
5303
6878
|
const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
|
|
5304
|
-
const latest =
|
|
6879
|
+
const latest = await npmVersion("@songsid/agend");
|
|
5305
6880
|
let target = latest;
|
|
5306
6881
|
if (currentVersion.includes("-beta")) {
|
|
5307
6882
|
// Beta users track the @beta channel (never fall back to @latest, which is
|
|
@@ -5309,7 +6884,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5309
6884
|
// of beta/latest is the newest.
|
|
5310
6885
|
let beta = "";
|
|
5311
6886
|
try {
|
|
5312
|
-
beta =
|
|
6887
|
+
beta = await npmVersion("@songsid/agend@beta");
|
|
5313
6888
|
}
|
|
5314
6889
|
catch { /* no beta tag */ }
|
|
5315
6890
|
target = beta || latest;
|
|
@@ -5383,28 +6958,22 @@ When users create specialized instances, suggest these configurations:
|
|
|
5383
6958
|
// ── Health HTTP endpoint ─────────────────────────────────────────────
|
|
5384
6959
|
startHealthServer(port) {
|
|
5385
6960
|
this.startedAt = Date.now();
|
|
5386
|
-
|
|
5387
|
-
this.
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
try {
|
|
5392
|
-
chmodSync(tokenPath, 0o600);
|
|
5393
|
-
}
|
|
5394
|
-
catch {
|
|
5395
|
-
// best-effort
|
|
5396
|
-
}
|
|
5397
|
-
// Separate read-only token for the /view page: grants terminal-view + profile
|
|
5398
|
-
// read, but never write (POSTs still require the full web token).
|
|
5399
|
-
this.viewToken = randomBytes(24).toString("hex");
|
|
5400
|
-
const viewTokenPath = join(this.dataDir, "view.token");
|
|
5401
|
-
writeFileSync(viewTokenPath, this.viewToken, { mode: 0o600 });
|
|
5402
|
-
try {
|
|
5403
|
-
chmodSync(viewTokenPath, 0o600);
|
|
5404
|
-
}
|
|
5405
|
-
catch { /* best-effort */ }
|
|
6961
|
+
this.healthServerListening = false;
|
|
6962
|
+
this.healthPortRetried = false;
|
|
6963
|
+
// Defensive for direct/unit callers; normal startup initializes these before adapters.
|
|
6964
|
+
if (!this.webToken || !this.viewToken)
|
|
6965
|
+
this.initializeWebAuthTokens();
|
|
5406
6966
|
this.healthServer = createServer((req, res) => {
|
|
5407
6967
|
res.setHeader("Content-Type", "application/json");
|
|
6968
|
+
const requestPath = new URL(req.url ?? "/", `http://localhost:${port}`).pathname;
|
|
6969
|
+
// Browsers request this automatically and AgEnD does not ship an icon.
|
|
6970
|
+
// It is neither user data nor an API route, so do not turn the harmless
|
|
6971
|
+
// probe into a noisy web-token 401 in the browser console.
|
|
6972
|
+
if (req.method === "GET" && requestPath === "/favicon.ico") {
|
|
6973
|
+
res.writeHead(204);
|
|
6974
|
+
res.end();
|
|
6975
|
+
return;
|
|
6976
|
+
}
|
|
5408
6977
|
// Public health probe — no auth required.
|
|
5409
6978
|
if (req.method === "GET" && req.url === "/health") {
|
|
5410
6979
|
// fallthrough to existing handler below
|
|
@@ -5416,6 +6985,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5416
6985
|
// /view routes accept the read-only view.token (or web.token) and do
|
|
5417
6986
|
// their own per-method auth in view-api.ts — skip the web-token gate.
|
|
5418
6987
|
}
|
|
6988
|
+
else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
|
|
6989
|
+
// /api/ai-usage is read-only GET data for the /view Usage panel — open
|
|
6990
|
+
// like the other /view data routes (usage-api.ts rejects non-GET).
|
|
6991
|
+
}
|
|
5419
6992
|
else {
|
|
5420
6993
|
// All other endpoints require a valid token (query ?token= or X-Agend-Token header).
|
|
5421
6994
|
// /ui/* will also re-check in web-api.ts, which is harmless.
|
|
@@ -5425,39 +6998,39 @@ When users create specialized instances, suggest these configurations:
|
|
|
5425
6998
|
?? (typeof headerToken === "string" ? headerToken : null);
|
|
5426
6999
|
if (!this.webToken || providedToken !== this.webToken) {
|
|
5427
7000
|
res.writeHead(401);
|
|
5428
|
-
res.end(JSON.stringify({ error:
|
|
7001
|
+
res.end(JSON.stringify({ error: WEB_TOKEN_INVALID_MESSAGE }));
|
|
5429
7002
|
return;
|
|
5430
7003
|
}
|
|
5431
7004
|
}
|
|
5432
7005
|
if (req.method === "GET" && req.url === "/health") {
|
|
5433
|
-
const
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
|
|
5437
|
-
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
5441
|
-
}));
|
|
7006
|
+
const health = this.getFleetHealth();
|
|
7007
|
+
// 503 when the fleet cannot do its job, so an external monitor sees it.
|
|
7008
|
+
// This used to always answer 200 "ok" with a count of CONFIGURED instances,
|
|
7009
|
+
// so every agent could be dead and every adapter down and it still looked
|
|
7010
|
+
// green.
|
|
7011
|
+
res.writeHead(health.status === "ok" ? 200 : 503);
|
|
7012
|
+
res.end(JSON.stringify(health));
|
|
5442
7013
|
return;
|
|
5443
7014
|
}
|
|
5444
7015
|
if (req.method === "GET" && req.url === "/status") {
|
|
5445
7016
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
|
|
5446
7017
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5447
|
-
let context_pct = 0;
|
|
5448
7018
|
let cost = 0;
|
|
5449
7019
|
try {
|
|
5450
7020
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5451
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5452
7021
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5453
7022
|
}
|
|
5454
7023
|
catch (err) {
|
|
5455
7024
|
this.logger.debug({ err, name }, "statusline.json read failed (/status)");
|
|
5456
7025
|
}
|
|
7026
|
+
const backend = this.fleetConfig?.instances[name]?.backend
|
|
7027
|
+
?? this.fleetConfig?.defaults?.backend
|
|
7028
|
+
?? "claude-code";
|
|
7029
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
5457
7030
|
return {
|
|
5458
7031
|
name,
|
|
5459
7032
|
status: this.getInstanceStatus(name),
|
|
5460
|
-
context_pct,
|
|
7033
|
+
context_pct: context ?? 0,
|
|
5461
7034
|
cost,
|
|
5462
7035
|
};
|
|
5463
7036
|
});
|
|
@@ -5563,7 +7136,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5563
7136
|
(async () => {
|
|
5564
7137
|
try {
|
|
5565
7138
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
5566
|
-
await this.startInstance(name, config, topicMode ?? false);
|
|
7139
|
+
await this.startInstance(name, config, topicMode ?? false, "fleet-topic", true);
|
|
5567
7140
|
this.emitSseEvent("status", this.getUiStatus());
|
|
5568
7141
|
res.writeHead(200);
|
|
5569
7142
|
res.end(JSON.stringify({ ok: true }));
|
|
@@ -5572,7 +7145,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5572
7145
|
res.writeHead(500);
|
|
5573
7146
|
res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
|
|
5574
7147
|
}
|
|
5575
|
-
|
|
7148
|
+
// The inner catch can itself throw (writeHead after a successful
|
|
7149
|
+
// writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
|
|
7150
|
+
// IIFE. Same for the two handlers below.
|
|
7151
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
|
|
5576
7152
|
return;
|
|
5577
7153
|
}
|
|
5578
7154
|
// Instance restart (immediate, no idle wait)
|
|
@@ -5593,7 +7169,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5593
7169
|
res.writeHead(status);
|
|
5594
7170
|
res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
|
|
5595
7171
|
}
|
|
5596
|
-
})();
|
|
7172
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
|
|
5597
7173
|
return;
|
|
5598
7174
|
}
|
|
5599
7175
|
if (req.method === "POST" && req.url?.startsWith("/stop/")) {
|
|
@@ -5616,7 +7192,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5616
7192
|
res.writeHead(500);
|
|
5617
7193
|
res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
|
|
5618
7194
|
}
|
|
5619
|
-
})();
|
|
7195
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
|
|
5620
7196
|
return;
|
|
5621
7197
|
}
|
|
5622
7198
|
// ── Agent CLI endpoint ─────
|
|
@@ -5628,6 +7204,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
5628
7204
|
const url = new URL(req.url ?? "/", `http://localhost:${port}`);
|
|
5629
7205
|
if (handleViewRequest(req, res, url, this))
|
|
5630
7206
|
return;
|
|
7207
|
+
if (handleUsageRequest(req, res, url, this))
|
|
7208
|
+
return;
|
|
5631
7209
|
if (handleSettingsRequest(req, res, url, this))
|
|
5632
7210
|
return;
|
|
5633
7211
|
if (handleWebRequest(req, res, url, this))
|
|
@@ -5635,10 +7213,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
5635
7213
|
res.writeHead(404);
|
|
5636
7214
|
res.end(JSON.stringify({ error: "not found" }));
|
|
5637
7215
|
});
|
|
7216
|
+
const markListening = (afterTakeover = false) => {
|
|
7217
|
+
this.healthServerListening = true;
|
|
7218
|
+
this.logger.info({ port }, afterTakeover
|
|
7219
|
+
? "Health endpoint listening (after takeover)"
|
|
7220
|
+
: "Health endpoint listening");
|
|
7221
|
+
this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
|
|
7222
|
+
this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
|
|
7223
|
+
};
|
|
5638
7224
|
this.healthServer.on("error", (err) => {
|
|
7225
|
+
this.healthServerListening = false;
|
|
5639
7226
|
if (err.code === "EADDRINUSE") {
|
|
5640
7227
|
if (this.healthPortRetried) {
|
|
5641
|
-
this.logger.
|
|
7228
|
+
this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
|
|
7229
|
+
this.notifyFleetError(`⚠️ Dashboard unavailable — health port ${port} is already in use. Stop the conflicting process or configure a different health_port.`);
|
|
5642
7230
|
return;
|
|
5643
7231
|
}
|
|
5644
7232
|
this.healthPortRetried = true;
|
|
@@ -5659,35 +7247,45 @@ When users create specialized instances, suggest these configurations:
|
|
|
5659
7247
|
setTimeout(() => {
|
|
5660
7248
|
if (!this.healthServer)
|
|
5661
7249
|
return;
|
|
5662
|
-
this.healthServer.listen(port, "127.0.0.1", () =>
|
|
5663
|
-
this.logger.info({ port }, "Health endpoint listening (after takeover)");
|
|
5664
|
-
});
|
|
7250
|
+
this.healthServer.listen(port, "127.0.0.1", () => markListening(true));
|
|
5665
7251
|
}, 1500);
|
|
5666
7252
|
return;
|
|
5667
7253
|
}
|
|
5668
7254
|
this.logger.error({ err, port }, "Health server error");
|
|
7255
|
+
this.notifyFleetError(`⚠️ Dashboard unavailable — health server failed: ${err.message}`);
|
|
5669
7256
|
});
|
|
5670
|
-
this.healthServer.listen(port, "127.0.0.1", () =>
|
|
5671
|
-
this.logger.info({ port }, "Health endpoint listening");
|
|
5672
|
-
});
|
|
5673
|
-
this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
|
|
5674
|
-
this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
|
|
7257
|
+
this.healthServer.listen(port, "127.0.0.1", () => markListening());
|
|
5675
7258
|
}
|
|
5676
7259
|
getUiStatus() {
|
|
5677
|
-
const
|
|
7260
|
+
const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
|
|
7261
|
+
// Classic rooms live only in classicBot.yaml — /api/profiles merges them into
|
|
7262
|
+
// the View roster, but previously getUiStatus skipped them so context_pct was
|
|
7263
|
+
// always 0 (live map miss → l?.context_pct ?? 0).
|
|
7264
|
+
const classicOnly = (this.classicChannels?.getAll() ?? [])
|
|
7265
|
+
.map(ch => ch.instanceName)
|
|
7266
|
+
.filter(name => !fleetNames.includes(name));
|
|
7267
|
+
const names = [...fleetNames, ...classicOnly];
|
|
7268
|
+
const instances = names.map(name => {
|
|
5678
7269
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5679
|
-
let context_pct = 0;
|
|
5680
7270
|
let cost = 0;
|
|
5681
7271
|
let model = "";
|
|
5682
7272
|
try {
|
|
5683
7273
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5684
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5685
7274
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5686
7275
|
model = data.model?.display_name ?? "";
|
|
5687
7276
|
}
|
|
5688
7277
|
catch (err) {
|
|
5689
7278
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
5690
7279
|
}
|
|
7280
|
+
// Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
|
|
7281
|
+
const classic = classicOnly.includes(name);
|
|
7282
|
+
const backend = classic
|
|
7283
|
+
? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
|
|
7284
|
+
: (this.fleetConfig?.instances[name]?.backend
|
|
7285
|
+
?? this.fleetConfig?.defaults?.backend
|
|
7286
|
+
?? "claude-code");
|
|
7287
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
7288
|
+
const context_pct = context ?? 0;
|
|
5691
7289
|
return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
|
|
5692
7290
|
});
|
|
5693
7291
|
return {
|