@songsid/agend 2.1.2-beta.4 → 2.1.2-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +1 -0
- package/dist/agent-cli.js +14 -0
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +25 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +20 -0
- package/dist/backend/antigravity.js +51 -4
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +59 -0
- package/dist/backend/claude-code.js +67 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +27 -0
- package/dist/backend/codex.js +93 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +25 -0
- package/dist/backend/grok.js +51 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +22 -0
- package/dist/backend/kiro.js +79 -1
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +58 -0
- package/dist/backend/types.js +13 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +100 -2
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +86 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +30 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +30 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +227 -87
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +21 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +282 -14
- package/dist/daemon.js +1062 -311
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +9 -0
- package/dist/fleet-lock.d.ts +28 -0
- package/dist/fleet-lock.js +130 -0
- package/dist/fleet-lock.js.map +1 -0
- package/dist/fleet-manager.d.ts +315 -2
- package/dist/fleet-manager.js +1414 -108
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +52 -3
- package/dist/instance-lifecycle.js +194 -48
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +5 -0
- package/dist/instructions.js +9 -11
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +5 -1
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -0
- package/dist/outbound-handlers.js +151 -27
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +11 -3
- package/dist/outbound-schemas.js +16 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/service-installer.d.ts +16 -0
- package/dist/service-installer.js +56 -1
- package/dist/service-installer.js.map +1 -1
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +40 -11
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +49 -9
- package/dist/topic-commands.js +267 -105
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +5 -1
- package/dist/tui-glyphs.js +6 -2
- package/dist/tui-glyphs.js.map +1 -1
- package/dist/types.d.ts +12 -2
- package/dist/ui/view.html +123 -2
- package/dist/update-marker.d.ts +15 -0
- package/dist/update-marker.js +69 -0
- package/dist/update-marker.js.map +1 -0
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +121 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/provider-alerts.d.ts +33 -0
- package/dist/usage/provider-alerts.js +57 -0
- package/dist/usage/provider-alerts.js.map +1 -0
- package/dist/usage/providers.d.ts +130 -0
- package/dist/usage/providers.js +1063 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/statusline-usage.d.ts +20 -0
- package/dist/usage/statusline-usage.js +120 -0
- package/dist/usage/statusline-usage.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +197 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.d.ts +3 -0
- package/dist/view-api.js +11 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -4,7 +4,10 @@ import { createServer } from "node:http";
|
|
|
4
4
|
import { join, dirname, basename } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
|
-
import {
|
|
7
|
+
import { clearUpdateMarker, isUpdateInProgress } from "./update-marker.js";
|
|
8
|
+
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
9
|
+
import { readFleetMemory } from "./process-memory.js";
|
|
10
|
+
import { ReplyDeduper } from "./reply-dedup.js";
|
|
8
11
|
import { isScalar, parseDocument } from "yaml";
|
|
9
12
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
13
|
const __dirname = dirname(__filename);
|
|
@@ -21,12 +24,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
|
|
|
21
24
|
import { createAdapter } from "./channel/factory.js";
|
|
22
25
|
import { createBackend } from "./backend/factory.js";
|
|
23
26
|
import { isModelCompatible } from "./backend/types.js";
|
|
24
|
-
import { createLogger } from "./logger.js";
|
|
27
|
+
import { createLogger, rotateLogIfNeeded } from "./logger.js";
|
|
25
28
|
import { processAttachments } from "./channel/attachment-handler.js";
|
|
26
29
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
27
30
|
import { Scheduler } from "./scheduler/index.js";
|
|
28
31
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
29
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
32
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
30
33
|
import { DailySummary } from "./daily-summary.js";
|
|
31
34
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
32
35
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -38,6 +41,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
38
41
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
39
42
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
40
43
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
44
|
+
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
41
45
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
42
46
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
43
47
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
@@ -45,6 +49,7 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
|
|
|
45
49
|
import { validateFleetConfig } from "./config-validator.js";
|
|
46
50
|
import { readLastInboundAt } from "./daemon.js";
|
|
47
51
|
import { clearPausedMarker } from "./pause-marker.js";
|
|
52
|
+
import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
48
53
|
import { getTmuxSession } from "./config.js";
|
|
49
54
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
50
55
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -83,9 +88,80 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
83
88
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
84
89
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
85
90
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
91
|
+
/**
|
|
92
|
+
* A queued turn can produce a very short idle edge while the CLI hands off to
|
|
93
|
+
* the next message. Do not retire the cancel button until that edge remains
|
|
94
|
+
* idle for this long; a working/stuck report during the grace cancels it.
|
|
95
|
+
*/
|
|
96
|
+
const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
|
|
97
|
+
/**
|
|
98
|
+
* How long after a reply an instance gets to resume working before its cancel
|
|
99
|
+
* button is retired. A short turn ends with a reply and never works again → the
|
|
100
|
+
* button disappears ~2 minutes after the answer. A multi-step run replies
|
|
101
|
+
* mid-flight and keeps going → the grace check sees "working" and leaves the
|
|
102
|
+
* button alone (the idle edge retires it when the run really ends).
|
|
103
|
+
*/
|
|
104
|
+
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
105
|
+
/**
|
|
106
|
+
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
107
|
+
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
108
|
+
* query each tick; a live daemon answers within milliseconds and refreshes the
|
|
109
|
+
* cache. When nothing has refreshed it for this long despite those pokes, the
|
|
110
|
+
* reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
|
|
111
|
+
* from 30 minutes ago proves nothing — the button may be retired.
|
|
112
|
+
*/
|
|
113
|
+
const STATE_REPORT_STALE_MS = 30 * 60_000;
|
|
114
|
+
/**
|
|
115
|
+
* Unconditional ceiling on a cancel button's life. Deliberately far beyond any
|
|
116
|
+
* legitimate run (multi-hour tasks are normal on this fleet): everything below
|
|
117
|
+
* this is decided by real state; a button that somehow survives a full day is
|
|
118
|
+
* wreckage, stuck or not.
|
|
119
|
+
*/
|
|
120
|
+
const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
|
|
121
|
+
/** A click on a button the fleet no longer tracks may fire at most this often. */
|
|
122
|
+
const STALE_CANCEL_CLICK_COOLDOWN_MS = 10_000;
|
|
123
|
+
/** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
|
|
124
|
+
const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
|
|
125
|
+
/**
|
|
126
|
+
* How often the cancel button's text is refreshed with elapsed working time.
|
|
127
|
+
*
|
|
128
|
+
* One edit per working instance per interval — at 60s that is trivial for both
|
|
129
|
+
* platforms' rate limits, and it reads as a live counter rather than a stale
|
|
130
|
+
* snapshot. Nothing new is posted, so the channel is never spammed: there is
|
|
131
|
+
* exactly one progress message per turn, and it is the cancel button itself.
|
|
132
|
+
*/
|
|
133
|
+
const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
|
|
134
|
+
/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
|
|
135
|
+
/**
|
|
136
|
+
* Default delay before the button starts showing elapsed time. Configurable via
|
|
137
|
+
* `defaults.progress_min_elapsed` (seconds) in fleet.yaml. 30s is the balance
|
|
138
|
+
* point: most quick answers finish inside it (no churn for ordinary turns),
|
|
139
|
+
* while anything real shows signs of life well before the old two minutes.
|
|
140
|
+
*/
|
|
141
|
+
const PROGRESS_MIN_ELAPSED_MS = 30_000;
|
|
142
|
+
/** How much of a tool summary the progress line will show before eliding. */
|
|
143
|
+
const PROGRESS_ACTIVITY_MAX_CHARS = 48;
|
|
144
|
+
/**
|
|
145
|
+
* Emoji AgEnD itself stamps on messages as the delivery-status ladder
|
|
146
|
+
* (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed). These are machine
|
|
147
|
+
* indicators, not opinions, so they never enter the reactions queue — from
|
|
148
|
+
* anyone. This exact-emoji filter is the ONLY bot filtering left: bot-to-bot
|
|
149
|
+
* reactions are otherwise delivered on purpose (agents signal each other), and
|
|
150
|
+
* 🫡 passes too — it reads as a deliberate acknowledgement, not plumbing.
|
|
151
|
+
*/
|
|
152
|
+
const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
153
|
+
/**
|
|
154
|
+
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
155
|
+
*
|
|
156
|
+
* Sized for a daemon restart (socket close → respawn → CLI ready), which is the
|
|
157
|
+
* event this exists for. Past it the delivery fails loudly as it always did.
|
|
158
|
+
*/
|
|
159
|
+
const IPC_RECONNECT_GRACE_MS = 30_000;
|
|
160
|
+
const IPC_RECONNECT_POLL_MS = 250;
|
|
86
161
|
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
87
162
|
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
88
163
|
const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
|
|
164
|
+
const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
|
|
89
165
|
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
90
166
|
export class FleetManager {
|
|
91
167
|
dataDir;
|
|
@@ -93,7 +169,7 @@ export class FleetManager {
|
|
|
93
169
|
static sighupHandlerInstalled = false;
|
|
94
170
|
children = new Map();
|
|
95
171
|
lifecycle;
|
|
96
|
-
/**
|
|
172
|
+
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
97
173
|
get daemons() { return this.lifecycle.daemons; }
|
|
98
174
|
fleetConfig = null;
|
|
99
175
|
rawFleetConfig = {};
|
|
@@ -151,6 +227,14 @@ export class FleetManager {
|
|
|
151
227
|
// reply, on cancel, or when a newer button supersedes it for the same
|
|
152
228
|
// instance. Per-button tracking means a failed delete never strands a button.
|
|
153
229
|
cancelButtons = new Map();
|
|
230
|
+
/** Pending idle-edge retirement, one timer per instance. */
|
|
231
|
+
cancelButtonIdleRetireTimers = new Map();
|
|
232
|
+
/** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
|
|
233
|
+
replyDeduper = new ReplyDeduper();
|
|
234
|
+
/** instanceName → what it is doing right now, when the backend can tell us. */
|
|
235
|
+
instanceActivity = new Map();
|
|
236
|
+
/** instanceName → tail of deliveries waiting for its IPC to come back. */
|
|
237
|
+
ipcWaitTails = new Map();
|
|
154
238
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
155
239
|
lastInboundMsg = new Map();
|
|
156
240
|
topicArchiver;
|
|
@@ -158,11 +242,17 @@ export class FleetManager {
|
|
|
158
242
|
classicChannels = null;
|
|
159
243
|
pendingClassicStarts = new Map();
|
|
160
244
|
/** In-flight /model selections, keyed by nonce (see handleModelSelection). */
|
|
245
|
+
/** In-flight /effort selections, same coordinator shape as pendingModelSelects. */
|
|
246
|
+
pendingEffortSelects = new Map();
|
|
161
247
|
pendingModelSelects = new Map();
|
|
162
248
|
// Model failover state
|
|
163
249
|
failoverActive = new Map(); // instance → current failover model
|
|
164
250
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
251
|
+
/** instance → when a click with no live button entry last fired a cancel. */
|
|
252
|
+
staleCancelClickAt = new Map();
|
|
165
253
|
ipcStoppingInstances = new Set();
|
|
254
|
+
/** Set the moment a graceful stop begins — see isPlannedRestart(). */
|
|
255
|
+
shuttingDown = false;
|
|
166
256
|
/** Coalesce concurrent connection attempts for the same daemon socket. */
|
|
167
257
|
ipcConnectInFlight = new Map();
|
|
168
258
|
/** At most one reconnect/backoff loop may exist per instance. */
|
|
@@ -176,6 +266,10 @@ export class FleetManager {
|
|
|
176
266
|
healthServer = null;
|
|
177
267
|
healthPortRetried = false;
|
|
178
268
|
updateCheckTimer = null;
|
|
269
|
+
eventLogPruneTimer = null;
|
|
270
|
+
logRotateTimer = null;
|
|
271
|
+
/** Days of event/activity history to keep. */
|
|
272
|
+
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
179
273
|
watchdogTimer = null;
|
|
180
274
|
startedAt = 0;
|
|
181
275
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
@@ -214,7 +308,13 @@ export class FleetManager {
|
|
|
214
308
|
}
|
|
215
309
|
this.reloadPending = false;
|
|
216
310
|
this.reconcileInFlight = this.reconcileInstances()
|
|
217
|
-
.catch(err =>
|
|
311
|
+
.catch(err => {
|
|
312
|
+
// Almost always a YAML parse error. Log-only meant the user edited
|
|
313
|
+
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
314
|
+
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
315
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
316
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
|
|
317
|
+
})
|
|
218
318
|
.finally(() => {
|
|
219
319
|
this.reconcileInFlight = null;
|
|
220
320
|
if (this.reloadPending && this.startupComplete) {
|
|
@@ -222,16 +322,55 @@ export class FleetManager {
|
|
|
222
322
|
}
|
|
223
323
|
});
|
|
224
324
|
}
|
|
325
|
+
/**
|
|
326
|
+
* Is the fleet going down (or coming back up) on purpose?
|
|
327
|
+
*
|
|
328
|
+
* Instances dying during a planned restart is the restart working, not an
|
|
329
|
+
* incident — but the code that notices a dead pane or a dead MCP server
|
|
330
|
+
* cannot tell the difference on its own. Two sources, because the noise
|
|
331
|
+
* starts before this process is even told to stop: `agend update` replaces
|
|
332
|
+
* the package on disk while this daemon is still running and still watching.
|
|
333
|
+
*/
|
|
334
|
+
isPlannedRestart() {
|
|
335
|
+
return this.shuttingDown || isUpdateInProgress(this.dataDir);
|
|
336
|
+
}
|
|
225
337
|
finishStartup() {
|
|
226
338
|
this.startupComplete = true;
|
|
339
|
+
// We are the post-update fleet: the update is over by definition. Clearing
|
|
340
|
+
// it here (rather than in the update command, which exits before the new
|
|
341
|
+
// fleet is up) is what keeps the quiet window from outliving the restart.
|
|
342
|
+
clearUpdateMarker(this.dataDir);
|
|
227
343
|
if (this.reloadPending)
|
|
228
344
|
this.scheduleReconcile();
|
|
345
|
+
void this.sweepOrphanedCancelButtons();
|
|
229
346
|
}
|
|
230
347
|
// ── ArchiverContext bridge ────────────────────────────────────────────
|
|
231
348
|
lastActivityMs(name) {
|
|
232
349
|
return this.lastActivity.get(name) ?? 0;
|
|
233
350
|
}
|
|
351
|
+
/**
|
|
352
|
+
* Is the instance between turns?
|
|
353
|
+
*
|
|
354
|
+
* Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
|
|
355
|
+
* the control client's raw 2-second output-silence heuristic. The raw heuristic
|
|
356
|
+
* reads every >2s output lull as idle — and long silent tools (a build, a test
|
|
357
|
+
* run) or an LLM pause produce those constantly mid-turn. That misreading is
|
|
358
|
+
* what retired cancel buttons in the middle of long work (the 5-minute backstop
|
|
359
|
+
* fired during a lull) and froze their progress text (ticker skipped "idle"
|
|
360
|
+
* ticks). The silence heuristic remains only as the fallback for instances
|
|
361
|
+
* whose daemon has not reported a state yet.
|
|
362
|
+
*/
|
|
234
363
|
getInstanceIdle(name) {
|
|
364
|
+
// A daemon that is not running cannot be mid-turn. This is what a stale
|
|
365
|
+
// "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
|
|
366
|
+
// report ever arrives) must not override.
|
|
367
|
+
if (this.getInstanceStatus(name) !== "running")
|
|
368
|
+
return true;
|
|
369
|
+
const state = this.getInstanceExecutionState(name);
|
|
370
|
+
if (state === "working" || state === "stuck")
|
|
371
|
+
return false;
|
|
372
|
+
if (state === "idle")
|
|
373
|
+
return true;
|
|
235
374
|
try {
|
|
236
375
|
const widFile = join(this.getInstanceDir(name), "window-id");
|
|
237
376
|
if (!existsSync(widFile))
|
|
@@ -243,6 +382,18 @@ export class FleetManager {
|
|
|
243
382
|
return true;
|
|
244
383
|
}
|
|
245
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* True when the instance claims working/stuck but nothing has refreshed that
|
|
387
|
+
* claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
|
|
388
|
+
* Measures the CACHE's age, not the button's — a healthy multi-hour run
|
|
389
|
+
* answers every query and never trips this.
|
|
390
|
+
*/
|
|
391
|
+
stateReportDead(name) {
|
|
392
|
+
const cached = this.instanceStateCache.get(name);
|
|
393
|
+
if (!cached)
|
|
394
|
+
return false; // no claim to distrust — getInstanceIdle owns this case
|
|
395
|
+
return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
|
|
396
|
+
}
|
|
246
397
|
// ── LifecycleContext bridge methods ──────────────────────────────────────
|
|
247
398
|
webhookEmit(event, name, data) {
|
|
248
399
|
this.webhookEmitter?.emit(event, name, data);
|
|
@@ -454,6 +605,10 @@ export class FleetManager {
|
|
|
454
605
|
getInstanceExecutionState(name) {
|
|
455
606
|
if (this.lifecycle.isPaused(name))
|
|
456
607
|
return null;
|
|
608
|
+
// Process status wins over a stale pane snapshot. A dead remain-on-exit pane
|
|
609
|
+
// can still contain the old ready marker and must never surface as Idle.
|
|
610
|
+
if (this.instanceProcessStatus.has(name))
|
|
611
|
+
return null;
|
|
457
612
|
return this.instanceStateCache.get(name)?.state ?? null;
|
|
458
613
|
}
|
|
459
614
|
isClassicInstance(name) {
|
|
@@ -471,14 +626,45 @@ export class FleetManager {
|
|
|
471
626
|
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
472
627
|
observedAt: numberOr(msg.observedAt, now),
|
|
473
628
|
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
629
|
+
// Fleet-manager receipt time, NOT the daemon's observation time: staleness
|
|
630
|
+
// asks "is anyone still reporting", which only the receiver can date.
|
|
631
|
+
receivedAt: now,
|
|
474
632
|
});
|
|
475
633
|
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
476
634
|
check();
|
|
477
635
|
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
478
636
|
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
479
637
|
// edge into idle, not on every idle heartbeat.
|
|
480
|
-
if (state === "idle" && previous?.state !== "idle")
|
|
638
|
+
if (state === "idle" && previous?.state !== "idle") {
|
|
481
639
|
this.enforceWarmCap();
|
|
640
|
+
// A queued message may turn this edge back into working almost
|
|
641
|
+
// immediately. Give that handoff a short grace before retiring the button.
|
|
642
|
+
this.scheduleIdleButtonRetirement(name);
|
|
643
|
+
}
|
|
644
|
+
else if (state !== "idle") {
|
|
645
|
+
this.cancelIdleButtonRetirement(name);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
cancelIdleButtonRetirement(name) {
|
|
649
|
+
const timer = this.cancelButtonIdleRetireTimers.get(name);
|
|
650
|
+
if (!timer)
|
|
651
|
+
return;
|
|
652
|
+
clearTimeout(timer);
|
|
653
|
+
this.cancelButtonIdleRetireTimers.delete(name);
|
|
654
|
+
}
|
|
655
|
+
scheduleIdleButtonRetirement(name) {
|
|
656
|
+
this.cancelIdleButtonRetirement(name);
|
|
657
|
+
const timer = setTimeout(() => {
|
|
658
|
+
// Ignore a superseded timer even if it was already queued to run.
|
|
659
|
+
if (this.cancelButtonIdleRetireTimers.get(name) !== timer)
|
|
660
|
+
return;
|
|
661
|
+
this.cancelButtonIdleRetireTimers.delete(name);
|
|
662
|
+
if (this.getInstanceExecutionState(name) === "idle") {
|
|
663
|
+
this.retireInstanceButtons(name);
|
|
664
|
+
}
|
|
665
|
+
}, CANCEL_BTN_IDLE_RETIRE_GRACE_MS);
|
|
666
|
+
timer.unref?.();
|
|
667
|
+
this.cancelButtonIdleRetireTimers.set(name, timer);
|
|
482
668
|
}
|
|
483
669
|
cacheInstanceProcessStatus(name, status) {
|
|
484
670
|
if (status === "running") {
|
|
@@ -487,6 +673,7 @@ export class FleetManager {
|
|
|
487
673
|
}
|
|
488
674
|
if (status !== "crashed" && status !== "stopped")
|
|
489
675
|
return;
|
|
676
|
+
this.cancelIdleButtonRetirement(name);
|
|
490
677
|
this.instanceProcessStatus.set(name, status);
|
|
491
678
|
// Never display the last ready prompt as current execution state after its
|
|
492
679
|
// owning CLI process has exited.
|
|
@@ -583,12 +770,70 @@ export class FleetManager {
|
|
|
583
770
|
if (!idle) {
|
|
584
771
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
585
772
|
}
|
|
586
|
-
|
|
587
|
-
if (!ipc?.connected)
|
|
588
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
589
|
-
ipc.send(payload);
|
|
773
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
590
774
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
591
775
|
}
|
|
776
|
+
/**
|
|
777
|
+
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
778
|
+
*
|
|
779
|
+
* A daemon that is restarting — `/restart`, crash recovery, a model switch —
|
|
780
|
+
* drops its socket for a few seconds. Any message arriving in that window used
|
|
781
|
+
* to fail instantly: the caller logged a warning, put ❌ on the user's message,
|
|
782
|
+
* and the message was gone. The user had to notice the ❌ and retype it. That is
|
|
783
|
+
* the "instance 訊息不容易掉" goal failing on the most predictable event there is.
|
|
784
|
+
*
|
|
785
|
+
* The wait is bounded. If the instance is genuinely down, this still throws and
|
|
786
|
+
* the ❌ still appears — just for a real failure rather than a restart.
|
|
787
|
+
*
|
|
788
|
+
* Ordering is preserved by serialising behind any waiter already queued for this
|
|
789
|
+
* instance, *including* when the socket happens to be up: otherwise a message
|
|
790
|
+
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
791
|
+
*/
|
|
792
|
+
async sendWhenConnected(instanceName, payload) {
|
|
793
|
+
const queued = this.ipcWaitTails.get(instanceName);
|
|
794
|
+
if (!queued) {
|
|
795
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
796
|
+
if (ipc?.connected && ipc.send(payload))
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
const attempt = (queued ?? Promise.resolve())
|
|
800
|
+
.catch(() => { })
|
|
801
|
+
.then(() => this.sendAfterIpcReturns(instanceName, payload));
|
|
802
|
+
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
803
|
+
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
804
|
+
const tail = attempt.catch(() => { });
|
|
805
|
+
this.ipcWaitTails.set(instanceName, tail);
|
|
806
|
+
try {
|
|
807
|
+
await attempt;
|
|
808
|
+
}
|
|
809
|
+
finally {
|
|
810
|
+
// Only the last waiter clears the chain; while a queue is still draining the
|
|
811
|
+
// map must keep pointing at it or ordering is lost.
|
|
812
|
+
if (this.ipcWaitTails.get(instanceName) === tail) {
|
|
813
|
+
this.ipcWaitTails.delete(instanceName);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
818
|
+
async sendAfterIpcReturns(instanceName, payload) {
|
|
819
|
+
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
820
|
+
let warned = false;
|
|
821
|
+
for (;;) {
|
|
822
|
+
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
823
|
+
// so a cached reference would stay dead forever.
|
|
824
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
825
|
+
if (ipc?.connected && ipc.send(payload))
|
|
826
|
+
return;
|
|
827
|
+
if (Date.now() >= deadline) {
|
|
828
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
829
|
+
}
|
|
830
|
+
if (!warned) {
|
|
831
|
+
warned = true;
|
|
832
|
+
this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
|
|
833
|
+
}
|
|
834
|
+
await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
|
|
835
|
+
}
|
|
836
|
+
}
|
|
592
837
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
593
838
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
594
839
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
@@ -604,10 +849,7 @@ export class FleetManager {
|
|
|
604
849
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
605
850
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
606
851
|
}
|
|
607
|
-
|
|
608
|
-
if (!ipc?.connected)
|
|
609
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
610
|
-
ipc.send(payload);
|
|
852
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
611
853
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
612
854
|
// working must not trust the stale idle snapshot from before the send.
|
|
613
855
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
@@ -655,7 +897,7 @@ export class FleetManager {
|
|
|
655
897
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
656
898
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
657
899
|
}
|
|
658
|
-
async startInstance(name, config, topicMode) {
|
|
900
|
+
async startInstance(name, config, topicMode, kind = "fleet-topic") {
|
|
659
901
|
if (this.lifecycle.isPaused(name)) {
|
|
660
902
|
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
661
903
|
return;
|
|
@@ -671,7 +913,11 @@ export class FleetManager {
|
|
|
671
913
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
672
914
|
}
|
|
673
915
|
this.instanceProcessStatus.delete(name);
|
|
674
|
-
await this.lifecycle.start(name, config, topicMode
|
|
916
|
+
await this.lifecycle.start(name, config, topicMode, {
|
|
917
|
+
kind,
|
|
918
|
+
backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
|
|
919
|
+
model: this.resolveInstanceModel(name).display,
|
|
920
|
+
});
|
|
675
921
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
676
922
|
await this.connectIpcToInstance(name);
|
|
677
923
|
}
|
|
@@ -772,6 +1018,7 @@ export class FleetManager {
|
|
|
772
1018
|
}
|
|
773
1019
|
async stopInstance(name) {
|
|
774
1020
|
this.failoverActive.delete(name);
|
|
1021
|
+
this.cancelIdleButtonRetirement(name);
|
|
775
1022
|
this.instanceStateCache.delete(name);
|
|
776
1023
|
this.instanceProcessStatus.delete(name);
|
|
777
1024
|
this.lastDeliveryAt.delete(name);
|
|
@@ -834,7 +1081,10 @@ export class FleetManager {
|
|
|
834
1081
|
const eqIdx = trimmed.indexOf("=");
|
|
835
1082
|
if (eqIdx < 0)
|
|
836
1083
|
continue;
|
|
837
|
-
|
|
1084
|
+
// Accept `export KEY=value` — the shell-style form people paste from their
|
|
1085
|
+
// .bashrc. Without this the variable landed in process.env under the key
|
|
1086
|
+
// "export KEY" and silently did nothing.
|
|
1087
|
+
const key = trimmed.slice(0, eqIdx).replace(/^export\s+/, "").trim();
|
|
838
1088
|
const raw = trimmed.slice(eqIdx + 1);
|
|
839
1089
|
const value = raw.replace(/^["'](.*)["']$/, '$1');
|
|
840
1090
|
// .env file always wins over inherited shell env vars, so that
|
|
@@ -846,10 +1096,12 @@ export class FleetManager {
|
|
|
846
1096
|
async startAll(configPath) {
|
|
847
1097
|
FleetManager.signalTarget = this;
|
|
848
1098
|
this.startupComplete = false;
|
|
1099
|
+
// Cleared here, not at the end of doStopAll: a stop has an async tail, and
|
|
1100
|
+
// anything arriving during it is still part of the stop.
|
|
1101
|
+
this.shuttingDown = false;
|
|
849
1102
|
this.configPath = configPath;
|
|
850
1103
|
this.loadEnvFile();
|
|
851
1104
|
// Rotate fleet.log if oversized (before any logging)
|
|
852
|
-
const { rotateLogIfNeeded } = await import("./logger.js");
|
|
853
1105
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
854
1106
|
const fleet = this.loadConfig(configPath);
|
|
855
1107
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
@@ -893,7 +1145,7 @@ export class FleetManager {
|
|
|
893
1145
|
}
|
|
894
1146
|
const pidPath = join(this.dataDir, "fleet.pid");
|
|
895
1147
|
writeFileSync(pidPath, String(process.pid), "utf-8");
|
|
896
|
-
this.eventLog =
|
|
1148
|
+
this.eventLog = this.openEventLog();
|
|
897
1149
|
// Initialize classic channel manager. The primary adapter (channels[0])
|
|
898
1150
|
// migrates legacy single-bot entries and names without a suffix. Classic
|
|
899
1151
|
// routing does NOT go through the routing engine (single-key, can't hold two
|
|
@@ -982,6 +1234,10 @@ export class FleetManager {
|
|
|
982
1234
|
// Rotate fleet.log daily too (besides the startup size check above), so a
|
|
983
1235
|
// long-running fleet doesn't accumulate an unbounded log.
|
|
984
1236
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1237
|
+
// Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
|
|
1238
|
+
// running instance's own log; this sweep is the safety net for every other
|
|
1239
|
+
// kind. One implementation, so the two cannot cover different sets.
|
|
1240
|
+
this.rotateAllInstanceLogs();
|
|
985
1241
|
}, () => {
|
|
986
1242
|
const instances = Object.keys(this.fleetConfig?.instances ?? {});
|
|
987
1243
|
const costMap = new Map();
|
|
@@ -1121,9 +1377,29 @@ export class FleetManager {
|
|
|
1121
1377
|
}
|
|
1122
1378
|
}
|
|
1123
1379
|
}
|
|
1124
|
-
//
|
|
1125
|
-
|
|
1380
|
+
// The systemd watchdog answers exactly one question: is this process still
|
|
1381
|
+
// turning its event loop? Pinging from a timer proves that, and after the
|
|
1382
|
+
// blocking child-process calls were made async it is a meaningful signal —
|
|
1383
|
+
// a deadlocked or frozen fleet stops pinging and systemd restarts it.
|
|
1384
|
+
//
|
|
1385
|
+
// It deliberately does NOT gate on fleet health. "No adapter connected" or
|
|
1386
|
+
// "an instance crashed" must not kill the process: the fleet would be restarted
|
|
1387
|
+
// into the same broken state, and a user who has legitimately stopped every
|
|
1388
|
+
// instance would get a restart loop. Those conditions surface through /health
|
|
1389
|
+
// (which now returns 503) and through the General-topic notifications instead.
|
|
1126
1390
|
this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
|
|
1391
|
+
// EventLog.prune() existed but was never called, so `events` and `activity`
|
|
1392
|
+
// grew without bound for the life of the install. Prune once at startup and
|
|
1393
|
+
// daily after that; the timer is unref'd so it never holds the loop open.
|
|
1394
|
+
this.pruneEventLog();
|
|
1395
|
+
this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
|
|
1396
|
+
this.eventLogPruneTimer.unref?.();
|
|
1397
|
+
// Same shape for pipe-pane logs, and for the same reason: the only sweep that
|
|
1398
|
+
// covered them lived inside the daily-summary callback, so it did not run at
|
|
1399
|
+
// all when summaries were off.
|
|
1400
|
+
this.rotateAllInstanceLogs();
|
|
1401
|
+
this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
|
|
1402
|
+
this.logRotateTimer.unref?.();
|
|
1127
1403
|
// Phase 2: Start remaining instances with staggered concurrency
|
|
1128
1404
|
if (others.length > 0) {
|
|
1129
1405
|
await this.startInstancesWithConcurrency(others, topicMode);
|
|
@@ -1259,6 +1535,15 @@ export class FleetManager {
|
|
|
1259
1535
|
// rest of startup finishes. Replay one coalesced reload only after all
|
|
1260
1536
|
// startup-owned lifecycle work and signal handlers are in place.
|
|
1261
1537
|
this.finishStartup();
|
|
1538
|
+
// Tell systemd we are ready only now. This used to fire right after the
|
|
1539
|
+
// generals started — before adapters, classic instances, topic creation and the
|
|
1540
|
+
// health server — so `systemctl start` returned success while the fleet was
|
|
1541
|
+
// still deaf: no path existed for a user message to arrive.
|
|
1542
|
+
sdNotify("READY=1");
|
|
1543
|
+
const health = this.getFleetHealth();
|
|
1544
|
+
if (health.status !== "ok") {
|
|
1545
|
+
this.logger.warn({ health }, "Fleet started with problems — see /health");
|
|
1546
|
+
}
|
|
1262
1547
|
}
|
|
1263
1548
|
/**
|
|
1264
1549
|
* Delete inbox files older than retentionDays (by mtime). Cleans the shared
|
|
@@ -1375,6 +1660,67 @@ export class FleetManager {
|
|
|
1375
1660
|
getAdapterStates() {
|
|
1376
1661
|
return this.adapterState;
|
|
1377
1662
|
}
|
|
1663
|
+
/**
|
|
1664
|
+
* Real, checkable fleet health for `/health` and the operator.
|
|
1665
|
+
*
|
|
1666
|
+
* `status` is:
|
|
1667
|
+
* - `ok` — at least one adapter connected and every configured instance
|
|
1668
|
+
* that should be running is running
|
|
1669
|
+
* - `degraded` — reachable, but something the operator should look at (an
|
|
1670
|
+
* adapter retrying, an instance crashed or stopped)
|
|
1671
|
+
* - `down` — the fleet cannot do its job: no adapter is connected, so no
|
|
1672
|
+
* message can arrive or be answered
|
|
1673
|
+
*
|
|
1674
|
+
* Deliberately does NOT gate the systemd watchdog — see the comment at the
|
|
1675
|
+
* WATCHDOG timer for why.
|
|
1676
|
+
*/
|
|
1677
|
+
getFleetHealth() {
|
|
1678
|
+
const names = Object.keys(this.fleetConfig?.instances ?? {});
|
|
1679
|
+
const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
|
|
1680
|
+
for (const name of names) {
|
|
1681
|
+
const state = this.getInstanceStatus(name);
|
|
1682
|
+
if (state === "running")
|
|
1683
|
+
counts.running++;
|
|
1684
|
+
else if (state === "crashed")
|
|
1685
|
+
counts.crashed++;
|
|
1686
|
+
else if (state === "paused")
|
|
1687
|
+
counts.paused++;
|
|
1688
|
+
else
|
|
1689
|
+
counts.stopped++;
|
|
1690
|
+
}
|
|
1691
|
+
const states = {};
|
|
1692
|
+
let connected = 0;
|
|
1693
|
+
for (const [id, state] of this.adapterState) {
|
|
1694
|
+
states[id] = state.status;
|
|
1695
|
+
if (state.status === "connected")
|
|
1696
|
+
connected++;
|
|
1697
|
+
}
|
|
1698
|
+
const problems = [];
|
|
1699
|
+
if (this.adapterState.size > 0 && connected === 0)
|
|
1700
|
+
problems.push("no channel adapter is connected");
|
|
1701
|
+
if (counts.crashed > 0)
|
|
1702
|
+
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
1703
|
+
for (const [id, state] of this.adapterState) {
|
|
1704
|
+
if (state.status !== "connected")
|
|
1705
|
+
problems.push(`adapter ${id} is ${state.status}`);
|
|
1706
|
+
}
|
|
1707
|
+
if (!this.startupComplete)
|
|
1708
|
+
problems.push("startup has not completed");
|
|
1709
|
+
// "down" is reserved for "cannot receive or answer a message at all". A fleet
|
|
1710
|
+
// with adapters configured but none connected is exactly that.
|
|
1711
|
+
const status = this.adapterState.size > 0 && connected === 0
|
|
1712
|
+
? "down"
|
|
1713
|
+
: problems.length > 0 ? "degraded" : "ok";
|
|
1714
|
+
return {
|
|
1715
|
+
status,
|
|
1716
|
+
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
1717
|
+
instances: counts,
|
|
1718
|
+
adapters: { total: this.adapterState.size, connected, states },
|
|
1719
|
+
startupComplete: this.startupComplete,
|
|
1720
|
+
memory: readFleetMemory(),
|
|
1721
|
+
problems,
|
|
1722
|
+
};
|
|
1723
|
+
}
|
|
1378
1724
|
/** Start the primary adapter (backward-compatible, sets this.adapter) */
|
|
1379
1725
|
async startSingleAdapter(fleet, channelConfig) {
|
|
1380
1726
|
const botToken = process.env[channelConfig.bot_token_env];
|
|
@@ -1401,11 +1747,16 @@ export class FleetManager {
|
|
|
1401
1747
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
1402
1748
|
await this.handleInboundMessage(msg);
|
|
1403
1749
|
}, this.logger, "adapter.message"));
|
|
1750
|
+
this.adapter.on("reaction", safeHandler(async (r) => {
|
|
1751
|
+
await this.handleInboundReaction(r);
|
|
1752
|
+
}, this.logger, "adapter.reaction"));
|
|
1404
1753
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1405
1754
|
if (await this.handleClassicBackendSelection(data))
|
|
1406
1755
|
return;
|
|
1407
1756
|
if (await this.handleModelSelection(data))
|
|
1408
1757
|
return;
|
|
1758
|
+
if (await this.handleEffortSelection(data))
|
|
1759
|
+
return;
|
|
1409
1760
|
if (data.callbackData.startsWith("hang:")) {
|
|
1410
1761
|
const parts = data.callbackData.split(":");
|
|
1411
1762
|
const action = parts[1];
|
|
@@ -1426,12 +1777,7 @@ export class FleetManager {
|
|
|
1426
1777
|
return;
|
|
1427
1778
|
}
|
|
1428
1779
|
if (data.callbackData.startsWith("cancel:")) {
|
|
1429
|
-
|
|
1430
|
-
// Idempotent: a button click only acts while the button is live. A
|
|
1431
|
-
// second click (entry already cleared) is a no-op — don't re-send the
|
|
1432
|
-
// interrupt key. (The /cancel command path calls cancelInstance directly.)
|
|
1433
|
-
if (this.hasCancelButton(instanceName))
|
|
1434
|
-
this.cancelInstance(instanceName);
|
|
1780
|
+
this.handleCancelClick(data.callbackData.slice("cancel:".length), this.adapter, data);
|
|
1435
1781
|
return;
|
|
1436
1782
|
}
|
|
1437
1783
|
}, this.logger, "adapter.callback_query"));
|
|
@@ -1511,6 +1857,9 @@ export class FleetManager {
|
|
|
1511
1857
|
else if (data.command === "model") {
|
|
1512
1858
|
await this.handleModelSlash(data, adapterId);
|
|
1513
1859
|
}
|
|
1860
|
+
else if (data.command === "effort") {
|
|
1861
|
+
await this.handleEffortSlash(data, adapterId);
|
|
1862
|
+
}
|
|
1514
1863
|
else if (data.command === "cancel") {
|
|
1515
1864
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1516
1865
|
if (!name) {
|
|
@@ -1575,19 +1924,29 @@ export class FleetManager {
|
|
|
1575
1924
|
await data.respond(t("not_authorized"));
|
|
1576
1925
|
return;
|
|
1577
1926
|
}
|
|
1927
|
+
await data.respond(await this.runBackendDoctor());
|
|
1928
|
+
}
|
|
1929
|
+
else if (data.command === "usage") {
|
|
1930
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
1931
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
1932
|
+
// channel either way.
|
|
1578
1933
|
try {
|
|
1579
|
-
const {
|
|
1580
|
-
const
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
await data.respond(clean || "No output");
|
|
1934
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
1935
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
1936
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
1937
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1584
1938
|
}
|
|
1585
1939
|
catch (err) {
|
|
1586
|
-
|
|
1587
|
-
await data.respond(output);
|
|
1940
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1588
1941
|
}
|
|
1589
1942
|
}
|
|
1590
1943
|
else if (data.command === "status") {
|
|
1944
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
1945
|
+
// instance's cost and IPC health.
|
|
1946
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
1947
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1591
1950
|
const text = await this.topicCommands.getStatusText();
|
|
1592
1951
|
await data.respond(text);
|
|
1593
1952
|
}
|
|
@@ -1696,11 +2055,16 @@ export class FleetManager {
|
|
|
1696
2055
|
adapter.on("message", safeHandler(async (msg) => {
|
|
1697
2056
|
await this.handleInboundMessage(msg);
|
|
1698
2057
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
2058
|
+
adapter.on("reaction", safeHandler(async (r) => {
|
|
2059
|
+
await this.handleInboundReaction(r);
|
|
2060
|
+
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
1699
2061
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1700
2062
|
if (await this.handleClassicBackendSelection(data))
|
|
1701
2063
|
return;
|
|
1702
2064
|
if (await this.handleModelSelection(data))
|
|
1703
2065
|
return;
|
|
2066
|
+
if (await this.handleEffortSelection(data))
|
|
2067
|
+
return;
|
|
1704
2068
|
if (data.callbackData.startsWith("hang:")) {
|
|
1705
2069
|
const parts = data.callbackData.split(":");
|
|
1706
2070
|
const action = parts[1];
|
|
@@ -1720,10 +2084,7 @@ export class FleetManager {
|
|
|
1720
2084
|
return;
|
|
1721
2085
|
}
|
|
1722
2086
|
if (data.callbackData.startsWith("cancel:")) {
|
|
1723
|
-
|
|
1724
|
-
// Idempotent: only the first click (while the button is live) acts.
|
|
1725
|
-
if (this.hasCancelButton(instanceName))
|
|
1726
|
-
this.cancelInstance(instanceName);
|
|
2087
|
+
this.handleCancelClick(data.callbackData.slice("cancel:".length), adapter, data);
|
|
1727
2088
|
return;
|
|
1728
2089
|
}
|
|
1729
2090
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
@@ -1793,6 +2154,9 @@ export class FleetManager {
|
|
|
1793
2154
|
else if (data.command === "model") {
|
|
1794
2155
|
await this.handleModelSlash(data, adapterId);
|
|
1795
2156
|
}
|
|
2157
|
+
else if (data.command === "effort") {
|
|
2158
|
+
await this.handleEffortSlash(data, adapterId);
|
|
2159
|
+
}
|
|
1796
2160
|
else if (data.command === "cancel") {
|
|
1797
2161
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1798
2162
|
if (!name) {
|
|
@@ -1857,19 +2221,29 @@ export class FleetManager {
|
|
|
1857
2221
|
await data.respond(t("not_authorized"));
|
|
1858
2222
|
return;
|
|
1859
2223
|
}
|
|
2224
|
+
await data.respond(await this.runBackendDoctor());
|
|
2225
|
+
}
|
|
2226
|
+
else if (data.command === "usage") {
|
|
2227
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
2228
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
2229
|
+
// channel either way.
|
|
1860
2230
|
try {
|
|
1861
|
-
const {
|
|
1862
|
-
const
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
await data.respond(clean || "No output");
|
|
2231
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2232
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2233
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2234
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1866
2235
|
}
|
|
1867
2236
|
catch (err) {
|
|
1868
|
-
|
|
1869
|
-
await data.respond(output);
|
|
2237
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1870
2238
|
}
|
|
1871
2239
|
}
|
|
1872
2240
|
else if (data.command === "status") {
|
|
2241
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
2242
|
+
// instance's cost and IPC health.
|
|
2243
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
2244
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
1873
2247
|
const text = await this.topicCommands.getStatusText();
|
|
1874
2248
|
await data.respond(text);
|
|
1875
2249
|
}
|
|
@@ -2025,6 +2399,9 @@ export class FleetManager {
|
|
|
2025
2399
|
else if (msg.type === "instance_process_state") {
|
|
2026
2400
|
this.cacheInstanceProcessStatus(name, msg.status);
|
|
2027
2401
|
}
|
|
2402
|
+
else if (msg.type === "instance_activity") {
|
|
2403
|
+
this.cacheInstanceActivity(name, msg.activity);
|
|
2404
|
+
}
|
|
2028
2405
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2029
2406
|
this.cacheInstanceExecutionState(name, msg);
|
|
2030
2407
|
if (msg.type === "instance_state_response") {
|
|
@@ -2095,9 +2472,22 @@ export class FleetManager {
|
|
|
2095
2472
|
if (existsSync(windowIdPath)) {
|
|
2096
2473
|
const windowId = readFileSync(windowIdPath, "utf-8").trim();
|
|
2097
2474
|
if (windowId) {
|
|
2475
|
+
// Async with an explicit timeout: this was execSync with NO timeout at
|
|
2476
|
+
// all, so a wedged tmux server blocked the whole fleet event loop
|
|
2477
|
+
// indefinitely — while we were here to diagnose a lost connection.
|
|
2478
|
+
// A timeout is also the correct signal: an unresponsive tmux server
|
|
2479
|
+
// means we cannot verify the pane, which is treated as dead (the same
|
|
2480
|
+
// conclusion the old code reached only by throwing).
|
|
2098
2481
|
try {
|
|
2099
|
-
const {
|
|
2100
|
-
|
|
2482
|
+
const { execFile } = await import("node:child_process");
|
|
2483
|
+
const { promisify } = await import("node:util");
|
|
2484
|
+
const { getTmuxSocketName } = await import("./paths.js");
|
|
2485
|
+
// Honour socket isolation: without -L this queried the user's default
|
|
2486
|
+
// tmux server instead of the fleet's, so under a custom AGEND_HOME the
|
|
2487
|
+
// check was meaningless (it reported every pane dead).
|
|
2488
|
+
const socket = getTmuxSocketName();
|
|
2489
|
+
const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
|
|
2490
|
+
await promisify(execFile)("tmux", args, { timeout: 5_000 });
|
|
2101
2491
|
}
|
|
2102
2492
|
catch {
|
|
2103
2493
|
// Pane dead — respawn
|
|
@@ -2118,6 +2508,12 @@ export class FleetManager {
|
|
|
2118
2508
|
if (this.adapterRestarting.has(id))
|
|
2119
2509
|
return;
|
|
2120
2510
|
this.adapterRestarting.add(id);
|
|
2511
|
+
// Reflect reality in adapterState throughout. This loop used to leave the state
|
|
2512
|
+
// untouched, so getAdapterStates() — and therefore /health and the dashboard —
|
|
2513
|
+
// kept reporting "connected" for an adapter that had been down for hours. An
|
|
2514
|
+
// adapter's true status was simply not knowable from inside the process.
|
|
2515
|
+
const previous = this.adapterState.get(id);
|
|
2516
|
+
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2121
2517
|
try {
|
|
2122
2518
|
for (let attempt = 1;; attempt++) {
|
|
2123
2519
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
@@ -2130,9 +2526,16 @@ export class FleetManager {
|
|
|
2130
2526
|
await adapter.stop().catch(() => { });
|
|
2131
2527
|
await adapter.start();
|
|
2132
2528
|
this.logger.info({ id, attempt }, "Adapter restarted successfully");
|
|
2529
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
2133
2530
|
return;
|
|
2134
2531
|
}
|
|
2135
|
-
catch {
|
|
2532
|
+
catch (err) {
|
|
2533
|
+
this.adapterState.set(id, {
|
|
2534
|
+
status: "retrying",
|
|
2535
|
+
retryCount: attempt,
|
|
2536
|
+
lastError: err?.message ?? String(err),
|
|
2537
|
+
});
|
|
2538
|
+
}
|
|
2136
2539
|
if (attempt % 10 === 0) {
|
|
2137
2540
|
this.logger.warn({ id, attempt }, "Adapter restart still failing");
|
|
2138
2541
|
}
|
|
@@ -2168,6 +2571,49 @@ export class FleetManager {
|
|
|
2168
2571
|
}
|
|
2169
2572
|
return generals[0];
|
|
2170
2573
|
}
|
|
2574
|
+
/**
|
|
2575
|
+
* A user reacted to one of the bot's messages (#408).
|
|
2576
|
+
*
|
|
2577
|
+
* A reaction is context, not a message (#432, reworking #413): it never triggers
|
|
2578
|
+
* an agent turn and never wakes anything. It is queued in the event log and rides
|
|
2579
|
+
* into the instance's NEXT real message as one compact leading line —
|
|
2580
|
+
* `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
|
|
2581
|
+
* No pending reactions → no line → zero context spent, which is the common case.
|
|
2582
|
+
*/
|
|
2583
|
+
async handleInboundReaction(r) {
|
|
2584
|
+
const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
|
|
2585
|
+
if (!instanceName) {
|
|
2586
|
+
this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
|
|
2587
|
+
return;
|
|
2588
|
+
}
|
|
2589
|
+
if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
|
|
2590
|
+
this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
|
|
2591
|
+
return;
|
|
2592
|
+
}
|
|
2593
|
+
this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
|
|
2594
|
+
if (r.action === "add") {
|
|
2595
|
+
this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2596
|
+
}
|
|
2597
|
+
else {
|
|
2598
|
+
// Withdrawn before anyone saw it → it never happened. See removeReaction.
|
|
2599
|
+
this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
/**
|
|
2603
|
+
* The queued-reaction summary for an instance's next real message, or {} when
|
|
2604
|
+
* nothing is pending (the common case must add zero context). The consume
|
|
2605
|
+
* callback is separate from the fetch so reactions are only marked once the
|
|
2606
|
+
* message actually went out — a failed delivery keeps them queued.
|
|
2607
|
+
*/
|
|
2608
|
+
pendingReactionsMeta(instanceName) {
|
|
2609
|
+
const pending = this.eventLog?.pendingReactions(instanceName);
|
|
2610
|
+
if (!pending)
|
|
2611
|
+
return { meta: {}, consume: () => { } };
|
|
2612
|
+
return {
|
|
2613
|
+
meta: { pending_reactions: pending.summary },
|
|
2614
|
+
consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
|
|
2615
|
+
};
|
|
2616
|
+
}
|
|
2171
2617
|
async handleInboundMessage(msg) {
|
|
2172
2618
|
const threadId = msg.threadId || undefined;
|
|
2173
2619
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
@@ -2505,6 +2951,7 @@ export class FleetManager {
|
|
|
2505
2951
|
}
|
|
2506
2952
|
this.warnIfRateLimited(generalInstance, msg);
|
|
2507
2953
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
|
|
2954
|
+
const generalReactions = this.pendingReactionsMeta(generalInstance);
|
|
2508
2955
|
try {
|
|
2509
2956
|
await this.deliverToInstance(generalInstance, {
|
|
2510
2957
|
type: "fleet_inbound",
|
|
@@ -2520,9 +2967,11 @@ export class FleetManager {
|
|
|
2520
2967
|
adapter_id: msg.adapterId,
|
|
2521
2968
|
source: msg.source,
|
|
2522
2969
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2970
|
+
...generalReactions.meta,
|
|
2523
2971
|
...extraMeta,
|
|
2524
2972
|
},
|
|
2525
2973
|
});
|
|
2974
|
+
generalReactions.consume();
|
|
2526
2975
|
this.lastInboundUser.set(generalInstance, msg.username);
|
|
2527
2976
|
this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
|
|
2528
2977
|
this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
|
|
@@ -2598,6 +3047,7 @@ export class FleetManager {
|
|
|
2598
3047
|
this.setTopicIcon(instanceName, "blue");
|
|
2599
3048
|
this.warnIfRateLimited(instanceName, msg);
|
|
2600
3049
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
|
|
3050
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
2601
3051
|
try {
|
|
2602
3052
|
await this.deliverToInstance(instanceName, {
|
|
2603
3053
|
type: "fleet_inbound",
|
|
@@ -2613,9 +3063,13 @@ export class FleetManager {
|
|
|
2613
3063
|
adapter_id: msg.adapterId,
|
|
2614
3064
|
source: msg.source,
|
|
2615
3065
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
3066
|
+
...reactions.meta,
|
|
2616
3067
|
...extraMeta,
|
|
2617
3068
|
},
|
|
2618
3069
|
});
|
|
3070
|
+
// Only after the message actually went out. A failed delivery keeps the
|
|
3071
|
+
// reactions queued for the retry / the next message.
|
|
3072
|
+
reactions.consume();
|
|
2619
3073
|
}
|
|
2620
3074
|
catch (err) {
|
|
2621
3075
|
this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
|
|
@@ -2704,25 +3158,33 @@ export class FleetManager {
|
|
|
2704
3158
|
delete args.thread_id;
|
|
2705
3159
|
threadId = undefined;
|
|
2706
3160
|
}
|
|
3161
|
+
// Reply dedup: retries land here when the agent was told a send failed
|
|
3162
|
+
// (daemon budget elapsed, shell tool killed) while the adapter send was
|
|
3163
|
+
// still in flight and about to succeed. One real send, everyone gets its
|
|
3164
|
+
// outcome; a genuinely failed send clears the entry so a retry passes.
|
|
3165
|
+
if (tool === "reply") {
|
|
3166
|
+
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3167
|
+
if (ticket.duplicate) {
|
|
3168
|
+
this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
|
|
3169
|
+
ticket.subscribe(respond);
|
|
3170
|
+
return;
|
|
3171
|
+
}
|
|
3172
|
+
const original = respond;
|
|
3173
|
+
const respondAndRecord = (result, error) => {
|
|
3174
|
+
ticket.complete(result, error);
|
|
3175
|
+
original(result, error);
|
|
3176
|
+
};
|
|
3177
|
+
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3178
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3179
|
+
return;
|
|
3180
|
+
}
|
|
3181
|
+
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
3182
|
+
ticket.complete(null, "reply not handled");
|
|
3183
|
+
original(null, "reply not handled");
|
|
3184
|
+
return;
|
|
3185
|
+
}
|
|
2707
3186
|
// Route standard channel tools (reply, react, edit_message, download_attachment)
|
|
2708
3187
|
if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
|
|
2709
|
-
if (tool === "reply") {
|
|
2710
|
-
// Agent answered — retire its pending cancel button and mark ✅ done.
|
|
2711
|
-
this.clearCancelButton(instanceName);
|
|
2712
|
-
this.reactDone(instanceName);
|
|
2713
|
-
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
2714
|
-
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
2715
|
-
this.emitSseEvent("message", {
|
|
2716
|
-
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
2717
|
-
text: (args.text ?? "").slice(0, 2000),
|
|
2718
|
-
ts: new Date().toISOString(),
|
|
2719
|
-
});
|
|
2720
|
-
// Log bot reply to classic instance chat-log
|
|
2721
|
-
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
2722
|
-
if (isClassic) {
|
|
2723
|
-
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
2724
|
-
}
|
|
2725
|
-
}
|
|
2726
3188
|
return;
|
|
2727
3189
|
}
|
|
2728
3190
|
// Log tool calls for activity visualization
|
|
@@ -2737,6 +3199,35 @@ export class FleetManager {
|
|
|
2737
3199
|
respond(null, `Unknown tool: ${tool}`);
|
|
2738
3200
|
}
|
|
2739
3201
|
}
|
|
3202
|
+
/** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
|
|
3203
|
+
afterReplyRouted(instanceName, args, senderSessionName) {
|
|
3204
|
+
// A reply is NOT proof the turn is over (#410) — but it is not proof of
|
|
3205
|
+
// more work either. Split the difference: an instance that is clearly
|
|
3206
|
+
// idle loses the button now; one that looks busy keeps it (re-posted
|
|
3207
|
+
// below the reply so it stays last in the channel), with a 2-minute
|
|
3208
|
+
// grace check — if it has NOT resumed working by then, the reply was the
|
|
3209
|
+
// end of the turn and the button goes. A multi-step run that keeps
|
|
3210
|
+
// working sails through the check and keeps its button.
|
|
3211
|
+
if (this.getInstanceIdle(instanceName)) {
|
|
3212
|
+
this.clearCancelButton(instanceName);
|
|
3213
|
+
}
|
|
3214
|
+
else {
|
|
3215
|
+
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
3216
|
+
}
|
|
3217
|
+
this.reactDone(instanceName);
|
|
3218
|
+
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
3219
|
+
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
3220
|
+
this.emitSseEvent("message", {
|
|
3221
|
+
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
3222
|
+
text: (args.text ?? "").slice(0, 2000),
|
|
3223
|
+
ts: new Date().toISOString(),
|
|
3224
|
+
});
|
|
3225
|
+
// Log bot reply to classic instance chat-log
|
|
3226
|
+
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
3227
|
+
if (isClassic) {
|
|
3228
|
+
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
2740
3231
|
/** Handle tool status update from a daemon instance */
|
|
2741
3232
|
handleToolStatusFromInstance(instanceName, msg) {
|
|
2742
3233
|
const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
@@ -3339,6 +3830,9 @@ export class FleetManager {
|
|
|
3339
3830
|
}
|
|
3340
3831
|
}
|
|
3341
3832
|
async removeInstance(name) {
|
|
3833
|
+
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
3834
|
+
// else evicted deleted entries, so it grew for the life of the process.
|
|
3835
|
+
forgetInstanceContext(name);
|
|
3342
3836
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
3343
3837
|
const config = this.fleetConfig?.instances[name];
|
|
3344
3838
|
if (this.scheduler && config?.topic_id) {
|
|
@@ -3444,6 +3938,171 @@ export class FleetManager {
|
|
|
3444
3938
|
this.collabInstances.add(instanceName);
|
|
3445
3939
|
return true;
|
|
3446
3940
|
}
|
|
3941
|
+
/**
|
|
3942
|
+
* Open the event log, tolerating a corrupt file.
|
|
3943
|
+
*
|
|
3944
|
+
* `events.db` holds history only — event rows and the activity feed. Nothing the
|
|
3945
|
+
* fleet needs to run depends on it, and every consumer already uses
|
|
3946
|
+
* `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
|
|
3947
|
+
* unreadable history file (a truncated WAL after a hard kill, a full disk)
|
|
3948
|
+
* threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
|
|
3949
|
+
* running agent for a file whose only job is reporting.
|
|
3950
|
+
*
|
|
3951
|
+
* So: try, move a bad file aside and retry once with a fresh one, and if even
|
|
3952
|
+
* that fails carry on without an event log.
|
|
3953
|
+
*/
|
|
3954
|
+
/**
|
|
3955
|
+
* Run `agend backend doctor` for the fleet's default backend and return its
|
|
3956
|
+
* cleaned output.
|
|
3957
|
+
*
|
|
3958
|
+
* Async on purpose: this was `execSync` with a 30s timeout, reachable by any
|
|
3959
|
+
* allowlisted user through `/doctor`. While it ran, the entire fleet event loop
|
|
3960
|
+
* was frozen — no IPC, no adapter, no message delivery, no health responses,
|
|
3961
|
+
* and critically no WATCHDOG ping, so a slow doctor could push past
|
|
3962
|
+
* WatchdogSec and have systemd SIGABRT the fleet.
|
|
3963
|
+
*/
|
|
3964
|
+
async runBackendDoctor() {
|
|
3965
|
+
const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
3966
|
+
const backend = this.fleetConfig?.defaults?.backend || "claude-code";
|
|
3967
|
+
try {
|
|
3968
|
+
const { execFile } = await import("node:child_process");
|
|
3969
|
+
const { promisify } = await import("node:util");
|
|
3970
|
+
// execFile with an argv array — no shell, so the backend name cannot be
|
|
3971
|
+
// interpreted as a command even if config is malformed.
|
|
3972
|
+
const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
|
|
3973
|
+
timeout: 30_000,
|
|
3974
|
+
encoding: "utf-8",
|
|
3975
|
+
});
|
|
3976
|
+
return stripAnsi(stdout) || "No output";
|
|
3977
|
+
}
|
|
3978
|
+
catch (err) {
|
|
3979
|
+
const e = err;
|
|
3980
|
+
return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
3984
|
+
/**
|
|
3985
|
+
* Cap every instance's pipe-pane log, walking the instances **directory** rather
|
|
3986
|
+
* than the config.
|
|
3987
|
+
*
|
|
3988
|
+
* A running instance rotates its own log on each health tick, so the ones that
|
|
3989
|
+
* need this are the ones nothing else looks at:
|
|
3990
|
+
*
|
|
3991
|
+
* - deleted instances, whose directory outlives the config entry. Nothing ever
|
|
3992
|
+
* touched these again. On the machine this was found on, one held 122 MB and
|
|
3993
|
+
* another 74 MB, out of 622 MB of pipe-pane logs in total.
|
|
3994
|
+
* - classic instances, which live in classicChannels, not fleetConfig.instances,
|
|
3995
|
+
* and so were never in the old config-driven loop at all.
|
|
3996
|
+
* - stopped instances, which have no health tick running.
|
|
3997
|
+
*
|
|
3998
|
+
* pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
|
|
3999
|
+
* at animation rate. Unbounded growth here fills the disk, which takes the whole
|
|
4000
|
+
* fleet down rather than one instance.
|
|
4001
|
+
*/
|
|
4002
|
+
rotateAllInstanceLogs() {
|
|
4003
|
+
const root = join(this.dataDir, "instances");
|
|
4004
|
+
let entries;
|
|
4005
|
+
try {
|
|
4006
|
+
entries = readdirSync(root, { withFileTypes: true });
|
|
4007
|
+
}
|
|
4008
|
+
catch {
|
|
4009
|
+
return; // no instances directory yet
|
|
4010
|
+
}
|
|
4011
|
+
for (const entry of entries) {
|
|
4012
|
+
if (!entry.isDirectory())
|
|
4013
|
+
continue;
|
|
4014
|
+
// rotateLogIfNeeded is already best-effort and returns early on a missing
|
|
4015
|
+
// file, so a directory without a pipe-pane log costs one stat.
|
|
4016
|
+
rotateLogIfNeeded(join(root, entry.name, "output.log"));
|
|
4017
|
+
}
|
|
4018
|
+
}
|
|
4019
|
+
pruneEventLog() {
|
|
4020
|
+
try {
|
|
4021
|
+
this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
|
|
4022
|
+
}
|
|
4023
|
+
catch (err) {
|
|
4024
|
+
this.logger.warn({ err }, "Event log prune failed");
|
|
4025
|
+
}
|
|
4026
|
+
}
|
|
4027
|
+
openEventLog() {
|
|
4028
|
+
const dbPath = join(this.dataDir, "events.db");
|
|
4029
|
+
try {
|
|
4030
|
+
return new EventLog(dbPath);
|
|
4031
|
+
}
|
|
4032
|
+
catch (err) {
|
|
4033
|
+
this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
|
|
4034
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
4035
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
4036
|
+
try {
|
|
4037
|
+
renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
|
|
4038
|
+
}
|
|
4039
|
+
catch { /* may not exist */ }
|
|
4040
|
+
}
|
|
4041
|
+
try {
|
|
4042
|
+
return new EventLog(dbPath);
|
|
4043
|
+
}
|
|
4044
|
+
catch (retryErr) {
|
|
4045
|
+
// History is worth losing; a fleet that won't start is not.
|
|
4046
|
+
this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
|
|
4047
|
+
return null;
|
|
4048
|
+
}
|
|
4049
|
+
}
|
|
4050
|
+
}
|
|
4051
|
+
/**
|
|
4052
|
+
* Report a fleet-level fault (not attributable to one instance) to the General
|
|
4053
|
+
* topic, so the operator learns about it without reading daemon.log.
|
|
4054
|
+
*
|
|
4055
|
+
* Throttled per distinct message: an unhandled rejection typically comes from a
|
|
4056
|
+
* loop (a poller, a repeating timer), and one channel message per occurrence
|
|
4057
|
+
* would bury the topic — which is worse than silence. First occurrence goes out
|
|
4058
|
+
* immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
|
|
4059
|
+
* count.
|
|
4060
|
+
*
|
|
4061
|
+
* The log line is written by the caller regardless: if every adapter is down,
|
|
4062
|
+
* the only notification path is the one that is broken.
|
|
4063
|
+
*/
|
|
4064
|
+
notifyFleetError(text) {
|
|
4065
|
+
const now = Date.now();
|
|
4066
|
+
const key = text.slice(0, 200);
|
|
4067
|
+
const seen = this.fleetErrorNotices.get(key);
|
|
4068
|
+
if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
|
|
4069
|
+
seen.suppressed++;
|
|
4070
|
+
return;
|
|
4071
|
+
}
|
|
4072
|
+
const suppressed = seen?.suppressed ?? 0;
|
|
4073
|
+
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
4074
|
+
// Bound the map: it is keyed by message text, and a message with a varying
|
|
4075
|
+
// suffix (a path, an id) would otherwise grow it without limit.
|
|
4076
|
+
if (this.fleetErrorNotices.size > 100) {
|
|
4077
|
+
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
4078
|
+
if (oldest !== undefined)
|
|
4079
|
+
this.fleetErrorNotices.delete(oldest);
|
|
4080
|
+
}
|
|
4081
|
+
const body = suppressed > 0
|
|
4082
|
+
? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
|
|
4083
|
+
: text;
|
|
4084
|
+
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4085
|
+
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
4086
|
+
// itself still exists, and notifyInstanceTopic only needs adapter + group +
|
|
4087
|
+
// topic_id to post into it.
|
|
4088
|
+
const general = Object.entries(this.fleetConfig?.instances ?? {})
|
|
4089
|
+
.find(([, config]) => config.general_topic === true)?.[0];
|
|
4090
|
+
if (general) {
|
|
4091
|
+
this.notifyInstanceTopic(general, body);
|
|
4092
|
+
return;
|
|
4093
|
+
}
|
|
4094
|
+
// No General instance — fall back to the primary channel's group.
|
|
4095
|
+
const channelCfg = this.getChannelConfig();
|
|
4096
|
+
const groupId = channelCfg?.group_id;
|
|
4097
|
+
if (this.adapter && groupId) {
|
|
4098
|
+
this.adapter.sendText(String(groupId), body)
|
|
4099
|
+
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
4100
|
+
return;
|
|
4101
|
+
}
|
|
4102
|
+
this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
|
|
4103
|
+
}
|
|
4104
|
+
static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
|
|
4105
|
+
fleetErrorNotices = new Map();
|
|
3447
4106
|
notifyInstanceTopic(instanceName, text, extraOpts) {
|
|
3448
4107
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3449
4108
|
if (!adapter)
|
|
@@ -3517,6 +4176,40 @@ export class FleetManager {
|
|
|
3517
4176
|
await data.respond(t("save.sent", cmd, target.name));
|
|
3518
4177
|
}
|
|
3519
4178
|
/** Whether the instance currently has at least one live cancel button. */
|
|
4179
|
+
/**
|
|
4180
|
+
* A click on a cancel button, whether or not the fleet still tracks it.
|
|
4181
|
+
*
|
|
4182
|
+
* The old rule was "act only while an entry is live", which made a click on a
|
|
4183
|
+
* button the fleet had forgotten a silent no-op — no cancel, no message, not
|
|
4184
|
+
* even a log line. That is indistinguishable from a broken button, and it is
|
|
4185
|
+
* what the "按鈕點了沒反應" reports were: the entry is briefly absent while a
|
|
4186
|
+
* button is being replaced, and a delete that fails leaves the message on
|
|
4187
|
+
* screen with no entry at all.
|
|
4188
|
+
*
|
|
4189
|
+
* So: honour the click if the instance is actually running, and say so plainly
|
|
4190
|
+
* if it is not. The stale-click path is rate-limited because the original
|
|
4191
|
+
* concern was real — a second click must not fire a second interrupt key at an
|
|
4192
|
+
* instance that has already started a new turn.
|
|
4193
|
+
*/
|
|
4194
|
+
handleCancelClick(instanceName, adapter, data) {
|
|
4195
|
+
if (this.hasCancelButton(instanceName)) {
|
|
4196
|
+
this.cancelInstance(instanceName);
|
|
4197
|
+
return;
|
|
4198
|
+
}
|
|
4199
|
+
const lastAt = this.staleCancelClickAt.get(instanceName) ?? 0;
|
|
4200
|
+
if (Date.now() - lastAt < STALE_CANCEL_CLICK_COOLDOWN_MS)
|
|
4201
|
+
return;
|
|
4202
|
+
this.staleCancelClickAt.set(instanceName, Date.now());
|
|
4203
|
+
// cancelInstance returns false when there is no daemon — i.e. nothing to
|
|
4204
|
+
// cancel, which is the one case where the button really is dead.
|
|
4205
|
+
if (this.cancelInstance(instanceName)) {
|
|
4206
|
+
this.logger.info({ instanceName }, "Cancel click honoured with no live button entry");
|
|
4207
|
+
return;
|
|
4208
|
+
}
|
|
4209
|
+
this.logger.info({ instanceName }, "Cancel click on an expired button — instance not running");
|
|
4210
|
+
adapter?.editMessage(data.chatId, data.messageId, t("cancel.button_stale", instanceName), data.threadId)
|
|
4211
|
+
.catch(() => { });
|
|
4212
|
+
}
|
|
3520
4213
|
hasCancelButton(instanceName) {
|
|
3521
4214
|
for (const e of this.cancelButtons.values()) {
|
|
3522
4215
|
if (e.instanceName === instanceName)
|
|
@@ -3525,15 +4218,22 @@ export class FleetManager {
|
|
|
3525
4218
|
return false;
|
|
3526
4219
|
}
|
|
3527
4220
|
async sendCancelButton(instanceName, correlationId) {
|
|
3528
|
-
//
|
|
3529
|
-
//
|
|
3530
|
-
//
|
|
3531
|
-
|
|
4221
|
+
// Post first, retire after (see the tail of this method). Retiring up front
|
|
4222
|
+
// meant that from the delete until the new message came back — a chat API
|
|
4223
|
+
// round trip, and every reply goes through here — the instance had NO live
|
|
4224
|
+
// entry, while the old button was still on screen. A click in that window
|
|
4225
|
+
// hit `hasCancelButton() === false` and was silently dropped: the reported
|
|
4226
|
+
// "按鈕失效". If notifyAlert then failed, the button was simply gone.
|
|
3532
4227
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3533
4228
|
if (!adapter)
|
|
3534
4229
|
return;
|
|
4230
|
+
// Resolve the group through the world fallback (first world when unbound),
|
|
4231
|
+
// NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
|
|
4232
|
+
// `channels:` worlds the primary `channel:` block is empty, so an instance
|
|
4233
|
+
// with no world binding yet (fresh restart, cross-instance delegation)
|
|
4234
|
+
// resolved group_id to undefined and the button silently never appeared.
|
|
3535
4235
|
const adapterId = this.instanceWorldBinding.get(instanceName);
|
|
3536
|
-
const groupId = this.
|
|
4236
|
+
const groupId = this.getGroupIdForInstance(instanceName) || undefined;
|
|
3537
4237
|
const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3538
4238
|
let chatId;
|
|
3539
4239
|
let threadId;
|
|
@@ -3549,8 +4249,12 @@ export class FleetManager {
|
|
|
3549
4249
|
if (!chatId && groupId)
|
|
3550
4250
|
chatId = String(groupId);
|
|
3551
4251
|
}
|
|
3552
|
-
if (!chatId)
|
|
4252
|
+
if (!chatId) {
|
|
4253
|
+
// A button that cannot be addressed must say so — this exact silence is how
|
|
4254
|
+
// "the cancel button sometimes never appears" stayed unreported-in-logs.
|
|
4255
|
+
this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
|
|
3553
4256
|
return;
|
|
4257
|
+
}
|
|
3554
4258
|
try {
|
|
3555
4259
|
const sent = await adapter.notifyAlert(chatId, {
|
|
3556
4260
|
type: "cancel",
|
|
@@ -3558,13 +4262,6 @@ export class FleetManager {
|
|
|
3558
4262
|
message: "👀 處理中…",
|
|
3559
4263
|
choices: [{ id: `cancel:${instanceName}`, label: t("cancel.button") }],
|
|
3560
4264
|
}, threadId ? { threadId } : undefined);
|
|
3561
|
-
// A concurrent sendCancelButton for the same instance may have posted its
|
|
3562
|
-
// own button while we awaited notifyAlert. Retire any other buttons for
|
|
3563
|
-
// this instance (not the one we just posted) so only the newest shows.
|
|
3564
|
-
for (const other of this.cancelButtons.values()) {
|
|
3565
|
-
if (other.instanceName === instanceName)
|
|
3566
|
-
this.retireButton(other);
|
|
3567
|
-
}
|
|
3568
4265
|
const entry = {
|
|
3569
4266
|
instanceName,
|
|
3570
4267
|
adapterId,
|
|
@@ -3573,7 +4270,16 @@ export class FleetManager {
|
|
|
3573
4270
|
threadId: sent.threadId ?? threadId,
|
|
3574
4271
|
correlationId,
|
|
3575
4272
|
retryCount: 0,
|
|
4273
|
+
// Elapsed time is measured from when this button was posted — i.e. from
|
|
4274
|
+
// when the work was handed over — not from the pane's working transition,
|
|
4275
|
+
// which resets if the CLI blips idle mid-turn.
|
|
4276
|
+
startedAt: Date.now(),
|
|
4277
|
+
// Matches the text notifyAlert just posted, so the first 60s tick does
|
|
4278
|
+
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
4279
|
+
// with nothing visibly changed.
|
|
4280
|
+
lastProgressText: "👀 處理中…",
|
|
3576
4281
|
};
|
|
4282
|
+
this.startProgressTicker(entry);
|
|
3577
4283
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
3578
4284
|
// button. Covers turns that end without hitting a clear trigger (reply /
|
|
3579
4285
|
// cancel / correlation). Cleared in discardButton when the entry is removed.
|
|
@@ -3582,18 +4288,167 @@ export class FleetManager {
|
|
|
3582
4288
|
clearInterval(entry.idleCheckTimer);
|
|
3583
4289
|
return;
|
|
3584
4290
|
}
|
|
3585
|
-
|
|
3586
|
-
this.
|
|
4291
|
+
const reason = this.getInstanceIdle(instanceName) ? "idle"
|
|
4292
|
+
: this.stateReportDead(instanceName) ? "state reports stopped"
|
|
4293
|
+
: Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
|
|
4294
|
+
: null;
|
|
4295
|
+
if (reason) {
|
|
4296
|
+
this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
|
|
3587
4297
|
this.retireButton(entry);
|
|
4298
|
+
return;
|
|
3588
4299
|
}
|
|
4300
|
+
// Still looks busy. The daemon only broadcasts on transitions, so ask for
|
|
4301
|
+
// a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
|
|
4302
|
+
// the staleness check honest; a dead one's silence is the evidence.
|
|
4303
|
+
this.instanceIpcClients.get(instanceName)?.send({
|
|
4304
|
+
type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
|
|
4305
|
+
});
|
|
3589
4306
|
}, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
|
|
3590
4307
|
this.cancelButtons.set(sent.messageId, entry);
|
|
4308
|
+
// Only now: at most one button per instance, but never zero. Covers both
|
|
4309
|
+
// the previous turn's button and any button a concurrent
|
|
4310
|
+
// sendCancelButton posted while we were awaiting notifyAlert.
|
|
4311
|
+
for (const other of [...this.cancelButtons.values()]) {
|
|
4312
|
+
if (other.instanceName === instanceName && other.messageId !== sent.messageId) {
|
|
4313
|
+
this.retireButton(other);
|
|
4314
|
+
}
|
|
4315
|
+
}
|
|
4316
|
+
this.persistCancelButtons();
|
|
3591
4317
|
this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
|
|
3592
4318
|
}
|
|
3593
4319
|
catch (e) {
|
|
3594
4320
|
this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
|
|
3595
4321
|
}
|
|
3596
4322
|
}
|
|
4323
|
+
/**
|
|
4324
|
+
* The cancel button's text for a given elapsed time.
|
|
4325
|
+
*
|
|
4326
|
+
* Below the threshold it keeps the original wording, so a normal quick answer
|
|
4327
|
+
* looks exactly as it did before. Past it, the button doubles as the live
|
|
4328
|
+
* progress indicator (#409) — the channel showed nothing at all during long work,
|
|
4329
|
+
* and once the agent had replied once there was no sign it was still going.
|
|
4330
|
+
*/
|
|
4331
|
+
static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
|
|
4332
|
+
if (elapsedMs < minElapsedMs)
|
|
4333
|
+
return "👀 處理中…";
|
|
4334
|
+
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
4335
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
4336
|
+
const seconds = totalSeconds % 60;
|
|
4337
|
+
const elapsed = minutes >= 60
|
|
4338
|
+
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
4339
|
+
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
4340
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
4341
|
+
return detail
|
|
4342
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
4343
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
4344
|
+
}
|
|
4345
|
+
/**
|
|
4346
|
+
* Make a tool summary safe to paste into a channel message.
|
|
4347
|
+
*
|
|
4348
|
+
* The text is agent-controlled (it is built from tool inputs — file paths,
|
|
4349
|
+
* shell commands), so it gets flattened to one line, capped, and stripped of
|
|
4350
|
+
* the two Discord mass-mention triggers. Neither channel renders it with a
|
|
4351
|
+
* parse mode, so no markup escaping is needed beyond that.
|
|
4352
|
+
*/
|
|
4353
|
+
static sanitizeActivity(activity) {
|
|
4354
|
+
if (!activity)
|
|
4355
|
+
return null;
|
|
4356
|
+
const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@$1").trim();
|
|
4357
|
+
if (!flat)
|
|
4358
|
+
return null;
|
|
4359
|
+
return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
|
|
4360
|
+
? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
|
|
4361
|
+
: flat;
|
|
4362
|
+
}
|
|
4363
|
+
/**
|
|
4364
|
+
* Remember what an instance is currently doing, for the progress line.
|
|
4365
|
+
*
|
|
4366
|
+
* Best-effort by design: only backends that expose a live activity feed report
|
|
4367
|
+
* anything, and the progress line simply omits the detail for the rest. It is
|
|
4368
|
+
* never used to decide anything — purely what the user is shown.
|
|
4369
|
+
*/
|
|
4370
|
+
cacheInstanceActivity(name, activity) {
|
|
4371
|
+
if (activity)
|
|
4372
|
+
this.instanceActivity.set(name, activity);
|
|
4373
|
+
else
|
|
4374
|
+
this.instanceActivity.delete(name);
|
|
4375
|
+
}
|
|
4376
|
+
/**
|
|
4377
|
+
* Refresh the button's text in place while the instance keeps working.
|
|
4378
|
+
*
|
|
4379
|
+
* Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
|
|
4380
|
+
* and the Bot API treats that as "clear the keyboard" — so editing with it would
|
|
4381
|
+
* delete the very cancel button this is trying to keep alive.
|
|
4382
|
+
*/
|
|
4383
|
+
/** Configured threshold before elapsed time appears, in ms. */
|
|
4384
|
+
progressMinElapsedMs() {
|
|
4385
|
+
const seconds = this.fleetConfig?.defaults
|
|
4386
|
+
?.progress_min_elapsed;
|
|
4387
|
+
if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) {
|
|
4388
|
+
return seconds * 1000;
|
|
4389
|
+
}
|
|
4390
|
+
return PROGRESS_MIN_ELAPSED_MS;
|
|
4391
|
+
}
|
|
4392
|
+
startProgressTicker(entry) {
|
|
4393
|
+
const tick = () => {
|
|
4394
|
+
if (!this.cancelButtons.has(entry.messageId)) {
|
|
4395
|
+
clearInterval(entry.progressTimer);
|
|
4396
|
+
return;
|
|
4397
|
+
}
|
|
4398
|
+
const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
|
|
4399
|
+
if (text === entry.lastProgressText)
|
|
4400
|
+
return; // nothing changed — skip the API call
|
|
4401
|
+
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
4402
|
+
if (!adapter?.editAlert)
|
|
4403
|
+
return;
|
|
4404
|
+
entry.lastProgressText = text;
|
|
4405
|
+
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
4406
|
+
type: "cancel",
|
|
4407
|
+
instanceName: entry.instanceName,
|
|
4408
|
+
message: text,
|
|
4409
|
+
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
4410
|
+
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
4411
|
+
.catch(err => {
|
|
4412
|
+
// A failed progress edit must never escalate: the button still works and
|
|
4413
|
+
// the next tick retries. Common causes are a deleted message or a
|
|
4414
|
+
// rate limit.
|
|
4415
|
+
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
4416
|
+
});
|
|
4417
|
+
};
|
|
4418
|
+
entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
|
|
4419
|
+
entry.progressTimer.unref?.();
|
|
4420
|
+
// One extra tick right when the threshold passes, so a 30s threshold shows
|
|
4421
|
+
// time at ~30s instead of waiting for the first 60s interval. Costs at most
|
|
4422
|
+
// one additional edit per turn that lives past the threshold.
|
|
4423
|
+
const firstAt = this.progressMinElapsedMs() - (Date.now() - (entry.startedAt ?? Date.now()));
|
|
4424
|
+
if (firstAt > 0 && firstAt < PROGRESS_UPDATE_INTERVAL_MS) {
|
|
4425
|
+
const firstTick = setTimeout(tick, firstAt);
|
|
4426
|
+
firstTick.unref?.();
|
|
4427
|
+
}
|
|
4428
|
+
}
|
|
4429
|
+
/**
|
|
4430
|
+
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
4431
|
+
* it has not, retire its button. Re-arming replaces the previous timer, so a
|
|
4432
|
+
* burst of replies ends with exactly one pending check.
|
|
4433
|
+
*/
|
|
4434
|
+
armReplyGrace(instanceName) {
|
|
4435
|
+
for (const entry of this.cancelButtons.values()) {
|
|
4436
|
+
if (entry.instanceName !== instanceName)
|
|
4437
|
+
continue;
|
|
4438
|
+
if (entry.replyGraceTimer)
|
|
4439
|
+
clearTimeout(entry.replyGraceTimer);
|
|
4440
|
+
entry.replyGraceTimer = setTimeout(() => {
|
|
4441
|
+
entry.replyGraceTimer = undefined;
|
|
4442
|
+
if (!this.cancelButtons.has(entry.messageId))
|
|
4443
|
+
return;
|
|
4444
|
+
if (!this.getInstanceIdle(instanceName))
|
|
4445
|
+
return; // resumed — a long run keeps its button
|
|
4446
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
4447
|
+
this.retireButton(entry);
|
|
4448
|
+
}, REPLY_RETIRE_GRACE_MS);
|
|
4449
|
+
entry.replyGraceTimer.unref?.();
|
|
4450
|
+
}
|
|
4451
|
+
}
|
|
3597
4452
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
3598
4453
|
retireInstanceButtons(instanceName) {
|
|
3599
4454
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -3625,7 +4480,70 @@ export class FleetManager {
|
|
|
3625
4480
|
clearTimeout(entry.retryTimer);
|
|
3626
4481
|
if (entry.idleCheckTimer)
|
|
3627
4482
|
clearInterval(entry.idleCheckTimer);
|
|
4483
|
+
if (entry.progressTimer)
|
|
4484
|
+
clearInterval(entry.progressTimer);
|
|
4485
|
+
if (entry.replyGraceTimer)
|
|
4486
|
+
clearTimeout(entry.replyGraceTimer);
|
|
3628
4487
|
this.cancelButtons.delete(entry.messageId);
|
|
4488
|
+
this.persistCancelButtons();
|
|
4489
|
+
}
|
|
4490
|
+
/**
|
|
4491
|
+
* Mirror the live buttons to disk. The map is memory-only, so before this a
|
|
4492
|
+
* fleet restart orphaned every button on screen: frozen "處理中…" text and a
|
|
4493
|
+
* click that did nothing, forever. The ledger is tiny (a handful of rows) and
|
|
4494
|
+
* written on every add/remove — no debounce needed at that rate.
|
|
4495
|
+
*/
|
|
4496
|
+
persistCancelButtons() {
|
|
4497
|
+
try {
|
|
4498
|
+
const rows = [...this.cancelButtons.values()].map(e => ({
|
|
4499
|
+
instanceName: e.instanceName,
|
|
4500
|
+
adapterId: e.adapterId,
|
|
4501
|
+
chatId: e.chatId,
|
|
4502
|
+
messageId: e.messageId,
|
|
4503
|
+
threadId: e.threadId,
|
|
4504
|
+
}));
|
|
4505
|
+
writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
|
|
4506
|
+
}
|
|
4507
|
+
catch (err) {
|
|
4508
|
+
this.logger.debug({ err }, "Cancel button ledger write failed");
|
|
4509
|
+
}
|
|
4510
|
+
}
|
|
4511
|
+
/**
|
|
4512
|
+
* Delete the previous process's buttons. Runs once adapters are up: nothing
|
|
4513
|
+
* from a previous fleet process can still be mid-turn from this process's
|
|
4514
|
+
* point of view, so every ledger row is an orphan by definition.
|
|
4515
|
+
*/
|
|
4516
|
+
async sweepOrphanedCancelButtons() {
|
|
4517
|
+
const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
|
|
4518
|
+
let rows;
|
|
4519
|
+
try {
|
|
4520
|
+
if (!existsSync(ledgerPath))
|
|
4521
|
+
return;
|
|
4522
|
+
rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
|
|
4523
|
+
}
|
|
4524
|
+
catch {
|
|
4525
|
+
try {
|
|
4526
|
+
unlinkSync(ledgerPath);
|
|
4527
|
+
}
|
|
4528
|
+
catch { /* corrupt ledger — drop it */ }
|
|
4529
|
+
return;
|
|
4530
|
+
}
|
|
4531
|
+
for (const row of rows) {
|
|
4532
|
+
const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
|
|
4533
|
+
?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
|
|
4534
|
+
if (!adapter?.deleteMessage)
|
|
4535
|
+
continue;
|
|
4536
|
+
try {
|
|
4537
|
+
await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
|
|
4538
|
+
this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
|
|
4539
|
+
}
|
|
4540
|
+
catch (err) {
|
|
4541
|
+
// Best effort: the message may already be gone, or too old to delete.
|
|
4542
|
+
this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
|
|
4543
|
+
}
|
|
4544
|
+
}
|
|
4545
|
+
// The current process owns the ledger from here on.
|
|
4546
|
+
this.persistCancelButtons();
|
|
3629
4547
|
}
|
|
3630
4548
|
/** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
|
|
3631
4549
|
scheduleButtonRetry(entry, err) {
|
|
@@ -4290,6 +5208,12 @@ When users create specialized instances, suggest these configurations:
|
|
|
4290
5208
|
meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
|
|
4291
5209
|
}
|
|
4292
5210
|
}
|
|
5211
|
+
// Classic channels queue reactions like everyone else (#432 stored them, but
|
|
5212
|
+
// this path never attached them — reactions in a ClassicBot channel went into
|
|
5213
|
+
// the DB and were never seen again). Same contract as the topic paths:
|
|
5214
|
+
// consumed only after the delivery succeeded.
|
|
5215
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
5216
|
+
Object.assign(meta, reactions.meta);
|
|
4293
5217
|
try {
|
|
4294
5218
|
await this.deliverToInstance(instanceName, {
|
|
4295
5219
|
type: "fleet_inbound",
|
|
@@ -4297,6 +5221,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4297
5221
|
targetSession: instanceName,
|
|
4298
5222
|
meta,
|
|
4299
5223
|
});
|
|
5224
|
+
reactions.consume();
|
|
4300
5225
|
}
|
|
4301
5226
|
catch (err) {
|
|
4302
5227
|
this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
|
|
@@ -4435,6 +5360,170 @@ When users create specialized instances, suggest these configurations:
|
|
|
4435
5360
|
return env?.models ?? [];
|
|
4436
5361
|
}
|
|
4437
5362
|
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
5363
|
+
/** Label an effort choice, marking the one currently configured. */
|
|
5364
|
+
effortChoiceLabel(level, current) {
|
|
5365
|
+
return level === current ? `✓ ${level}` : level;
|
|
5366
|
+
}
|
|
5367
|
+
effortMenuHeader(instanceName) {
|
|
5368
|
+
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
5369
|
+
if (!effort)
|
|
5370
|
+
return "Current effort: (CLI default)";
|
|
5371
|
+
return source === "fleet-default"
|
|
5372
|
+
? `Current effort: ${effort} (fleet default)`
|
|
5373
|
+
: `Current effort: ${effort}`;
|
|
5374
|
+
}
|
|
5375
|
+
/** `/effort` — DC Select Menu, or apply directly when a level is given. */
|
|
5376
|
+
async handleEffortSlash(data, adapterId) {
|
|
5377
|
+
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
5378
|
+
await data.respond(t("permission.denied"));
|
|
5379
|
+
return;
|
|
5380
|
+
}
|
|
5381
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
5382
|
+
if (!name) {
|
|
5383
|
+
await data.respond(t("classic.no_agent"));
|
|
5384
|
+
return;
|
|
5385
|
+
}
|
|
5386
|
+
const requested = (typeof data.options?.level === "string" ? data.options.level.trim() : "")
|
|
5387
|
+
|| (data.text?.trim() ?? "");
|
|
5388
|
+
if (requested) {
|
|
5389
|
+
await data.respond(await this.applyEffort(name, requested));
|
|
5390
|
+
return;
|
|
5391
|
+
}
|
|
5392
|
+
const levels = this.effortLevelsFor(name);
|
|
5393
|
+
if (levels.length === 0) {
|
|
5394
|
+
await data.respond(`❌ ${this.backendNameForInstance(name)} has no reasoning-effort setting.`);
|
|
5395
|
+
return;
|
|
5396
|
+
}
|
|
5397
|
+
if (!data.respondChoices) {
|
|
5398
|
+
await data.respond(`Usage: /effort <${levels.join("|")}>`);
|
|
5399
|
+
return;
|
|
5400
|
+
}
|
|
5401
|
+
const current = this.resolveInstanceEffort(name).effort;
|
|
5402
|
+
const nonce = randomBytes(6).toString("hex");
|
|
5403
|
+
const choices = levels.map(l => ({
|
|
5404
|
+
id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
|
|
5405
|
+
label: this.effortChoiceLabel(l, current),
|
|
5406
|
+
}));
|
|
5407
|
+
const timer = setTimeout(() => this.pendingEffortSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
5408
|
+
timer.unref?.();
|
|
5409
|
+
this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
5410
|
+
try {
|
|
5411
|
+
await data.respondChoices(`${this.effortMenuHeader(name)}\nSelect a new effort level:`, choices);
|
|
5412
|
+
}
|
|
5413
|
+
catch (err) {
|
|
5414
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5415
|
+
clearTimeout(timer);
|
|
5416
|
+
this.logger.warn({ err, instanceName: name }, "effort menu failed");
|
|
5417
|
+
await data.respond(`Usage: /effort <${levels.join("|")}>`);
|
|
5418
|
+
}
|
|
5419
|
+
}
|
|
5420
|
+
/** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
|
|
5421
|
+
async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
5422
|
+
const levels = this.effortLevelsFor(instanceName);
|
|
5423
|
+
if (levels.length === 0) {
|
|
5424
|
+
return `❌ ${this.backendNameForInstance(instanceName)} has no reasoning-effort setting.`;
|
|
5425
|
+
}
|
|
5426
|
+
const current = this.resolveInstanceEffort(instanceName).effort;
|
|
5427
|
+
const nonce = randomBytes(6).toString("hex");
|
|
5428
|
+
const choices = levels.map(l => ({
|
|
5429
|
+
id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
|
|
5430
|
+
label: this.effortChoiceLabel(l, current),
|
|
5431
|
+
}));
|
|
5432
|
+
const respond = async (text) => {
|
|
5433
|
+
await adapter.sendText(chatId, text, { threadId });
|
|
5434
|
+
return undefined;
|
|
5435
|
+
};
|
|
5436
|
+
const timer = setTimeout(() => {
|
|
5437
|
+
const p = this.pendingEffortSelects.get(nonce);
|
|
5438
|
+
if (p) {
|
|
5439
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5440
|
+
p.respond("⏰ Effort selection expired.").catch(() => { });
|
|
5441
|
+
}
|
|
5442
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
5443
|
+
timer.unref?.();
|
|
5444
|
+
this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
5445
|
+
try {
|
|
5446
|
+
const menuMessageId = await adapter.promptUser(chatId, `${this.effortMenuHeader(instanceName)}\nSelect a new effort level:`, choices, { threadId });
|
|
5447
|
+
const pending = this.pendingEffortSelects.get(nonce);
|
|
5448
|
+
if (pending)
|
|
5449
|
+
pending.menuMessageId = menuMessageId;
|
|
5450
|
+
return null;
|
|
5451
|
+
}
|
|
5452
|
+
catch (err) {
|
|
5453
|
+
this.pendingEffortSelects.delete(nonce);
|
|
5454
|
+
clearTimeout(timer);
|
|
5455
|
+
this.logger.warn({ err, instanceName }, "TG effort menu failed");
|
|
5456
|
+
return `Usage: /effort <${levels.join("|")}>`;
|
|
5457
|
+
}
|
|
5458
|
+
}
|
|
5459
|
+
/** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
|
|
5460
|
+
async handleEffortSelection(data) {
|
|
5461
|
+
if (!data.callbackData.startsWith(EFFORT_SELECT_CALLBACK_PREFIX))
|
|
5462
|
+
return false;
|
|
5463
|
+
const match = data.callbackData.match(/^effort-select:([0-9a-f]+):(.+)$/);
|
|
5464
|
+
if (!match)
|
|
5465
|
+
return true;
|
|
5466
|
+
const pending = this.pendingEffortSelects.get(match[1]);
|
|
5467
|
+
if (!pending)
|
|
5468
|
+
return true;
|
|
5469
|
+
if (data.userId && data.userId !== pending.userId)
|
|
5470
|
+
return true;
|
|
5471
|
+
const cbChannel = data.threadId ?? data.chatId;
|
|
5472
|
+
if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
|
|
5473
|
+
return true;
|
|
5474
|
+
this.pendingEffortSelects.delete(match[1]);
|
|
5475
|
+
clearTimeout(pending.timer);
|
|
5476
|
+
const level = match[2];
|
|
5477
|
+
const progressText = `⏳ Setting ${pending.instanceName} effort to \`${level}\`…`;
|
|
5478
|
+
let progressMsgId;
|
|
5479
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
5480
|
+
const menuMessageId = pending.menuMessageId ?? data.messageId;
|
|
5481
|
+
if (menuMessageId && pending.adapter.editMessageRemoveButtons) {
|
|
5482
|
+
try {
|
|
5483
|
+
await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, menuMessageId, progressText, pending.adapterThreadId);
|
|
5484
|
+
progressMsgId = menuMessageId;
|
|
5485
|
+
}
|
|
5486
|
+
catch { /* fall back to a new message */ }
|
|
5487
|
+
}
|
|
5488
|
+
if (!progressMsgId) {
|
|
5489
|
+
try {
|
|
5490
|
+
const sent = await pending.adapter.sendText(pending.adapterChatId, progressText, { threadId: pending.adapterThreadId });
|
|
5491
|
+
progressMsgId = sent.messageId;
|
|
5492
|
+
}
|
|
5493
|
+
catch { /* non-fatal */ }
|
|
5494
|
+
}
|
|
5495
|
+
}
|
|
5496
|
+
else {
|
|
5497
|
+
await pending.respond(progressText).catch(() => { });
|
|
5498
|
+
}
|
|
5499
|
+
// Background-applied and guarded for the same reason as the model path: a
|
|
5500
|
+
// restart backend respawns the instance here, and an unguarded rejection
|
|
5501
|
+
// from a menu click must not take the fleet down.
|
|
5502
|
+
void (async () => {
|
|
5503
|
+
let result;
|
|
5504
|
+
try {
|
|
5505
|
+
result = await this.applyEffort(pending.instanceName, level);
|
|
5506
|
+
}
|
|
5507
|
+
catch (err) {
|
|
5508
|
+
this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
|
|
5509
|
+
result = `Effort switch to \`${level}\` failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
5510
|
+
}
|
|
5511
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
5512
|
+
if (progressMsgId) {
|
|
5513
|
+
pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
|
|
5514
|
+
pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
|
|
5515
|
+
});
|
|
5516
|
+
}
|
|
5517
|
+
else {
|
|
5518
|
+
pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
|
|
5519
|
+
}
|
|
5520
|
+
}
|
|
5521
|
+
else {
|
|
5522
|
+
await pending.respond(result).catch(() => { });
|
|
5523
|
+
}
|
|
5524
|
+
})();
|
|
5525
|
+
return true;
|
|
5526
|
+
}
|
|
4438
5527
|
async handleModelSlash(data, adapterId) {
|
|
4439
5528
|
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
4440
5529
|
await data.respond(t("permission.denied"));
|
|
@@ -4571,9 +5660,19 @@ When users create specialized instances, suggest these configurations:
|
|
|
4571
5660
|
// DC path: respond immediately with progress text
|
|
4572
5661
|
await pending.respond(progressText).catch(() => { });
|
|
4573
5662
|
}
|
|
4574
|
-
// Apply model in background — don't await here (keeps callback handler fast)
|
|
5663
|
+
// Apply model in background — don't await here (keeps callback handler fast).
|
|
5664
|
+
// Guarded: applyModel() restarts the instance, and an unguarded rejection here
|
|
5665
|
+
// meant a user picking from the /model menu could take the whole fleet down.
|
|
5666
|
+
// On failure the user gets told, rather than the click silently doing nothing.
|
|
4575
5667
|
void (async () => {
|
|
4576
|
-
|
|
5668
|
+
let result;
|
|
5669
|
+
try {
|
|
5670
|
+
result = await this.applyModel(pending.instanceName, model);
|
|
5671
|
+
}
|
|
5672
|
+
catch (err) {
|
|
5673
|
+
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
5674
|
+
result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
5675
|
+
}
|
|
4577
5676
|
if (pending.adapter && pending.adapterChatId) {
|
|
4578
5677
|
if (progressMsgId) {
|
|
4579
5678
|
pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
|
|
@@ -4591,6 +5690,110 @@ When users create specialized instances, suggest these configurations:
|
|
|
4591
5690
|
return true;
|
|
4592
5691
|
}
|
|
4593
5692
|
/** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
|
|
5693
|
+
/** AgEnD's canonical effort ladder, low → max. Backends expose a subset. */
|
|
5694
|
+
static EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"];
|
|
5695
|
+
/** How this instance's backend applies an effort change. */
|
|
5696
|
+
effortStrategyFor(instanceName) {
|
|
5697
|
+
try {
|
|
5698
|
+
const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
|
|
5699
|
+
const strategy = backend.getEffortStrategy?.() ?? "unsupported";
|
|
5700
|
+
// A backend claiming support but listing no levels is unusable either way.
|
|
5701
|
+
return strategy !== "unsupported" && (backend.getEffortLevels?.() ?? []).length > 0
|
|
5702
|
+
? strategy
|
|
5703
|
+
: "unsupported";
|
|
5704
|
+
}
|
|
5705
|
+
catch {
|
|
5706
|
+
return "unsupported";
|
|
5707
|
+
}
|
|
5708
|
+
}
|
|
5709
|
+
/** Effort levels this instance's backend actually accepts (empty = unsupported). */
|
|
5710
|
+
effortLevelsFor(instanceName) {
|
|
5711
|
+
try {
|
|
5712
|
+
const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
|
|
5713
|
+
if ((backend.getEffortStrategy?.() ?? "unsupported") === "unsupported")
|
|
5714
|
+
return [];
|
|
5715
|
+
return backend.getEffortLevels?.() ?? [];
|
|
5716
|
+
}
|
|
5717
|
+
catch {
|
|
5718
|
+
return [];
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
/** Configured effort for an instance: per-instance, else fleet default, else none. */
|
|
5722
|
+
resolveInstanceEffort(instanceName) {
|
|
5723
|
+
const own = this.fleetConfig?.instances[instanceName]?.effort;
|
|
5724
|
+
if (own)
|
|
5725
|
+
return { effort: own, source: "instance" };
|
|
5726
|
+
const fallback = this.fleetConfig?.defaults?.effort;
|
|
5727
|
+
if (fallback)
|
|
5728
|
+
return { effort: fallback, source: "fleet-default" };
|
|
5729
|
+
return { effort: null, source: "unset" };
|
|
5730
|
+
}
|
|
5731
|
+
/**
|
|
5732
|
+
* Clamp a canonical level to the nearest one this backend supports.
|
|
5733
|
+
*
|
|
5734
|
+
* Clamping DOWN the ladder, never up: asking for `max` on a CLI that stops at
|
|
5735
|
+
* `high` should get high, not silently fall to low. The caller reports the
|
|
5736
|
+
* clamp — a user who asks for max and quietly receives high has been told the
|
|
5737
|
+
* request succeeded when it did not.
|
|
5738
|
+
*/
|
|
5739
|
+
static clampEffort(level, supported) {
|
|
5740
|
+
if (supported.includes(level))
|
|
5741
|
+
return level;
|
|
5742
|
+
const ladder = FleetManager.EFFORT_LEVELS;
|
|
5743
|
+
const wanted = ladder.indexOf(level);
|
|
5744
|
+
if (wanted < 0)
|
|
5745
|
+
return null;
|
|
5746
|
+
for (let i = wanted - 1; i >= 0; i--) {
|
|
5747
|
+
if (supported.includes(ladder[i]))
|
|
5748
|
+
return ladder[i];
|
|
5749
|
+
}
|
|
5750
|
+
return supported[0] ?? null;
|
|
5751
|
+
}
|
|
5752
|
+
/**
|
|
5753
|
+
* Apply a reasoning-effort level, mirroring applyModel's shape.
|
|
5754
|
+
*
|
|
5755
|
+
* runtime backends take `/effort <level>` in the pane and keep working;
|
|
5756
|
+
* restart backends only read it at launch, so it is persisted and the
|
|
5757
|
+
* instance respawns.
|
|
5758
|
+
*/
|
|
5759
|
+
async applyEffort(instanceName, requested) {
|
|
5760
|
+
const level = requested.trim().toLowerCase();
|
|
5761
|
+
const backendName = this.backendNameForInstance(instanceName);
|
|
5762
|
+
let strategy = "unsupported";
|
|
5763
|
+
let supported = [];
|
|
5764
|
+
try {
|
|
5765
|
+
const backend = createBackend(backendName, this.getInstanceDir(instanceName));
|
|
5766
|
+
strategy = backend.getEffortStrategy?.() ?? "unsupported";
|
|
5767
|
+
supported = backend.getEffortLevels?.() ?? [];
|
|
5768
|
+
}
|
|
5769
|
+
catch { /* treated as unsupported below */ }
|
|
5770
|
+
if (strategy === "unsupported" || supported.length === 0) {
|
|
5771
|
+
return `❌ ${backendName} has no reasoning-effort setting.`;
|
|
5772
|
+
}
|
|
5773
|
+
if (!FleetManager.EFFORT_LEVELS.includes(level)) {
|
|
5774
|
+
return `❌ Unknown effort level \`${level}\`. Use: ${FleetManager.EFFORT_LEVELS.join(", ")}.`;
|
|
5775
|
+
}
|
|
5776
|
+
const applied = FleetManager.clampEffort(level, supported);
|
|
5777
|
+
if (!applied)
|
|
5778
|
+
return `❌ ${backendName} accepts none of the canonical effort levels.`;
|
|
5779
|
+
const warn = applied === level
|
|
5780
|
+
? ""
|
|
5781
|
+
: `⚠️ Clamped to \`${applied}\` (\`${level}\` not supported by ${backendName}).\n`;
|
|
5782
|
+
// Persist either way: a runtime switch must survive the next respawn too,
|
|
5783
|
+
// or the instance silently reverts on restart.
|
|
5784
|
+
if (this.fleetConfig?.instances[instanceName]) {
|
|
5785
|
+
this.fleetConfig.instances[instanceName].effort = applied;
|
|
5786
|
+
this.saveFleetConfig();
|
|
5787
|
+
}
|
|
5788
|
+
if (strategy === "runtime") {
|
|
5789
|
+
if (!this.instanceIpcClients.get(instanceName))
|
|
5790
|
+
return `${warn}❌ ${instanceName} is not running.`;
|
|
5791
|
+
this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
|
|
5792
|
+
return `${warn}✅ Set ${instanceName} effort to \`${applied}\` (runtime).`;
|
|
5793
|
+
}
|
|
5794
|
+
await this.restartSingleInstance(instanceName);
|
|
5795
|
+
return `${warn}✅ Set ${instanceName} effort to \`${applied}\` and restarted.`;
|
|
5796
|
+
}
|
|
4594
5797
|
async applyModel(instanceName, model) {
|
|
4595
5798
|
const backendName = this.backendNameForInstance(instanceName);
|
|
4596
5799
|
let strategy = "restart";
|
|
@@ -4603,7 +5806,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4603
5806
|
if (!this.instanceIpcClients.get(instanceName))
|
|
4604
5807
|
return `${warn}❌ ${instanceName} is not running.`;
|
|
4605
5808
|
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
4606
|
-
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime)
|
|
5809
|
+
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).${this.effortSuffix(instanceName)}`;
|
|
4607
5810
|
}
|
|
4608
5811
|
// restart: persist the model so the respawned CLI launches with it.
|
|
4609
5812
|
let persisted = false;
|
|
@@ -4618,7 +5821,24 @@ When users create specialized instances, suggest these configurations:
|
|
|
4618
5821
|
if (!persisted)
|
|
4619
5822
|
return `${warn}❌ Could not set model for ${instanceName}.`;
|
|
4620
5823
|
await this.restartSingleInstance(instanceName);
|
|
4621
|
-
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted
|
|
5824
|
+
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.${this.effortSuffix(instanceName)}`;
|
|
5825
|
+
}
|
|
5826
|
+
/**
|
|
5827
|
+
* The trailing "Current effort: …" line for a /model reply.
|
|
5828
|
+
*
|
|
5829
|
+
* Model and effort interact (a cheaper model at max effort is a different
|
|
5830
|
+
* trade than a bigger one at low), so showing the effort in force right after
|
|
5831
|
+
* a switch saves the round trip of asking. Empty when the backend has none.
|
|
5832
|
+
*/
|
|
5833
|
+
effortSuffix(instanceName) {
|
|
5834
|
+
if (this.effortLevelsFor(instanceName).length === 0)
|
|
5835
|
+
return "";
|
|
5836
|
+
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
5837
|
+
if (!effort)
|
|
5838
|
+
return "\nCurrent effort: (CLI default)";
|
|
5839
|
+
return source === "fleet-default"
|
|
5840
|
+
? `\nCurrent effort: ${effort} (fleet default)`
|
|
5841
|
+
: `\nCurrent effort: ${effort}`;
|
|
4622
5842
|
}
|
|
4623
5843
|
/** Read recent chat log for agent context */
|
|
4624
5844
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
@@ -4811,7 +6031,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4811
6031
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
4812
6032
|
};
|
|
4813
6033
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
4814
|
-
await this.startInstance(instanceName, config, topicMode);
|
|
6034
|
+
await this.startInstance(instanceName, config, topicMode, "classic");
|
|
4815
6035
|
}
|
|
4816
6036
|
/** Handle /start slash command — register classic channel */
|
|
4817
6037
|
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
@@ -4853,11 +6073,28 @@ When users create specialized instances, suggest these configurations:
|
|
|
4853
6073
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
4854
6074
|
return t("classic.stopped");
|
|
4855
6075
|
}
|
|
4856
|
-
|
|
6076
|
+
/**
|
|
6077
|
+
* Idempotent while in flight: SIGINT and SIGTERM share one handler and the
|
|
6078
|
+
* uncaughtException path calls this too, so overlapping runs were possible —
|
|
6079
|
+
* each snapshotting the daemon map and calling stop() on the same daemons
|
|
6080
|
+
* concurrently. Deliberately NOT `async`, so callers receive the same promise
|
|
6081
|
+
* object rather than a fresh wrapper around it. The latch clears when the run
|
|
6082
|
+
* settles, so a later genuine stop (after a restart) still does the work.
|
|
6083
|
+
*/
|
|
6084
|
+
stopAll() {
|
|
6085
|
+
this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
|
|
6086
|
+
return this.stopAllInFlight;
|
|
6087
|
+
}
|
|
6088
|
+
stopAllInFlight = null;
|
|
6089
|
+
async doStopAll() {
|
|
4857
6090
|
this.startupComplete = false;
|
|
4858
6091
|
this.reloadPending = false;
|
|
6092
|
+
// Before anything is stopped: everything that dies from here on dies
|
|
6093
|
+
// because we asked it to. Set synchronously — doStopAll runs to its first
|
|
6094
|
+
// await in the same tick as the signal handler, so no event can slip in.
|
|
6095
|
+
this.shuttingDown = true;
|
|
4859
6096
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
4860
|
-
|
|
6097
|
+
sdNotifyBlocking("STOPPING=1");
|
|
4861
6098
|
if (this.watchdogTimer) {
|
|
4862
6099
|
clearInterval(this.watchdogTimer);
|
|
4863
6100
|
this.watchdogTimer = null;
|
|
@@ -4877,6 +6114,29 @@ When users create specialized instances, suggest these configurations:
|
|
|
4877
6114
|
clearInterval(this.updateCheckTimer);
|
|
4878
6115
|
this.updateCheckTimer = null;
|
|
4879
6116
|
}
|
|
6117
|
+
if (this.eventLogPruneTimer) {
|
|
6118
|
+
clearInterval(this.eventLogPruneTimer);
|
|
6119
|
+
this.eventLogPruneTimer = null;
|
|
6120
|
+
}
|
|
6121
|
+
if (this.logRotateTimer) {
|
|
6122
|
+
clearInterval(this.logRotateTimer);
|
|
6123
|
+
this.logRotateTimer = null;
|
|
6124
|
+
}
|
|
6125
|
+
// Cancel-button timers were never cleared here. The idle-check interval is not
|
|
6126
|
+
// unref'd, so it held the event loop open past shutdown and kept retrying
|
|
6127
|
+
// deletes against an adapter that was already gone.
|
|
6128
|
+
for (const entry of [...this.cancelButtons.values()]) {
|
|
6129
|
+
if (entry.retryTimer)
|
|
6130
|
+
clearTimeout(entry.retryTimer);
|
|
6131
|
+
if (entry.idleCheckTimer)
|
|
6132
|
+
clearInterval(entry.idleCheckTimer);
|
|
6133
|
+
if (entry.progressTimer)
|
|
6134
|
+
clearInterval(entry.progressTimer);
|
|
6135
|
+
}
|
|
6136
|
+
this.cancelButtons.clear();
|
|
6137
|
+
for (const timer of this.cancelButtonIdleRetireTimers.values())
|
|
6138
|
+
clearTimeout(timer);
|
|
6139
|
+
this.cancelButtonIdleRetireTimers.clear();
|
|
4880
6140
|
if (this.topicCleanupTimer) {
|
|
4881
6141
|
clearInterval(this.topicCleanupTimer);
|
|
4882
6142
|
this.topicCleanupTimer = null;
|
|
@@ -4944,6 +6204,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
4944
6204
|
catch (e) {
|
|
4945
6205
|
this.logger.debug({ err: e }, "Failed to remove fleet PID file");
|
|
4946
6206
|
}
|
|
6207
|
+
// The lock contains a nonce, so an older/shutting-down process can never
|
|
6208
|
+
// remove a lock acquired by a newer fleet owner.
|
|
6209
|
+
releaseProcessFleetLock();
|
|
4947
6210
|
}
|
|
4948
6211
|
/**
|
|
4949
6212
|
* Prune stale external sessions by re-querying each daemon for live sessions.
|
|
@@ -5095,6 +6358,15 @@ When users create specialized instances, suggest these configurations:
|
|
|
5095
6358
|
removedRatio,
|
|
5096
6359
|
validationErrors: validation.errors,
|
|
5097
6360
|
}, "Refusing unsafe fleet config reload; running configuration was kept");
|
|
6361
|
+
// Tell the operator. A silently ignored config edit is the most confusing
|
|
6362
|
+
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
6363
|
+
// with no explanation anywhere they are looking.
|
|
6364
|
+
const why = !validation.valid
|
|
6365
|
+
? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
|
|
6366
|
+
: unsafeEmpty
|
|
6367
|
+
? `it removed every instance (${oldCount} → 0)`
|
|
6368
|
+
: `it removed more than half the instances (${oldCount} → ${newCount})`;
|
|
6369
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
|
|
5098
6370
|
return;
|
|
5099
6371
|
}
|
|
5100
6372
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -5294,10 +6566,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
5294
6566
|
// ── Update check ────────────────────────────────────────────────────
|
|
5295
6567
|
async checkForUpdates() {
|
|
5296
6568
|
try {
|
|
5297
|
-
|
|
6569
|
+
// Both npm lookups are async: as execSync they froze the fleet event loop for
|
|
6570
|
+
// up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
|
|
6571
|
+
// past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
|
|
6572
|
+
// for a background version check.
|
|
6573
|
+
const { execFile } = await import("node:child_process");
|
|
6574
|
+
const { promisify } = await import("node:util");
|
|
6575
|
+
const execFileP = promisify(execFile);
|
|
6576
|
+
const npmVersion = async (spec) => {
|
|
6577
|
+
const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
|
|
6578
|
+
return stdout.toString().trim();
|
|
6579
|
+
};
|
|
5298
6580
|
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
5299
6581
|
const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
|
|
5300
|
-
const latest =
|
|
6582
|
+
const latest = await npmVersion("@songsid/agend");
|
|
5301
6583
|
let target = latest;
|
|
5302
6584
|
if (currentVersion.includes("-beta")) {
|
|
5303
6585
|
// Beta users track the @beta channel (never fall back to @latest, which is
|
|
@@ -5305,7 +6587,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5305
6587
|
// of beta/latest is the newest.
|
|
5306
6588
|
let beta = "";
|
|
5307
6589
|
try {
|
|
5308
|
-
beta =
|
|
6590
|
+
beta = await npmVersion("@songsid/agend@beta");
|
|
5309
6591
|
}
|
|
5310
6592
|
catch { /* no beta tag */ }
|
|
5311
6593
|
target = beta || latest;
|
|
@@ -5412,6 +6694,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5412
6694
|
// /view routes accept the read-only view.token (or web.token) and do
|
|
5413
6695
|
// their own per-method auth in view-api.ts — skip the web-token gate.
|
|
5414
6696
|
}
|
|
6697
|
+
else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
|
|
6698
|
+
// /api/ai-usage is read-only GET data for the /view Usage panel — open
|
|
6699
|
+
// like the other /view data routes (usage-api.ts rejects non-GET).
|
|
6700
|
+
}
|
|
5415
6701
|
else {
|
|
5416
6702
|
// All other endpoints require a valid token (query ?token= or X-Agend-Token header).
|
|
5417
6703
|
// /ui/* will also re-check in web-api.ts, which is harmless.
|
|
@@ -5426,34 +6712,34 @@ When users create specialized instances, suggest these configurations:
|
|
|
5426
6712
|
}
|
|
5427
6713
|
}
|
|
5428
6714
|
if (req.method === "GET" && req.url === "/health") {
|
|
5429
|
-
const
|
|
5430
|
-
|
|
5431
|
-
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5435
|
-
|
|
5436
|
-
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
5437
|
-
}));
|
|
6715
|
+
const health = this.getFleetHealth();
|
|
6716
|
+
// 503 when the fleet cannot do its job, so an external monitor sees it.
|
|
6717
|
+
// This used to always answer 200 "ok" with a count of CONFIGURED instances,
|
|
6718
|
+
// so every agent could be dead and every adapter down and it still looked
|
|
6719
|
+
// green.
|
|
6720
|
+
res.writeHead(health.status === "ok" ? 200 : 503);
|
|
6721
|
+
res.end(JSON.stringify(health));
|
|
5438
6722
|
return;
|
|
5439
6723
|
}
|
|
5440
6724
|
if (req.method === "GET" && req.url === "/status") {
|
|
5441
6725
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
|
|
5442
6726
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5443
|
-
let context_pct = 0;
|
|
5444
6727
|
let cost = 0;
|
|
5445
6728
|
try {
|
|
5446
6729
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5447
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5448
6730
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5449
6731
|
}
|
|
5450
6732
|
catch (err) {
|
|
5451
6733
|
this.logger.debug({ err, name }, "statusline.json read failed (/status)");
|
|
5452
6734
|
}
|
|
6735
|
+
const backend = this.fleetConfig?.instances[name]?.backend
|
|
6736
|
+
?? this.fleetConfig?.defaults?.backend
|
|
6737
|
+
?? "claude-code";
|
|
6738
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
5453
6739
|
return {
|
|
5454
6740
|
name,
|
|
5455
6741
|
status: this.getInstanceStatus(name),
|
|
5456
|
-
context_pct,
|
|
6742
|
+
context_pct: context ?? 0,
|
|
5457
6743
|
cost,
|
|
5458
6744
|
};
|
|
5459
6745
|
});
|
|
@@ -5568,7 +6854,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5568
6854
|
res.writeHead(500);
|
|
5569
6855
|
res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
|
|
5570
6856
|
}
|
|
5571
|
-
|
|
6857
|
+
// The inner catch can itself throw (writeHead after a successful
|
|
6858
|
+
// writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
|
|
6859
|
+
// IIFE. Same for the two handlers below.
|
|
6860
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
|
|
5572
6861
|
return;
|
|
5573
6862
|
}
|
|
5574
6863
|
// Instance restart (immediate, no idle wait)
|
|
@@ -5589,7 +6878,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5589
6878
|
res.writeHead(status);
|
|
5590
6879
|
res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
|
|
5591
6880
|
}
|
|
5592
|
-
})();
|
|
6881
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
|
|
5593
6882
|
return;
|
|
5594
6883
|
}
|
|
5595
6884
|
if (req.method === "POST" && req.url?.startsWith("/stop/")) {
|
|
@@ -5612,7 +6901,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5612
6901
|
res.writeHead(500);
|
|
5613
6902
|
res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
|
|
5614
6903
|
}
|
|
5615
|
-
})();
|
|
6904
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
|
|
5616
6905
|
return;
|
|
5617
6906
|
}
|
|
5618
6907
|
// ── Agent CLI endpoint ─────
|
|
@@ -5624,6 +6913,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
5624
6913
|
const url = new URL(req.url ?? "/", `http://localhost:${port}`);
|
|
5625
6914
|
if (handleViewRequest(req, res, url, this))
|
|
5626
6915
|
return;
|
|
6916
|
+
if (handleUsageRequest(req, res, url, this))
|
|
6917
|
+
return;
|
|
5627
6918
|
if (handleSettingsRequest(req, res, url, this))
|
|
5628
6919
|
return;
|
|
5629
6920
|
if (handleWebRequest(req, res, url, this))
|
|
@@ -5670,20 +6961,35 @@ When users create specialized instances, suggest these configurations:
|
|
|
5670
6961
|
this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
|
|
5671
6962
|
}
|
|
5672
6963
|
getUiStatus() {
|
|
5673
|
-
const
|
|
6964
|
+
const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
|
|
6965
|
+
// Classic rooms live only in classicBot.yaml — /api/profiles merges them into
|
|
6966
|
+
// the View roster, but previously getUiStatus skipped them so context_pct was
|
|
6967
|
+
// always 0 (live map miss → l?.context_pct ?? 0).
|
|
6968
|
+
const classicOnly = (this.classicChannels?.getAll() ?? [])
|
|
6969
|
+
.map(ch => ch.instanceName)
|
|
6970
|
+
.filter(name => !fleetNames.includes(name));
|
|
6971
|
+
const names = [...fleetNames, ...classicOnly];
|
|
6972
|
+
const instances = names.map(name => {
|
|
5674
6973
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5675
|
-
let context_pct = 0;
|
|
5676
6974
|
let cost = 0;
|
|
5677
6975
|
let model = "";
|
|
5678
6976
|
try {
|
|
5679
6977
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5680
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5681
6978
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5682
6979
|
model = data.model?.display_name ?? "";
|
|
5683
6980
|
}
|
|
5684
6981
|
catch (err) {
|
|
5685
6982
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
5686
6983
|
}
|
|
6984
|
+
// Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
|
|
6985
|
+
const classic = classicOnly.includes(name);
|
|
6986
|
+
const backend = classic
|
|
6987
|
+
? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
|
|
6988
|
+
: (this.fleetConfig?.instances[name]?.backend
|
|
6989
|
+
?? this.fleetConfig?.defaults?.backend
|
|
6990
|
+
?? "claude-code");
|
|
6991
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
6992
|
+
const context_pct = context ?? 0;
|
|
5687
6993
|
return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
|
|
5688
6994
|
});
|
|
5689
6995
|
return {
|