@songsid/agend 2.1.2-beta.2 → 2.1.2-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +17 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +18 -0
- package/dist/backend/antigravity.js +32 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +37 -0
- package/dist/backend/claude-code.js +43 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +1 -0
- package/dist/backend/codex.js +25 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +23 -0
- package/dist/backend/grok.js +45 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +20 -0
- package/dist/backend/kiro.js +77 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +34 -0
- package/dist/backend/types.js +1 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +91 -2
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +82 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +17 -1
- package/dist/channel/ipc-bridge.js +53 -17
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +30 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +28 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +132 -52
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +9 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +219 -13
- package/dist/daemon.js +919 -303
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-manager.d.ts +238 -2
- package/dist/fleet-manager.js +1008 -90
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +13 -1
- package/dist/instance-lifecycle.js +84 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +5 -0
- package/dist/instructions.js +9 -11
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +2 -1
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -0
- package/dist/outbound-handlers.js +121 -27
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +8 -3
- package/dist/outbound-schemas.js +13 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +40 -11
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +50 -10
- package/dist/topic-commands.js +242 -106
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +25 -0
- package/dist/tui-glyphs.js +26 -0
- package/dist/tui-glyphs.js.map +1 -0
- package/dist/types.d.ts +6 -0
- package/dist/ui/view.html +123 -2
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +108 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/providers.d.ts +97 -0
- package/dist/usage/providers.js +834 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +111 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.d.ts +3 -0
- package/dist/view-api.js +11 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -4,7 +4,9 @@ import { createServer } from "node:http";
|
|
|
4
4
|
import { join, dirname, basename } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
|
-
import { sdNotify } from "./sd-notify.js";
|
|
7
|
+
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
8
|
+
import { readFleetMemory } from "./process-memory.js";
|
|
9
|
+
import { ReplyDeduper } from "./reply-dedup.js";
|
|
8
10
|
import { isScalar, parseDocument } from "yaml";
|
|
9
11
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
12
|
const __dirname = dirname(__filename);
|
|
@@ -21,12 +23,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
|
|
|
21
23
|
import { createAdapter } from "./channel/factory.js";
|
|
22
24
|
import { createBackend } from "./backend/factory.js";
|
|
23
25
|
import { isModelCompatible } from "./backend/types.js";
|
|
24
|
-
import { createLogger } from "./logger.js";
|
|
26
|
+
import { createLogger, rotateLogIfNeeded } from "./logger.js";
|
|
25
27
|
import { processAttachments } from "./channel/attachment-handler.js";
|
|
26
28
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
27
29
|
import { Scheduler } from "./scheduler/index.js";
|
|
28
30
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
29
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
31
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
30
32
|
import { DailySummary } from "./daily-summary.js";
|
|
31
33
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
32
34
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -38,6 +40,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
38
40
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
39
41
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
40
42
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
43
|
+
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
41
44
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
42
45
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
43
46
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
@@ -83,6 +86,63 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
83
86
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
84
87
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
85
88
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
89
|
+
/**
|
|
90
|
+
* How long after a reply an instance gets to resume working before its cancel
|
|
91
|
+
* button is retired. A short turn ends with a reply and never works again → the
|
|
92
|
+
* button disappears ~2 minutes after the answer. A multi-step run replies
|
|
93
|
+
* mid-flight and keeps going → the grace check sees "working" and leaves the
|
|
94
|
+
* button alone (the idle edge retires it when the run really ends).
|
|
95
|
+
*/
|
|
96
|
+
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
97
|
+
/**
|
|
98
|
+
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
99
|
+
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
100
|
+
* query each tick; a live daemon answers within milliseconds and refreshes the
|
|
101
|
+
* cache. When nothing has refreshed it for this long despite those pokes, the
|
|
102
|
+
* reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
|
|
103
|
+
* from 30 minutes ago proves nothing — the button may be retired.
|
|
104
|
+
*/
|
|
105
|
+
const STATE_REPORT_STALE_MS = 30 * 60_000;
|
|
106
|
+
/**
|
|
107
|
+
* Unconditional ceiling on a cancel button's life. Deliberately far beyond any
|
|
108
|
+
* legitimate run (multi-hour tasks are normal on this fleet): everything below
|
|
109
|
+
* this is decided by real state; a button that somehow survives a full day is
|
|
110
|
+
* wreckage, stuck or not.
|
|
111
|
+
*/
|
|
112
|
+
const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
|
|
113
|
+
/** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
|
|
114
|
+
const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
|
|
115
|
+
/**
|
|
116
|
+
* How often the cancel button's text is refreshed with elapsed working time.
|
|
117
|
+
*
|
|
118
|
+
* One edit per working instance per interval — at 60s that is trivial for both
|
|
119
|
+
* platforms' rate limits, and it reads as a live counter rather than a stale
|
|
120
|
+
* snapshot. Nothing new is posted, so the channel is never spammed: there is
|
|
121
|
+
* exactly one progress message per turn, and it is the cancel button itself.
|
|
122
|
+
*/
|
|
123
|
+
const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
|
|
124
|
+
/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
|
|
125
|
+
const PROGRESS_MIN_ELAPSED_MS = 2 * 60_000;
|
|
126
|
+
/** How much of a tool summary the progress line will show before eliding. */
|
|
127
|
+
const PROGRESS_ACTIVITY_MAX_CHARS = 48;
|
|
128
|
+
/**
|
|
129
|
+
* Emoji AgEnD itself stamps on messages as machine signals: the delivery-status
|
|
130
|
+
* ladder (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed) and the 🫡 ack. They
|
|
131
|
+
* are indicators, not opinions, so they never belong in the reactions queue —
|
|
132
|
+
* regardless of who they arrive from, because the adapter-level bot filter can
|
|
133
|
+
* miss (Discord partial users may not carry the `bot` flag). The cost is that a
|
|
134
|
+
* human genuinely reacting ✅ is also dropped; with four bot-stamped meanings
|
|
135
|
+
* already attached to these glyphs, that reading is ambiguous anyway.
|
|
136
|
+
*/
|
|
137
|
+
const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌", "🫡"]);
|
|
138
|
+
/**
|
|
139
|
+
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
140
|
+
*
|
|
141
|
+
* Sized for a daemon restart (socket close → respawn → CLI ready), which is the
|
|
142
|
+
* event this exists for. Past it the delivery fails loudly as it always did.
|
|
143
|
+
*/
|
|
144
|
+
const IPC_RECONNECT_GRACE_MS = 30_000;
|
|
145
|
+
const IPC_RECONNECT_POLL_MS = 250;
|
|
86
146
|
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
87
147
|
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
88
148
|
const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
|
|
@@ -93,7 +153,7 @@ export class FleetManager {
|
|
|
93
153
|
static sighupHandlerInstalled = false;
|
|
94
154
|
children = new Map();
|
|
95
155
|
lifecycle;
|
|
96
|
-
/**
|
|
156
|
+
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
97
157
|
get daemons() { return this.lifecycle.daemons; }
|
|
98
158
|
fleetConfig = null;
|
|
99
159
|
rawFleetConfig = {};
|
|
@@ -151,6 +211,12 @@ export class FleetManager {
|
|
|
151
211
|
// reply, on cancel, or when a newer button supersedes it for the same
|
|
152
212
|
// instance. Per-button tracking means a failed delete never strands a button.
|
|
153
213
|
cancelButtons = new Map();
|
|
214
|
+
/** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
|
|
215
|
+
replyDeduper = new ReplyDeduper();
|
|
216
|
+
/** instanceName → what it is doing right now, when the backend can tell us. */
|
|
217
|
+
instanceActivity = new Map();
|
|
218
|
+
/** instanceName → tail of deliveries waiting for its IPC to come back. */
|
|
219
|
+
ipcWaitTails = new Map();
|
|
154
220
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
155
221
|
lastInboundMsg = new Map();
|
|
156
222
|
topicArchiver;
|
|
@@ -163,6 +229,10 @@ export class FleetManager {
|
|
|
163
229
|
failoverActive = new Map(); // instance → current failover model
|
|
164
230
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
165
231
|
ipcStoppingInstances = new Set();
|
|
232
|
+
/** Coalesce concurrent connection attempts for the same daemon socket. */
|
|
233
|
+
ipcConnectInFlight = new Map();
|
|
234
|
+
/** At most one reconnect/backoff loop may exist per instance. */
|
|
235
|
+
ipcReconnectInFlight = new Map();
|
|
166
236
|
// Adapter restart: prevents re-entrant restart attempts
|
|
167
237
|
adapterRestarting = new Set();
|
|
168
238
|
// Adapter isolation: track state per adapter for retry + visibility
|
|
@@ -172,6 +242,10 @@ export class FleetManager {
|
|
|
172
242
|
healthServer = null;
|
|
173
243
|
healthPortRetried = false;
|
|
174
244
|
updateCheckTimer = null;
|
|
245
|
+
eventLogPruneTimer = null;
|
|
246
|
+
logRotateTimer = null;
|
|
247
|
+
/** Days of event/activity history to keep. */
|
|
248
|
+
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
175
249
|
watchdogTimer = null;
|
|
176
250
|
startedAt = 0;
|
|
177
251
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
@@ -210,7 +284,13 @@ export class FleetManager {
|
|
|
210
284
|
}
|
|
211
285
|
this.reloadPending = false;
|
|
212
286
|
this.reconcileInFlight = this.reconcileInstances()
|
|
213
|
-
.catch(err =>
|
|
287
|
+
.catch(err => {
|
|
288
|
+
// Almost always a YAML parse error. Log-only meant the user edited
|
|
289
|
+
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
290
|
+
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
291
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
292
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
|
|
293
|
+
})
|
|
214
294
|
.finally(() => {
|
|
215
295
|
this.reconcileInFlight = null;
|
|
216
296
|
if (this.reloadPending && this.startupComplete) {
|
|
@@ -222,12 +302,35 @@ export class FleetManager {
|
|
|
222
302
|
this.startupComplete = true;
|
|
223
303
|
if (this.reloadPending)
|
|
224
304
|
this.scheduleReconcile();
|
|
305
|
+
void this.sweepOrphanedCancelButtons();
|
|
225
306
|
}
|
|
226
307
|
// ── ArchiverContext bridge ────────────────────────────────────────────
|
|
227
308
|
lastActivityMs(name) {
|
|
228
309
|
return this.lastActivity.get(name) ?? 0;
|
|
229
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Is the instance between turns?
|
|
313
|
+
*
|
|
314
|
+
* Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
|
|
315
|
+
* the control client's raw 2-second output-silence heuristic. The raw heuristic
|
|
316
|
+
* reads every >2s output lull as idle — and long silent tools (a build, a test
|
|
317
|
+
* run) or an LLM pause produce those constantly mid-turn. That misreading is
|
|
318
|
+
* what retired cancel buttons in the middle of long work (the 5-minute backstop
|
|
319
|
+
* fired during a lull) and froze their progress text (ticker skipped "idle"
|
|
320
|
+
* ticks). The silence heuristic remains only as the fallback for instances
|
|
321
|
+
* whose daemon has not reported a state yet.
|
|
322
|
+
*/
|
|
230
323
|
getInstanceIdle(name) {
|
|
324
|
+
// A daemon that is not running cannot be mid-turn. This is what a stale
|
|
325
|
+
// "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
|
|
326
|
+
// report ever arrives) must not override.
|
|
327
|
+
if (this.getInstanceStatus(name) !== "running")
|
|
328
|
+
return true;
|
|
329
|
+
const state = this.getInstanceExecutionState(name);
|
|
330
|
+
if (state === "working" || state === "stuck")
|
|
331
|
+
return false;
|
|
332
|
+
if (state === "idle")
|
|
333
|
+
return true;
|
|
231
334
|
try {
|
|
232
335
|
const widFile = join(this.getInstanceDir(name), "window-id");
|
|
233
336
|
if (!existsSync(widFile))
|
|
@@ -239,6 +342,18 @@ export class FleetManager {
|
|
|
239
342
|
return true;
|
|
240
343
|
}
|
|
241
344
|
}
|
|
345
|
+
/**
|
|
346
|
+
* True when the instance claims working/stuck but nothing has refreshed that
|
|
347
|
+
* claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
|
|
348
|
+
* Measures the CACHE's age, not the button's — a healthy multi-hour run
|
|
349
|
+
* answers every query and never trips this.
|
|
350
|
+
*/
|
|
351
|
+
stateReportDead(name) {
|
|
352
|
+
const cached = this.instanceStateCache.get(name);
|
|
353
|
+
if (!cached)
|
|
354
|
+
return false; // no claim to distrust — getInstanceIdle owns this case
|
|
355
|
+
return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
|
|
356
|
+
}
|
|
242
357
|
// ── LifecycleContext bridge methods ──────────────────────────────────────
|
|
243
358
|
webhookEmit(event, name, data) {
|
|
244
359
|
this.webhookEmitter?.emit(event, name, data);
|
|
@@ -467,14 +582,21 @@ export class FleetManager {
|
|
|
467
582
|
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
468
583
|
observedAt: numberOr(msg.observedAt, now),
|
|
469
584
|
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
585
|
+
// Fleet-manager receipt time, NOT the daemon's observation time: staleness
|
|
586
|
+
// asks "is anyone still reporting", which only the receiver can date.
|
|
587
|
+
receivedAt: now,
|
|
470
588
|
});
|
|
471
589
|
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
472
590
|
check();
|
|
473
591
|
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
474
592
|
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
475
593
|
// edge into idle, not on every idle heartbeat.
|
|
476
|
-
if (state === "idle" && previous?.state !== "idle")
|
|
594
|
+
if (state === "idle" && previous?.state !== "idle") {
|
|
477
595
|
this.enforceWarmCap();
|
|
596
|
+
// The turn is genuinely over — retire the cancel/progress button now rather
|
|
597
|
+
// than waiting for the 5-minute idle backstop to notice.
|
|
598
|
+
this.retireInstanceButtons(name);
|
|
599
|
+
}
|
|
478
600
|
}
|
|
479
601
|
cacheInstanceProcessStatus(name, status) {
|
|
480
602
|
if (status === "running") {
|
|
@@ -579,12 +701,70 @@ export class FleetManager {
|
|
|
579
701
|
if (!idle) {
|
|
580
702
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
581
703
|
}
|
|
582
|
-
|
|
583
|
-
if (!ipc?.connected)
|
|
584
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
585
|
-
ipc.send(payload);
|
|
704
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
586
705
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
587
706
|
}
|
|
707
|
+
/**
|
|
708
|
+
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
709
|
+
*
|
|
710
|
+
* A daemon that is restarting — `/restart`, crash recovery, a model switch —
|
|
711
|
+
* drops its socket for a few seconds. Any message arriving in that window used
|
|
712
|
+
* to fail instantly: the caller logged a warning, put ❌ on the user's message,
|
|
713
|
+
* and the message was gone. The user had to notice the ❌ and retype it. That is
|
|
714
|
+
* the "instance 訊息不容易掉" goal failing on the most predictable event there is.
|
|
715
|
+
*
|
|
716
|
+
* The wait is bounded. If the instance is genuinely down, this still throws and
|
|
717
|
+
* the ❌ still appears — just for a real failure rather than a restart.
|
|
718
|
+
*
|
|
719
|
+
* Ordering is preserved by serialising behind any waiter already queued for this
|
|
720
|
+
* instance, *including* when the socket happens to be up: otherwise a message
|
|
721
|
+
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
722
|
+
*/
|
|
723
|
+
async sendWhenConnected(instanceName, payload) {
|
|
724
|
+
const queued = this.ipcWaitTails.get(instanceName);
|
|
725
|
+
if (!queued) {
|
|
726
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
727
|
+
if (ipc?.connected && ipc.send(payload))
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
const attempt = (queued ?? Promise.resolve())
|
|
731
|
+
.catch(() => { })
|
|
732
|
+
.then(() => this.sendAfterIpcReturns(instanceName, payload));
|
|
733
|
+
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
734
|
+
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
735
|
+
const tail = attempt.catch(() => { });
|
|
736
|
+
this.ipcWaitTails.set(instanceName, tail);
|
|
737
|
+
try {
|
|
738
|
+
await attempt;
|
|
739
|
+
}
|
|
740
|
+
finally {
|
|
741
|
+
// Only the last waiter clears the chain; while a queue is still draining the
|
|
742
|
+
// map must keep pointing at it or ordering is lost.
|
|
743
|
+
if (this.ipcWaitTails.get(instanceName) === tail) {
|
|
744
|
+
this.ipcWaitTails.delete(instanceName);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
749
|
+
async sendAfterIpcReturns(instanceName, payload) {
|
|
750
|
+
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
751
|
+
let warned = false;
|
|
752
|
+
for (;;) {
|
|
753
|
+
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
754
|
+
// so a cached reference would stay dead forever.
|
|
755
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
756
|
+
if (ipc?.connected && ipc.send(payload))
|
|
757
|
+
return;
|
|
758
|
+
if (Date.now() >= deadline) {
|
|
759
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
760
|
+
}
|
|
761
|
+
if (!warned) {
|
|
762
|
+
warned = true;
|
|
763
|
+
this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
|
|
764
|
+
}
|
|
765
|
+
await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
|
|
766
|
+
}
|
|
767
|
+
}
|
|
588
768
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
589
769
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
590
770
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
@@ -600,10 +780,7 @@ export class FleetManager {
|
|
|
600
780
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
601
781
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
602
782
|
}
|
|
603
|
-
|
|
604
|
-
if (!ipc?.connected)
|
|
605
|
-
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
606
|
-
ipc.send(payload);
|
|
783
|
+
await this.sendWhenConnected(instanceName, payload);
|
|
607
784
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
608
785
|
// working must not trust the stale idle snapshot from before the send.
|
|
609
786
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
@@ -651,7 +828,7 @@ export class FleetManager {
|
|
|
651
828
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
652
829
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
653
830
|
}
|
|
654
|
-
async startInstance(name, config, topicMode) {
|
|
831
|
+
async startInstance(name, config, topicMode, kind = "fleet-topic") {
|
|
655
832
|
if (this.lifecycle.isPaused(name)) {
|
|
656
833
|
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
657
834
|
return;
|
|
@@ -667,7 +844,11 @@ export class FleetManager {
|
|
|
667
844
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
668
845
|
}
|
|
669
846
|
this.instanceProcessStatus.delete(name);
|
|
670
|
-
await this.lifecycle.start(name, config, topicMode
|
|
847
|
+
await this.lifecycle.start(name, config, topicMode, {
|
|
848
|
+
kind,
|
|
849
|
+
backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
|
|
850
|
+
model: this.resolveInstanceModel(name).display,
|
|
851
|
+
});
|
|
671
852
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
672
853
|
await this.connectIpcToInstance(name);
|
|
673
854
|
}
|
|
@@ -845,7 +1026,6 @@ export class FleetManager {
|
|
|
845
1026
|
this.configPath = configPath;
|
|
846
1027
|
this.loadEnvFile();
|
|
847
1028
|
// Rotate fleet.log if oversized (before any logging)
|
|
848
|
-
const { rotateLogIfNeeded } = await import("./logger.js");
|
|
849
1029
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
850
1030
|
const fleet = this.loadConfig(configPath);
|
|
851
1031
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
@@ -889,7 +1069,7 @@ export class FleetManager {
|
|
|
889
1069
|
}
|
|
890
1070
|
const pidPath = join(this.dataDir, "fleet.pid");
|
|
891
1071
|
writeFileSync(pidPath, String(process.pid), "utf-8");
|
|
892
|
-
this.eventLog =
|
|
1072
|
+
this.eventLog = this.openEventLog();
|
|
893
1073
|
// Initialize classic channel manager. The primary adapter (channels[0])
|
|
894
1074
|
// migrates legacy single-bot entries and names without a suffix. Classic
|
|
895
1075
|
// routing does NOT go through the routing engine (single-key, can't hold two
|
|
@@ -978,6 +1158,10 @@ export class FleetManager {
|
|
|
978
1158
|
// Rotate fleet.log daily too (besides the startup size check above), so a
|
|
979
1159
|
// long-running fleet doesn't accumulate an unbounded log.
|
|
980
1160
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1161
|
+
// Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
|
|
1162
|
+
// running instance's own log; this sweep is the safety net for every other
|
|
1163
|
+
// kind. One implementation, so the two cannot cover different sets.
|
|
1164
|
+
this.rotateAllInstanceLogs();
|
|
981
1165
|
}, () => {
|
|
982
1166
|
const instances = Object.keys(this.fleetConfig?.instances ?? {});
|
|
983
1167
|
const costMap = new Map();
|
|
@@ -1117,9 +1301,29 @@ export class FleetManager {
|
|
|
1117
1301
|
}
|
|
1118
1302
|
}
|
|
1119
1303
|
}
|
|
1120
|
-
//
|
|
1121
|
-
|
|
1304
|
+
// The systemd watchdog answers exactly one question: is this process still
|
|
1305
|
+
// turning its event loop? Pinging from a timer proves that, and after the
|
|
1306
|
+
// blocking child-process calls were made async it is a meaningful signal —
|
|
1307
|
+
// a deadlocked or frozen fleet stops pinging and systemd restarts it.
|
|
1308
|
+
//
|
|
1309
|
+
// It deliberately does NOT gate on fleet health. "No adapter connected" or
|
|
1310
|
+
// "an instance crashed" must not kill the process: the fleet would be restarted
|
|
1311
|
+
// into the same broken state, and a user who has legitimately stopped every
|
|
1312
|
+
// instance would get a restart loop. Those conditions surface through /health
|
|
1313
|
+
// (which now returns 503) and through the General-topic notifications instead.
|
|
1122
1314
|
this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
|
|
1315
|
+
// EventLog.prune() existed but was never called, so `events` and `activity`
|
|
1316
|
+
// grew without bound for the life of the install. Prune once at startup and
|
|
1317
|
+
// daily after that; the timer is unref'd so it never holds the loop open.
|
|
1318
|
+
this.pruneEventLog();
|
|
1319
|
+
this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
|
|
1320
|
+
this.eventLogPruneTimer.unref?.();
|
|
1321
|
+
// Same shape for pipe-pane logs, and for the same reason: the only sweep that
|
|
1322
|
+
// covered them lived inside the daily-summary callback, so it did not run at
|
|
1323
|
+
// all when summaries were off.
|
|
1324
|
+
this.rotateAllInstanceLogs();
|
|
1325
|
+
this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
|
|
1326
|
+
this.logRotateTimer.unref?.();
|
|
1123
1327
|
// Phase 2: Start remaining instances with staggered concurrency
|
|
1124
1328
|
if (others.length > 0) {
|
|
1125
1329
|
await this.startInstancesWithConcurrency(others, topicMode);
|
|
@@ -1255,6 +1459,15 @@ export class FleetManager {
|
|
|
1255
1459
|
// rest of startup finishes. Replay one coalesced reload only after all
|
|
1256
1460
|
// startup-owned lifecycle work and signal handlers are in place.
|
|
1257
1461
|
this.finishStartup();
|
|
1462
|
+
// Tell systemd we are ready only now. This used to fire right after the
|
|
1463
|
+
// generals started — before adapters, classic instances, topic creation and the
|
|
1464
|
+
// health server — so `systemctl start` returned success while the fleet was
|
|
1465
|
+
// still deaf: no path existed for a user message to arrive.
|
|
1466
|
+
sdNotify("READY=1");
|
|
1467
|
+
const health = this.getFleetHealth();
|
|
1468
|
+
if (health.status !== "ok") {
|
|
1469
|
+
this.logger.warn({ health }, "Fleet started with problems — see /health");
|
|
1470
|
+
}
|
|
1258
1471
|
}
|
|
1259
1472
|
/**
|
|
1260
1473
|
* Delete inbox files older than retentionDays (by mtime). Cleans the shared
|
|
@@ -1371,6 +1584,67 @@ export class FleetManager {
|
|
|
1371
1584
|
getAdapterStates() {
|
|
1372
1585
|
return this.adapterState;
|
|
1373
1586
|
}
|
|
1587
|
+
/**
|
|
1588
|
+
* Real, checkable fleet health for `/health` and the operator.
|
|
1589
|
+
*
|
|
1590
|
+
* `status` is:
|
|
1591
|
+
* - `ok` — at least one adapter connected and every configured instance
|
|
1592
|
+
* that should be running is running
|
|
1593
|
+
* - `degraded` — reachable, but something the operator should look at (an
|
|
1594
|
+
* adapter retrying, an instance crashed or stopped)
|
|
1595
|
+
* - `down` — the fleet cannot do its job: no adapter is connected, so no
|
|
1596
|
+
* message can arrive or be answered
|
|
1597
|
+
*
|
|
1598
|
+
* Deliberately does NOT gate the systemd watchdog — see the comment at the
|
|
1599
|
+
* WATCHDOG timer for why.
|
|
1600
|
+
*/
|
|
1601
|
+
getFleetHealth() {
|
|
1602
|
+
const names = Object.keys(this.fleetConfig?.instances ?? {});
|
|
1603
|
+
const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
|
|
1604
|
+
for (const name of names) {
|
|
1605
|
+
const state = this.getInstanceStatus(name);
|
|
1606
|
+
if (state === "running")
|
|
1607
|
+
counts.running++;
|
|
1608
|
+
else if (state === "crashed")
|
|
1609
|
+
counts.crashed++;
|
|
1610
|
+
else if (state === "paused")
|
|
1611
|
+
counts.paused++;
|
|
1612
|
+
else
|
|
1613
|
+
counts.stopped++;
|
|
1614
|
+
}
|
|
1615
|
+
const states = {};
|
|
1616
|
+
let connected = 0;
|
|
1617
|
+
for (const [id, state] of this.adapterState) {
|
|
1618
|
+
states[id] = state.status;
|
|
1619
|
+
if (state.status === "connected")
|
|
1620
|
+
connected++;
|
|
1621
|
+
}
|
|
1622
|
+
const problems = [];
|
|
1623
|
+
if (this.adapterState.size > 0 && connected === 0)
|
|
1624
|
+
problems.push("no channel adapter is connected");
|
|
1625
|
+
if (counts.crashed > 0)
|
|
1626
|
+
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
1627
|
+
for (const [id, state] of this.adapterState) {
|
|
1628
|
+
if (state.status !== "connected")
|
|
1629
|
+
problems.push(`adapter ${id} is ${state.status}`);
|
|
1630
|
+
}
|
|
1631
|
+
if (!this.startupComplete)
|
|
1632
|
+
problems.push("startup has not completed");
|
|
1633
|
+
// "down" is reserved for "cannot receive or answer a message at all". A fleet
|
|
1634
|
+
// with adapters configured but none connected is exactly that.
|
|
1635
|
+
const status = this.adapterState.size > 0 && connected === 0
|
|
1636
|
+
? "down"
|
|
1637
|
+
: problems.length > 0 ? "degraded" : "ok";
|
|
1638
|
+
return {
|
|
1639
|
+
status,
|
|
1640
|
+
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
1641
|
+
instances: counts,
|
|
1642
|
+
adapters: { total: this.adapterState.size, connected, states },
|
|
1643
|
+
startupComplete: this.startupComplete,
|
|
1644
|
+
memory: readFleetMemory(),
|
|
1645
|
+
problems,
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1374
1648
|
/** Start the primary adapter (backward-compatible, sets this.adapter) */
|
|
1375
1649
|
async startSingleAdapter(fleet, channelConfig) {
|
|
1376
1650
|
const botToken = process.env[channelConfig.bot_token_env];
|
|
@@ -1397,6 +1671,9 @@ export class FleetManager {
|
|
|
1397
1671
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
1398
1672
|
await this.handleInboundMessage(msg);
|
|
1399
1673
|
}, this.logger, "adapter.message"));
|
|
1674
|
+
this.adapter.on("reaction", safeHandler(async (r) => {
|
|
1675
|
+
await this.handleInboundReaction(r);
|
|
1676
|
+
}, this.logger, "adapter.reaction"));
|
|
1400
1677
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1401
1678
|
if (await this.handleClassicBackendSelection(data))
|
|
1402
1679
|
return;
|
|
@@ -1571,19 +1848,29 @@ export class FleetManager {
|
|
|
1571
1848
|
await data.respond(t("not_authorized"));
|
|
1572
1849
|
return;
|
|
1573
1850
|
}
|
|
1851
|
+
await data.respond(await this.runBackendDoctor());
|
|
1852
|
+
}
|
|
1853
|
+
else if (data.command === "usage") {
|
|
1854
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
1855
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
1856
|
+
// channel either way.
|
|
1574
1857
|
try {
|
|
1575
|
-
const {
|
|
1576
|
-
const
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
await data.respond(clean || "No output");
|
|
1858
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
1859
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
1860
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
1861
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1580
1862
|
}
|
|
1581
1863
|
catch (err) {
|
|
1582
|
-
|
|
1583
|
-
await data.respond(output);
|
|
1864
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1584
1865
|
}
|
|
1585
1866
|
}
|
|
1586
1867
|
else if (data.command === "status") {
|
|
1868
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
1869
|
+
// instance's cost and IPC health.
|
|
1870
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
1871
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
1872
|
+
return;
|
|
1873
|
+
}
|
|
1587
1874
|
const text = await this.topicCommands.getStatusText();
|
|
1588
1875
|
await data.respond(text);
|
|
1589
1876
|
}
|
|
@@ -1692,6 +1979,9 @@ export class FleetManager {
|
|
|
1692
1979
|
adapter.on("message", safeHandler(async (msg) => {
|
|
1693
1980
|
await this.handleInboundMessage(msg);
|
|
1694
1981
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
1982
|
+
adapter.on("reaction", safeHandler(async (r) => {
|
|
1983
|
+
await this.handleInboundReaction(r);
|
|
1984
|
+
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
1695
1985
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1696
1986
|
if (await this.handleClassicBackendSelection(data))
|
|
1697
1987
|
return;
|
|
@@ -1853,19 +2143,29 @@ export class FleetManager {
|
|
|
1853
2143
|
await data.respond(t("not_authorized"));
|
|
1854
2144
|
return;
|
|
1855
2145
|
}
|
|
2146
|
+
await data.respond(await this.runBackendDoctor());
|
|
2147
|
+
}
|
|
2148
|
+
else if (data.command === "usage") {
|
|
2149
|
+
// Same permission level as /ctx (none). The reply is still ephemeral —
|
|
2150
|
+
// the adapter defers non-chat commands that way — so it never spams the
|
|
2151
|
+
// channel either way.
|
|
1856
2152
|
try {
|
|
1857
|
-
const {
|
|
1858
|
-
const
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
await data.respond(clean || "No output");
|
|
2153
|
+
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2154
|
+
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2155
|
+
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2156
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
1862
2157
|
}
|
|
1863
2158
|
catch (err) {
|
|
1864
|
-
|
|
1865
|
-
await data.respond(output);
|
|
2159
|
+
await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
|
|
1866
2160
|
}
|
|
1867
2161
|
}
|
|
1868
2162
|
else if (data.command === "status") {
|
|
2163
|
+
// Admin-gated (like the topic path): the merged table shows every
|
|
2164
|
+
// instance's cost and IPC health.
|
|
2165
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
2166
|
+
await data.respond(t("cmd.admin_required", "/status"));
|
|
2167
|
+
return;
|
|
2168
|
+
}
|
|
1869
2169
|
const text = await this.topicCommands.getStatusText();
|
|
1870
2170
|
await data.respond(text);
|
|
1871
2171
|
}
|
|
@@ -1931,19 +2231,38 @@ export class FleetManager {
|
|
|
1931
2231
|
this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
|
|
1932
2232
|
}
|
|
1933
2233
|
/** Connect IPC to a single instance with all handlers */
|
|
1934
|
-
|
|
2234
|
+
connectIpcToInstance(name) {
|
|
2235
|
+
const inFlight = this.ipcConnectInFlight.get(name);
|
|
2236
|
+
if (inFlight)
|
|
2237
|
+
return inFlight;
|
|
2238
|
+
const connection = this.connectIpcToInstanceInternal(name)
|
|
2239
|
+
.finally(() => {
|
|
2240
|
+
if (this.ipcConnectInFlight.get(name) === connection) {
|
|
2241
|
+
this.ipcConnectInFlight.delete(name);
|
|
2242
|
+
}
|
|
2243
|
+
});
|
|
2244
|
+
this.ipcConnectInFlight.set(name, connection);
|
|
2245
|
+
return connection;
|
|
2246
|
+
}
|
|
2247
|
+
async connectIpcToInstanceInternal(name) {
|
|
1935
2248
|
// Close existing client to prevent socket leak on reconnect
|
|
1936
2249
|
const existing = this.instanceIpcClients.get(name);
|
|
1937
2250
|
if (existing) {
|
|
1938
|
-
|
|
2251
|
+
// Remove application listeners before destroying the socket. Even if a
|
|
2252
|
+
// future regression creates two clients, the replaced one cannot keep
|
|
2253
|
+
// handling fleet_outbound messages as an orphan.
|
|
2254
|
+
existing.removeAllListeners();
|
|
1939
2255
|
try {
|
|
1940
|
-
existing.close();
|
|
2256
|
+
await existing.close();
|
|
1941
2257
|
}
|
|
1942
2258
|
catch (err) {
|
|
1943
2259
|
this.logger.debug({ err, name }, "IPC client close failed (likely already closed)");
|
|
1944
2260
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
2261
|
+
finally {
|
|
2262
|
+
if (this.instanceIpcClients.get(name) === existing) {
|
|
2263
|
+
this.instanceIpcClients.delete(name);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
1947
2266
|
}
|
|
1948
2267
|
const sockPath = join(this.getInstanceDir(name), "channel.sock");
|
|
1949
2268
|
if (!existsSync(sockPath))
|
|
@@ -2002,6 +2321,9 @@ export class FleetManager {
|
|
|
2002
2321
|
else if (msg.type === "instance_process_state") {
|
|
2003
2322
|
this.cacheInstanceProcessStatus(name, msg.status);
|
|
2004
2323
|
}
|
|
2324
|
+
else if (msg.type === "instance_activity") {
|
|
2325
|
+
this.cacheInstanceActivity(name, msg.activity);
|
|
2326
|
+
}
|
|
2005
2327
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2006
2328
|
this.cacheInstanceExecutionState(name, msg);
|
|
2007
2329
|
if (msg.type === "instance_state_response") {
|
|
@@ -2021,6 +2343,10 @@ export class FleetManager {
|
|
|
2021
2343
|
}
|
|
2022
2344
|
// Auto-reconnect on disconnect (unless intentionally stopping)
|
|
2023
2345
|
ipc.on("disconnect", () => {
|
|
2346
|
+
// A delayed event from a replaced/stale client must never delete the
|
|
2347
|
+
// current connection or start another reconnect loop.
|
|
2348
|
+
if (this.instanceIpcClients.get(name) !== ipc)
|
|
2349
|
+
return;
|
|
2024
2350
|
this.instanceIpcClients.delete(name);
|
|
2025
2351
|
if (this.ipcStoppingInstances.has(name))
|
|
2026
2352
|
return;
|
|
@@ -2032,7 +2358,20 @@ export class FleetManager {
|
|
|
2032
2358
|
}
|
|
2033
2359
|
}
|
|
2034
2360
|
/** Attempt IPC reconnection with exponential backoff */
|
|
2035
|
-
|
|
2361
|
+
ipcReconnect(name) {
|
|
2362
|
+
const inFlight = this.ipcReconnectInFlight.get(name);
|
|
2363
|
+
if (inFlight)
|
|
2364
|
+
return inFlight;
|
|
2365
|
+
const reconnect = this.runIpcReconnect(name)
|
|
2366
|
+
.finally(() => {
|
|
2367
|
+
if (this.ipcReconnectInFlight.get(name) === reconnect) {
|
|
2368
|
+
this.ipcReconnectInFlight.delete(name);
|
|
2369
|
+
}
|
|
2370
|
+
});
|
|
2371
|
+
this.ipcReconnectInFlight.set(name, reconnect);
|
|
2372
|
+
return reconnect;
|
|
2373
|
+
}
|
|
2374
|
+
async runIpcReconnect(name) {
|
|
2036
2375
|
for (let attempt = 1;; attempt++) {
|
|
2037
2376
|
if (this.ipcStoppingInstances.has(name) || !this.daemons.has(name))
|
|
2038
2377
|
return;
|
|
@@ -2055,9 +2394,22 @@ export class FleetManager {
|
|
|
2055
2394
|
if (existsSync(windowIdPath)) {
|
|
2056
2395
|
const windowId = readFileSync(windowIdPath, "utf-8").trim();
|
|
2057
2396
|
if (windowId) {
|
|
2397
|
+
// Async with an explicit timeout: this was execSync with NO timeout at
|
|
2398
|
+
// all, so a wedged tmux server blocked the whole fleet event loop
|
|
2399
|
+
// indefinitely — while we were here to diagnose a lost connection.
|
|
2400
|
+
// A timeout is also the correct signal: an unresponsive tmux server
|
|
2401
|
+
// means we cannot verify the pane, which is treated as dead (the same
|
|
2402
|
+
// conclusion the old code reached only by throwing).
|
|
2058
2403
|
try {
|
|
2059
|
-
const {
|
|
2060
|
-
|
|
2404
|
+
const { execFile } = await import("node:child_process");
|
|
2405
|
+
const { promisify } = await import("node:util");
|
|
2406
|
+
const { getTmuxSocketName } = await import("./paths.js");
|
|
2407
|
+
// Honour socket isolation: without -L this queried the user's default
|
|
2408
|
+
// tmux server instead of the fleet's, so under a custom AGEND_HOME the
|
|
2409
|
+
// check was meaningless (it reported every pane dead).
|
|
2410
|
+
const socket = getTmuxSocketName();
|
|
2411
|
+
const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
|
|
2412
|
+
await promisify(execFile)("tmux", args, { timeout: 5_000 });
|
|
2061
2413
|
}
|
|
2062
2414
|
catch {
|
|
2063
2415
|
// Pane dead — respawn
|
|
@@ -2078,6 +2430,12 @@ export class FleetManager {
|
|
|
2078
2430
|
if (this.adapterRestarting.has(id))
|
|
2079
2431
|
return;
|
|
2080
2432
|
this.adapterRestarting.add(id);
|
|
2433
|
+
// Reflect reality in adapterState throughout. This loop used to leave the state
|
|
2434
|
+
// untouched, so getAdapterStates() — and therefore /health and the dashboard —
|
|
2435
|
+
// kept reporting "connected" for an adapter that had been down for hours. An
|
|
2436
|
+
// adapter's true status was simply not knowable from inside the process.
|
|
2437
|
+
const previous = this.adapterState.get(id);
|
|
2438
|
+
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2081
2439
|
try {
|
|
2082
2440
|
for (let attempt = 1;; attempt++) {
|
|
2083
2441
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
@@ -2090,9 +2448,16 @@ export class FleetManager {
|
|
|
2090
2448
|
await adapter.stop().catch(() => { });
|
|
2091
2449
|
await adapter.start();
|
|
2092
2450
|
this.logger.info({ id, attempt }, "Adapter restarted successfully");
|
|
2451
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
2093
2452
|
return;
|
|
2094
2453
|
}
|
|
2095
|
-
catch {
|
|
2454
|
+
catch (err) {
|
|
2455
|
+
this.adapterState.set(id, {
|
|
2456
|
+
status: "retrying",
|
|
2457
|
+
retryCount: attempt,
|
|
2458
|
+
lastError: err?.message ?? String(err),
|
|
2459
|
+
});
|
|
2460
|
+
}
|
|
2096
2461
|
if (attempt % 10 === 0) {
|
|
2097
2462
|
this.logger.warn({ id, attempt }, "Adapter restart still failing");
|
|
2098
2463
|
}
|
|
@@ -2128,6 +2493,49 @@ export class FleetManager {
|
|
|
2128
2493
|
}
|
|
2129
2494
|
return generals[0];
|
|
2130
2495
|
}
|
|
2496
|
+
/**
|
|
2497
|
+
* A user reacted to one of the bot's messages (#408).
|
|
2498
|
+
*
|
|
2499
|
+
* A reaction is context, not a message (#432, reworking #413): it never triggers
|
|
2500
|
+
* an agent turn and never wakes anything. It is queued in the event log and rides
|
|
2501
|
+
* into the instance's NEXT real message as one compact leading line —
|
|
2502
|
+
* `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
|
|
2503
|
+
* No pending reactions → no line → zero context spent, which is the common case.
|
|
2504
|
+
*/
|
|
2505
|
+
async handleInboundReaction(r) {
|
|
2506
|
+
const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
|
|
2507
|
+
if (!instanceName) {
|
|
2508
|
+
this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
|
|
2509
|
+
return;
|
|
2510
|
+
}
|
|
2511
|
+
if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
|
|
2512
|
+
this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
|
|
2513
|
+
return;
|
|
2514
|
+
}
|
|
2515
|
+
this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
|
|
2516
|
+
if (r.action === "add") {
|
|
2517
|
+
this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2518
|
+
}
|
|
2519
|
+
else {
|
|
2520
|
+
// Withdrawn before anyone saw it → it never happened. See removeReaction.
|
|
2521
|
+
this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
/**
|
|
2525
|
+
* The queued-reaction summary for an instance's next real message, or {} when
|
|
2526
|
+
* nothing is pending (the common case must add zero context). The consume
|
|
2527
|
+
* callback is separate from the fetch so reactions are only marked once the
|
|
2528
|
+
* message actually went out — a failed delivery keeps them queued.
|
|
2529
|
+
*/
|
|
2530
|
+
pendingReactionsMeta(instanceName) {
|
|
2531
|
+
const pending = this.eventLog?.pendingReactions(instanceName);
|
|
2532
|
+
if (!pending)
|
|
2533
|
+
return { meta: {}, consume: () => { } };
|
|
2534
|
+
return {
|
|
2535
|
+
meta: { pending_reactions: pending.summary },
|
|
2536
|
+
consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
|
|
2537
|
+
};
|
|
2538
|
+
}
|
|
2131
2539
|
async handleInboundMessage(msg) {
|
|
2132
2540
|
const threadId = msg.threadId || undefined;
|
|
2133
2541
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
@@ -2465,6 +2873,7 @@ export class FleetManager {
|
|
|
2465
2873
|
}
|
|
2466
2874
|
this.warnIfRateLimited(generalInstance, msg);
|
|
2467
2875
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
|
|
2876
|
+
const generalReactions = this.pendingReactionsMeta(generalInstance);
|
|
2468
2877
|
try {
|
|
2469
2878
|
await this.deliverToInstance(generalInstance, {
|
|
2470
2879
|
type: "fleet_inbound",
|
|
@@ -2480,9 +2889,11 @@ export class FleetManager {
|
|
|
2480
2889
|
adapter_id: msg.adapterId,
|
|
2481
2890
|
source: msg.source,
|
|
2482
2891
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2892
|
+
...generalReactions.meta,
|
|
2483
2893
|
...extraMeta,
|
|
2484
2894
|
},
|
|
2485
2895
|
});
|
|
2896
|
+
generalReactions.consume();
|
|
2486
2897
|
this.lastInboundUser.set(generalInstance, msg.username);
|
|
2487
2898
|
this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
|
|
2488
2899
|
this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
|
|
@@ -2558,6 +2969,7 @@ export class FleetManager {
|
|
|
2558
2969
|
this.setTopicIcon(instanceName, "blue");
|
|
2559
2970
|
this.warnIfRateLimited(instanceName, msg);
|
|
2560
2971
|
const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
|
|
2972
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
2561
2973
|
try {
|
|
2562
2974
|
await this.deliverToInstance(instanceName, {
|
|
2563
2975
|
type: "fleet_inbound",
|
|
@@ -2573,9 +2985,13 @@ export class FleetManager {
|
|
|
2573
2985
|
adapter_id: msg.adapterId,
|
|
2574
2986
|
source: msg.source,
|
|
2575
2987
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2988
|
+
...reactions.meta,
|
|
2576
2989
|
...extraMeta,
|
|
2577
2990
|
},
|
|
2578
2991
|
});
|
|
2992
|
+
// Only after the message actually went out. A failed delivery keeps the
|
|
2993
|
+
// reactions queued for the retry / the next message.
|
|
2994
|
+
reactions.consume();
|
|
2579
2995
|
}
|
|
2580
2996
|
catch (err) {
|
|
2581
2997
|
this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
|
|
@@ -2664,25 +3080,33 @@ export class FleetManager {
|
|
|
2664
3080
|
delete args.thread_id;
|
|
2665
3081
|
threadId = undefined;
|
|
2666
3082
|
}
|
|
3083
|
+
// Reply dedup: retries land here when the agent was told a send failed
|
|
3084
|
+
// (daemon budget elapsed, shell tool killed) while the adapter send was
|
|
3085
|
+
// still in flight and about to succeed. One real send, everyone gets its
|
|
3086
|
+
// outcome; a genuinely failed send clears the entry so a retry passes.
|
|
3087
|
+
if (tool === "reply") {
|
|
3088
|
+
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3089
|
+
if (ticket.duplicate) {
|
|
3090
|
+
this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
|
|
3091
|
+
ticket.subscribe(respond);
|
|
3092
|
+
return;
|
|
3093
|
+
}
|
|
3094
|
+
const original = respond;
|
|
3095
|
+
const respondAndRecord = (result, error) => {
|
|
3096
|
+
ticket.complete(result, error);
|
|
3097
|
+
original(result, error);
|
|
3098
|
+
};
|
|
3099
|
+
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3100
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3101
|
+
return;
|
|
3102
|
+
}
|
|
3103
|
+
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
3104
|
+
ticket.complete(null, "reply not handled");
|
|
3105
|
+
original(null, "reply not handled");
|
|
3106
|
+
return;
|
|
3107
|
+
}
|
|
2667
3108
|
// Route standard channel tools (reply, react, edit_message, download_attachment)
|
|
2668
3109
|
if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
|
|
2669
|
-
if (tool === "reply") {
|
|
2670
|
-
// Agent answered — retire its pending cancel button and mark ✅ done.
|
|
2671
|
-
this.clearCancelButton(instanceName);
|
|
2672
|
-
this.reactDone(instanceName);
|
|
2673
|
-
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
2674
|
-
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
2675
|
-
this.emitSseEvent("message", {
|
|
2676
|
-
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
2677
|
-
text: (args.text ?? "").slice(0, 2000),
|
|
2678
|
-
ts: new Date().toISOString(),
|
|
2679
|
-
});
|
|
2680
|
-
// Log bot reply to classic instance chat-log
|
|
2681
|
-
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
2682
|
-
if (isClassic) {
|
|
2683
|
-
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
2684
|
-
}
|
|
2685
|
-
}
|
|
2686
3110
|
return;
|
|
2687
3111
|
}
|
|
2688
3112
|
// Log tool calls for activity visualization
|
|
@@ -2697,6 +3121,35 @@ export class FleetManager {
|
|
|
2697
3121
|
respond(null, `Unknown tool: ${tool}`);
|
|
2698
3122
|
}
|
|
2699
3123
|
}
|
|
3124
|
+
/** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
|
|
3125
|
+
afterReplyRouted(instanceName, args, senderSessionName) {
|
|
3126
|
+
// A reply is NOT proof the turn is over (#410) — but it is not proof of
|
|
3127
|
+
// more work either. Split the difference: an instance that is clearly
|
|
3128
|
+
// idle loses the button now; one that looks busy keeps it (re-posted
|
|
3129
|
+
// below the reply so it stays last in the channel), with a 2-minute
|
|
3130
|
+
// grace check — if it has NOT resumed working by then, the reply was the
|
|
3131
|
+
// end of the turn and the button goes. A multi-step run that keeps
|
|
3132
|
+
// working sails through the check and keeps its button.
|
|
3133
|
+
if (this.getInstanceIdle(instanceName)) {
|
|
3134
|
+
this.clearCancelButton(instanceName);
|
|
3135
|
+
}
|
|
3136
|
+
else {
|
|
3137
|
+
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
3138
|
+
}
|
|
3139
|
+
this.reactDone(instanceName);
|
|
3140
|
+
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
3141
|
+
this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
|
|
3142
|
+
this.emitSseEvent("message", {
|
|
3143
|
+
instance: instanceName, sender: senderSessionName ?? instanceName,
|
|
3144
|
+
text: (args.text ?? "").slice(0, 2000),
|
|
3145
|
+
ts: new Date().toISOString(),
|
|
3146
|
+
});
|
|
3147
|
+
// Log bot reply to classic instance chat-log
|
|
3148
|
+
const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
|
|
3149
|
+
if (isClassic) {
|
|
3150
|
+
ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
|
|
3151
|
+
}
|
|
3152
|
+
}
|
|
2700
3153
|
/** Handle tool status update from a daemon instance */
|
|
2701
3154
|
handleToolStatusFromInstance(instanceName, msg) {
|
|
2702
3155
|
const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
@@ -3299,6 +3752,9 @@ export class FleetManager {
|
|
|
3299
3752
|
}
|
|
3300
3753
|
}
|
|
3301
3754
|
async removeInstance(name) {
|
|
3755
|
+
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
3756
|
+
// else evicted deleted entries, so it grew for the life of the process.
|
|
3757
|
+
forgetInstanceContext(name);
|
|
3302
3758
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
3303
3759
|
const config = this.fleetConfig?.instances[name];
|
|
3304
3760
|
if (this.scheduler && config?.topic_id) {
|
|
@@ -3404,6 +3860,171 @@ export class FleetManager {
|
|
|
3404
3860
|
this.collabInstances.add(instanceName);
|
|
3405
3861
|
return true;
|
|
3406
3862
|
}
|
|
3863
|
+
/**
|
|
3864
|
+
* Open the event log, tolerating a corrupt file.
|
|
3865
|
+
*
|
|
3866
|
+
* `events.db` holds history only — event rows and the activity feed. Nothing the
|
|
3867
|
+
* fleet needs to run depends on it, and every consumer already uses
|
|
3868
|
+
* `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
|
|
3869
|
+
* unreadable history file (a truncated WAL after a hard kill, a full disk)
|
|
3870
|
+
* threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
|
|
3871
|
+
* running agent for a file whose only job is reporting.
|
|
3872
|
+
*
|
|
3873
|
+
* So: try, move a bad file aside and retry once with a fresh one, and if even
|
|
3874
|
+
* that fails carry on without an event log.
|
|
3875
|
+
*/
|
|
3876
|
+
/**
|
|
3877
|
+
* Run `agend backend doctor` for the fleet's default backend and return its
|
|
3878
|
+
* cleaned output.
|
|
3879
|
+
*
|
|
3880
|
+
* Async on purpose: this was `execSync` with a 30s timeout, reachable by any
|
|
3881
|
+
* allowlisted user through `/doctor`. While it ran, the entire fleet event loop
|
|
3882
|
+
* was frozen — no IPC, no adapter, no message delivery, no health responses,
|
|
3883
|
+
* and critically no WATCHDOG ping, so a slow doctor could push past
|
|
3884
|
+
* WatchdogSec and have systemd SIGABRT the fleet.
|
|
3885
|
+
*/
|
|
3886
|
+
async runBackendDoctor() {
|
|
3887
|
+
const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
3888
|
+
const backend = this.fleetConfig?.defaults?.backend || "claude-code";
|
|
3889
|
+
try {
|
|
3890
|
+
const { execFile } = await import("node:child_process");
|
|
3891
|
+
const { promisify } = await import("node:util");
|
|
3892
|
+
// execFile with an argv array — no shell, so the backend name cannot be
|
|
3893
|
+
// interpreted as a command even if config is malformed.
|
|
3894
|
+
const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
|
|
3895
|
+
timeout: 30_000,
|
|
3896
|
+
encoding: "utf-8",
|
|
3897
|
+
});
|
|
3898
|
+
return stripAnsi(stdout) || "No output";
|
|
3899
|
+
}
|
|
3900
|
+
catch (err) {
|
|
3901
|
+
const e = err;
|
|
3902
|
+
return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
|
|
3903
|
+
}
|
|
3904
|
+
}
|
|
3905
|
+
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
3906
|
+
/**
|
|
3907
|
+
* Cap every instance's pipe-pane log, walking the instances **directory** rather
|
|
3908
|
+
* than the config.
|
|
3909
|
+
*
|
|
3910
|
+
* A running instance rotates its own log on each health tick, so the ones that
|
|
3911
|
+
* need this are the ones nothing else looks at:
|
|
3912
|
+
*
|
|
3913
|
+
* - deleted instances, whose directory outlives the config entry. Nothing ever
|
|
3914
|
+
* touched these again. On the machine this was found on, one held 122 MB and
|
|
3915
|
+
* another 74 MB, out of 622 MB of pipe-pane logs in total.
|
|
3916
|
+
* - classic instances, which live in classicChannels, not fleetConfig.instances,
|
|
3917
|
+
* and so were never in the old config-driven loop at all.
|
|
3918
|
+
* - stopped instances, which have no health tick running.
|
|
3919
|
+
*
|
|
3920
|
+
* pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
|
|
3921
|
+
* at animation rate. Unbounded growth here fills the disk, which takes the whole
|
|
3922
|
+
* fleet down rather than one instance.
|
|
3923
|
+
*/
|
|
3924
|
+
rotateAllInstanceLogs() {
|
|
3925
|
+
const root = join(this.dataDir, "instances");
|
|
3926
|
+
let entries;
|
|
3927
|
+
try {
|
|
3928
|
+
entries = readdirSync(root, { withFileTypes: true });
|
|
3929
|
+
}
|
|
3930
|
+
catch {
|
|
3931
|
+
return; // no instances directory yet
|
|
3932
|
+
}
|
|
3933
|
+
for (const entry of entries) {
|
|
3934
|
+
if (!entry.isDirectory())
|
|
3935
|
+
continue;
|
|
3936
|
+
// rotateLogIfNeeded is already best-effort and returns early on a missing
|
|
3937
|
+
// file, so a directory without a pipe-pane log costs one stat.
|
|
3938
|
+
rotateLogIfNeeded(join(root, entry.name, "output.log"));
|
|
3939
|
+
}
|
|
3940
|
+
}
|
|
3941
|
+
pruneEventLog() {
|
|
3942
|
+
try {
|
|
3943
|
+
this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
|
|
3944
|
+
}
|
|
3945
|
+
catch (err) {
|
|
3946
|
+
this.logger.warn({ err }, "Event log prune failed");
|
|
3947
|
+
}
|
|
3948
|
+
}
|
|
3949
|
+
openEventLog() {
|
|
3950
|
+
const dbPath = join(this.dataDir, "events.db");
|
|
3951
|
+
try {
|
|
3952
|
+
return new EventLog(dbPath);
|
|
3953
|
+
}
|
|
3954
|
+
catch (err) {
|
|
3955
|
+
this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
|
|
3956
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
3957
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
3958
|
+
try {
|
|
3959
|
+
renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
|
|
3960
|
+
}
|
|
3961
|
+
catch { /* may not exist */ }
|
|
3962
|
+
}
|
|
3963
|
+
try {
|
|
3964
|
+
return new EventLog(dbPath);
|
|
3965
|
+
}
|
|
3966
|
+
catch (retryErr) {
|
|
3967
|
+
// History is worth losing; a fleet that won't start is not.
|
|
3968
|
+
this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
|
|
3969
|
+
return null;
|
|
3970
|
+
}
|
|
3971
|
+
}
|
|
3972
|
+
}
|
|
3973
|
+
/**
|
|
3974
|
+
* Report a fleet-level fault (not attributable to one instance) to the General
|
|
3975
|
+
* topic, so the operator learns about it without reading daemon.log.
|
|
3976
|
+
*
|
|
3977
|
+
* Throttled per distinct message: an unhandled rejection typically comes from a
|
|
3978
|
+
* loop (a poller, a repeating timer), and one channel message per occurrence
|
|
3979
|
+
* would bury the topic — which is worse than silence. First occurrence goes out
|
|
3980
|
+
* immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
|
|
3981
|
+
* count.
|
|
3982
|
+
*
|
|
3983
|
+
* The log line is written by the caller regardless: if every adapter is down,
|
|
3984
|
+
* the only notification path is the one that is broken.
|
|
3985
|
+
*/
|
|
3986
|
+
notifyFleetError(text) {
|
|
3987
|
+
const now = Date.now();
|
|
3988
|
+
const key = text.slice(0, 200);
|
|
3989
|
+
const seen = this.fleetErrorNotices.get(key);
|
|
3990
|
+
if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
|
|
3991
|
+
seen.suppressed++;
|
|
3992
|
+
return;
|
|
3993
|
+
}
|
|
3994
|
+
const suppressed = seen?.suppressed ?? 0;
|
|
3995
|
+
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
3996
|
+
// Bound the map: it is keyed by message text, and a message with a varying
|
|
3997
|
+
// suffix (a path, an id) would otherwise grow it without limit.
|
|
3998
|
+
if (this.fleetErrorNotices.size > 100) {
|
|
3999
|
+
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
4000
|
+
if (oldest !== undefined)
|
|
4001
|
+
this.fleetErrorNotices.delete(oldest);
|
|
4002
|
+
}
|
|
4003
|
+
const body = suppressed > 0
|
|
4004
|
+
? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
|
|
4005
|
+
: text;
|
|
4006
|
+
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4007
|
+
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
4008
|
+
// itself still exists, and notifyInstanceTopic only needs adapter + group +
|
|
4009
|
+
// topic_id to post into it.
|
|
4010
|
+
const general = Object.entries(this.fleetConfig?.instances ?? {})
|
|
4011
|
+
.find(([, config]) => config.general_topic === true)?.[0];
|
|
4012
|
+
if (general) {
|
|
4013
|
+
this.notifyInstanceTopic(general, body);
|
|
4014
|
+
return;
|
|
4015
|
+
}
|
|
4016
|
+
// No General instance — fall back to the primary channel's group.
|
|
4017
|
+
const channelCfg = this.getChannelConfig();
|
|
4018
|
+
const groupId = channelCfg?.group_id;
|
|
4019
|
+
if (this.adapter && groupId) {
|
|
4020
|
+
this.adapter.sendText(String(groupId), body)
|
|
4021
|
+
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
4022
|
+
return;
|
|
4023
|
+
}
|
|
4024
|
+
this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
|
|
4025
|
+
}
|
|
4026
|
+
static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
|
|
4027
|
+
fleetErrorNotices = new Map();
|
|
3407
4028
|
notifyInstanceTopic(instanceName, text, extraOpts) {
|
|
3408
4029
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3409
4030
|
if (!adapter)
|
|
@@ -3492,8 +4113,13 @@ export class FleetManager {
|
|
|
3492
4113
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3493
4114
|
if (!adapter)
|
|
3494
4115
|
return;
|
|
4116
|
+
// Resolve the group through the world fallback (first world when unbound),
|
|
4117
|
+
// NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
|
|
4118
|
+
// `channels:` worlds the primary `channel:` block is empty, so an instance
|
|
4119
|
+
// with no world binding yet (fresh restart, cross-instance delegation)
|
|
4120
|
+
// resolved group_id to undefined and the button silently never appeared.
|
|
3495
4121
|
const adapterId = this.instanceWorldBinding.get(instanceName);
|
|
3496
|
-
const groupId = this.
|
|
4122
|
+
const groupId = this.getGroupIdForInstance(instanceName) || undefined;
|
|
3497
4123
|
const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3498
4124
|
let chatId;
|
|
3499
4125
|
let threadId;
|
|
@@ -3509,8 +4135,12 @@ export class FleetManager {
|
|
|
3509
4135
|
if (!chatId && groupId)
|
|
3510
4136
|
chatId = String(groupId);
|
|
3511
4137
|
}
|
|
3512
|
-
if (!chatId)
|
|
4138
|
+
if (!chatId) {
|
|
4139
|
+
// A button that cannot be addressed must say so — this exact silence is how
|
|
4140
|
+
// "the cancel button sometimes never appears" stayed unreported-in-logs.
|
|
4141
|
+
this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
|
|
3513
4142
|
return;
|
|
4143
|
+
}
|
|
3514
4144
|
try {
|
|
3515
4145
|
const sent = await adapter.notifyAlert(chatId, {
|
|
3516
4146
|
type: "cancel",
|
|
@@ -3533,7 +4163,16 @@ export class FleetManager {
|
|
|
3533
4163
|
threadId: sent.threadId ?? threadId,
|
|
3534
4164
|
correlationId,
|
|
3535
4165
|
retryCount: 0,
|
|
4166
|
+
// Elapsed time is measured from when this button was posted — i.e. from
|
|
4167
|
+
// when the work was handed over — not from the pane's working transition,
|
|
4168
|
+
// which resets if the CLI blips idle mid-turn.
|
|
4169
|
+
startedAt: Date.now(),
|
|
4170
|
+
// Matches the text notifyAlert just posted, so the first 60s tick does
|
|
4171
|
+
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
4172
|
+
// with nothing visibly changed.
|
|
4173
|
+
lastProgressText: "👀 處理中…",
|
|
3536
4174
|
};
|
|
4175
|
+
this.startProgressTicker(entry);
|
|
3537
4176
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
3538
4177
|
// button. Covers turns that end without hitting a clear trigger (reply /
|
|
3539
4178
|
// cancel / correlation). Cleared in discardButton when the entry is removed.
|
|
@@ -3542,18 +4181,141 @@ export class FleetManager {
|
|
|
3542
4181
|
clearInterval(entry.idleCheckTimer);
|
|
3543
4182
|
return;
|
|
3544
4183
|
}
|
|
3545
|
-
|
|
3546
|
-
this.
|
|
4184
|
+
const reason = this.getInstanceIdle(instanceName) ? "idle"
|
|
4185
|
+
: this.stateReportDead(instanceName) ? "state reports stopped"
|
|
4186
|
+
: Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
|
|
4187
|
+
: null;
|
|
4188
|
+
if (reason) {
|
|
4189
|
+
this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
|
|
3547
4190
|
this.retireButton(entry);
|
|
4191
|
+
return;
|
|
3548
4192
|
}
|
|
4193
|
+
// Still looks busy. The daemon only broadcasts on transitions, so ask for
|
|
4194
|
+
// a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
|
|
4195
|
+
// the staleness check honest; a dead one's silence is the evidence.
|
|
4196
|
+
this.instanceIpcClients.get(instanceName)?.send({
|
|
4197
|
+
type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
|
|
4198
|
+
});
|
|
3549
4199
|
}, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
|
|
3550
4200
|
this.cancelButtons.set(sent.messageId, entry);
|
|
4201
|
+
this.persistCancelButtons();
|
|
3551
4202
|
this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
|
|
3552
4203
|
}
|
|
3553
4204
|
catch (e) {
|
|
3554
4205
|
this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
|
|
3555
4206
|
}
|
|
3556
4207
|
}
|
|
4208
|
+
/**
|
|
4209
|
+
* The cancel button's text for a given elapsed time.
|
|
4210
|
+
*
|
|
4211
|
+
* Below the threshold it keeps the original wording, so a normal quick answer
|
|
4212
|
+
* looks exactly as it did before. Past it, the button doubles as the live
|
|
4213
|
+
* progress indicator (#409) — the channel showed nothing at all during long work,
|
|
4214
|
+
* and once the agent had replied once there was no sign it was still going.
|
|
4215
|
+
*/
|
|
4216
|
+
static progressText(elapsedMs, activity) {
|
|
4217
|
+
if (elapsedMs < PROGRESS_MIN_ELAPSED_MS)
|
|
4218
|
+
return "👀 處理中…";
|
|
4219
|
+
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
4220
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
4221
|
+
const seconds = totalSeconds % 60;
|
|
4222
|
+
const elapsed = minutes >= 60
|
|
4223
|
+
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
4224
|
+
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
4225
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
4226
|
+
return detail
|
|
4227
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
4228
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
4229
|
+
}
|
|
4230
|
+
/**
|
|
4231
|
+
* Make a tool summary safe to paste into a channel message.
|
|
4232
|
+
*
|
|
4233
|
+
* The text is agent-controlled (it is built from tool inputs — file paths,
|
|
4234
|
+
* shell commands), so it gets flattened to one line, capped, and stripped of
|
|
4235
|
+
* the two Discord mass-mention triggers. Neither channel renders it with a
|
|
4236
|
+
* parse mode, so no markup escaping is needed beyond that.
|
|
4237
|
+
*/
|
|
4238
|
+
static sanitizeActivity(activity) {
|
|
4239
|
+
if (!activity)
|
|
4240
|
+
return null;
|
|
4241
|
+
const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@$1").trim();
|
|
4242
|
+
if (!flat)
|
|
4243
|
+
return null;
|
|
4244
|
+
return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
|
|
4245
|
+
? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
|
|
4246
|
+
: flat;
|
|
4247
|
+
}
|
|
4248
|
+
/**
|
|
4249
|
+
* Remember what an instance is currently doing, for the progress line.
|
|
4250
|
+
*
|
|
4251
|
+
* Best-effort by design: only backends that expose a live activity feed report
|
|
4252
|
+
* anything, and the progress line simply omits the detail for the rest. It is
|
|
4253
|
+
* never used to decide anything — purely what the user is shown.
|
|
4254
|
+
*/
|
|
4255
|
+
cacheInstanceActivity(name, activity) {
|
|
4256
|
+
if (activity)
|
|
4257
|
+
this.instanceActivity.set(name, activity);
|
|
4258
|
+
else
|
|
4259
|
+
this.instanceActivity.delete(name);
|
|
4260
|
+
}
|
|
4261
|
+
/**
|
|
4262
|
+
* Refresh the button's text in place while the instance keeps working.
|
|
4263
|
+
*
|
|
4264
|
+
* Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
|
|
4265
|
+
* and the Bot API treats that as "clear the keyboard" — so editing with it would
|
|
4266
|
+
* delete the very cancel button this is trying to keep alive.
|
|
4267
|
+
*/
|
|
4268
|
+
startProgressTicker(entry) {
|
|
4269
|
+
entry.progressTimer = setInterval(() => {
|
|
4270
|
+
if (!this.cancelButtons.has(entry.messageId)) {
|
|
4271
|
+
clearInterval(entry.progressTimer);
|
|
4272
|
+
return;
|
|
4273
|
+
}
|
|
4274
|
+
const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName));
|
|
4275
|
+
if (text === entry.lastProgressText)
|
|
4276
|
+
return; // nothing changed — skip the API call
|
|
4277
|
+
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
4278
|
+
if (!adapter?.editAlert)
|
|
4279
|
+
return;
|
|
4280
|
+
entry.lastProgressText = text;
|
|
4281
|
+
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
4282
|
+
type: "cancel",
|
|
4283
|
+
instanceName: entry.instanceName,
|
|
4284
|
+
message: text,
|
|
4285
|
+
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
4286
|
+
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
4287
|
+
.catch(err => {
|
|
4288
|
+
// A failed progress edit must never escalate: the button still works and
|
|
4289
|
+
// the next tick retries. Common causes are a deleted message or a
|
|
4290
|
+
// rate limit.
|
|
4291
|
+
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
4292
|
+
});
|
|
4293
|
+
}, PROGRESS_UPDATE_INTERVAL_MS);
|
|
4294
|
+
entry.progressTimer.unref?.();
|
|
4295
|
+
}
|
|
4296
|
+
/**
|
|
4297
|
+
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
4298
|
+
* it has not, retire its button. Re-arming replaces the previous timer, so a
|
|
4299
|
+
* burst of replies ends with exactly one pending check.
|
|
4300
|
+
*/
|
|
4301
|
+
armReplyGrace(instanceName) {
|
|
4302
|
+
for (const entry of this.cancelButtons.values()) {
|
|
4303
|
+
if (entry.instanceName !== instanceName)
|
|
4304
|
+
continue;
|
|
4305
|
+
if (entry.replyGraceTimer)
|
|
4306
|
+
clearTimeout(entry.replyGraceTimer);
|
|
4307
|
+
entry.replyGraceTimer = setTimeout(() => {
|
|
4308
|
+
entry.replyGraceTimer = undefined;
|
|
4309
|
+
if (!this.cancelButtons.has(entry.messageId))
|
|
4310
|
+
return;
|
|
4311
|
+
if (!this.getInstanceIdle(instanceName))
|
|
4312
|
+
return; // resumed — a long run keeps its button
|
|
4313
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
4314
|
+
this.retireButton(entry);
|
|
4315
|
+
}, REPLY_RETIRE_GRACE_MS);
|
|
4316
|
+
entry.replyGraceTimer.unref?.();
|
|
4317
|
+
}
|
|
4318
|
+
}
|
|
3557
4319
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
3558
4320
|
retireInstanceButtons(instanceName) {
|
|
3559
4321
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -3585,7 +4347,70 @@ export class FleetManager {
|
|
|
3585
4347
|
clearTimeout(entry.retryTimer);
|
|
3586
4348
|
if (entry.idleCheckTimer)
|
|
3587
4349
|
clearInterval(entry.idleCheckTimer);
|
|
4350
|
+
if (entry.progressTimer)
|
|
4351
|
+
clearInterval(entry.progressTimer);
|
|
4352
|
+
if (entry.replyGraceTimer)
|
|
4353
|
+
clearTimeout(entry.replyGraceTimer);
|
|
3588
4354
|
this.cancelButtons.delete(entry.messageId);
|
|
4355
|
+
this.persistCancelButtons();
|
|
4356
|
+
}
|
|
4357
|
+
/**
|
|
4358
|
+
* Mirror the live buttons to disk. The map is memory-only, so before this a
|
|
4359
|
+
* fleet restart orphaned every button on screen: frozen "處理中…" text and a
|
|
4360
|
+
* click that did nothing, forever. The ledger is tiny (a handful of rows) and
|
|
4361
|
+
* written on every add/remove — no debounce needed at that rate.
|
|
4362
|
+
*/
|
|
4363
|
+
persistCancelButtons() {
|
|
4364
|
+
try {
|
|
4365
|
+
const rows = [...this.cancelButtons.values()].map(e => ({
|
|
4366
|
+
instanceName: e.instanceName,
|
|
4367
|
+
adapterId: e.adapterId,
|
|
4368
|
+
chatId: e.chatId,
|
|
4369
|
+
messageId: e.messageId,
|
|
4370
|
+
threadId: e.threadId,
|
|
4371
|
+
}));
|
|
4372
|
+
writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
|
|
4373
|
+
}
|
|
4374
|
+
catch (err) {
|
|
4375
|
+
this.logger.debug({ err }, "Cancel button ledger write failed");
|
|
4376
|
+
}
|
|
4377
|
+
}
|
|
4378
|
+
/**
|
|
4379
|
+
* Delete the previous process's buttons. Runs once adapters are up: nothing
|
|
4380
|
+
* from a previous fleet process can still be mid-turn from this process's
|
|
4381
|
+
* point of view, so every ledger row is an orphan by definition.
|
|
4382
|
+
*/
|
|
4383
|
+
async sweepOrphanedCancelButtons() {
|
|
4384
|
+
const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
|
|
4385
|
+
let rows;
|
|
4386
|
+
try {
|
|
4387
|
+
if (!existsSync(ledgerPath))
|
|
4388
|
+
return;
|
|
4389
|
+
rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
|
|
4390
|
+
}
|
|
4391
|
+
catch {
|
|
4392
|
+
try {
|
|
4393
|
+
unlinkSync(ledgerPath);
|
|
4394
|
+
}
|
|
4395
|
+
catch { /* corrupt ledger — drop it */ }
|
|
4396
|
+
return;
|
|
4397
|
+
}
|
|
4398
|
+
for (const row of rows) {
|
|
4399
|
+
const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
|
|
4400
|
+
?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
|
|
4401
|
+
if (!adapter?.deleteMessage)
|
|
4402
|
+
continue;
|
|
4403
|
+
try {
|
|
4404
|
+
await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
|
|
4405
|
+
this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
|
|
4406
|
+
}
|
|
4407
|
+
catch (err) {
|
|
4408
|
+
// Best effort: the message may already be gone, or too old to delete.
|
|
4409
|
+
this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
|
|
4410
|
+
}
|
|
4411
|
+
}
|
|
4412
|
+
// The current process owns the ledger from here on.
|
|
4413
|
+
this.persistCancelButtons();
|
|
3589
4414
|
}
|
|
3590
4415
|
/** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
|
|
3591
4416
|
scheduleButtonRetry(entry, err) {
|
|
@@ -4250,6 +5075,12 @@ When users create specialized instances, suggest these configurations:
|
|
|
4250
5075
|
meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
|
|
4251
5076
|
}
|
|
4252
5077
|
}
|
|
5078
|
+
// Classic channels queue reactions like everyone else (#432 stored them, but
|
|
5079
|
+
// this path never attached them — reactions in a ClassicBot channel went into
|
|
5080
|
+
// the DB and were never seen again). Same contract as the topic paths:
|
|
5081
|
+
// consumed only after the delivery succeeded.
|
|
5082
|
+
const reactions = this.pendingReactionsMeta(instanceName);
|
|
5083
|
+
Object.assign(meta, reactions.meta);
|
|
4253
5084
|
try {
|
|
4254
5085
|
await this.deliverToInstance(instanceName, {
|
|
4255
5086
|
type: "fleet_inbound",
|
|
@@ -4257,6 +5088,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4257
5088
|
targetSession: instanceName,
|
|
4258
5089
|
meta,
|
|
4259
5090
|
});
|
|
5091
|
+
reactions.consume();
|
|
4260
5092
|
}
|
|
4261
5093
|
catch (err) {
|
|
4262
5094
|
this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
|
|
@@ -4531,9 +5363,19 @@ When users create specialized instances, suggest these configurations:
|
|
|
4531
5363
|
// DC path: respond immediately with progress text
|
|
4532
5364
|
await pending.respond(progressText).catch(() => { });
|
|
4533
5365
|
}
|
|
4534
|
-
// Apply model in background — don't await here (keeps callback handler fast)
|
|
5366
|
+
// Apply model in background — don't await here (keeps callback handler fast).
|
|
5367
|
+
// Guarded: applyModel() restarts the instance, and an unguarded rejection here
|
|
5368
|
+
// meant a user picking from the /model menu could take the whole fleet down.
|
|
5369
|
+
// On failure the user gets told, rather than the click silently doing nothing.
|
|
4535
5370
|
void (async () => {
|
|
4536
|
-
|
|
5371
|
+
let result;
|
|
5372
|
+
try {
|
|
5373
|
+
result = await this.applyModel(pending.instanceName, model);
|
|
5374
|
+
}
|
|
5375
|
+
catch (err) {
|
|
5376
|
+
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
5377
|
+
result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
5378
|
+
}
|
|
4537
5379
|
if (pending.adapter && pending.adapterChatId) {
|
|
4538
5380
|
if (progressMsgId) {
|
|
4539
5381
|
pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
|
|
@@ -4771,7 +5613,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4771
5613
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
4772
5614
|
};
|
|
4773
5615
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
4774
|
-
await this.startInstance(instanceName, config, topicMode);
|
|
5616
|
+
await this.startInstance(instanceName, config, topicMode, "classic");
|
|
4775
5617
|
}
|
|
4776
5618
|
/** Handle /start slash command — register classic channel */
|
|
4777
5619
|
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
@@ -4813,11 +5655,24 @@ When users create specialized instances, suggest these configurations:
|
|
|
4813
5655
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
4814
5656
|
return t("classic.stopped");
|
|
4815
5657
|
}
|
|
4816
|
-
|
|
5658
|
+
/**
|
|
5659
|
+
* Idempotent while in flight: SIGINT and SIGTERM share one handler and the
|
|
5660
|
+
* uncaughtException path calls this too, so overlapping runs were possible —
|
|
5661
|
+
* each snapshotting the daemon map and calling stop() on the same daemons
|
|
5662
|
+
* concurrently. Deliberately NOT `async`, so callers receive the same promise
|
|
5663
|
+
* object rather than a fresh wrapper around it. The latch clears when the run
|
|
5664
|
+
* settles, so a later genuine stop (after a restart) still does the work.
|
|
5665
|
+
*/
|
|
5666
|
+
stopAll() {
|
|
5667
|
+
this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
|
|
5668
|
+
return this.stopAllInFlight;
|
|
5669
|
+
}
|
|
5670
|
+
stopAllInFlight = null;
|
|
5671
|
+
async doStopAll() {
|
|
4817
5672
|
this.startupComplete = false;
|
|
4818
5673
|
this.reloadPending = false;
|
|
4819
5674
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
4820
|
-
|
|
5675
|
+
sdNotifyBlocking("STOPPING=1");
|
|
4821
5676
|
if (this.watchdogTimer) {
|
|
4822
5677
|
clearInterval(this.watchdogTimer);
|
|
4823
5678
|
this.watchdogTimer = null;
|
|
@@ -4837,6 +5692,26 @@ When users create specialized instances, suggest these configurations:
|
|
|
4837
5692
|
clearInterval(this.updateCheckTimer);
|
|
4838
5693
|
this.updateCheckTimer = null;
|
|
4839
5694
|
}
|
|
5695
|
+
if (this.eventLogPruneTimer) {
|
|
5696
|
+
clearInterval(this.eventLogPruneTimer);
|
|
5697
|
+
this.eventLogPruneTimer = null;
|
|
5698
|
+
}
|
|
5699
|
+
if (this.logRotateTimer) {
|
|
5700
|
+
clearInterval(this.logRotateTimer);
|
|
5701
|
+
this.logRotateTimer = null;
|
|
5702
|
+
}
|
|
5703
|
+
// Cancel-button timers were never cleared here. The idle-check interval is not
|
|
5704
|
+
// unref'd, so it held the event loop open past shutdown and kept retrying
|
|
5705
|
+
// deletes against an adapter that was already gone.
|
|
5706
|
+
for (const entry of [...this.cancelButtons.values()]) {
|
|
5707
|
+
if (entry.retryTimer)
|
|
5708
|
+
clearTimeout(entry.retryTimer);
|
|
5709
|
+
if (entry.idleCheckTimer)
|
|
5710
|
+
clearInterval(entry.idleCheckTimer);
|
|
5711
|
+
if (entry.progressTimer)
|
|
5712
|
+
clearInterval(entry.progressTimer);
|
|
5713
|
+
}
|
|
5714
|
+
this.cancelButtons.clear();
|
|
4840
5715
|
if (this.topicCleanupTimer) {
|
|
4841
5716
|
clearInterval(this.topicCleanupTimer);
|
|
4842
5717
|
this.topicCleanupTimer = null;
|
|
@@ -5055,6 +5930,15 @@ When users create specialized instances, suggest these configurations:
|
|
|
5055
5930
|
removedRatio,
|
|
5056
5931
|
validationErrors: validation.errors,
|
|
5057
5932
|
}, "Refusing unsafe fleet config reload; running configuration was kept");
|
|
5933
|
+
// Tell the operator. A silently ignored config edit is the most confusing
|
|
5934
|
+
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
5935
|
+
// with no explanation anywhere they are looking.
|
|
5936
|
+
const why = !validation.valid
|
|
5937
|
+
? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
|
|
5938
|
+
: unsafeEmpty
|
|
5939
|
+
? `it removed every instance (${oldCount} → 0)`
|
|
5940
|
+
: `it removed more than half the instances (${oldCount} → ${newCount})`;
|
|
5941
|
+
this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
|
|
5058
5942
|
return;
|
|
5059
5943
|
}
|
|
5060
5944
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -5254,10 +6138,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
5254
6138
|
// ── Update check ────────────────────────────────────────────────────
|
|
5255
6139
|
async checkForUpdates() {
|
|
5256
6140
|
try {
|
|
5257
|
-
|
|
6141
|
+
// Both npm lookups are async: as execSync they froze the fleet event loop for
|
|
6142
|
+
// up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
|
|
6143
|
+
// past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
|
|
6144
|
+
// for a background version check.
|
|
6145
|
+
const { execFile } = await import("node:child_process");
|
|
6146
|
+
const { promisify } = await import("node:util");
|
|
6147
|
+
const execFileP = promisify(execFile);
|
|
6148
|
+
const npmVersion = async (spec) => {
|
|
6149
|
+
const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
|
|
6150
|
+
return stdout.toString().trim();
|
|
6151
|
+
};
|
|
5258
6152
|
const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
5259
6153
|
const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
|
|
5260
|
-
const latest =
|
|
6154
|
+
const latest = await npmVersion("@songsid/agend");
|
|
5261
6155
|
let target = latest;
|
|
5262
6156
|
if (currentVersion.includes("-beta")) {
|
|
5263
6157
|
// Beta users track the @beta channel (never fall back to @latest, which is
|
|
@@ -5265,7 +6159,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5265
6159
|
// of beta/latest is the newest.
|
|
5266
6160
|
let beta = "";
|
|
5267
6161
|
try {
|
|
5268
|
-
beta =
|
|
6162
|
+
beta = await npmVersion("@songsid/agend@beta");
|
|
5269
6163
|
}
|
|
5270
6164
|
catch { /* no beta tag */ }
|
|
5271
6165
|
target = beta || latest;
|
|
@@ -5372,6 +6266,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5372
6266
|
// /view routes accept the read-only view.token (or web.token) and do
|
|
5373
6267
|
// their own per-method auth in view-api.ts — skip the web-token gate.
|
|
5374
6268
|
}
|
|
6269
|
+
else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
|
|
6270
|
+
// /api/ai-usage is read-only GET data for the /view Usage panel — open
|
|
6271
|
+
// like the other /view data routes (usage-api.ts rejects non-GET).
|
|
6272
|
+
}
|
|
5375
6273
|
else {
|
|
5376
6274
|
// All other endpoints require a valid token (query ?token= or X-Agend-Token header).
|
|
5377
6275
|
// /ui/* will also re-check in web-api.ts, which is harmless.
|
|
@@ -5386,34 +6284,34 @@ When users create specialized instances, suggest these configurations:
|
|
|
5386
6284
|
}
|
|
5387
6285
|
}
|
|
5388
6286
|
if (req.method === "GET" && req.url === "/health") {
|
|
5389
|
-
const
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
5397
|
-
}));
|
|
6287
|
+
const health = this.getFleetHealth();
|
|
6288
|
+
// 503 when the fleet cannot do its job, so an external monitor sees it.
|
|
6289
|
+
// This used to always answer 200 "ok" with a count of CONFIGURED instances,
|
|
6290
|
+
// so every agent could be dead and every adapter down and it still looked
|
|
6291
|
+
// green.
|
|
6292
|
+
res.writeHead(health.status === "ok" ? 200 : 503);
|
|
6293
|
+
res.end(JSON.stringify(health));
|
|
5398
6294
|
return;
|
|
5399
6295
|
}
|
|
5400
6296
|
if (req.method === "GET" && req.url === "/status") {
|
|
5401
6297
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
|
|
5402
6298
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5403
|
-
let context_pct = 0;
|
|
5404
6299
|
let cost = 0;
|
|
5405
6300
|
try {
|
|
5406
6301
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5407
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5408
6302
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5409
6303
|
}
|
|
5410
6304
|
catch (err) {
|
|
5411
6305
|
this.logger.debug({ err, name }, "statusline.json read failed (/status)");
|
|
5412
6306
|
}
|
|
6307
|
+
const backend = this.fleetConfig?.instances[name]?.backend
|
|
6308
|
+
?? this.fleetConfig?.defaults?.backend
|
|
6309
|
+
?? "claude-code";
|
|
6310
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
5413
6311
|
return {
|
|
5414
6312
|
name,
|
|
5415
6313
|
status: this.getInstanceStatus(name),
|
|
5416
|
-
context_pct,
|
|
6314
|
+
context_pct: context ?? 0,
|
|
5417
6315
|
cost,
|
|
5418
6316
|
};
|
|
5419
6317
|
});
|
|
@@ -5528,7 +6426,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
5528
6426
|
res.writeHead(500);
|
|
5529
6427
|
res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
|
|
5530
6428
|
}
|
|
5531
|
-
|
|
6429
|
+
// The inner catch can itself throw (writeHead after a successful
|
|
6430
|
+
// writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
|
|
6431
|
+
// IIFE. Same for the two handlers below.
|
|
6432
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
|
|
5532
6433
|
return;
|
|
5533
6434
|
}
|
|
5534
6435
|
// Instance restart (immediate, no idle wait)
|
|
@@ -5549,7 +6450,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5549
6450
|
res.writeHead(status);
|
|
5550
6451
|
res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
|
|
5551
6452
|
}
|
|
5552
|
-
})();
|
|
6453
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
|
|
5553
6454
|
return;
|
|
5554
6455
|
}
|
|
5555
6456
|
if (req.method === "POST" && req.url?.startsWith("/stop/")) {
|
|
@@ -5572,7 +6473,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
5572
6473
|
res.writeHead(500);
|
|
5573
6474
|
res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
|
|
5574
6475
|
}
|
|
5575
|
-
})();
|
|
6476
|
+
})().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
|
|
5576
6477
|
return;
|
|
5577
6478
|
}
|
|
5578
6479
|
// ── Agent CLI endpoint ─────
|
|
@@ -5584,6 +6485,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
5584
6485
|
const url = new URL(req.url ?? "/", `http://localhost:${port}`);
|
|
5585
6486
|
if (handleViewRequest(req, res, url, this))
|
|
5586
6487
|
return;
|
|
6488
|
+
if (handleUsageRequest(req, res, url, this))
|
|
6489
|
+
return;
|
|
5587
6490
|
if (handleSettingsRequest(req, res, url, this))
|
|
5588
6491
|
return;
|
|
5589
6492
|
if (handleWebRequest(req, res, url, this))
|
|
@@ -5630,20 +6533,35 @@ When users create specialized instances, suggest these configurations:
|
|
|
5630
6533
|
this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
|
|
5631
6534
|
}
|
|
5632
6535
|
getUiStatus() {
|
|
5633
|
-
const
|
|
6536
|
+
const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
|
|
6537
|
+
// Classic rooms live only in classicBot.yaml — /api/profiles merges them into
|
|
6538
|
+
// the View roster, but previously getUiStatus skipped them so context_pct was
|
|
6539
|
+
// always 0 (live map miss → l?.context_pct ?? 0).
|
|
6540
|
+
const classicOnly = (this.classicChannels?.getAll() ?? [])
|
|
6541
|
+
.map(ch => ch.instanceName)
|
|
6542
|
+
.filter(name => !fleetNames.includes(name));
|
|
6543
|
+
const names = [...fleetNames, ...classicOnly];
|
|
6544
|
+
const instances = names.map(name => {
|
|
5634
6545
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
5635
|
-
let context_pct = 0;
|
|
5636
6546
|
let cost = 0;
|
|
5637
6547
|
let model = "";
|
|
5638
6548
|
try {
|
|
5639
6549
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
5640
|
-
context_pct = data.context_window?.used_percentage ?? 0;
|
|
5641
6550
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
5642
6551
|
model = data.model?.display_name ?? "";
|
|
5643
6552
|
}
|
|
5644
6553
|
catch (err) {
|
|
5645
6554
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
5646
6555
|
}
|
|
6556
|
+
// Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
|
|
6557
|
+
const classic = classicOnly.includes(name);
|
|
6558
|
+
const backend = classic
|
|
6559
|
+
? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
|
|
6560
|
+
: (this.fleetConfig?.instances[name]?.backend
|
|
6561
|
+
?? this.fleetConfig?.defaults?.backend
|
|
6562
|
+
?? "claude-code");
|
|
6563
|
+
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
6564
|
+
const context_pct = context ?? 0;
|
|
5647
6565
|
return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
|
|
5648
6566
|
});
|
|
5649
6567
|
return {
|