@songsid/agend 2.1.4-beta.8 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-cli-instructions.md +2 -2
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +12 -8
- package/dist/backend/antigravity.js +189 -42
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +87 -23
- package/dist/backend/claude-code.js +361 -46
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +139 -5
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +52 -2
- package/dist/backend/kiro.js +267 -12
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.js +17 -4
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +122 -3
- package/dist/backend/types.js +5 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/backend-outage.d.ts +61 -0
- package/dist/backend-outage.js +71 -0
- package/dist/backend-outage.js.map +1 -0
- package/dist/channel/adapters/discord.d.ts +44 -2
- package/dist/channel/adapters/discord.js +529 -89
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +30 -0
- package/dist/channel/adapters/telegram.js +221 -19
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/ipc-bridge.d.ts +2 -1
- package/dist/channel/ipc-bridge.js +33 -4
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/markdown-chunk.d.ts +42 -0
- package/dist/channel/markdown-chunk.js +205 -0
- package/dist/channel/markdown-chunk.js.map +1 -0
- package/dist/channel/mcp-server.js +34 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +5 -3
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.js +64 -8
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/types.d.ts +31 -1
- package/dist/classic-channel-manager.d.ts +65 -0
- package/dist/classic-channel-manager.js +172 -3
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +124 -83
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +16 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +12 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +6 -0
- package/dist/cross-instance-envelope.js +30 -0
- package/dist/cross-instance-envelope.js.map +1 -0
- package/dist/daemon.d.ts +538 -11
- package/dist/daemon.js +2272 -288
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +41 -0
- package/dist/doctor.js +267 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +63 -0
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +424 -11
- package/dist/fleet-manager.js +2753 -444
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +63 -16
- package/dist/general-knowledge/skills/session-management/SKILL.md +171 -8
- package/dist/instance-lifecycle.d.ts +114 -1
- package/dist/instance-lifecycle.js +404 -14
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +42 -4
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +1024 -310
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +91 -0
- package/dist/login-flows.js +170 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/network-family.d.ts +18 -0
- package/dist/network-family.js +20 -0
- package/dist/network-family.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -1
- package/dist/outbound-handlers.js +174 -40
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +10 -3
- package/dist/outbound-schemas.js +10 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +52 -0
- package/dist/pane-input-residue.js +107 -0
- package/dist/pane-input-residue.js.map +1 -0
- package/dist/process-memory.d.ts +6 -0
- package/dist/process-memory.js +45 -0
- package/dist/process-memory.js.map +1 -1
- package/dist/reply-dedup.d.ts +7 -8
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/spawn-gate.d.ts +30 -0
- package/dist/spawn-gate.js +79 -0
- package/dist/spawn-gate.js.map +1 -0
- package/dist/steer-capability.d.ts +1 -0
- package/dist/steer-capability.js +6 -0
- package/dist/steer-capability.js.map +1 -0
- package/dist/storm-window.d.ts +83 -0
- package/dist/storm-window.js +251 -0
- package/dist/storm-window.js.map +1 -0
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +16 -0
- package/dist/tmux-manager.js +110 -25
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-progress.d.ts +40 -0
- package/dist/tool-progress.js +289 -0
- package/dist/tool-progress.js.map +1 -0
- package/dist/topic-commands.d.ts +45 -6
- package/dist/topic-commands.js +394 -66
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.d.ts +15 -2
- package/dist/transcript-monitor.js +63 -17
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/transcript-sources.d.ts +131 -0
- package/dist/transcript-sources.js +580 -0
- package/dist/transcript-sources.js.map +1 -0
- package/dist/types.d.ts +11 -0
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +200 -60
- package/dist/ui/view.html +147 -30
- package/dist/usage/format-rich.d.ts +1 -1
- package/dist/usage/format-rich.js +28 -24
- package/dist/usage/format-rich.js.map +1 -1
- package/dist/usage/i18n-keys.d.ts +7 -0
- package/dist/usage/i18n-keys.js +34 -0
- package/dist/usage/i18n-keys.js.map +1 -0
- package/dist/usage/i18n.d.ts +5 -0
- package/dist/usage/i18n.js +27 -0
- package/dist/usage/i18n.js.map +1 -0
- package/dist/usage/providers.d.ts +21 -0
- package/dist/usage/providers.js +153 -75
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +11 -3
- package/dist/usage/usage-api.js +61 -24
- package/dist/usage/usage-api.js.map +1 -1
- package/dist/workflow-templates/default.md +2 -1
- package/package.json +2 -2
package/dist/fleet-manager.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { freemem, totalmem } from "node:os";
|
|
3
5
|
import { createServer } from "node:http";
|
|
4
6
|
import { join, dirname, basename } from "node:path";
|
|
5
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
10
|
import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
|
|
8
11
|
import { formatUpdateProgress } from "./update-progress.js";
|
|
9
12
|
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
10
13
|
import { readFleetMemory } from "./process-memory.js";
|
|
11
14
|
import { ReplyDeduper } from "./reply-dedup.js";
|
|
12
|
-
import { isScalar, parseDocument } from "yaml";
|
|
15
|
+
import { isMap, isScalar, parseDocument } from "yaml";
|
|
13
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
14
17
|
const __dirname = dirname(__filename);
|
|
15
18
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
@@ -30,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
30
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
31
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
32
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
33
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE,
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
34
37
|
import { DailySummary } from "./daily-summary.js";
|
|
35
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
36
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -42,9 +45,11 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
42
45
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
43
46
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
44
47
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
45
|
-
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
48
|
+
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
|
+
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
|
+
import { LoginSession } from "./login-manager.js";
|
|
46
51
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
47
|
-
import { setLocale, detectLocale, t } from "./locale.js";
|
|
52
|
+
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
48
53
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
49
54
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
50
55
|
import { validateFleetConfig } from "./config-validator.js";
|
|
@@ -54,6 +59,11 @@ import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
|
54
59
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
55
60
|
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
56
61
|
import { RestartProgress } from "./restart-progress.js";
|
|
62
|
+
import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
|
|
63
|
+
import { StormWindow } from "./storm-window.js";
|
|
64
|
+
import { SpawnGate } from "./spawn-gate.js";
|
|
65
|
+
import { BackendOutageTracker } from "./backend-outage.js";
|
|
66
|
+
import { canUnlockAdvancedTips, DailyTipScheduler, selectTip, visibleTipLevels, } from "./tips.js";
|
|
57
67
|
import { getTmuxSession } from "./config.js";
|
|
58
68
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
59
69
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -106,6 +116,8 @@ const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
|
|
|
106
116
|
* button alone (the idle edge retires it when the run really ends).
|
|
107
117
|
*/
|
|
108
118
|
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
119
|
+
/** Bound for the daemon to capture the pane and answer a post-reply state query. */
|
|
120
|
+
const REPLY_STATE_REFRESH_TIMEOUT_MS = 2_000;
|
|
109
121
|
/**
|
|
110
122
|
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
111
123
|
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
@@ -135,6 +147,8 @@ const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
|
|
|
135
147
|
* exactly one progress message per turn, and it is the cancel button itself.
|
|
136
148
|
*/
|
|
137
149
|
const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
|
|
150
|
+
/** Floor between tool-progress-driven bubble edits (Telegram flood safety). */
|
|
151
|
+
const TOOL_PROGRESS_EDIT_MIN_MS = 4_000;
|
|
138
152
|
/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
|
|
139
153
|
/**
|
|
140
154
|
* Default delay before the button starts showing elapsed time. Configurable via
|
|
@@ -160,6 +174,30 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
160
174
|
* emoji never changes the documented delivery-state protocol.
|
|
161
175
|
*/
|
|
162
176
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
177
|
+
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
178
|
+
"tool_progress",
|
|
179
|
+
"mcp_proxy_reply",
|
|
180
|
+
"auto_pause_after",
|
|
181
|
+
"warm_cap",
|
|
182
|
+
"display_name",
|
|
183
|
+
"description",
|
|
184
|
+
"tags",
|
|
185
|
+
"log_level",
|
|
186
|
+
]);
|
|
187
|
+
function splitHotColdConfig(config) {
|
|
188
|
+
const hot = {};
|
|
189
|
+
const cold = {};
|
|
190
|
+
for (const [key, value] of Object.entries(config)) {
|
|
191
|
+
(HOT_INSTANCE_CONFIG_KEYS.has(key) ? hot : cold)[key] = value;
|
|
192
|
+
}
|
|
193
|
+
return { hot, cold };
|
|
194
|
+
}
|
|
195
|
+
function hotConfigUpdate(config) {
|
|
196
|
+
const update = {};
|
|
197
|
+
for (const key of HOT_INSTANCE_CONFIG_KEYS)
|
|
198
|
+
update[key] = config[key] ?? null;
|
|
199
|
+
return update;
|
|
200
|
+
}
|
|
163
201
|
/**
|
|
164
202
|
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
165
203
|
*
|
|
@@ -176,9 +214,18 @@ const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
|
|
|
176
214
|
const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
|
|
177
215
|
const HANG_CALLBACK_PREFIX = "hang:";
|
|
178
216
|
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
217
|
+
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
218
|
+
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
219
|
+
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
220
|
+
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
221
|
+
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
222
|
+
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
223
|
+
const LOGIN_CONFIRM_CALLBACK_PREFIX = "login-confirm:";
|
|
224
|
+
const INSTALL_LOGIN_CALLBACK_PREFIX = "install-login:";
|
|
179
225
|
const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
|
|
180
226
|
/** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
|
|
181
227
|
const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
|
|
228
|
+
const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
|
|
182
229
|
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
183
230
|
export class FleetManager {
|
|
184
231
|
dataDir;
|
|
@@ -186,6 +233,10 @@ export class FleetManager {
|
|
|
186
233
|
static sighupHandlerInstalled = false;
|
|
187
234
|
children = new Map();
|
|
188
235
|
lifecycle;
|
|
236
|
+
stormWindow;
|
|
237
|
+
spawnGate;
|
|
238
|
+
/** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
|
|
239
|
+
backendOutage = new BackendOutageTracker();
|
|
189
240
|
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
190
241
|
get daemons() { return this.lifecycle.daemons; }
|
|
191
242
|
fleetConfig = null;
|
|
@@ -225,6 +276,7 @@ export class FleetManager {
|
|
|
225
276
|
costGuard = null;
|
|
226
277
|
statuslineWatcher;
|
|
227
278
|
dailySummary = null;
|
|
279
|
+
dailyTipScheduler = null;
|
|
228
280
|
webhookEmitter = null;
|
|
229
281
|
// Topic icon + auto-archive state
|
|
230
282
|
topicIcons = {};
|
|
@@ -237,6 +289,13 @@ export class FleetManager {
|
|
|
237
289
|
warmCapEvicting = new Set();
|
|
238
290
|
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
239
291
|
idleGatedDeliveryTails = new Map();
|
|
292
|
+
/**
|
|
293
|
+
* Per-instance cancellation epoch for deliveries which have not reached the
|
|
294
|
+
* daemon yet. A cancel advances the epoch; queued work captures the old value
|
|
295
|
+
* and becomes a no-op, while messages arriving after the click capture the new
|
|
296
|
+
* value and remain deliverable.
|
|
297
|
+
*/
|
|
298
|
+
deliveryEpochs = new Map();
|
|
240
299
|
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
241
300
|
lastDeliveryAt = new Map();
|
|
242
301
|
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
@@ -253,10 +312,34 @@ export class FleetManager {
|
|
|
253
312
|
replyDeduper = new ReplyDeduper();
|
|
254
313
|
/** instanceName → what it is doing right now, when the backend can tell us. */
|
|
255
314
|
instanceActivity = new Map();
|
|
315
|
+
/** instanceName → this turn's tool list (multi-line), for the bubble. */
|
|
316
|
+
instanceProgress = new Map();
|
|
256
317
|
/** instanceName → tail of deliveries waiting for its IPC to come back. */
|
|
257
318
|
ipcWaitTails = new Map();
|
|
258
319
|
/** instanceName → restart currently executing; concurrent callers join it. */
|
|
259
320
|
restartsInFlight = new Map();
|
|
321
|
+
/**
|
|
322
|
+
* Delayed automatic retries for instances whose startup failed. Before this a
|
|
323
|
+
* failed start was logged once and the instance stayed `stopped` until an
|
|
324
|
+
* operator noticed (2026-09-03: 4 kiro instances, all victims of the same
|
|
325
|
+
* backend outage during a post-update herd). instanceName → pending retry.
|
|
326
|
+
*/
|
|
327
|
+
startupRetries = new Map();
|
|
328
|
+
/** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
|
|
329
|
+
explicitStopGeneration = new Map();
|
|
330
|
+
/** instanceName → outage hand-off currently executing; a repeat joins it. */
|
|
331
|
+
handOffsInFlight = new Map();
|
|
332
|
+
/** instanceName → explicit stop currently executing; the outage hand-off waits for it. */
|
|
333
|
+
stopsInFlight = new Map();
|
|
334
|
+
/** Aggregation window for the "N instances failed to start" notice. */
|
|
335
|
+
startupRetryNotices = new Map();
|
|
336
|
+
/** Backoff between automatic startup retries; the last step repeats while the backend is down. */
|
|
337
|
+
static STARTUP_RETRY_BACKOFF_MS = [60_000, 5 * 60_000, 15 * 60_000];
|
|
338
|
+
/** Hard cap on automatic startup retries (3 backoff steps + up to 3 more during a backend outage). */
|
|
339
|
+
static STARTUP_RETRY_MAX_ATTEMPTS = 6;
|
|
340
|
+
/** Re-check interval when a retry is due but the tmux storm window still blocks spawns. */
|
|
341
|
+
static STARTUP_RETRY_STORM_DEFER_MS = 60_000;
|
|
342
|
+
static STARTUP_RETRY_NOTICE_AGGREGATE_MS = 1_000;
|
|
260
343
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
261
344
|
lastInboundMsg = new Map();
|
|
262
345
|
topicArchiver;
|
|
@@ -293,12 +376,14 @@ export class FleetManager {
|
|
|
293
376
|
updateProgressTimer = null;
|
|
294
377
|
updateProgressEditRunning = false;
|
|
295
378
|
lastUpdateProgressText = null;
|
|
379
|
+
updateCompletionTipText = null;
|
|
296
380
|
eventLogPruneTimer = null;
|
|
297
381
|
logRotateTimer = null;
|
|
298
382
|
/** Days of event/activity history to keep. */
|
|
299
383
|
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
300
384
|
watchdogTimer = null;
|
|
301
385
|
startedAt = 0;
|
|
386
|
+
stormOpenNotifyTimer = null;
|
|
302
387
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
303
388
|
mirrorBuffer = [];
|
|
304
389
|
mirrorTimer = null;
|
|
@@ -314,11 +399,63 @@ export class FleetManager {
|
|
|
314
399
|
process.on("SIGHUP", () => FleetManager.signalTarget?.handleSighup());
|
|
315
400
|
FleetManager.sighupHandlerInstalled = true;
|
|
316
401
|
}
|
|
402
|
+
this.stormWindow = new StormWindow();
|
|
403
|
+
this.spawnGate = new SpawnGate({
|
|
404
|
+
storm: this.stormWindow,
|
|
405
|
+
concurrency: () => this.spawnConcurrency(),
|
|
406
|
+
staggerMs: () => this.fleetConfig?.defaults?.startup?.stagger_delay_ms ?? 500,
|
|
407
|
+
});
|
|
408
|
+
this.bindStormWindowEvents();
|
|
317
409
|
this.lifecycle = new InstanceLifecycle(this);
|
|
318
410
|
this.topicCommands = new TopicCommands(this);
|
|
319
411
|
this.topicArchiver = new TopicArchiver(this);
|
|
320
412
|
this.statuslineWatcher = new StatuslineWatcher(this);
|
|
321
413
|
}
|
|
414
|
+
spawnConcurrency() {
|
|
415
|
+
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
416
|
+
if (explicit != null)
|
|
417
|
+
return Math.max(1, Math.min(20, explicit));
|
|
418
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
419
|
+
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
420
|
+
}
|
|
421
|
+
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
422
|
+
bindStormWindowEvents() {
|
|
423
|
+
this.stormWindow.on("opened", (snapshot) => {
|
|
424
|
+
for (const [name, daemon] of this.daemons) {
|
|
425
|
+
if (!daemon.isPaused)
|
|
426
|
+
this.stormWindow.addAffected(name);
|
|
427
|
+
}
|
|
428
|
+
this.logger.error({ ...snapshot }, "tmux server storm opened — holding respawns and delivery");
|
|
429
|
+
if (this.stormOpenNotifyTimer)
|
|
430
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
431
|
+
this.stormOpenNotifyTimer = setTimeout(() => {
|
|
432
|
+
this.stormOpenNotifyTimer = null;
|
|
433
|
+
const current = this.stormWindow.snapshot();
|
|
434
|
+
this.notifyFleetError(t("storm.opened", current.affected.length, this.formatStormDelay(current.backoffMs)));
|
|
435
|
+
}, 1_000);
|
|
436
|
+
this.stormOpenNotifyTimer.unref?.();
|
|
437
|
+
});
|
|
438
|
+
this.stormWindow.on("extended", (snapshot) => {
|
|
439
|
+
this.logger.error({ ...snapshot }, "tmux server storm repeated — backoff extended");
|
|
440
|
+
this.notifyFleetError(t("storm.extended", snapshot.crashCount, this.formatStormDelay(snapshot.backoffMs)));
|
|
441
|
+
});
|
|
442
|
+
this.stormWindow.on("recovery_due", (snapshot) => {
|
|
443
|
+
this.logger.warn({ ...snapshot }, "tmux storm backoff elapsed — rolling recovery started");
|
|
444
|
+
});
|
|
445
|
+
this.stormWindow.on("closed", (snapshot, reason) => {
|
|
446
|
+
const unresolved = snapshot.affected.filter(name => !snapshot.recovered.includes(name));
|
|
447
|
+
this.logger.info({ ...snapshot, reason, unresolved }, "tmux server storm closed");
|
|
448
|
+
this.notifyFleetError(t(reason === "timeout" ? "storm.timed_out" : "storm.recovered", snapshot.recovered.length, snapshot.affected.length, unresolved.length > 0 ? unresolved.join(", ") : t("storm.none")));
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
formatStormDelay(ms) {
|
|
452
|
+
if (ms >= 60_000)
|
|
453
|
+
return `${Math.round(ms / 60_000)}m`;
|
|
454
|
+
return `${Math.round(ms / 1_000)}s`;
|
|
455
|
+
}
|
|
456
|
+
stormSuppressed(kind) {
|
|
457
|
+
return this.stormWindow.shouldSuppress(kind);
|
|
458
|
+
}
|
|
322
459
|
handleSighup() {
|
|
323
460
|
this.logger.info("Received SIGHUP, hot-reloading config...");
|
|
324
461
|
if (!this.startupComplete) {
|
|
@@ -341,7 +478,7 @@ export class FleetManager {
|
|
|
341
478
|
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
342
479
|
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
343
480
|
const message = err instanceof Error ? err.message : String(err);
|
|
344
|
-
this.notifyFleetError(
|
|
481
|
+
this.notifyFleetError(t("fleet.reload_failed", message));
|
|
345
482
|
})
|
|
346
483
|
.finally(() => {
|
|
347
484
|
this.reconcileInFlight = null;
|
|
@@ -430,7 +567,8 @@ export class FleetManager {
|
|
|
430
567
|
getSysInfo() {
|
|
431
568
|
const mem = process.memoryUsage();
|
|
432
569
|
const toMB = (b) => Math.round(b / 1024 / 1024 * 10) / 10;
|
|
433
|
-
|
|
570
|
+
// Fleet instances (fleet.yaml)
|
|
571
|
+
const fleetInstances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
434
572
|
name,
|
|
435
573
|
status: this.getInstanceStatus(name),
|
|
436
574
|
state: this.getInstanceExecutionState(name),
|
|
@@ -438,12 +576,46 @@ export class FleetManager {
|
|
|
438
576
|
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
439
577
|
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
440
578
|
}));
|
|
579
|
+
// Classic instances (classicBot.yaml) — dedupe against fleet
|
|
580
|
+
const fleetNames = new Set(fleetInstances.map(i => i.name));
|
|
581
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
582
|
+
.filter(ch => !fleetNames.has(ch.instanceName))
|
|
583
|
+
.map(ch => ({
|
|
584
|
+
name: ch.instanceName,
|
|
585
|
+
status: this.getInstanceStatus(ch.instanceName),
|
|
586
|
+
state: this.getInstanceExecutionState(ch.instanceName),
|
|
587
|
+
ipc: this.instanceIpcClients.has(ch.instanceName),
|
|
588
|
+
costCents: this.costGuard?.getDailyCostCents(ch.instanceName) ?? 0,
|
|
589
|
+
rateLimits: this.statuslineWatcher.getRateLimits(ch.instanceName) ?? null,
|
|
590
|
+
}));
|
|
591
|
+
// Combined roster (matches /api/fleet and agend ls)
|
|
592
|
+
const allInstances = [...fleetInstances, ...classicInstances];
|
|
593
|
+
// Fleet summary counts (fleet + Classic combined)
|
|
594
|
+
const running_count = allInstances.filter(i => i.status === "running").length;
|
|
595
|
+
const paused_count = allInstances.filter(i => i.status === "paused").length;
|
|
596
|
+
// System memory (GB, 1 decimal)
|
|
597
|
+
const totalGB = totalmem() / (1024 ** 3);
|
|
598
|
+
const usedGB = (totalmem() - freemem()) / (1024 ** 3);
|
|
599
|
+
const system_mem_gb = {
|
|
600
|
+
used: Math.round(usedGB * 10) / 10,
|
|
601
|
+
total: Math.round(totalGB * 10) / 10,
|
|
602
|
+
};
|
|
603
|
+
// Fleet memory: O(1) cgroup read (includes entire service tree: fleet + CLIs + MCP servers)
|
|
604
|
+
// This avoids blocking the event loop with per-instance tree scans.
|
|
605
|
+
const fleetMem = readFleetMemory();
|
|
606
|
+
const fleet_mem_mb = fleetMem.cgroupAnonBytes != null
|
|
607
|
+
? Math.round(fleetMem.cgroupAnonBytes / (1024 * 1024) * 10) / 10
|
|
608
|
+
: null;
|
|
441
609
|
return {
|
|
442
610
|
uptime_seconds: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
443
611
|
memory_mb: { rss: toMB(mem.rss), heapUsed: toMB(mem.heapUsed), heapTotal: toMB(mem.heapTotal) },
|
|
444
|
-
instances,
|
|
612
|
+
instances: fleetInstances, // SysInfo.instances is fleet-only; /api/fleet enriches with Classic
|
|
445
613
|
fleet_cost_cents: this.costGuard?.getFleetTotalCents() ?? 0,
|
|
446
614
|
fleet_cost_limit_cents: this.costGuard?.getLimitCents() ?? 0,
|
|
615
|
+
running_count,
|
|
616
|
+
paused_count,
|
|
617
|
+
fleet_mem_mb,
|
|
618
|
+
system_mem_gb,
|
|
447
619
|
};
|
|
448
620
|
}
|
|
449
621
|
/** Load fleet.yaml and build routing table */
|
|
@@ -591,6 +763,57 @@ export class FleetManager {
|
|
|
591
763
|
child.once("error", err => this.failUpdateProgress(err.message));
|
|
592
764
|
child.unref();
|
|
593
765
|
}
|
|
766
|
+
async handleTipsSlash(data, adapterId) {
|
|
767
|
+
if (!this.fleetConfig)
|
|
768
|
+
return;
|
|
769
|
+
const mode = typeof data.options?.mode === "string" ? data.options.mode : "";
|
|
770
|
+
if (mode === "advanced on") {
|
|
771
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
772
|
+
await data.respond(t("permission.denied"));
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
await data.respond(this.unlockAdvancedTips(data.userId)
|
|
776
|
+
? t("tips.advanced.unlocked")
|
|
777
|
+
: t("tips.unavailable"));
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
if (mode === "on" || mode === "off") {
|
|
781
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
782
|
+
await data.respond(t("permission.denied"));
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
this.fleetConfig.defaults.tips = mode === "on";
|
|
786
|
+
this.saveFleetConfig();
|
|
787
|
+
await data.respond(t(mode === "on" ? "tips.enabled" : "tips.disabled"));
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
791
|
+
if (!adapter) {
|
|
792
|
+
await data.respond(t("tips.unavailable"));
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const targetName = this.resolveSlashTarget(data.channelId, adapterId)
|
|
796
|
+
?? this.findGeneralInstance(adapterId)
|
|
797
|
+
?? "general";
|
|
798
|
+
// Discord slash interactions are deferred ephemerally, but the Tip itself
|
|
799
|
+
// belongs in the channel where /tips was invoked. Post there directly,
|
|
800
|
+
// then delete the empty acknowledgement instead of leaving a slash corpse.
|
|
801
|
+
//
|
|
802
|
+
// The callback binding uses (chatId, threadId) that match what the adapter
|
|
803
|
+
// emits on callback_query. Discord emits (guildId, channelId); Telegram
|
|
804
|
+
// emits (supergroup chatId, topic threadId). For /tips invoked in an
|
|
805
|
+
// arbitrary channel, pass the canonical group id as chatId and the
|
|
806
|
+
// invocation channel as threadId, exactly as sendTipToGeneral() does.
|
|
807
|
+
const chatId = this.getGroupIdForInstance(targetName) || data.channelId;
|
|
808
|
+
const result = await this.promptTip(targetName, adapter, chatId, data.channelId);
|
|
809
|
+
if (result === "posted" && data.dismissResponse) {
|
|
810
|
+
await data.dismissResponse();
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
await data.respond(t(result === "posted"
|
|
814
|
+
? "tips.posted"
|
|
815
|
+
: result === "empty" ? "tips.empty" : "tips.unavailable"));
|
|
816
|
+
}
|
|
594
817
|
/** Admin-only full conversation reset for fleet-topic and Classic instances. */
|
|
595
818
|
async handleClearSlash(data, adapterId) {
|
|
596
819
|
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
@@ -743,6 +966,39 @@ export class FleetManager {
|
|
|
743
966
|
return null;
|
|
744
967
|
return this.instanceStateCache.get(name)?.state ?? null;
|
|
745
968
|
}
|
|
969
|
+
/**
|
|
970
|
+
* Subscription providers currently relevant to this fleet. Stopped/crashed
|
|
971
|
+
* rows do not keep a usage card alive, while persisted paused instances do.
|
|
972
|
+
* Classic channels use their effective backend merge chain, not the raw row.
|
|
973
|
+
*/
|
|
974
|
+
getActiveUsageProviderIds() {
|
|
975
|
+
const providers = new Set();
|
|
976
|
+
for (const backend of this.getActiveBackendIds()) {
|
|
977
|
+
const provider = usageProviderIdForBackend(backend);
|
|
978
|
+
if (provider)
|
|
979
|
+
providers.add(provider);
|
|
980
|
+
}
|
|
981
|
+
return providers;
|
|
982
|
+
}
|
|
983
|
+
/** Effective backends with a running or persisted-paused fleet/Classic instance. */
|
|
984
|
+
getActiveBackendIds() {
|
|
985
|
+
const backends = new Set();
|
|
986
|
+
const add = (name, backend) => {
|
|
987
|
+
const status = this.getInstanceStatus(name);
|
|
988
|
+
if (status !== "running" && status !== "paused")
|
|
989
|
+
return;
|
|
990
|
+
if (backend)
|
|
991
|
+
backends.add(backend);
|
|
992
|
+
};
|
|
993
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
994
|
+
// loadFleetConfig() has already merged the fleet default into each row.
|
|
995
|
+
add(name, config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
996
|
+
}
|
|
997
|
+
for (const channel of this.classicChannels?.getAll() ?? []) {
|
|
998
|
+
add(channel.instanceName, this.classicChannels?.getBackendByInstance(channel.instanceName, this.fleetConfig?.defaults?.backend));
|
|
999
|
+
}
|
|
1000
|
+
return backends;
|
|
1001
|
+
}
|
|
746
1002
|
isClassicInstance(name) {
|
|
747
1003
|
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
748
1004
|
}
|
|
@@ -855,7 +1111,7 @@ export class FleetManager {
|
|
|
855
1111
|
.finally(() => this.warmCapEvicting.delete(victim));
|
|
856
1112
|
}
|
|
857
1113
|
}
|
|
858
|
-
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
1114
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0, cancelled) {
|
|
859
1115
|
const isReady = () => {
|
|
860
1116
|
const snapshot = this.instanceStateCache.get(instanceName);
|
|
861
1117
|
return snapshot?.state === "idle"
|
|
@@ -877,8 +1133,12 @@ export class FleetManager {
|
|
|
877
1133
|
this.instanceIdleWaiters.delete(instanceName);
|
|
878
1134
|
resolve(idle);
|
|
879
1135
|
};
|
|
880
|
-
const check = () => {
|
|
881
|
-
|
|
1136
|
+
const check = () => {
|
|
1137
|
+
if (cancelled?.())
|
|
1138
|
+
finish(false);
|
|
1139
|
+
else if (isReady())
|
|
1140
|
+
finish(true);
|
|
1141
|
+
};
|
|
882
1142
|
const query = () => {
|
|
883
1143
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
884
1144
|
if (ipc?.connected) {
|
|
@@ -894,24 +1154,90 @@ export class FleetManager {
|
|
|
894
1154
|
query();
|
|
895
1155
|
});
|
|
896
1156
|
}
|
|
897
|
-
|
|
1157
|
+
/**
|
|
1158
|
+
* Ask the daemon to capture the pane before answering, then wait for any state
|
|
1159
|
+
* report produced after this request. The ordinary query is intentionally
|
|
1160
|
+
* cache-only (idle gates call it frequently); this opt-in refresh is reserved
|
|
1161
|
+
* for lifecycle decisions where a stale state would strand UI.
|
|
1162
|
+
*/
|
|
1163
|
+
refreshInstanceExecutionState(instanceName, timeoutMs) {
|
|
1164
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1165
|
+
if (!ipc?.connected)
|
|
1166
|
+
return Promise.resolve(false);
|
|
1167
|
+
const previous = this.instanceStateCache.get(instanceName);
|
|
1168
|
+
return new Promise(resolve => {
|
|
1169
|
+
let settled = false;
|
|
1170
|
+
const finish = (refreshed) => {
|
|
1171
|
+
if (settled)
|
|
1172
|
+
return;
|
|
1173
|
+
settled = true;
|
|
1174
|
+
clearTimeout(timeout);
|
|
1175
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
1176
|
+
waiters?.delete(check);
|
|
1177
|
+
if (waiters?.size === 0)
|
|
1178
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
1179
|
+
resolve(refreshed);
|
|
1180
|
+
};
|
|
1181
|
+
const check = () => {
|
|
1182
|
+
if (this.instanceStateCache.get(instanceName) !== previous)
|
|
1183
|
+
finish(true);
|
|
1184
|
+
};
|
|
1185
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
1186
|
+
waiters.add(check);
|
|
1187
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
1188
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
1189
|
+
timeout.unref?.();
|
|
1190
|
+
const sent = ipc.send({
|
|
1191
|
+
type: "query_instance_state",
|
|
1192
|
+
requestId: `reply-grace-${Date.now()}`,
|
|
1193
|
+
refresh: true,
|
|
1194
|
+
});
|
|
1195
|
+
if (!sent)
|
|
1196
|
+
finish(false);
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
|
|
1200
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1201
|
+
return;
|
|
1202
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1203
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1204
|
+
return;
|
|
898
1205
|
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
899
1206
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
900
1207
|
const wakeStartedAt = Date.now();
|
|
901
1208
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1209
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1210
|
+
return;
|
|
902
1211
|
// Waking added one to the warm count — make room by evicting a different
|
|
903
1212
|
// LRU idle instance (never this one; it's about to work).
|
|
904
1213
|
this.enforceWarmCap(instanceName);
|
|
905
1214
|
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
906
1215
|
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
907
1216
|
}
|
|
908
|
-
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
1217
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
|
|
1218
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1219
|
+
this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
// A server crash can land while waitForInstanceIdle is pending. Re-check
|
|
1223
|
+
// immediately before the old timeout path would force text into a boot UI.
|
|
1224
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1225
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1226
|
+
return;
|
|
909
1227
|
if (!idle) {
|
|
910
1228
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
911
1229
|
}
|
|
912
|
-
await this.sendWhenConnected(instanceName, payload);
|
|
1230
|
+
await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
|
|
1231
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1232
|
+
return;
|
|
913
1233
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
914
1234
|
}
|
|
1235
|
+
async holdDeliveryForStorm(instanceName, deliveryEpoch) {
|
|
1236
|
+
if (!this.stormWindow.isDeliveryHeld(instanceName))
|
|
1237
|
+
return;
|
|
1238
|
+
this.logger.warn({ instanceName, deliveryEpoch }, "Delivery held during tmux server recovery");
|
|
1239
|
+
await this.stormWindow.waitForDeliveryAllowed(instanceName);
|
|
1240
|
+
}
|
|
915
1241
|
/**
|
|
916
1242
|
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
917
1243
|
*
|
|
@@ -928,7 +1254,9 @@ export class FleetManager {
|
|
|
928
1254
|
* instance, *including* when the socket happens to be up: otherwise a message
|
|
929
1255
|
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
930
1256
|
*/
|
|
931
|
-
async sendWhenConnected(instanceName, payload) {
|
|
1257
|
+
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
1258
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1259
|
+
return;
|
|
932
1260
|
const queued = this.ipcWaitTails.get(instanceName);
|
|
933
1261
|
if (!queued) {
|
|
934
1262
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -937,7 +1265,11 @@ export class FleetManager {
|
|
|
937
1265
|
}
|
|
938
1266
|
const attempt = (queued ?? Promise.resolve())
|
|
939
1267
|
.catch(() => { })
|
|
940
|
-
.then(() =>
|
|
1268
|
+
.then(() => {
|
|
1269
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1270
|
+
return;
|
|
1271
|
+
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
|
|
1272
|
+
});
|
|
941
1273
|
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
942
1274
|
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
943
1275
|
const tail = attempt.catch(() => { });
|
|
@@ -954,10 +1286,12 @@ export class FleetManager {
|
|
|
954
1286
|
}
|
|
955
1287
|
}
|
|
956
1288
|
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
957
|
-
async sendAfterIpcReturns(instanceName, payload) {
|
|
1289
|
+
async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
958
1290
|
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
959
1291
|
let warned = false;
|
|
960
1292
|
for (;;) {
|
|
1293
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1294
|
+
return;
|
|
961
1295
|
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
962
1296
|
// so a cached reference would stay dead forever.
|
|
963
1297
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -973,8 +1307,14 @@ export class FleetManager {
|
|
|
973
1307
|
await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
|
|
974
1308
|
}
|
|
975
1309
|
}
|
|
1310
|
+
/** Whether this target already has an ordinary non-user delivery in its FIFO. */
|
|
1311
|
+
hasPendingIdleGatedDelivery(instanceName) {
|
|
1312
|
+
return this.idleGatedDeliveryTails.has(instanceName);
|
|
1313
|
+
}
|
|
976
1314
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
977
1315
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
1316
|
+
const deliveryEpoch = this.getDeliveryEpoch(instanceName);
|
|
1317
|
+
const deliveryPayload = { ...payload, delivery_epoch: deliveryEpoch };
|
|
978
1318
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
979
1319
|
? payload.meta
|
|
980
1320
|
: undefined;
|
|
@@ -986,16 +1326,20 @@ export class FleetManager {
|
|
|
986
1326
|
if (!waitForIdle) {
|
|
987
1327
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
988
1328
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1329
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1330
|
+
return;
|
|
989
1331
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
990
1332
|
}
|
|
991
|
-
await this.sendWhenConnected(instanceName,
|
|
1333
|
+
await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
|
|
1334
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1335
|
+
return;
|
|
992
1336
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
993
1337
|
// working must not trust the stale idle snapshot from before the send.
|
|
994
1338
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
995
1339
|
return;
|
|
996
1340
|
}
|
|
997
1341
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
998
|
-
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName,
|
|
1342
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
|
|
999
1343
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
1000
1344
|
try {
|
|
1001
1345
|
await delivery;
|
|
@@ -1029,6 +1373,7 @@ export class FleetManager {
|
|
|
1029
1373
|
throw new Error("Classic channel manager not initialized");
|
|
1030
1374
|
const wasRunning = this.daemons.has(instanceName);
|
|
1031
1375
|
this.classicChannels.reloadFromDisk();
|
|
1376
|
+
this.reportClassicUnrecoverableIds();
|
|
1032
1377
|
this.reregisterClassicChannels();
|
|
1033
1378
|
const channel = this.classicChannels.getAll().find(item => item.instanceName === instanceName);
|
|
1034
1379
|
if (!channel)
|
|
@@ -1046,6 +1391,9 @@ export class FleetManager {
|
|
|
1046
1391
|
* across a fleet restart.
|
|
1047
1392
|
*/
|
|
1048
1393
|
resumePaused = false) {
|
|
1394
|
+
// Any start supersedes a pending automatic retry (it would otherwise fire
|
|
1395
|
+
// into a running instance — harmless, but noisy — or race this start).
|
|
1396
|
+
this.cancelStartupRetry(name);
|
|
1049
1397
|
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
1050
1398
|
await this.lifecycle.wake(name, 30_000);
|
|
1051
1399
|
// A successful wake clears the persisted pause marker and produces a
|
|
@@ -1076,12 +1424,6 @@ export class FleetManager {
|
|
|
1076
1424
|
}
|
|
1077
1425
|
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
1078
1426
|
if (config.general_topic) {
|
|
1079
|
-
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
1080
|
-
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
1081
|
-
if (backend === "antigravity") {
|
|
1082
|
-
this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
|
|
1083
|
-
this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
|
|
1084
|
-
}
|
|
1085
1427
|
this.ensureGeneralInstructions(config.working_directory, backend, name);
|
|
1086
1428
|
}
|
|
1087
1429
|
else if (kind === "fleet-topic") {
|
|
@@ -1147,86 +1489,243 @@ export class FleetManager {
|
|
|
1147
1489
|
}
|
|
1148
1490
|
if (runnableEntries.length === 0)
|
|
1149
1491
|
return;
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1492
|
+
if (this.fleetConfig?.defaults?.startup?.concurrency == null) {
|
|
1493
|
+
this.logger.info({
|
|
1494
|
+
concurrency: this.spawnConcurrency(),
|
|
1495
|
+
freeMemMB: Math.round(freemem() / (1024 * 1024)),
|
|
1496
|
+
totalInstances: runnableEntries.length,
|
|
1497
|
+
}, "Adaptive startup concurrency");
|
|
1498
|
+
}
|
|
1499
|
+
await Promise.all(runnableEntries.map(([name, config]) => this.spawnGate.run({
|
|
1500
|
+
instanceName: name,
|
|
1501
|
+
workingDirectory: config.working_directory,
|
|
1502
|
+
reason: "startup",
|
|
1503
|
+
}, async () => {
|
|
1504
|
+
if (await this.startInstanceUnattended(name, config, topicMode, "instance"))
|
|
1505
|
+
onReady?.(name);
|
|
1506
|
+
})));
|
|
1507
|
+
}
|
|
1508
|
+
/**
|
|
1509
|
+
* Start from an UNATTENDED path (fleet startup, full restart, config
|
|
1510
|
+
* reconcile): nobody is watching the result, so a failure is logged and
|
|
1511
|
+
* handed to the delayed automatic retry instead of leaving the instance
|
|
1512
|
+
* `stopped` forever. Explicit operator/API starts call startInstance directly
|
|
1513
|
+
* and keep their synchronous error. Returns whether the instance is up.
|
|
1514
|
+
*/
|
|
1515
|
+
async startInstanceUnattended(name, config, topicMode, what) {
|
|
1516
|
+
try {
|
|
1517
|
+
await this.startInstance(name, config, topicMode);
|
|
1518
|
+
return this.daemons.has(name);
|
|
1160
1519
|
}
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1520
|
+
catch (err) {
|
|
1521
|
+
this.logger.error({ err, name }, `Failed to start ${what}`);
|
|
1522
|
+
this.scheduleStartupRetry(name, 0);
|
|
1523
|
+
return false;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
// ── Delayed automatic startup retries ──────────────────────────────────
|
|
1527
|
+
/**
|
|
1528
|
+
* Schedule attempt `attempt` (0-based) of the automatic startup retry for an
|
|
1529
|
+
* instance whose start just failed. Backoff 1 → 5 → 15 min; while the
|
|
1530
|
+
* instance's backend is known to be unreachable the 15-min step repeats up to
|
|
1531
|
+
* STARTUP_RETRY_MAX_ATTEMPTS, after which we give up with one notice. Each
|
|
1532
|
+
* retry re-checks the world (still configured, not running, not paused, no
|
|
1533
|
+
* tmux storm) and runs through the SpawnGate as "recovery" — so a herd of
|
|
1534
|
+
* failed instances comes back at the gate's concurrency, never all at once.
|
|
1535
|
+
*/
|
|
1536
|
+
scheduleStartupRetry(name, attempt) {
|
|
1537
|
+
if (this.shuttingDown)
|
|
1538
|
+
return;
|
|
1539
|
+
if (this.startupRetries.has(name))
|
|
1540
|
+
return;
|
|
1541
|
+
const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
|
|
1542
|
+
const outage = this.backendOutage.isActive(this.backendNameOf(name));
|
|
1543
|
+
const exhausted = attempt >= backoff.length && !(outage && attempt < FleetManager.STARTUP_RETRY_MAX_ATTEMPTS);
|
|
1544
|
+
if (attempt >= FleetManager.STARTUP_RETRY_MAX_ATTEMPTS || exhausted) {
|
|
1545
|
+
this.logger.error({ name, attempts: attempt }, "Giving up automatic startup retries");
|
|
1546
|
+
this.queueStartupRetryNotice("gave_up", name, 0);
|
|
1547
|
+
return;
|
|
1548
|
+
}
|
|
1549
|
+
const delayMs = backoff[Math.min(attempt, backoff.length - 1)];
|
|
1550
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, delayMs);
|
|
1551
|
+
timer.unref?.();
|
|
1552
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1553
|
+
this.logger.warn({ name, attempt: attempt + 1, delayMs, backendOutage: outage }, "Startup failed — automatic retry scheduled");
|
|
1554
|
+
if (attempt === 0)
|
|
1555
|
+
this.queueStartupRetryNotice("scheduled", name, delayMs);
|
|
1556
|
+
}
|
|
1557
|
+
/** Drop a pending automatic retry (an operator start/stop/restart supersedes it). */
|
|
1558
|
+
cancelStartupRetry(name) {
|
|
1559
|
+
const pending = this.startupRetries.get(name);
|
|
1560
|
+
if (!pending)
|
|
1561
|
+
return;
|
|
1562
|
+
clearTimeout(pending.timer);
|
|
1563
|
+
this.startupRetries.delete(name);
|
|
1564
|
+
this.logger.info({ name }, "Pending automatic startup retry cancelled");
|
|
1565
|
+
}
|
|
1566
|
+
/** Pending automatic retry, if any (status display / tests). */
|
|
1567
|
+
pendingStartupRetry(name) {
|
|
1568
|
+
const pending = this.startupRetries.get(name);
|
|
1569
|
+
return pending ? { attempt: pending.attempt } : null;
|
|
1570
|
+
}
|
|
1571
|
+
/**
|
|
1572
|
+
* A daemon's crash-respawn hit the backend outage (startup_backend_unreachable):
|
|
1573
|
+
* stop THAT daemon and schedule the delayed retry. Serialized against the
|
|
1574
|
+
* operator paths: an in-flight restart is awaited first (it replaces the
|
|
1575
|
+
* daemon itself), the stop is identity-checked so a fresh daemon registered
|
|
1576
|
+
* meanwhile is never deleted, and an explicit stop/restart that began during
|
|
1577
|
+
* the hand-off (generation bump) owns the outcome — no retry is scheduled
|
|
1578
|
+
* behind an operator's back.
|
|
1579
|
+
*/
|
|
1580
|
+
async handOffToStartupRetry(name, daemon) {
|
|
1581
|
+
const joined = this.handOffsInFlight.get(name);
|
|
1582
|
+
if (joined)
|
|
1583
|
+
return joined;
|
|
1584
|
+
const run = (async () => {
|
|
1585
|
+
// Let any operator restart/stop that is already executing finish first:
|
|
1586
|
+
// a restart replaces the daemon itself, a stop removes it — either way
|
|
1587
|
+
// the identity check below then sees "not ours any more" and we do
|
|
1588
|
+
// nothing. Loop: one operation can be chained behind another.
|
|
1589
|
+
for (;;) {
|
|
1590
|
+
const inFlight = this.restartsInFlight.get(name) ?? this.stopsInFlight.get(name);
|
|
1591
|
+
if (!inFlight)
|
|
1592
|
+
break;
|
|
1593
|
+
await inFlight.catch(() => { });
|
|
1594
|
+
}
|
|
1595
|
+
const stopGen = this.explicitStopGeneration.get(name) ?? 0;
|
|
1596
|
+
if (this.lifecycle.daemons.get(name) !== daemon)
|
|
1597
|
+
return; // already replaced or stopped
|
|
1598
|
+
await this.lifecycle.stopIfCurrent(name, daemon);
|
|
1599
|
+
if (this.shuttingDown)
|
|
1600
|
+
return;
|
|
1601
|
+
if ((this.explicitStopGeneration.get(name) ?? 0) !== stopGen) {
|
|
1602
|
+
this.logger.info({ name }, "Outage hand-off superseded by an explicit stop/restart — no automatic retry");
|
|
1181
1603
|
return;
|
|
1182
1604
|
}
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1605
|
+
if (this.lifecycle.daemons.has(name) || this.lifecycle.isPaused(name))
|
|
1606
|
+
return;
|
|
1607
|
+
this.scheduleStartupRetry(name, 0);
|
|
1608
|
+
})().finally(() => this.handOffsInFlight.delete(name));
|
|
1609
|
+
this.handOffsInFlight.set(name, run);
|
|
1610
|
+
return run;
|
|
1611
|
+
}
|
|
1612
|
+
async runStartupRetry(name, attempt) {
|
|
1613
|
+
this.startupRetries.delete(name);
|
|
1614
|
+
if (this.shuttingDown)
|
|
1615
|
+
return;
|
|
1616
|
+
if (!this.isConfiguredInstance(name))
|
|
1617
|
+
return; // removed from fleet.yaml / classic channel meanwhile
|
|
1618
|
+
if (this.lifecycle.isPaused(name) || this.daemons.has(name))
|
|
1619
|
+
return; // operator acted meanwhile
|
|
1620
|
+
if (this.stormWindow.isSpawnBlocked()) {
|
|
1621
|
+
// The tmux server is being restarted; joining that herd is what we are
|
|
1622
|
+
// trying to avoid. Same attempt again after the storm's own recovery.
|
|
1623
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, FleetManager.STARTUP_RETRY_STORM_DEFER_MS);
|
|
1624
|
+
timer.unref?.();
|
|
1625
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1626
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Startup retry deferred — tmux storm window is blocking spawns");
|
|
1627
|
+
return;
|
|
1628
|
+
}
|
|
1629
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
1630
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
1631
|
+
try {
|
|
1632
|
+
await this.spawnGate.run({
|
|
1633
|
+
instanceName: name,
|
|
1634
|
+
workingDirectory: this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name),
|
|
1635
|
+
reason: "recovery",
|
|
1636
|
+
}, () => this.startConfiguredInstance(name, topicMode));
|
|
1637
|
+
if (this.daemons.has(name)) {
|
|
1638
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Automatic startup retry succeeded");
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
catch (err) {
|
|
1642
|
+
this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
|
|
1643
|
+
this.scheduleStartupRetry(name, attempt + 1);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
/** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
|
|
1647
|
+
isConfiguredInstance(name) {
|
|
1648
|
+
if (this.fleetConfig?.instances[name])
|
|
1649
|
+
return true;
|
|
1650
|
+
return !!this.classicChannels?.getAll().some(channel => channel.instanceName === name);
|
|
1651
|
+
}
|
|
1652
|
+
/**
|
|
1653
|
+
* Kind-aware start for the automatic retry: fleet-topic instances come from
|
|
1654
|
+
* fleet.yaml, ClassicBot instances exist only in the classic channel manager
|
|
1655
|
+
* and must be rebuilt through startClassicInstance (a fleet.yaml lookup alone
|
|
1656
|
+
* silently dropped them — the retry timer fired and nothing happened).
|
|
1657
|
+
*/
|
|
1658
|
+
async startConfiguredInstance(name, topicMode) {
|
|
1659
|
+
const config = this.fleetConfig?.instances[name];
|
|
1660
|
+
if (config) {
|
|
1661
|
+
await this.startInstance(name, config, topicMode);
|
|
1662
|
+
return;
|
|
1663
|
+
}
|
|
1664
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
1665
|
+
if (!channel || !this.classicChannels)
|
|
1666
|
+
throw new Error(`Instance '${name}' is no longer configured`);
|
|
1667
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1668
|
+
}
|
|
1669
|
+
/**
|
|
1670
|
+
* Unattended ClassicBot start (fleet startup batch, full-restart batch, the
|
|
1671
|
+
* classicBot.yaml reconcile): same contract as startInstanceUnattended —
|
|
1672
|
+
* failures are logged and handed to the delayed automatic retry, whose
|
|
1673
|
+
* kind-aware startConfiguredInstance rebuilds the Classic instance. Returns
|
|
1674
|
+
* whether the instance is up.
|
|
1675
|
+
*/
|
|
1676
|
+
async startClassicInstanceUnattended(ch, what) {
|
|
1677
|
+
try {
|
|
1678
|
+
await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1679
|
+
return this.daemons.has(ch.instanceName);
|
|
1680
|
+
}
|
|
1681
|
+
catch (err) {
|
|
1682
|
+
this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
|
|
1683
|
+
this.scheduleStartupRetry(ch.instanceName, 0);
|
|
1684
|
+
return false;
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
backendNameOf(name) {
|
|
1688
|
+
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1689
|
+
const configured = this.fleetConfig?.instances[name]?.backend;
|
|
1690
|
+
if (configured)
|
|
1691
|
+
return configured;
|
|
1692
|
+
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1693
|
+
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1694
|
+
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1695
|
+
}
|
|
1696
|
+
return fleetDefault ?? "claude-code";
|
|
1697
|
+
}
|
|
1698
|
+
/**
|
|
1699
|
+
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1700
|
+
* fails many instances within the same second. Two notices per incident at
|
|
1701
|
+
* most — "N failed, retrying in X" and, if it comes to that, "gave up on N".
|
|
1702
|
+
*/
|
|
1703
|
+
queueStartupRetryNotice(kind, name, delayMs) {
|
|
1704
|
+
const pending = this.startupRetryNotices.get(kind);
|
|
1705
|
+
if (pending) {
|
|
1706
|
+
if (!pending.names.includes(name))
|
|
1707
|
+
pending.names.push(name);
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
const timer = setTimeout(() => {
|
|
1711
|
+
const entry = this.startupRetryNotices.get(kind);
|
|
1712
|
+
this.startupRetryNotices.delete(kind);
|
|
1713
|
+
if (!entry)
|
|
1714
|
+
return;
|
|
1715
|
+
const list = entry.names.join(", ");
|
|
1716
|
+
if (kind === "scheduled") {
|
|
1717
|
+
let text = t("fleet.startup_retry_scheduled", entry.names.length, list, this.formatStormDelay(entry.delayMs));
|
|
1718
|
+
const downBackends = [...new Set(entry.names.map(n => this.backendNameOf(n)))].filter(b => this.backendOutage.isActive(b));
|
|
1719
|
+
if (downBackends.length)
|
|
1720
|
+
text += `\n${t("fleet.startup_retry_outage", downBackends.join(", "))}`;
|
|
1721
|
+
this.notifyFleetError(text);
|
|
1722
|
+
}
|
|
1723
|
+
else {
|
|
1724
|
+
this.notifyFleetError(t("fleet.startup_retry_gave_up", entry.names.length, list));
|
|
1725
|
+
}
|
|
1726
|
+
}, FleetManager.STARTUP_RETRY_NOTICE_AGGREGATE_MS);
|
|
1727
|
+
timer.unref?.();
|
|
1728
|
+
this.startupRetryNotices.set(kind, { names: [name], delayMs, timer });
|
|
1230
1729
|
}
|
|
1231
1730
|
runnableStartupCount(fleet, includeClassic) {
|
|
1232
1731
|
const names = this.configuredStartupInstanceNames(fleet, includeClassic);
|
|
@@ -1261,6 +1760,8 @@ export class FleetManager {
|
|
|
1261
1760
|
};
|
|
1262
1761
|
}
|
|
1263
1762
|
async stopInstance(name) {
|
|
1763
|
+
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
1764
|
+
this.cancelStartupRetry(name);
|
|
1264
1765
|
this.failoverActive.delete(name);
|
|
1265
1766
|
this.cancelIdleButtonRetirement(name);
|
|
1266
1767
|
this.instanceStateCache.delete(name);
|
|
@@ -1272,12 +1773,20 @@ export class FleetManager {
|
|
|
1272
1773
|
// interactive-prompt / clean-exit events whose handlers post fresh prompts
|
|
1273
1774
|
// while the stop is still awaiting (the TOCTOU sol's review called out).
|
|
1274
1775
|
this.clearNoncePromptsForInstance(name);
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1776
|
+
// Published so the outage hand-off can wait for an explicit stop that began
|
|
1777
|
+
// BEFORE it (the generation fence alone only catches stops that begin after
|
|
1778
|
+
// the hand-off snapshotted it).
|
|
1779
|
+
const run = (async () => {
|
|
1780
|
+
try {
|
|
1781
|
+
await this.lifecycle.stop(name);
|
|
1782
|
+
}
|
|
1783
|
+
finally {
|
|
1784
|
+
this.clearNoncePromptsForInstance(name);
|
|
1785
|
+
}
|
|
1786
|
+
})().finally(() => { if (this.stopsInFlight.get(name) === run)
|
|
1787
|
+
this.stopsInFlight.delete(name); });
|
|
1788
|
+
this.stopsInFlight.set(name, run);
|
|
1789
|
+
return run;
|
|
1281
1790
|
}
|
|
1282
1791
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
1283
1792
|
async restartSingleInstance(name, opts) {
|
|
@@ -1290,7 +1799,12 @@ export class FleetManager {
|
|
|
1290
1799
|
this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
|
|
1291
1800
|
return inFlight;
|
|
1292
1801
|
}
|
|
1293
|
-
const
|
|
1802
|
+
const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
|
|
1803
|
+
const run = this.spawnGate.run({
|
|
1804
|
+
instanceName: name,
|
|
1805
|
+
workingDirectory,
|
|
1806
|
+
reason: "restart",
|
|
1807
|
+
}, () => this.doRestartSingleInstance(name, opts))
|
|
1294
1808
|
.finally(() => this.restartsInFlight.delete(name));
|
|
1295
1809
|
this.restartsInFlight.set(name, run);
|
|
1296
1810
|
return run;
|
|
@@ -1308,6 +1822,7 @@ export class FleetManager {
|
|
|
1308
1822
|
this.writeFreshStartMarker(name);
|
|
1309
1823
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
1310
1824
|
await this.startInstance(name, config, topicMode ?? false);
|
|
1825
|
+
this.stormWindow.markRecovered(name);
|
|
1311
1826
|
return;
|
|
1312
1827
|
}
|
|
1313
1828
|
// Classic instance fallback
|
|
@@ -1320,6 +1835,7 @@ export class FleetManager {
|
|
|
1320
1835
|
if (opts?.freshStart)
|
|
1321
1836
|
this.writeFreshStartMarker(name);
|
|
1322
1837
|
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1838
|
+
this.stormWindow.markRecovered(name);
|
|
1323
1839
|
return;
|
|
1324
1840
|
}
|
|
1325
1841
|
throw new Error(`Instance not found: ${name}`);
|
|
@@ -1385,6 +1901,7 @@ export class FleetManager {
|
|
|
1385
1901
|
messageId,
|
|
1386
1902
|
});
|
|
1387
1903
|
this.lastUpdateProgressText = null;
|
|
1904
|
+
this.updateCompletionTipText = null;
|
|
1388
1905
|
this.startUpdateProgressMonitor(adapter);
|
|
1389
1906
|
}
|
|
1390
1907
|
failUpdateProgress(message) {
|
|
@@ -1408,7 +1925,17 @@ export class FleetManager {
|
|
|
1408
1925
|
const adapter = this.adapters.get(target.adapterId) ?? (initialAdapter?.id === target.adapterId ? initialAdapter : undefined);
|
|
1409
1926
|
if (!adapter)
|
|
1410
1927
|
return;
|
|
1411
|
-
|
|
1928
|
+
let text = formatUpdateProgress(marker);
|
|
1929
|
+
// No-restart updates complete in the old fleet process, so they never
|
|
1930
|
+
// reach RestartProgress.finish(). Append the same optional tip here.
|
|
1931
|
+
if (marker.progress.stage === "complete" && this.tipsEnabled()) {
|
|
1932
|
+
if (this.updateCompletionTipText === null) {
|
|
1933
|
+
const tip = this.pickAvailableTip();
|
|
1934
|
+
this.updateCompletionTipText = tip ? this.formatTip(tip) : "";
|
|
1935
|
+
}
|
|
1936
|
+
if (this.updateCompletionTipText)
|
|
1937
|
+
text += `\n\n${this.updateCompletionTipText}`;
|
|
1938
|
+
}
|
|
1412
1939
|
if (text === this.lastUpdateProgressText)
|
|
1413
1940
|
return;
|
|
1414
1941
|
this.updateProgressEditRunning = true;
|
|
@@ -1449,6 +1976,7 @@ export class FleetManager {
|
|
|
1449
1976
|
// Rotate fleet.log if oversized (before any logging)
|
|
1450
1977
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1451
1978
|
const fleet = this.loadConfig(configPath);
|
|
1979
|
+
this.slimFleetConfigAtStartup();
|
|
1452
1980
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
1453
1981
|
const savedUpdateProgress = readUpdateProgress(this.dataDir);
|
|
1454
1982
|
const pendingUpdateProgress = savedUpdateProgress
|
|
@@ -1465,6 +1993,10 @@ export class FleetManager {
|
|
|
1465
1993
|
const { getTmuxSocketName: getSocket } = await import("./paths.js");
|
|
1466
1994
|
TmuxManager.setSocketName(getSocket());
|
|
1467
1995
|
await TmuxManager.ensureSession(getTmuxSession());
|
|
1996
|
+
// Pre-flight (advisory, fire-and-forget): warm the per-backend auth cache
|
|
1997
|
+
// before instances spawn, so a CLI that comes up on its sign-in screen gets
|
|
1998
|
+
// an instant 🔑 verdict instead of decaying into crash/MCP-died noise.
|
|
1999
|
+
this.lifecycle.primeAuthVerification(Object.values(fleet.instances).map(config => config.backend ?? fleet.defaults?.backend ?? "claude-code"));
|
|
1468
2000
|
// Start tmux control mode client for idle detection
|
|
1469
2001
|
if (!this.controlClient) {
|
|
1470
2002
|
this.controlClient = new TmuxControlClient(getTmuxSession(), 2000, this.logger);
|
|
@@ -1508,6 +2040,11 @@ export class FleetManager {
|
|
|
1508
2040
|
this.classicChannels = new ClassicChannelManager(this.dataDir, this.logger);
|
|
1509
2041
|
const classicAdapters = fleet.channels?.length ? fleet.channels : (fleet.channel ? [fleet.channel] : []);
|
|
1510
2042
|
this.classicChannels.configureAdapters(classicAdapters);
|
|
2043
|
+
// The unrecoverable-id report is deliberately NOT sent here: no adapter
|
|
2044
|
+
// exists yet at this point in startAll(), so notifyFleetError would find
|
|
2045
|
+
// nothing to deliver through, drop the message silently, AND burn its
|
|
2046
|
+
// 10-minute throttle key on the way out. It is sent once the shared adapter
|
|
2047
|
+
// is up — see the adapterStartup continuation below.
|
|
1511
2048
|
// Restore the persisted bot binding so replies/cancel go through the right
|
|
1512
2049
|
// bot after a restart (before this, inbound would re-bind lazily).
|
|
1513
2050
|
for (const ch of this.classicChannels.getAll()) {
|
|
@@ -1531,6 +2068,9 @@ export class FleetManager {
|
|
|
1531
2068
|
}
|
|
1532
2069
|
if (!this.classicChannels.checkReload())
|
|
1533
2070
|
return;
|
|
2071
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2072
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
2073
|
+
this.reportClassicUnrecoverableIds();
|
|
1534
2074
|
this.reregisterClassicChannels();
|
|
1535
2075
|
for (const ch of this.classicChannels.getAll()) {
|
|
1536
2076
|
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
@@ -1544,7 +2084,10 @@ export class FleetManager {
|
|
|
1544
2084
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1545
2085
|
// Small delay to let tmux window clean up
|
|
1546
2086
|
await new Promise(r => setTimeout(r, 2000));
|
|
1547
|
-
|
|
2087
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
2088
|
+
// unattended helper reads them from it and schedules the delayed
|
|
2089
|
+
// retry on failure like every other unattended start.
|
|
2090
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1548
2091
|
}
|
|
1549
2092
|
}
|
|
1550
2093
|
}
|
|
@@ -1695,6 +2238,13 @@ export class FleetManager {
|
|
|
1695
2238
|
this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule) => this.handleScheduleTrigger(schedule), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
|
|
1696
2239
|
this.scheduler.init();
|
|
1697
2240
|
this.logger.info("Scheduler initialized");
|
|
2241
|
+
// Tips share the daily-report clock but remain an independent internal
|
|
2242
|
+
// job: disabling daily_summary must not disable tips (and vice versa).
|
|
2243
|
+
// The callback checks defaults.tips at fire time, so /tips on|off is hot.
|
|
2244
|
+
this.dailyTipScheduler = new DailyTipScheduler(summaryConfig, costGuardConfig.timezone, () => this.sendTipToGeneral().catch(err => {
|
|
2245
|
+
this.logger.warn({ err }, "Failed to send daily tip");
|
|
2246
|
+
}));
|
|
2247
|
+
this.dailyTipScheduler.start();
|
|
1698
2248
|
// Inject active decisions as env var for MCP instructions.
|
|
1699
2249
|
// Snapshotted at startup — new decisions via post_decision are available
|
|
1700
2250
|
// through list_decisions tool but not auto-injected until restart.
|
|
@@ -1724,6 +2274,9 @@ export class FleetManager {
|
|
|
1724
2274
|
}
|
|
1725
2275
|
catch (err) {
|
|
1726
2276
|
this.logger.error({ err, name }, "Failed to start general instance");
|
|
2277
|
+
// General is the most important instance to bring back: it also gets
|
|
2278
|
+
// the delayed automatic retry (the topic notice below still goes out).
|
|
2279
|
+
this.scheduleStartupRetry(name, 0);
|
|
1727
2280
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1728
2281
|
const topicId = cfg.topic_id ? String(cfg.topic_id) : undefined;
|
|
1729
2282
|
if (this.adapter && topicId) {
|
|
@@ -1752,6 +2305,10 @@ export class FleetManager {
|
|
|
1752
2305
|
catch (err) {
|
|
1753
2306
|
this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
|
|
1754
2307
|
}
|
|
2308
|
+
// Now that there is somewhere to deliver: a classicBot.yaml that already
|
|
2309
|
+
// holds an unmatchable id at boot is the COMMON case, and reporting it
|
|
2310
|
+
// before the adapter existed meant the operator heard nothing at all.
|
|
2311
|
+
this.reportClassicUnrecoverableIds();
|
|
1755
2312
|
})();
|
|
1756
2313
|
progressStart = adapterStartup.then(() => {
|
|
1757
2314
|
if (pendingUpdateProgress) {
|
|
@@ -1843,14 +2400,8 @@ export class FleetManager {
|
|
|
1843
2400
|
while (idx < channels.length) {
|
|
1844
2401
|
const batch = channels.slice(idx, idx + concurrency);
|
|
1845
2402
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
if (this.daemons.has(ch.instanceName))
|
|
1849
|
-
startupProgress.markReady();
|
|
1850
|
-
}
|
|
1851
|
-
catch (err) {
|
|
1852
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
1853
|
-
}
|
|
2403
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
2404
|
+
startupProgress.markReady();
|
|
1854
2405
|
}));
|
|
1855
2406
|
idx += concurrency;
|
|
1856
2407
|
}
|
|
@@ -1877,6 +2428,12 @@ export class FleetManager {
|
|
|
1877
2428
|
version: agendVersion,
|
|
1878
2429
|
pausedNames,
|
|
1879
2430
|
failedNames,
|
|
2431
|
+
tipText: pendingUpdateProgress && this.tipsEnabled()
|
|
2432
|
+
? (() => {
|
|
2433
|
+
const tip = this.pickAvailableTip();
|
|
2434
|
+
return tip ? this.formatTip(tip) : undefined;
|
|
2435
|
+
})()
|
|
2436
|
+
: undefined,
|
|
1880
2437
|
});
|
|
1881
2438
|
if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
|
|
1882
2439
|
let text;
|
|
@@ -2038,20 +2595,33 @@ export class FleetManager {
|
|
|
2038
2595
|
notifyAdapterFailure(adapterId, error) {
|
|
2039
2596
|
const generalId = this.findGeneralInstance();
|
|
2040
2597
|
if (generalId) {
|
|
2041
|
-
this.notifyInstanceTopic(generalId,
|
|
2598
|
+
this.notifyInstanceTopic(generalId, t("adapter.start_failed", adapterId, error));
|
|
2042
2599
|
}
|
|
2043
2600
|
}
|
|
2044
2601
|
/** Notify admin that a retried adapter reconnected. */
|
|
2045
2602
|
notifyAdapterRecovery(adapterId, attempts) {
|
|
2046
2603
|
const generalId = this.findGeneralInstance();
|
|
2047
2604
|
if (generalId) {
|
|
2048
|
-
this.notifyInstanceTopic(generalId,
|
|
2605
|
+
this.notifyInstanceTopic(generalId, t("adapter.reconnected", adapterId, attempts));
|
|
2049
2606
|
}
|
|
2050
2607
|
}
|
|
2051
2608
|
/** Get adapter states for /status visibility. */
|
|
2052
2609
|
getAdapterStates() {
|
|
2053
2610
|
return this.adapterState;
|
|
2054
2611
|
}
|
|
2612
|
+
bindAdapterHealth(adapter, adapterId) {
|
|
2613
|
+
adapter.on("gateway_health", (snapshot) => {
|
|
2614
|
+
const previous = this.adapterState.get(adapterId);
|
|
2615
|
+
const status = snapshot.status === "connected" ? "connected"
|
|
2616
|
+
: snapshot.status === "stopped" ? "failed"
|
|
2617
|
+
: "retrying";
|
|
2618
|
+
this.adapterState.set(adapterId, {
|
|
2619
|
+
status,
|
|
2620
|
+
retryCount: previous?.retryCount ?? 0,
|
|
2621
|
+
lastError: status === "connected" ? undefined : snapshot.lastReconnectReason ?? previous?.lastError,
|
|
2622
|
+
});
|
|
2623
|
+
});
|
|
2624
|
+
}
|
|
2055
2625
|
/**
|
|
2056
2626
|
* Real, checkable fleet health for `/health` and the operator.
|
|
2057
2627
|
*
|
|
@@ -2081,10 +2651,17 @@ export class FleetManager {
|
|
|
2081
2651
|
counts.stopped++;
|
|
2082
2652
|
}
|
|
2083
2653
|
const states = {};
|
|
2654
|
+
const details = {};
|
|
2084
2655
|
let connected = 0;
|
|
2085
2656
|
for (const [id, state] of this.adapterState) {
|
|
2086
|
-
|
|
2087
|
-
if (
|
|
2657
|
+
const snapshot = this.adapters.get(id)?.getHealthSnapshot?.();
|
|
2658
|
+
if (snapshot)
|
|
2659
|
+
details[id] = snapshot;
|
|
2660
|
+
const effectiveStatus = snapshot?.status === "connected" ? "connected"
|
|
2661
|
+
: snapshot && snapshot.status !== "stopped" ? "retrying"
|
|
2662
|
+
: state.status;
|
|
2663
|
+
states[id] = effectiveStatus;
|
|
2664
|
+
if (effectiveStatus === "connected")
|
|
2088
2665
|
connected++;
|
|
2089
2666
|
}
|
|
2090
2667
|
const problems = [];
|
|
@@ -2092,9 +2669,9 @@ export class FleetManager {
|
|
|
2092
2669
|
problems.push("no channel adapter is connected");
|
|
2093
2670
|
if (counts.crashed > 0)
|
|
2094
2671
|
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
2095
|
-
for (const [id, state] of
|
|
2096
|
-
if (state
|
|
2097
|
-
problems.push(`adapter ${id} is ${state
|
|
2672
|
+
for (const [id, state] of Object.entries(states)) {
|
|
2673
|
+
if (state !== "connected")
|
|
2674
|
+
problems.push(`adapter ${id} is ${state}`);
|
|
2098
2675
|
}
|
|
2099
2676
|
if (!this.startupComplete)
|
|
2100
2677
|
problems.push("startup has not completed");
|
|
@@ -2107,7 +2684,7 @@ export class FleetManager {
|
|
|
2107
2684
|
status,
|
|
2108
2685
|
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
2109
2686
|
instances: counts,
|
|
2110
|
-
adapters: { total: this.adapterState.size, connected, states },
|
|
2687
|
+
adapters: { total: this.adapterState.size, connected, states, details },
|
|
2111
2688
|
startupComplete: this.startupComplete,
|
|
2112
2689
|
memory: readFleetMemory(),
|
|
2113
2690
|
problems,
|
|
@@ -2136,6 +2713,7 @@ export class FleetManager {
|
|
|
2136
2713
|
const world = new AdapterWorld(adapterId, this.adapter, accessManager, channelConfig);
|
|
2137
2714
|
this.worlds.set(adapterId, world);
|
|
2138
2715
|
this.adapters.set(adapterId, this.adapter);
|
|
2716
|
+
this.bindAdapterHealth(this.adapter, adapterId);
|
|
2139
2717
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2140
2718
|
await this.handleInboundMessage(msg);
|
|
2141
2719
|
}, this.logger, "adapter.message"));
|
|
@@ -2143,6 +2721,22 @@ export class FleetManager {
|
|
|
2143
2721
|
await this.handleInboundReaction(r);
|
|
2144
2722
|
}, this.logger, "adapter.reaction"));
|
|
2145
2723
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
2724
|
+
if (await this.handleTipDismiss(data, adapterId, this.adapter ?? undefined))
|
|
2725
|
+
return;
|
|
2726
|
+
if (await this.handleTipUnlock(data, adapterId, this.adapter ?? undefined))
|
|
2727
|
+
return;
|
|
2728
|
+
if (await this.handleLoginBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2729
|
+
return;
|
|
2730
|
+
if (await this.handleInstallBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2731
|
+
return;
|
|
2732
|
+
if (await this.handleClassicApproval(data, adapterId, this.adapter ?? undefined))
|
|
2733
|
+
return;
|
|
2734
|
+
if (await this.handleLoginMenuSelect(data, adapterId, this.adapter ?? undefined))
|
|
2735
|
+
return;
|
|
2736
|
+
if (await this.handleLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2737
|
+
return;
|
|
2738
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2739
|
+
return;
|
|
2146
2740
|
if (await this.handleClearConfirmation(data, adapterId, this.adapter ?? undefined))
|
|
2147
2741
|
return;
|
|
2148
2742
|
if (await this.handleExitRestartPrompt(data, adapterId, this.adapter ?? undefined))
|
|
@@ -2235,6 +2829,49 @@ export class FleetManager {
|
|
|
2235
2829
|
const result = await this.topicCommands.sendCompact(name);
|
|
2236
2830
|
await data.respond(result);
|
|
2237
2831
|
}
|
|
2832
|
+
else if (data.command === "steer") {
|
|
2833
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2834
|
+
if (!name) {
|
|
2835
|
+
await data.respond(t("classic.no_agent"));
|
|
2836
|
+
return;
|
|
2837
|
+
}
|
|
2838
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
2839
|
+
if (!steerText) {
|
|
2840
|
+
await data.respond(t("steer.usage"));
|
|
2841
|
+
return;
|
|
2842
|
+
}
|
|
2843
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
2844
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
2845
|
+
// rerouting the instance's replies to the slash context.
|
|
2846
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
2847
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2848
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2849
|
+
});
|
|
2850
|
+
await data.respond(result);
|
|
2851
|
+
}
|
|
2852
|
+
else if (data.command === "btw") {
|
|
2853
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2854
|
+
if (!name) {
|
|
2855
|
+
await data.respond(t("classic.no_agent"));
|
|
2856
|
+
return;
|
|
2857
|
+
}
|
|
2858
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
2859
|
+
if (!btwText) {
|
|
2860
|
+
await data.respond(t("btw.usage"));
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
2864
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2865
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2866
|
+
});
|
|
2867
|
+
await data.respond(result);
|
|
2868
|
+
}
|
|
2869
|
+
else if (data.command === "login") {
|
|
2870
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
2871
|
+
}
|
|
2872
|
+
else if (data.command === "install-cli") {
|
|
2873
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
2874
|
+
}
|
|
2238
2875
|
else if (data.command === "clear") {
|
|
2239
2876
|
await this.handleClearSlash(data, adapterId);
|
|
2240
2877
|
}
|
|
@@ -2308,12 +2945,15 @@ export class FleetManager {
|
|
|
2308
2945
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2309
2946
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2310
2947
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2311
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
2948
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2312
2949
|
}
|
|
2313
2950
|
catch (err) {
|
|
2314
|
-
await data.respond(
|
|
2951
|
+
await data.respond(t("usage.failed", err.message));
|
|
2315
2952
|
}
|
|
2316
2953
|
}
|
|
2954
|
+
else if (data.command === "tips") {
|
|
2955
|
+
await this.handleTipsSlash(data, adapterId);
|
|
2956
|
+
}
|
|
2317
2957
|
else if (data.command === "status") {
|
|
2318
2958
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2319
2959
|
// instance's cost and IPC health.
|
|
@@ -2325,7 +2965,8 @@ export class FleetManager {
|
|
|
2325
2965
|
await data.respond(text);
|
|
2326
2966
|
}
|
|
2327
2967
|
else if (data.command === "sysinfo") {
|
|
2328
|
-
|
|
2968
|
+
// Slash commands are Discord-only; use plain lines (no markdown table)
|
|
2969
|
+
await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
|
|
2329
2970
|
}
|
|
2330
2971
|
else if (data.command === "dashboard") {
|
|
2331
2972
|
// Reply is ephemeral (adapter defers non-chat commands ephemerally), so
|
|
@@ -2359,6 +3000,49 @@ export class FleetManager {
|
|
|
2359
3000
|
const result = await this.topicCommands.sendCompact(name);
|
|
2360
3001
|
await data.respond(result);
|
|
2361
3002
|
}
|
|
3003
|
+
else if (data.command === "steer") {
|
|
3004
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3005
|
+
if (!name) {
|
|
3006
|
+
await data.respond(t("classic.no_agent"));
|
|
3007
|
+
return;
|
|
3008
|
+
}
|
|
3009
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
3010
|
+
if (!steerText) {
|
|
3011
|
+
await data.respond(t("steer.usage"));
|
|
3012
|
+
return;
|
|
3013
|
+
}
|
|
3014
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
3015
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
3016
|
+
// rerouting the instance's replies to the slash context.
|
|
3017
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
3018
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3019
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3020
|
+
});
|
|
3021
|
+
await data.respond(result);
|
|
3022
|
+
}
|
|
3023
|
+
else if (data.command === "btw") {
|
|
3024
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3025
|
+
if (!name) {
|
|
3026
|
+
await data.respond(t("classic.no_agent"));
|
|
3027
|
+
return;
|
|
3028
|
+
}
|
|
3029
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
3030
|
+
if (!btwText) {
|
|
3031
|
+
await data.respond(t("btw.usage"));
|
|
3032
|
+
return;
|
|
3033
|
+
}
|
|
3034
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
3035
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3036
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3037
|
+
});
|
|
3038
|
+
await data.respond(result);
|
|
3039
|
+
}
|
|
3040
|
+
else if (data.command === "login") {
|
|
3041
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
3042
|
+
}
|
|
3043
|
+
else if (data.command === "install-cli") {
|
|
3044
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
3045
|
+
}
|
|
2362
3046
|
}, this.logger, "adapter.slash_command"));
|
|
2363
3047
|
await this.topicCommands.registerBotCommands().catch(e => this.logger.warn({ err: e }, "registerBotCommands failed (non-fatal)"));
|
|
2364
3048
|
// Background-probe each backend's CLI env (version/models) → cli-env cache.
|
|
@@ -2385,13 +3069,14 @@ export class FleetManager {
|
|
|
2385
3069
|
}, this.logger, "adapter.handler_error"));
|
|
2386
3070
|
this.adapter.on("error", (err) => {
|
|
2387
3071
|
this.logger.error({ err }, "Primary adapter fatal error");
|
|
2388
|
-
this.restartAdapter(this.adapter,
|
|
3072
|
+
this.restartAdapter(this.adapter, adapterId).catch(() => { });
|
|
2389
3073
|
});
|
|
2390
|
-
this.adapter.on("new_group_detected", safeHandler((data) => {
|
|
3074
|
+
this.adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2391
3075
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2392
3076
|
const generalId = this.findGeneralInstance();
|
|
3077
|
+
// No user to promote: the bot was just added, nobody has run /start yet.
|
|
2393
3078
|
if (generalId)
|
|
2394
|
-
this.
|
|
3079
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2395
3080
|
}, this.logger, "adapter.new_group_detected"));
|
|
2396
3081
|
// Start adapter AFTER all event listeners are registered (started event sets botUsername)
|
|
2397
3082
|
await this.adapter.start();
|
|
@@ -2427,6 +3112,7 @@ export class FleetManager {
|
|
|
2427
3112
|
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2428
3113
|
this.worlds.set(adapterId, world);
|
|
2429
3114
|
this.adapters.set(adapterId, adapter);
|
|
3115
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2430
3116
|
// Wire up event handlers (same as primary, routes through shared handleInboundMessage)
|
|
2431
3117
|
adapter.on("message", safeHandler(async (msg) => {
|
|
2432
3118
|
await this.handleInboundMessage(msg);
|
|
@@ -2435,6 +3121,22 @@ export class FleetManager {
|
|
|
2435
3121
|
await this.handleInboundReaction(r);
|
|
2436
3122
|
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
2437
3123
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
3124
|
+
if (await this.handleTipDismiss(data, adapterId, adapter))
|
|
3125
|
+
return;
|
|
3126
|
+
if (await this.handleTipUnlock(data, adapterId, adapter))
|
|
3127
|
+
return;
|
|
3128
|
+
if (await this.handleLoginBackendSelect(data, adapterId, adapter))
|
|
3129
|
+
return;
|
|
3130
|
+
if (await this.handleInstallBackendSelect(data, adapterId, adapter))
|
|
3131
|
+
return;
|
|
3132
|
+
if (await this.handleClassicApproval(data, adapterId, adapter))
|
|
3133
|
+
return;
|
|
3134
|
+
if (await this.handleLoginMenuSelect(data, adapterId, adapter))
|
|
3135
|
+
return;
|
|
3136
|
+
if (await this.handleLoginConfirm(data, adapterId, adapter))
|
|
3137
|
+
return;
|
|
3138
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, adapter))
|
|
3139
|
+
return;
|
|
2438
3140
|
if (await this.handleClearConfirmation(data, adapterId, adapter))
|
|
2439
3141
|
return;
|
|
2440
3142
|
if (await this.handleExitRestartPrompt(data, adapterId, adapter))
|
|
@@ -2590,12 +3292,15 @@ export class FleetManager {
|
|
|
2590
3292
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2591
3293
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2592
3294
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2593
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
3295
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2594
3296
|
}
|
|
2595
3297
|
catch (err) {
|
|
2596
|
-
await data.respond(
|
|
3298
|
+
await data.respond(t("usage.failed", err.message));
|
|
2597
3299
|
}
|
|
2598
3300
|
}
|
|
3301
|
+
else if (data.command === "tips") {
|
|
3302
|
+
await this.handleTipsSlash(data, adapterId);
|
|
3303
|
+
}
|
|
2599
3304
|
else if (data.command === "status") {
|
|
2600
3305
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2601
3306
|
// instance's cost and IPC health.
|
|
@@ -2607,7 +3312,8 @@ export class FleetManager {
|
|
|
2607
3312
|
await data.respond(text);
|
|
2608
3313
|
}
|
|
2609
3314
|
else if (data.command === "sysinfo") {
|
|
2610
|
-
|
|
3315
|
+
// Slash commands are Discord-only; use plain lines (no markdown table)
|
|
3316
|
+
await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
|
|
2611
3317
|
}
|
|
2612
3318
|
else if (data.command === "dashboard") {
|
|
2613
3319
|
// Reply is ephemeral (adapter defers non-chat commands ephemerally), so
|
|
@@ -2641,11 +3347,50 @@ export class FleetManager {
|
|
|
2641
3347
|
const result = await this.topicCommands.sendCompact(name);
|
|
2642
3348
|
await data.respond(result);
|
|
2643
3349
|
}
|
|
3350
|
+
else if (data.command === "steer") {
|
|
3351
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3352
|
+
if (!name) {
|
|
3353
|
+
await data.respond(t("classic.no_agent"));
|
|
3354
|
+
return;
|
|
3355
|
+
}
|
|
3356
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
3357
|
+
if (!steerText) {
|
|
3358
|
+
await data.respond(t("steer.usage"));
|
|
3359
|
+
return;
|
|
3360
|
+
}
|
|
3361
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
3362
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
3363
|
+
// rerouting the instance's replies to the slash context.
|
|
3364
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
3365
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3366
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3367
|
+
});
|
|
3368
|
+
await data.respond(result);
|
|
3369
|
+
}
|
|
3370
|
+
else if (data.command === "btw") {
|
|
3371
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3372
|
+
if (!name) {
|
|
3373
|
+
await data.respond(t("classic.no_agent"));
|
|
3374
|
+
return;
|
|
3375
|
+
}
|
|
3376
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
3377
|
+
if (!btwText) {
|
|
3378
|
+
await data.respond(t("btw.usage"));
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
3381
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
3382
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3383
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3384
|
+
});
|
|
3385
|
+
await data.respond(result);
|
|
3386
|
+
}
|
|
3387
|
+
else if (data.command === "login") {
|
|
3388
|
+
await this.handleLoginSlash(data, adapterId, adapter);
|
|
3389
|
+
}
|
|
3390
|
+
else if (data.command === "install-cli") {
|
|
3391
|
+
await this.handleInstallCliSlash(data, adapterId, adapter);
|
|
3392
|
+
}
|
|
2644
3393
|
}, this.logger, `adapter[${adapterId}].slash_command`));
|
|
2645
|
-
await adapter.start();
|
|
2646
|
-
if (channelConfig.group_id) {
|
|
2647
|
-
adapter.setChatId(String(channelConfig.group_id));
|
|
2648
|
-
}
|
|
2649
3394
|
adapter.on("started", safeHandler((username, userId) => {
|
|
2650
3395
|
this.logger.info(`[${adapterId}] Bot @${username} polling started.`);
|
|
2651
3396
|
const world = this.worlds.get(adapterId);
|
|
@@ -2655,16 +3400,21 @@ export class FleetManager {
|
|
|
2655
3400
|
world.botUserId = userId;
|
|
2656
3401
|
}
|
|
2657
3402
|
}, this.logger, `adapter[${adapterId}].started`));
|
|
2658
|
-
adapter.on("new_group_detected", safeHandler((data) => {
|
|
3403
|
+
adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2659
3404
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2660
3405
|
const generalId = this.findGeneralInstance(adapterId);
|
|
2661
3406
|
if (generalId)
|
|
2662
|
-
this.
|
|
3407
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2663
3408
|
}, this.logger, `adapter[${adapterId}].new_group_detected`));
|
|
2664
3409
|
adapter.on("error", (err) => {
|
|
2665
3410
|
this.logger.error({ err, adapterId }, "Additional adapter fatal error");
|
|
2666
3411
|
this.restartAdapter(adapter, adapterId).catch(() => { });
|
|
2667
3412
|
});
|
|
3413
|
+
// Register lifecycle listeners before login; a fast ready/error must not be lost.
|
|
3414
|
+
await adapter.start();
|
|
3415
|
+
if (channelConfig.group_id) {
|
|
3416
|
+
adapter.setChatId(String(channelConfig.group_id));
|
|
3417
|
+
}
|
|
2668
3418
|
this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
|
|
2669
3419
|
}
|
|
2670
3420
|
/** Connect IPC to a single instance with all handlers */
|
|
@@ -2758,6 +3508,42 @@ export class FleetManager {
|
|
|
2758
3508
|
else if (msg.type === "instance_activity") {
|
|
2759
3509
|
this.cacheInstanceActivity(name, msg.activity);
|
|
2760
3510
|
}
|
|
3511
|
+
else if (msg.type === "instance_progress") {
|
|
3512
|
+
this.cacheInstanceProgress(name, msg.progress || null);
|
|
3513
|
+
}
|
|
3514
|
+
else if (msg.type === "cross_instance_delivery_failed") {
|
|
3515
|
+
const senderSession = String(msg.senderSession ?? "");
|
|
3516
|
+
const targetInstance = String(msg.targetInstance ?? name);
|
|
3517
|
+
const correlationId = String(msg.correlationId ?? "unknown");
|
|
3518
|
+
const error = String(msg.error ?? "unknown delivery failure");
|
|
3519
|
+
const senderInstance = this.instanceIpcClients.has(senderSession)
|
|
3520
|
+
? senderSession
|
|
3521
|
+
: this.sessionRegistry.get(senderSession);
|
|
3522
|
+
this.logger.error({ senderSession, targetInstance, correlationId, error }, "Cross-instance pane delivery failed after queue acceptance");
|
|
3523
|
+
this.eventLog?.insert(targetInstance, "cross_instance_delivery_failed", {
|
|
3524
|
+
from: senderSession, correlation_id: correlationId, error,
|
|
3525
|
+
});
|
|
3526
|
+
if (senderInstance) {
|
|
3527
|
+
this.notifyInstanceTopic(senderInstance, t("cross_instance.delivery_failed_notice", targetInstance, error, correlationId));
|
|
3528
|
+
void this.deliverToInstance(senderInstance, {
|
|
3529
|
+
type: "fleet_inbound",
|
|
3530
|
+
targetSession: senderSession,
|
|
3531
|
+
content: t("cross_instance.delivery_failed_agent", targetInstance, error, correlationId),
|
|
3532
|
+
meta: {
|
|
3533
|
+
chat_id: "",
|
|
3534
|
+
message_id: `delivery-failed-${Date.now()}`,
|
|
3535
|
+
user: "AgEnD",
|
|
3536
|
+
user_id: "system:agend",
|
|
3537
|
+
ts: new Date().toISOString(),
|
|
3538
|
+
thread_id: "",
|
|
3539
|
+
request_kind: "update",
|
|
3540
|
+
correlation_id: correlationId,
|
|
3541
|
+
},
|
|
3542
|
+
}, { waitForIdle: true }).catch(deliveryError => {
|
|
3543
|
+
this.logger.error({ err: deliveryError, senderSession, correlationId }, "Could not deliver cross-instance failure status back to sender");
|
|
3544
|
+
});
|
|
3545
|
+
}
|
|
3546
|
+
}
|
|
2761
3547
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2762
3548
|
this.cacheInstanceExecutionState(name, msg);
|
|
2763
3549
|
if (msg.type === "instance_state_response") {
|
|
@@ -2871,6 +3657,26 @@ export class FleetManager {
|
|
|
2871
3657
|
const previous = this.adapterState.get(id);
|
|
2872
3658
|
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2873
3659
|
try {
|
|
3660
|
+
if (adapter.reconnectGateway) {
|
|
3661
|
+
try {
|
|
3662
|
+
// Discord requires a fresh Client after destroy(). Its adapter owns the
|
|
3663
|
+
// single-flight, generation fence and bounded IDENTIFY backoff, so all
|
|
3664
|
+
// watchdog/manual/error triggers must converge here instead of stop/start.
|
|
3665
|
+
await adapter.reconnectGateway(previous?.lastError ?? "fleet adapter restart");
|
|
3666
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
3667
|
+
this.logger.info({ id }, "Adapter gateway rebuilt successfully");
|
|
3668
|
+
}
|
|
3669
|
+
catch (err) {
|
|
3670
|
+
if (!this.ipcStoppingInstances.has("__fleet_stopping__")) {
|
|
3671
|
+
this.adapterState.set(id, {
|
|
3672
|
+
status: "failed",
|
|
3673
|
+
retryCount: previous?.retryCount ?? 0,
|
|
3674
|
+
lastError: err?.message ?? String(err),
|
|
3675
|
+
});
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
return;
|
|
3679
|
+
}
|
|
2874
3680
|
for (let attempt = 1;; attempt++) {
|
|
2875
3681
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
2876
3682
|
return;
|
|
@@ -2974,9 +3780,148 @@ export class FleetManager {
|
|
|
2974
3780
|
consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
|
|
2975
3781
|
};
|
|
2976
3782
|
}
|
|
3783
|
+
/**
|
|
3784
|
+
* Which adapter's access policy governs an inbound message.
|
|
3785
|
+
*
|
|
3786
|
+
* `authoritative` means the thread resolved to an owning instance, so that
|
|
3787
|
+
* adapter's policy is the only one entitled to decide. A resolved owner whose
|
|
3788
|
+
* policy cannot be read is NOT the same as having no owner: the owner's world
|
|
3789
|
+
* may not exist yet (an adapter that fails its token check returns before its
|
|
3790
|
+
* world and AccessManager are created) while sibling bots are already live and
|
|
3791
|
+
* receiving copies. Falling back to a sibling's policy — or to the fleet-wide
|
|
3792
|
+
* one — would let the owner's rules be decided by another adapter, so callers
|
|
3793
|
+
* must refuse instead.
|
|
3794
|
+
*
|
|
3795
|
+
* Without an owner (classic channels, the no-thread Telegram path, unrouted
|
|
3796
|
+
* threads, no adapter identity) the receiving adapter's policy applies, as
|
|
3797
|
+
* before.
|
|
3798
|
+
*/
|
|
3799
|
+
governingAccess(msg, threadId) {
|
|
3800
|
+
if (threadId && !this.classicChannels?.hasChannel(threadId)) {
|
|
3801
|
+
const target = this.routing.resolve(threadId);
|
|
3802
|
+
if (target) {
|
|
3803
|
+
const owner = this.getInstanceAdapterId(target.name);
|
|
3804
|
+
if (owner)
|
|
3805
|
+
return { adapterId: owner, authoritative: true };
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
return { adapterId: msg.adapterId, authoritative: false };
|
|
3809
|
+
}
|
|
3810
|
+
/**
|
|
3811
|
+
* Why this adapter should leave an inbound message to another adapter, or null
|
|
3812
|
+
* to handle it.
|
|
3813
|
+
*
|
|
3814
|
+
* A fleet topic is served by exactly one instance, and that instance is
|
|
3815
|
+
* answered by exactly one adapter. When several bots share a guild they each
|
|
3816
|
+
* receive their own copy of every message, so the copy that is acted on must
|
|
3817
|
+
* be chosen by ownership rather than by which inbound happened to fire first.
|
|
3818
|
+
* Ownership is per-instance — the adapter the instance is bound to, falling
|
|
3819
|
+
* back to channels[0] when it is not bound — so an instance answered by a
|
|
3820
|
+
* non-default bot still receives its own traffic, and the access policy that
|
|
3821
|
+
* applies is that instance's own adapter's.
|
|
3822
|
+
*
|
|
3823
|
+
* Callers must ask before claiming the shared dedup key: a copy that is going
|
|
3824
|
+
* to be left alone must not consume the key, or the owner's copy would be
|
|
3825
|
+
* discarded as a duplicate and the message lost.
|
|
3826
|
+
*
|
|
3827
|
+
* Exempt: classic channels (each bot owns its own agent there and handles its
|
|
3828
|
+
* own copy, which is why their dedup key is adapter-scoped), the no-thread
|
|
3829
|
+
* Telegram path (no thread to resolve an instance from), and messages with no
|
|
3830
|
+
* adapter id (single-adapter fleets).
|
|
3831
|
+
*/
|
|
3832
|
+
topicOwnerDropReason(msg, threadId) {
|
|
3833
|
+
if (!threadId || !msg.adapterId)
|
|
3834
|
+
return null;
|
|
3835
|
+
if (this.classicChannels?.hasChannel(threadId))
|
|
3836
|
+
return null;
|
|
3837
|
+
const target = this.routing.resolve(threadId);
|
|
3838
|
+
if (!target)
|
|
3839
|
+
return null;
|
|
3840
|
+
const ownerAdapterId = this.getInstanceAdapterId(target.name);
|
|
3841
|
+
if (ownerAdapterId && msg.adapterId !== ownerAdapterId) {
|
|
3842
|
+
return `not the adapter bound to ${target.name} (that is ${ownerAdapterId})`;
|
|
3843
|
+
}
|
|
3844
|
+
return null;
|
|
3845
|
+
}
|
|
3846
|
+
/**
|
|
3847
|
+
* Why this adapter must ignore a bot/webhook message, or null to accept it.
|
|
3848
|
+
* Pure: callers rely on being able to ask before claiming the dedup key.
|
|
3849
|
+
*/
|
|
3850
|
+
botMessageDropReason(msg, threadId) {
|
|
3851
|
+
if (!threadId) {
|
|
3852
|
+
// TG classic: allow if the bot @mentions our bot or access mode is open
|
|
3853
|
+
const world = this.worlds.get(msg.adapterId ?? "");
|
|
3854
|
+
const botUser = world?.botUsername;
|
|
3855
|
+
const isOpen = this.getChannelConfig(msg.adapterId)?.access?.mode === "open";
|
|
3856
|
+
const mentionsUs = !!(botUser && msg.text?.toLowerCase().includes(`@${botUser.toLowerCase()}`));
|
|
3857
|
+
return (!isOpen && !mentionsUs) ? "tg-classic: not open and does not mention us" : null;
|
|
3858
|
+
}
|
|
3859
|
+
if (this.classicChannels?.hasChannel(threadId)) {
|
|
3860
|
+
// Classic channel (per-bot): bot messages only when THIS bot owns an
|
|
3861
|
+
// agent here and collab is on for it.
|
|
3862
|
+
if (!this.classicChannels.getInstanceByChannel(threadId, msg.adapterId)) {
|
|
3863
|
+
return "classic: this bot owns no agent in the channel";
|
|
3864
|
+
}
|
|
3865
|
+
if (!this.classicChannels.isCollab(threadId, msg.adapterId))
|
|
3866
|
+
return "classic: collab off";
|
|
3867
|
+
return null;
|
|
3868
|
+
}
|
|
3869
|
+
const target = this.routing.resolve(threadId);
|
|
3870
|
+
if (!target)
|
|
3871
|
+
return "fleet topic: no instance routed for this thread";
|
|
3872
|
+
// Fleet topic: allow if collab enabled OR access mode is open
|
|
3873
|
+
const isOpen = this.getChannelConfig(msg.adapterId)?.access?.mode === "open";
|
|
3874
|
+
if (!isOpen && !this.collabInstances.has(target.name)) {
|
|
3875
|
+
return `fleet topic: adapter not open and collab off for ${target.name}`;
|
|
3876
|
+
}
|
|
3877
|
+
return null;
|
|
3878
|
+
}
|
|
2977
3879
|
async handleInboundMessage(msg) {
|
|
2978
3880
|
const threadId = msg.threadId || undefined;
|
|
2979
3881
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
3882
|
+
// Ownership and per-adapter filtering both run before the dedup claim below.
|
|
3883
|
+
// When several bots share a guild they all receive the same message; a copy
|
|
3884
|
+
// this adapter is going to leave alone must not consume the shared dedup key,
|
|
3885
|
+
// or the owning adapter's copy would be discarded as a duplicate and the
|
|
3886
|
+
// message lost entirely. Claiming the key only on surviving copies also keeps
|
|
3887
|
+
// delivery single when more than one adapter would otherwise accept it.
|
|
3888
|
+
// Only bot messages are settled by ownership: a bot answering in a fleet
|
|
3889
|
+
// topic would speak as the wrong identity, whereas a human message is just
|
|
3890
|
+
// input for the instance and its reply is canonicalized to the owning
|
|
3891
|
+
// adapter downstream — so a topic only a non-owner bot can see still works.
|
|
3892
|
+
const drop = msg.isBotMessage
|
|
3893
|
+
? (this.topicOwnerDropReason(msg, threadId) ?? this.botMessageDropReason(msg, threadId))
|
|
3894
|
+
: null;
|
|
3895
|
+
if (drop) {
|
|
3896
|
+
this.logger.debug({ adapterId: msg.adapterId, threadId: threadId ?? null, messageId: msg.messageId, reason: drop }, "Inbound message dropped before the dedup claim — key not consumed");
|
|
3897
|
+
return;
|
|
3898
|
+
}
|
|
3899
|
+
// Access control — classic channels are open to all, others require an allowed
|
|
3900
|
+
// user. This runs before the dedup claim, and is judged by the adapter that
|
|
3901
|
+
// owns the topic rather than the one that happened to receive the message:
|
|
3902
|
+
// otherwise a sibling bot could settle the message under its own policy —
|
|
3903
|
+
// claiming the shared key and then refusing it, or admitting a user the
|
|
3904
|
+
// owning adapter does not allow — purely by arriving first.
|
|
3905
|
+
const governing = this.governingAccess(msg, threadId);
|
|
3906
|
+
const ownerWorld = governing.adapterId ? this.worlds.get(governing.adapterId) : undefined;
|
|
3907
|
+
if (governing.authoritative && !ownerWorld) {
|
|
3908
|
+
this.logger.warn({ adapterId: msg.adapterId, owner: governing.adapterId, threadId, messageId: msg.messageId }, "Refusing inbound: the owning adapter is not running, so its access policy cannot be applied");
|
|
3909
|
+
return;
|
|
3910
|
+
}
|
|
3911
|
+
const am = governing.authoritative
|
|
3912
|
+
? ownerWorld?.accessManager
|
|
3913
|
+
: (ownerWorld?.accessManager ?? this.accessManager);
|
|
3914
|
+
if (am && !am.isAllowed(msg.userId)) {
|
|
3915
|
+
const adapterGroupId = String(this.getChannelConfig(msg.adapterId)?.group_id ?? "");
|
|
3916
|
+
const isTelegramClassicCandidate = msg.source === "telegram" && msg.chatId !== adapterGroupId && !threadId;
|
|
3917
|
+
if (!isTelegramClassicCandidate) {
|
|
3918
|
+
// Classic channels are open to all; check per-bot ownership (or fleet topic).
|
|
3919
|
+
const isClassic = !!(threadId && this.classicChannels?.hasChannel(threadId));
|
|
3920
|
+
this.logger.info({ userId: msg.userId, threadId, isClassic, owner: governing.adapterId }, "Access DENIED for non-allowed user");
|
|
3921
|
+
if (!isClassic)
|
|
3922
|
+
return;
|
|
3923
|
+
}
|
|
3924
|
+
}
|
|
2980
3925
|
// Multi-adapter dedup: when several bots share a guild, each adapter fires
|
|
2981
3926
|
// its own "message" event for the same underlying message. Process it once.
|
|
2982
3927
|
// Routing (by topic/channel) and reply-adapter selection (by channel_id
|
|
@@ -3005,55 +3950,6 @@ export class FleetManager {
|
|
|
3005
3950
|
this.recentMessageIds.delete(oldest);
|
|
3006
3951
|
}
|
|
3007
3952
|
}
|
|
3008
|
-
// Bot messages: only allow in collab channels or TG classic with @mention
|
|
3009
|
-
if (msg.isBotMessage) {
|
|
3010
|
-
if (!threadId) {
|
|
3011
|
-
// TG classic: allow if bot @mentions our bot or access mode is open
|
|
3012
|
-
const world = this.worlds.get(msg.adapterId ?? "");
|
|
3013
|
-
const botUser = world?.botUsername;
|
|
3014
|
-
const channelCfg = this.getChannelConfig(msg.adapterId);
|
|
3015
|
-
const isOpen = channelCfg?.access?.mode === "open";
|
|
3016
|
-
const mentionsUs = !!(botUser && msg.text?.toLowerCase().includes(`@${botUser.toLowerCase()}`));
|
|
3017
|
-
this.logger.debug({ botUser, mentionsUs, isOpen, isBotMessage: true, threadId: null }, "Bot message filter (no threadId path)");
|
|
3018
|
-
if (!isOpen && !mentionsUs)
|
|
3019
|
-
return;
|
|
3020
|
-
// Fall through to TG classic handling below
|
|
3021
|
-
}
|
|
3022
|
-
else if (this.classicChannels?.hasChannel(threadId)) {
|
|
3023
|
-
// Classic channel (per-bot): bot messages only when THIS bot owns an
|
|
3024
|
-
// agent here and collab is on for it.
|
|
3025
|
-
const classicName = this.classicChannels.getInstanceByChannel(threadId, msg.adapterId);
|
|
3026
|
-
if (!classicName)
|
|
3027
|
-
return;
|
|
3028
|
-
if (!this.classicChannels.isCollab(threadId, msg.adapterId))
|
|
3029
|
-
return;
|
|
3030
|
-
// Fall through to channel handling
|
|
3031
|
-
}
|
|
3032
|
-
else {
|
|
3033
|
-
const target = this.routing.resolve(threadId);
|
|
3034
|
-
if (!target)
|
|
3035
|
-
return;
|
|
3036
|
-
// Fleet topic: allow if collab enabled OR access mode is open
|
|
3037
|
-
const channelCfg = this.getChannelConfig(msg.adapterId);
|
|
3038
|
-
const isOpen = channelCfg?.access?.mode === "open";
|
|
3039
|
-
if (!isOpen && !this.collabInstances.has(target.name))
|
|
3040
|
-
return;
|
|
3041
|
-
// Fall through to channel handling
|
|
3042
|
-
}
|
|
3043
|
-
}
|
|
3044
|
-
// Access control — classic channels are open to all, others require allowed user
|
|
3045
|
-
const am = (msg.adapterId ? this.worlds.get(msg.adapterId)?.accessManager : undefined) ?? this.accessManager;
|
|
3046
|
-
if (am && !am.isAllowed(msg.userId)) {
|
|
3047
|
-
const adapterGroupId = String(this.getChannelConfig(msg.adapterId)?.group_id ?? "");
|
|
3048
|
-
const isTelegramClassicCandidate = msg.source === "telegram" && msg.chatId !== adapterGroupId && !threadId;
|
|
3049
|
-
if (!isTelegramClassicCandidate) {
|
|
3050
|
-
// Classic channels are open to all; check per-bot ownership (or fleet topic).
|
|
3051
|
-
const isClassic = !!(threadId && this.classicChannels?.hasChannel(threadId));
|
|
3052
|
-
this.logger.info({ userId: msg.userId, threadId, isClassic }, "Access DENIED for non-allowed user");
|
|
3053
|
-
if (!isClassic)
|
|
3054
|
-
return;
|
|
3055
|
-
}
|
|
3056
|
-
}
|
|
3057
3953
|
if (threadId == null) {
|
|
3058
3954
|
// ── Telegram Classic Mode ──
|
|
3059
3955
|
// Messages from chats other than the primary forum group are classic mode candidates.
|
|
@@ -3108,7 +4004,13 @@ export class FleetManager {
|
|
|
3108
4004
|
const adminMsg = t("alert.new_group", groupTitle, chatId, msg.username, msg.userId, msg.source);
|
|
3109
4005
|
const generalId = this.findGeneralInstance(msg.adapterId);
|
|
3110
4006
|
if (generalId) {
|
|
3111
|
-
this.
|
|
4007
|
+
await this.promptClassicApproval({
|
|
4008
|
+
// Guarded by isGroupAllowed → this belongs in allowed_groups,
|
|
4009
|
+
// NOT allowed_guilds; writing the latter would change the file
|
|
4010
|
+
// without unblocking the group.
|
|
4011
|
+
generalName: generalId, message: adminMsg, groupId: String(chatId),
|
|
4012
|
+
scope: "group", userId: msg.userId,
|
|
4013
|
+
});
|
|
3112
4014
|
}
|
|
3113
4015
|
await msgAdapter?.sendText(chatId, t("classic.access_requested"));
|
|
3114
4016
|
return;
|
|
@@ -3205,6 +4107,42 @@ export class FleetManager {
|
|
|
3205
4107
|
await msgAdapter?.sendText(chatId, result);
|
|
3206
4108
|
return;
|
|
3207
4109
|
}
|
|
4110
|
+
// /steer — interject into the running turn. Not admin-gated: anyone who
|
|
4111
|
+
// can talk to this agent can send it a message; steer only changes when
|
|
4112
|
+
// it lands, and it keeps the full [user:] formatting (unlike /raw).
|
|
4113
|
+
if (text === "/steer" || text.startsWith("/steer ") || text.startsWith("/steer@")) {
|
|
4114
|
+
const steerName = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
4115
|
+
if (!steerName) {
|
|
4116
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
4117
|
+
return;
|
|
4118
|
+
}
|
|
4119
|
+
const steerContent = text.replace(/^\/steer(@\S+)?/, "").trim();
|
|
4120
|
+
if (!steerContent) {
|
|
4121
|
+
await msgAdapter?.sendText(chatId, t("steer.usage"));
|
|
4122
|
+
return;
|
|
4123
|
+
}
|
|
4124
|
+
const result = this.topicCommands.sendSteer(steerName, steerContent, msg);
|
|
4125
|
+
await msgAdapter?.sendText(chatId, result);
|
|
4126
|
+
return;
|
|
4127
|
+
}
|
|
4128
|
+
// /btw — Claude Code's native side-thread command. Like /steer it is
|
|
4129
|
+
// not admin-gated, but it remains a distinct raw CLI command so Claude
|
|
4130
|
+
// does not fold the question into the active task.
|
|
4131
|
+
if (text === "/btw" || text.startsWith("/btw ") || text.startsWith("/btw@")) {
|
|
4132
|
+
const btwName = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
4133
|
+
if (!btwName) {
|
|
4134
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
4135
|
+
return;
|
|
4136
|
+
}
|
|
4137
|
+
const btwContent = text.replace(/^\/btw(@\S+)?/, "").trim();
|
|
4138
|
+
if (!btwContent) {
|
|
4139
|
+
await msgAdapter?.sendText(chatId, t("btw.usage"));
|
|
4140
|
+
return;
|
|
4141
|
+
}
|
|
4142
|
+
const result = this.topicCommands.sendBtw(btwName, btwContent, msg);
|
|
4143
|
+
await msgAdapter?.sendText(chatId, result);
|
|
4144
|
+
return;
|
|
4145
|
+
}
|
|
3208
4146
|
// Handle /clear command (admin only) — unlike /compact this starts a
|
|
3209
4147
|
// fresh conversation and intentionally discards the current history.
|
|
3210
4148
|
if (text === "/clear" || text.startsWith("/clear@")) {
|
|
@@ -3232,7 +4170,7 @@ export class FleetManager {
|
|
|
3232
4170
|
return;
|
|
3233
4171
|
}
|
|
3234
4172
|
const ok = this.cancelInstance(cancelName);
|
|
3235
|
-
await msgAdapter?.sendText(chatId, ok ?
|
|
4173
|
+
await msgAdapter?.sendText(chatId, ok ? t("cancel.sent", cancelName) : t("cancel.not_running", cancelName));
|
|
3236
4174
|
return;
|
|
3237
4175
|
}
|
|
3238
4176
|
// Handle /ctx command
|
|
@@ -3269,7 +4207,7 @@ export class FleetManager {
|
|
|
3269
4207
|
const backend = this.classicChannels.getBackendByInstance(saveName, this.fleetConfig?.defaults?.backend);
|
|
3270
4208
|
const cmd = saveCommandForBackend(backend, filename);
|
|
3271
4209
|
if (!cmd) {
|
|
3272
|
-
await msgAdapter?.sendText(chatId,
|
|
4210
|
+
await msgAdapter?.sendText(chatId, t("save.unsupported"));
|
|
3273
4211
|
return;
|
|
3274
4212
|
}
|
|
3275
4213
|
this.pasteRawToClassicInstance(saveName, cmd);
|
|
@@ -3487,10 +4425,10 @@ export class FleetManager {
|
|
|
3487
4425
|
return;
|
|
3488
4426
|
let warning = "";
|
|
3489
4427
|
if (rl.five_hour_pct >= 95) {
|
|
3490
|
-
warning =
|
|
4428
|
+
warning = t("rate_limit.five_hour", instanceName, Math.round(rl.five_hour_pct));
|
|
3491
4429
|
}
|
|
3492
4430
|
else if (rl.seven_day_pct >= 95) {
|
|
3493
|
-
warning =
|
|
4431
|
+
warning = t("rate_limit.weekly", instanceName, Math.round(rl.seven_day_pct));
|
|
3494
4432
|
}
|
|
3495
4433
|
if (!warning)
|
|
3496
4434
|
return;
|
|
@@ -3505,8 +4443,6 @@ export class FleetManager {
|
|
|
3505
4443
|
}
|
|
3506
4444
|
/** Handle outbound tool calls from a daemon instance */
|
|
3507
4445
|
async handleOutboundFromInstance(instanceName, msg) {
|
|
3508
|
-
if (this.worlds.size === 0)
|
|
3509
|
-
return;
|
|
3510
4446
|
this.touchActivity(instanceName);
|
|
3511
4447
|
this.setTopicIcon(instanceName, "green");
|
|
3512
4448
|
const tool = msg.tool;
|
|
@@ -3516,13 +4452,21 @@ export class FleetManager {
|
|
|
3516
4452
|
const senderSessionName = msg.senderSessionName;
|
|
3517
4453
|
const respond = (result, error) => {
|
|
3518
4454
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
4455
|
+
let sent = false;
|
|
3519
4456
|
if (fleetRequestId) {
|
|
3520
|
-
ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error });
|
|
4457
|
+
sent = ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error }) ?? false;
|
|
3521
4458
|
}
|
|
3522
4459
|
else {
|
|
3523
|
-
ipc?.send({ type: "fleet_outbound_response", requestId, result, error });
|
|
4460
|
+
sent = ipc?.send({ type: "fleet_outbound_response", requestId, result, error }) ?? false;
|
|
4461
|
+
}
|
|
4462
|
+
if (!sent) {
|
|
4463
|
+
this.logger.warn({ instanceName, tool, requestId, fleetRequestId, error }, "Fleet outbound result could not be returned — instance IPC is disconnected");
|
|
3524
4464
|
}
|
|
3525
4465
|
};
|
|
4466
|
+
if (this.worlds.size === 0) {
|
|
4467
|
+
respond(null, "Channel adapters are not ready — retry shortly");
|
|
4468
|
+
return;
|
|
4469
|
+
}
|
|
3526
4470
|
// Resolve threadId: use sender's topic_id if sender is a known fleet instance,
|
|
3527
4471
|
// fall back to general topic if sender is unknown, or IPC owner if no sender.
|
|
3528
4472
|
const senderInstanceName = senderSessionName && this.fleetConfig?.instances[senderSessionName]
|
|
@@ -3567,17 +4511,30 @@ export class FleetManager {
|
|
|
3567
4511
|
if (tool === "reply") {
|
|
3568
4512
|
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3569
4513
|
if (ticket.duplicate) {
|
|
3570
|
-
this.logger.info({ instanceName }, "
|
|
4514
|
+
this.logger.info({ instanceName }, "Concurrent duplicate reply joined — awaiting the original send's outcome");
|
|
3571
4515
|
ticket.subscribe(respond);
|
|
3572
4516
|
return;
|
|
3573
4517
|
}
|
|
3574
4518
|
const original = respond;
|
|
3575
4519
|
const respondAndRecord = (result, error) => {
|
|
3576
4520
|
ticket.complete(result, error);
|
|
4521
|
+
// Return the platform outcome first. Bookkeeping below must never turn
|
|
4522
|
+
// a confirmed Discord/Telegram POST into a tool error if a secondary
|
|
4523
|
+
// log/button side effect happens to throw.
|
|
3577
4524
|
original(result, error);
|
|
4525
|
+
// The adapter resolves only after the platform POST returns. Keep
|
|
4526
|
+
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4527
|
+
// a routed-but-failed reply is not a delivered reply.
|
|
4528
|
+
if (!error && result != null) {
|
|
4529
|
+
try {
|
|
4530
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4531
|
+
}
|
|
4532
|
+
catch (err) {
|
|
4533
|
+
this.logger.warn({ err, instanceName }, "Reply delivered but post-delivery bookkeeping failed");
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
3578
4536
|
};
|
|
3579
4537
|
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3580
|
-
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3581
4538
|
return;
|
|
3582
4539
|
}
|
|
3583
4540
|
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
@@ -3614,7 +4571,7 @@ export class FleetManager {
|
|
|
3614
4571
|
this.clearCancelButton(instanceName);
|
|
3615
4572
|
}
|
|
3616
4573
|
else {
|
|
3617
|
-
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
4574
|
+
void this.sendCancelButton(instanceName, undefined, true).then(() => this.armReplyGrace(instanceName));
|
|
3618
4575
|
}
|
|
3619
4576
|
this.reactDone(instanceName);
|
|
3620
4577
|
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
@@ -3651,7 +4608,11 @@ export class FleetManager {
|
|
|
3651
4608
|
if (silent) {
|
|
3652
4609
|
const ipc = this.instanceIpcClients.get(target);
|
|
3653
4610
|
if (ipc) {
|
|
3654
|
-
ipc.send({
|
|
4611
|
+
ipc.send({
|
|
4612
|
+
type: "raw_paste",
|
|
4613
|
+
content: message,
|
|
4614
|
+
delivery_epoch: this.getDeliveryEpoch(target),
|
|
4615
|
+
});
|
|
3655
4616
|
this.scheduler.recordRun(id, "delivered");
|
|
3656
4617
|
this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
|
|
3657
4618
|
}
|
|
@@ -3666,10 +4627,19 @@ export class FleetManager {
|
|
|
3666
4627
|
const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
|
|
3667
4628
|
const deliver = async () => {
|
|
3668
4629
|
try {
|
|
4630
|
+
// A schedule has no live inbound adapter context. Seed the daemon with
|
|
4631
|
+
// the target instance's configured world so replies use its persona
|
|
4632
|
+
// after a fresh start instead of falling back to channels[0].
|
|
4633
|
+
const adapterId = this.getInstanceAdapterId(target);
|
|
3669
4634
|
await this.deliverToInstance(target, {
|
|
3670
4635
|
type: "fleet_schedule_trigger",
|
|
3671
4636
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
3672
|
-
meta: {
|
|
4637
|
+
meta: {
|
|
4638
|
+
chat_id: reply_chat_id,
|
|
4639
|
+
thread_id: reply_thread_id,
|
|
4640
|
+
user: "scheduler",
|
|
4641
|
+
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4642
|
+
},
|
|
3673
4643
|
}, { waitForIdle: true });
|
|
3674
4644
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
3675
4645
|
void this.sendCancelButton(target);
|
|
@@ -3817,7 +4787,29 @@ export class FleetManager {
|
|
|
3817
4787
|
}
|
|
3818
4788
|
/** Resolve display name for an instance, fallback to instance name. */
|
|
3819
4789
|
resolveDisplayName(instanceName) {
|
|
3820
|
-
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4790
|
+
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4791
|
+
?? this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName)?.displayName
|
|
4792
|
+
?? instanceName;
|
|
4793
|
+
}
|
|
4794
|
+
/** Persist identity to the instance's actual config store. Classic instances
|
|
4795
|
+
* are registry rows in classicBot.yaml, not fleet.yaml instance entries. */
|
|
4796
|
+
setInstanceDisplayName(instanceName, displayName) {
|
|
4797
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4798
|
+
if (fleetInstance) {
|
|
4799
|
+
fleetInstance.display_name = displayName;
|
|
4800
|
+
this.saveFleetConfig();
|
|
4801
|
+
return true;
|
|
4802
|
+
}
|
|
4803
|
+
return this.classicChannels?.setDisplayNameByInstance(instanceName, displayName) ?? false;
|
|
4804
|
+
}
|
|
4805
|
+
setInstanceDescription(instanceName, description) {
|
|
4806
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4807
|
+
if (fleetInstance) {
|
|
4808
|
+
fleetInstance.description = description;
|
|
4809
|
+
this.saveFleetConfig();
|
|
4810
|
+
return true;
|
|
4811
|
+
}
|
|
4812
|
+
return this.classicChannels?.setDescriptionByInstance(instanceName, description) ?? false;
|
|
3821
4813
|
}
|
|
3822
4814
|
handleSetDisplayName(instanceName, msg) {
|
|
3823
4815
|
const fleetRequestId = msg.fleetRequestId;
|
|
@@ -3830,8 +4822,10 @@ export class FleetManager {
|
|
|
3830
4822
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: "Name must be 1-30 characters" });
|
|
3831
4823
|
return;
|
|
3832
4824
|
}
|
|
3833
|
-
this.
|
|
3834
|
-
|
|
4825
|
+
if (!this.setInstanceDisplayName(instanceName, displayName)) {
|
|
4826
|
+
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4827
|
+
return;
|
|
4828
|
+
}
|
|
3835
4829
|
this.logger.info({ instanceName, displayName }, "Display name set");
|
|
3836
4830
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, result: { display_name: displayName } });
|
|
3837
4831
|
}
|
|
@@ -3846,8 +4840,10 @@ export class FleetManager {
|
|
|
3846
4840
|
ipc.send({ type: "fleet_description_response", fleetRequestId, error: "Description cannot be empty" });
|
|
3847
4841
|
return;
|
|
3848
4842
|
}
|
|
3849
|
-
this.
|
|
3850
|
-
|
|
4843
|
+
if (!this.setInstanceDescription(instanceName, description)) {
|
|
4844
|
+
ipc.send({ type: "fleet_description_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4845
|
+
return;
|
|
4846
|
+
}
|
|
3851
4847
|
this.logger.info({ instanceName, description: description.slice(0, 80) }, "Description set");
|
|
3852
4848
|
ipc.send({ type: "fleet_description_response", fleetRequestId, result: { description } });
|
|
3853
4849
|
}
|
|
@@ -3976,8 +4972,8 @@ export class FleetManager {
|
|
|
3976
4972
|
return { error: "Fleet config not available" };
|
|
3977
4973
|
if (!name || name.length > 30)
|
|
3978
4974
|
return { error: "Name must be 1-30 characters" };
|
|
3979
|
-
this.
|
|
3980
|
-
|
|
4975
|
+
if (!this.setInstanceDisplayName(instance, name))
|
|
4976
|
+
return { error: `Instance '${instance}' not found` };
|
|
3981
4977
|
return { display_name: name };
|
|
3982
4978
|
}
|
|
3983
4979
|
async handleSetDescriptionHttp(instance, description) {
|
|
@@ -3985,8 +4981,8 @@ export class FleetManager {
|
|
|
3985
4981
|
return { error: "Fleet config not available" };
|
|
3986
4982
|
if (!description)
|
|
3987
4983
|
return { error: "Description cannot be empty" };
|
|
3988
|
-
this.
|
|
3989
|
-
|
|
4984
|
+
if (!this.setInstanceDescription(instance, description))
|
|
4985
|
+
return { error: `Instance '${instance}' not found` };
|
|
3990
4986
|
return { description };
|
|
3991
4987
|
}
|
|
3992
4988
|
summarizeToolCall(tool, args) {
|
|
@@ -4118,7 +5114,8 @@ export class FleetManager {
|
|
|
4118
5114
|
}
|
|
4119
5115
|
/**
|
|
4120
5116
|
* Patch only values changed in the effective config into the original YAML
|
|
4121
|
-
* document. Unknown keys
|
|
5117
|
+
* document. Unknown keys and comments remain untouched; redundant
|
|
5118
|
+
* non-identity instance leaves are canonicalized to inheritance afterward.
|
|
4122
5119
|
*/
|
|
4123
5120
|
saveFleetConfig(explicitPatches = []) {
|
|
4124
5121
|
if (!this.fleetConfig || !this.configPath)
|
|
@@ -4134,9 +5131,9 @@ export class FleetManager {
|
|
|
4134
5131
|
}
|
|
4135
5132
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4136
5133
|
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
4137
|
-
// Settings edits are expressed against the raw config
|
|
4138
|
-
//
|
|
4139
|
-
//
|
|
5134
|
+
// Settings edits are expressed against the raw config, so apply them before
|
|
5135
|
+
// canonicalization. A non-identity value equal to its inherited default is
|
|
5136
|
+
// intentionally stored as inheritance rather than an explicit duplicate.
|
|
4140
5137
|
for (const patch of explicitPatches) {
|
|
4141
5138
|
if (patch.remove) {
|
|
4142
5139
|
// YAML's deleteIn throws when an inherited nested key has no raw parent
|
|
@@ -4151,7 +5148,24 @@ export class FleetManager {
|
|
|
4151
5148
|
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
4152
5149
|
}
|
|
4153
5150
|
}
|
|
5151
|
+
const rawAfterPatches = this.rawFleetDocument.toJS();
|
|
5152
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(rawAfterPatches);
|
|
5153
|
+
for (const path of redundantPaths) {
|
|
5154
|
+
if (this.rawFleetDocument.hasIn(path))
|
|
5155
|
+
this.rawFleetDocument.deleteIn(path);
|
|
5156
|
+
// Avoid leaving empty operational maps such as `terminal: {}` while
|
|
5157
|
+
// preserving the instance mapping itself and all surrounding comments.
|
|
5158
|
+
for (let depth = path.length - 1; depth > 2; depth--) {
|
|
5159
|
+
const parentPath = path.slice(0, depth);
|
|
5160
|
+
const parent = this.rawFleetDocument.getIn(parentPath, true);
|
|
5161
|
+
if (!isMap(parent) || parent.items.length > 0)
|
|
5162
|
+
break;
|
|
5163
|
+
this.rawFleetDocument.deleteIn(parentPath);
|
|
5164
|
+
}
|
|
5165
|
+
}
|
|
4154
5166
|
const output = String(this.rawFleetDocument);
|
|
5167
|
+
if (redundantPaths.length > 0)
|
|
5168
|
+
this.writeFleetConfigBackup(source);
|
|
4155
5169
|
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
4156
5170
|
writeFileSync(tempPath, output, "utf-8");
|
|
4157
5171
|
if (existsSync(this.configPath))
|
|
@@ -4159,7 +5173,36 @@ export class FleetManager {
|
|
|
4159
5173
|
renameSync(tempPath, this.configPath);
|
|
4160
5174
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4161
5175
|
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
4162
|
-
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
5176
|
+
this.logger.info({ path: this.configPath, strippedDefaults: redundantPaths.length }, "Saved fleet config (lossless patch)");
|
|
5177
|
+
}
|
|
5178
|
+
/** One-time upgrade migration; invalid YAML is never rewritten. */
|
|
5179
|
+
slimFleetConfigAtStartup() {
|
|
5180
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(this.rawFleetConfig);
|
|
5181
|
+
if (redundantPaths.length === 0)
|
|
5182
|
+
return;
|
|
5183
|
+
const validation = validateFleetConfig(this.rawFleetConfig);
|
|
5184
|
+
if (!validation.valid) {
|
|
5185
|
+
this.logger.warn({ errors: validation.errors, redundantDefaults: redundantPaths.length }, "Skipping fleet.yaml default slimming because the raw config is invalid");
|
|
5186
|
+
return;
|
|
5187
|
+
}
|
|
5188
|
+
try {
|
|
5189
|
+
this.saveFleetConfig();
|
|
5190
|
+
this.logger.info({ strippedDefaults: redundantPaths.length, backup: `${this.configPath}.bak` }, "Slimmed redundant instance defaults in fleet.yaml");
|
|
5191
|
+
}
|
|
5192
|
+
catch (err) {
|
|
5193
|
+
// A migration must not turn a previously bootable fleet into an outage.
|
|
5194
|
+
this.logger.warn({ err }, "Could not slim fleet.yaml; continuing with the original config");
|
|
5195
|
+
}
|
|
5196
|
+
}
|
|
5197
|
+
writeFleetConfigBackup(source) {
|
|
5198
|
+
if (!this.configPath)
|
|
5199
|
+
return;
|
|
5200
|
+
const backupPath = `${this.configPath}.bak`;
|
|
5201
|
+
const tempPath = `${backupPath}.tmp-${process.pid}`;
|
|
5202
|
+
writeFileSync(tempPath, source, "utf-8");
|
|
5203
|
+
if (existsSync(this.configPath))
|
|
5204
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
5205
|
+
renameSync(tempPath, backupPath);
|
|
4163
5206
|
}
|
|
4164
5207
|
patchFleetDocument(document, path, before, after) {
|
|
4165
5208
|
if (Object.is(before, after))
|
|
@@ -4211,7 +5254,12 @@ export class FleetManager {
|
|
|
4211
5254
|
currentNode.value = after;
|
|
4212
5255
|
}
|
|
4213
5256
|
else {
|
|
4214
|
-
|
|
5257
|
+
// Use a YAML collection node for newly-added objects. Passing the raw
|
|
5258
|
+
// object creates a scalar wrapper: it serializes, but nested hasIn /
|
|
5259
|
+
// deleteIn cannot traverse it (notably when slimming create_instance).
|
|
5260
|
+
document.setIn(path, after !== null && typeof after === "object"
|
|
5261
|
+
? document.createNode(after)
|
|
5262
|
+
: after);
|
|
4215
5263
|
}
|
|
4216
5264
|
}
|
|
4217
5265
|
}
|
|
@@ -4302,7 +5350,7 @@ export class FleetManager {
|
|
|
4302
5350
|
from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct,
|
|
4303
5351
|
});
|
|
4304
5352
|
this.webhookEmitter?.emit("model_failover", name, { from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct });
|
|
4305
|
-
this.notifyInstanceTopic(name,
|
|
5353
|
+
this.notifyInstanceTopic(name, t("failover.triggered", fiveHourPct, fallbackModel, primaryModel));
|
|
4306
5354
|
}
|
|
4307
5355
|
else if (fiveHourPct < FleetManager.FAILOVER_RECOVER_PCT && currentFailover) {
|
|
4308
5356
|
// Recover: switch back to primary
|
|
@@ -4313,7 +5361,7 @@ export class FleetManager {
|
|
|
4313
5361
|
restored: primaryModel, five_hour_pct: fiveHourPct,
|
|
4314
5362
|
});
|
|
4315
5363
|
this.webhookEmitter?.emit("model_recovered", name, { restored: primaryModel, five_hour_pct: fiveHourPct });
|
|
4316
|
-
this.notifyInstanceTopic(name,
|
|
5364
|
+
this.notifyInstanceTopic(name, t("failover.recovered", fiveHourPct, primaryModel));
|
|
4317
5365
|
}
|
|
4318
5366
|
}
|
|
4319
5367
|
toggleFleetCollab(instanceName) {
|
|
@@ -4367,9 +5415,9 @@ export class FleetManager {
|
|
|
4367
5415
|
if (output)
|
|
4368
5416
|
return output;
|
|
4369
5417
|
if (e.code === "ENOENT") {
|
|
4370
|
-
return "
|
|
5418
|
+
return t("doctor.cli_missing");
|
|
4371
5419
|
}
|
|
4372
|
-
return stripAnsi(e.message ?? "").trim() || "
|
|
5420
|
+
return stripAnsi(e.message ?? "").trim() || t("doctor.failed");
|
|
4373
5421
|
}
|
|
4374
5422
|
}
|
|
4375
5423
|
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
@@ -4453,6 +5501,31 @@ export class FleetManager {
|
|
|
4453
5501
|
* The log line is written by the caller regardless: if every adapter is down,
|
|
4454
5502
|
* the only notification path is the one that is broken.
|
|
4455
5503
|
*/
|
|
5504
|
+
/**
|
|
5505
|
+
* Surface ids that can never match into the operator's General topic.
|
|
5506
|
+
*
|
|
5507
|
+
* The load-time log is still silence for anyone not reading daemon.log, and a
|
|
5508
|
+
* chat that never gets in with nothing said anywhere is exactly the failure
|
|
5509
|
+
* this line of work exists to remove. notifyFleetError throttles by message
|
|
5510
|
+
* text for 10 minutes and resolves its target from config, so the 30s reload
|
|
5511
|
+
* poll cannot flood the topic and a down General does not swallow it.
|
|
5512
|
+
*/
|
|
5513
|
+
reportClassicUnrecoverableIds() {
|
|
5514
|
+
const bad = this.classicChannels?.getUnrecoverableIds?.() ?? [];
|
|
5515
|
+
if (bad.length === 0)
|
|
5516
|
+
return;
|
|
5517
|
+
// notifyFleetError claims its throttle key BEFORE attempting delivery, so
|
|
5518
|
+
// calling it with no adapter would silence this message for ten minutes
|
|
5519
|
+
// without anyone having seen it. Log and leave the key unspent; a later
|
|
5520
|
+
// call, once an adapter exists, still gets through.
|
|
5521
|
+
if (!this.adapter && this.adapters.size === 0) {
|
|
5522
|
+
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match — deferring the operator notice until an adapter is up");
|
|
5523
|
+
return;
|
|
5524
|
+
}
|
|
5525
|
+
const list = bad.map(e => `${e.field}: ${e.value}`).join(", ");
|
|
5526
|
+
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5527
|
+
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5528
|
+
}
|
|
4456
5529
|
notifyFleetError(text) {
|
|
4457
5530
|
const now = Date.now();
|
|
4458
5531
|
const key = text.slice(0, 200);
|
|
@@ -4462,16 +5535,8 @@ export class FleetManager {
|
|
|
4462
5535
|
return;
|
|
4463
5536
|
}
|
|
4464
5537
|
const suppressed = seen?.suppressed ?? 0;
|
|
4465
|
-
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
4466
|
-
// Bound the map: it is keyed by message text, and a message with a varying
|
|
4467
|
-
// suffix (a path, an id) would otherwise grow it without limit.
|
|
4468
|
-
if (this.fleetErrorNotices.size > 100) {
|
|
4469
|
-
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
4470
|
-
if (oldest !== undefined)
|
|
4471
|
-
this.fleetErrorNotices.delete(oldest);
|
|
4472
|
-
}
|
|
4473
5538
|
const body = suppressed > 0
|
|
4474
|
-
? `${text}\n(
|
|
5539
|
+
? `${text}\n${t("fleet.error_suppressed", suppressed, Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000))}`
|
|
4475
5540
|
: text;
|
|
4476
5541
|
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4477
5542
|
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
@@ -4479,26 +5544,60 @@ export class FleetManager {
|
|
|
4479
5544
|
// topic_id to post into it.
|
|
4480
5545
|
const general = Object.entries(this.fleetConfig?.instances ?? {})
|
|
4481
5546
|
.find(([, config]) => config.general_topic === true)?.[0];
|
|
5547
|
+
// "dispatched", not "delivered": see notifyInstanceTopic. A platform that
|
|
5548
|
+
// rejects the message afterwards still consumes the throttle window, which
|
|
5549
|
+
// is the pre-existing semantic and not something this change alters.
|
|
5550
|
+
let dispatched = false;
|
|
4482
5551
|
if (general) {
|
|
4483
|
-
this.notifyInstanceTopic(general, body);
|
|
4484
|
-
return;
|
|
5552
|
+
dispatched = this.notifyInstanceTopic(general, body);
|
|
4485
5553
|
}
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
5554
|
+
else {
|
|
5555
|
+
// No General instance — fall back to the primary channel's group.
|
|
5556
|
+
const groupId = this.getChannelConfig()?.group_id;
|
|
5557
|
+
if (this.adapter && groupId) {
|
|
5558
|
+
this.adapter.sendText(String(groupId), body)
|
|
5559
|
+
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5560
|
+
dispatched = true;
|
|
5561
|
+
}
|
|
5562
|
+
}
|
|
5563
|
+
if (!dispatched) {
|
|
5564
|
+
// Do NOT record the throttle key. It used to be claimed before delivery
|
|
5565
|
+
// was attempted, so a message nobody could receive — every fleet error
|
|
5566
|
+
// raised before adapters exist, which is when startup faults happen —
|
|
5567
|
+
// also suppressed the next ten minutes of identical ones. Leaving the key
|
|
5568
|
+
// unspent lets the next caller, once an adapter is up, actually get
|
|
5569
|
+
// through. The counter is not reset either: nothing was shown.
|
|
5570
|
+
this.logger.warn({ text: body }, "Fleet error could not be delivered (no adapter or no target yet) — not consuming the throttle window");
|
|
4492
5571
|
return;
|
|
4493
5572
|
}
|
|
4494
|
-
this.
|
|
5573
|
+
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
5574
|
+
// Bound the map: it is keyed by message text, and a message with a varying
|
|
5575
|
+
// suffix (a path, an id) would otherwise grow it without limit.
|
|
5576
|
+
if (this.fleetErrorNotices.size > 100) {
|
|
5577
|
+
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
5578
|
+
if (oldest !== undefined)
|
|
5579
|
+
this.fleetErrorNotices.delete(oldest);
|
|
5580
|
+
}
|
|
4495
5581
|
}
|
|
4496
5582
|
static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
|
|
4497
5583
|
fleetErrorNotices = new Map();
|
|
5584
|
+
/**
|
|
5585
|
+
* Post into an instance's topic. Returns whether a send was DISPATCHED — a
|
|
5586
|
+
* target was resolved and sendText was called — so a caller that must not
|
|
5587
|
+
* lose the message (notifyFleetError, which spends a throttle key) can tell
|
|
5588
|
+
* that from silence. Existing callers ignore the result and are unaffected.
|
|
5589
|
+
*
|
|
5590
|
+
* Not a delivery guarantee: sendText is fire-and-forget, and a platform-side
|
|
5591
|
+
* rejection surfaces only as a warn in its .catch. True delivery confirmation
|
|
5592
|
+
* would have to make this async and change every caller.
|
|
5593
|
+
*/
|
|
4498
5594
|
notifyInstanceTopic(instanceName, text, extraOpts) {
|
|
4499
5595
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
4500
|
-
if (!adapter)
|
|
4501
|
-
|
|
5596
|
+
if (!adapter) {
|
|
5597
|
+
// Early startup: adapters are created well after the fleet object exists.
|
|
5598
|
+
this.logger.warn({ instanceName }, "No adapter yet — instance topic notification not sent");
|
|
5599
|
+
return false;
|
|
5600
|
+
}
|
|
4502
5601
|
const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
|
|
4503
5602
|
const groupId = channelCfg?.group_id;
|
|
4504
5603
|
// Fleet topic instance
|
|
@@ -4506,26 +5605,30 @@ export class FleetManager {
|
|
|
4506
5605
|
if (threadId != null && groupId) {
|
|
4507
5606
|
adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
|
|
4508
5607
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
|
|
4509
|
-
return;
|
|
5608
|
+
return true;
|
|
4510
5609
|
}
|
|
4511
5610
|
// Classic instance: find its channelId from the classic manager
|
|
4512
5611
|
const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
|
|
4513
5612
|
if (classicChatId) {
|
|
4514
5613
|
adapter.sendText(classicChatId, text, extraOpts)
|
|
4515
5614
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
4516
|
-
return;
|
|
5615
|
+
return true;
|
|
4517
5616
|
}
|
|
4518
5617
|
// Fallback: send to group without threadId
|
|
4519
5618
|
if (groupId) {
|
|
4520
5619
|
adapter.sendText(String(groupId), text, extraOpts)
|
|
4521
5620
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5621
|
+
return true;
|
|
4522
5622
|
}
|
|
5623
|
+
this.logger.warn({ instanceName }, "No group id — instance topic notification not sent");
|
|
5624
|
+
return false;
|
|
4523
5625
|
}
|
|
4524
5626
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
4525
5627
|
//
|
|
4526
5628
|
// One shared lifecycle for every "notification with decision buttons":
|
|
4527
5629
|
// post with a 128-bit nonce, arm a bounded expiry, bind the click to the
|
|
4528
|
-
// exact adapter+chat+thread+message that created it, require fleet admin
|
|
5630
|
+
// exact adapter+chat+thread+message that created it, require fleet admin for
|
|
5631
|
+
// actions that mutate runtime state (Tips are the harmless exception),
|
|
4529
5632
|
// consume exactly once. The features differ only in what they post
|
|
4530
5633
|
// and what a consumed click does.
|
|
4531
5634
|
/**
|
|
@@ -4536,8 +5639,14 @@ export class FleetManager {
|
|
|
4536
5639
|
*/
|
|
4537
5640
|
async postNonceButtonPrompt(opts) {
|
|
4538
5641
|
// 16 bytes = the 128-bit capability the design claims. Telegram's 64-byte
|
|
4539
|
-
// callback_data cap still holds
|
|
4540
|
-
// + 32 hex + ":confirm" (8) = 59
|
|
5642
|
+
// callback_data cap still holds, with two prefixes tied at the longest:
|
|
5643
|
+
// "interactive-assist:" (19) + 32 hex + ":confirm" (8) = 59, and
|
|
5644
|
+
// "install-select:" (15) + 32 hex + ":" + the longest backend name
|
|
5645
|
+
// ("claude-code"/"antigravity", 11) = 59. The longest overall is
|
|
5646
|
+
// "classic-approve:" (16) + 32 hex + ":allow-admin" (12) = 60, leaving 4
|
|
5647
|
+
// bytes. A longer prefix or action would be silently rejected by Telegram —
|
|
5648
|
+
// both sets are pinned by callback_data assertions in
|
|
5649
|
+
// install-backend-menu.test.ts and classic-approve-buttons.test.ts.
|
|
4541
5650
|
const nonce = randomBytes(16).toString("hex");
|
|
4542
5651
|
const entry = {
|
|
4543
5652
|
prefix: opts.prefix,
|
|
@@ -4567,6 +5676,17 @@ export class FleetManager {
|
|
|
4567
5676
|
message: opts.message,
|
|
4568
5677
|
choices: opts.choices.map(c => ({ id: `${opts.prefix}${nonce}:${c.action}`, label: c.label })),
|
|
4569
5678
|
}, opts.threadId ? { threadId: opts.threadId } : undefined);
|
|
5679
|
+
// Bind the nonce to the provider's canonical delivery address, not the
|
|
5680
|
+
// logical routing input. Telegram, for example, represents General as
|
|
5681
|
+
// AgEnD topic "1" on input but omits message_thread_id on the wire and in
|
|
5682
|
+
// callback queries. Exact callback matching below remains fail-closed.
|
|
5683
|
+
entry.chatId = sent.chatId;
|
|
5684
|
+
// Preserve compatibility with adapters that predate SentMessage.threadId
|
|
5685
|
+
// and omit the optional property altogether. An explicit undefined is a
|
|
5686
|
+
// canonical flat/root delivery context (notably Telegram General).
|
|
5687
|
+
if (Object.prototype.hasOwnProperty.call(sent, "threadId")) {
|
|
5688
|
+
entry.threadId = sent.threadId;
|
|
5689
|
+
}
|
|
4570
5690
|
entry.messageId = sent.messageId;
|
|
4571
5691
|
return nonce;
|
|
4572
5692
|
}
|
|
@@ -4579,19 +5699,18 @@ export class FleetManager {
|
|
|
4579
5699
|
}
|
|
4580
5700
|
}
|
|
4581
5701
|
/**
|
|
4582
|
-
*
|
|
5702
|
+
* Validate a nonce-armed callback and claim it exactly once.
|
|
4583
5703
|
*
|
|
4584
5704
|
* Returns:
|
|
4585
5705
|
* - null — the callback is not for this prefix; try the next handler
|
|
4586
5706
|
* - "consumed" — for this prefix but stale/denied/malformed; stop dispatch
|
|
4587
|
-
* - the entry+action — the click is authorized and
|
|
4588
|
-
* from the map before any await, so double clicks cannot act twice)
|
|
5707
|
+
* - the entry+action — the click is authorized and claimed before any await
|
|
4589
5708
|
*
|
|
4590
5709
|
* A stale click (expired nonce, or a pre-upgrade button whose payload no
|
|
4591
5710
|
* longer parses) collapses the clicked message so the dead button stops
|
|
4592
5711
|
* inviting clicks — the same courtesy the cancel button extends.
|
|
4593
5712
|
*/
|
|
4594
|
-
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter) {
|
|
5713
|
+
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter, staleHandling) {
|
|
4595
5714
|
if (!data.callbackData.startsWith(prefix))
|
|
4596
5715
|
return null;
|
|
4597
5716
|
const match = data.callbackData.match(actionRe);
|
|
@@ -4603,27 +5722,52 @@ export class FleetManager {
|
|
|
4603
5722
|
pending = undefined;
|
|
4604
5723
|
if (!match || !pending) {
|
|
4605
5724
|
const adapter = receivingAdapter ?? this.adapter;
|
|
4606
|
-
|
|
5725
|
+
if (staleHandling?.keepText && adapter?.removeMessageButtons) {
|
|
5726
|
+
adapter.removeMessageButtons(data.chatId, data.messageId, data.threadId)
|
|
5727
|
+
.catch(() => { });
|
|
5728
|
+
}
|
|
5729
|
+
else {
|
|
5730
|
+
adapter?.editMessageRemoveButtons?.(data.chatId, data.messageId, t("buttons.stale"), data.threadId).catch(() => { });
|
|
5731
|
+
}
|
|
5732
|
+
if (staleHandling?.notice) {
|
|
5733
|
+
adapter?.sendText(data.chatId, staleHandling.notice, { threadId: data.threadId })
|
|
5734
|
+
.catch(() => { });
|
|
5735
|
+
}
|
|
4607
5736
|
return "consumed";
|
|
4608
5737
|
}
|
|
4609
5738
|
// Bind the capability to the exact message/world that created it. Telegram
|
|
4610
|
-
// keyboards are visible to everyone, so
|
|
5739
|
+
// keyboards are visible to everyone, so mutating actions require fleet admin;
|
|
5740
|
+
// a Tip acknowledgement only records that the shared content was read.
|
|
4611
5741
|
const isAuthorized = data.userId
|
|
4612
|
-
? pending.
|
|
4613
|
-
?
|
|
4614
|
-
:
|
|
5742
|
+
? pending.allowAnyUser
|
|
5743
|
+
? true
|
|
5744
|
+
: pending.authChannelId
|
|
5745
|
+
? this.isModelAdmin(data.userId, pending.authChannelId, callbackAdapterId)
|
|
5746
|
+
: this.isFleetAdmin(data.userId, callbackAdapterId)
|
|
4615
5747
|
: false;
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
5748
|
+
const mismatchedFields = [];
|
|
5749
|
+
if (pending.adapterId !== callbackAdapterId)
|
|
5750
|
+
mismatchedFields.push("adapterId");
|
|
5751
|
+
if (data.chatId !== pending.chatId)
|
|
5752
|
+
mismatchedFields.push("chatId");
|
|
5753
|
+
if (pending.threadId != null && data.threadId !== pending.threadId)
|
|
5754
|
+
mismatchedFields.push("threadId");
|
|
5755
|
+
if (pending.messageId != null && data.messageId !== pending.messageId)
|
|
5756
|
+
mismatchedFields.push("messageId");
|
|
5757
|
+
if (!isAuthorized)
|
|
5758
|
+
mismatchedFields.push("authorization");
|
|
5759
|
+
if (mismatchedFields.length > 0) {
|
|
4621
5760
|
// Deliberately does NOT consume the nonce: the real admin can still click.
|
|
4622
|
-
this.logger.warn({
|
|
5761
|
+
this.logger.warn({
|
|
5762
|
+
instanceName: pending.instanceName,
|
|
5763
|
+
prefix,
|
|
5764
|
+
userId: data.userId,
|
|
5765
|
+
mismatchedFields: mismatchedFields.join(","),
|
|
5766
|
+
}, "Rejected unauthorized or mismatched button callback");
|
|
4623
5767
|
return "consumed";
|
|
4624
5768
|
}
|
|
4625
|
-
// Claim before any await: double clicks and duplicate callback delivery
|
|
4626
|
-
// never act twice.
|
|
5769
|
+
// Claim before any await: double clicks and duplicate callback delivery
|
|
5770
|
+
// can never act twice for state-changing actions.
|
|
4627
5771
|
this.pendingNonceButtons.delete(match[1]);
|
|
4628
5772
|
if (pending.timer)
|
|
4629
5773
|
clearTimeout(pending.timer);
|
|
@@ -4644,10 +5788,223 @@ export class FleetManager {
|
|
|
4644
5788
|
this.logger.warn({ err, instanceName: pending.instanceName }, "Failed to retire prompt buttons");
|
|
4645
5789
|
}
|
|
4646
5790
|
}
|
|
5791
|
+
tipsEnabled() {
|
|
5792
|
+
return this.fleetConfig?.defaults.tips !== false;
|
|
5793
|
+
}
|
|
5794
|
+
readTipState() {
|
|
5795
|
+
if (!this.scheduler)
|
|
5796
|
+
return null;
|
|
5797
|
+
try {
|
|
5798
|
+
return {
|
|
5799
|
+
dismissed: this.scheduler.db.listDismissedTipIds(),
|
|
5800
|
+
advancedUnlocked: this.scheduler.db.isAdvancedTipsUnlocked(),
|
|
5801
|
+
};
|
|
5802
|
+
}
|
|
5803
|
+
catch (err) {
|
|
5804
|
+
// Tips are additive. A damaged/locked scheduler DB must never block fleet
|
|
5805
|
+
// startup, update completion, or a General command.
|
|
5806
|
+
this.logger.warn({ err }, "Failed to read tip state");
|
|
5807
|
+
return null;
|
|
5808
|
+
}
|
|
5809
|
+
}
|
|
5810
|
+
pickAvailableTip() {
|
|
5811
|
+
const state = this.readTipState();
|
|
5812
|
+
return state
|
|
5813
|
+
? selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds())
|
|
5814
|
+
: null;
|
|
5815
|
+
}
|
|
5816
|
+
tipText(tip) {
|
|
5817
|
+
return getLocale() === "zh-TW" ? tip.text_zh : tip.text_en;
|
|
5818
|
+
}
|
|
5819
|
+
formatTip(tip) {
|
|
5820
|
+
return `💡 ${t("tips.label")}: ${this.tipText(tip)}`;
|
|
5821
|
+
}
|
|
5822
|
+
/** Persistently enable advanced tips through the admin-only slash command. */
|
|
5823
|
+
unlockAdvancedTips(userId) {
|
|
5824
|
+
if (!this.scheduler)
|
|
5825
|
+
return false;
|
|
5826
|
+
try {
|
|
5827
|
+
this.scheduler.db.unlockAdvancedTips(userId);
|
|
5828
|
+
this.eventLog?.insert(this.findGeneralInstance() ?? "general", "tips_advanced_unlocked", {
|
|
5829
|
+
userId,
|
|
5830
|
+
source: "command",
|
|
5831
|
+
});
|
|
5832
|
+
return true;
|
|
5833
|
+
}
|
|
5834
|
+
catch (err) {
|
|
5835
|
+
this.logger.warn({ err, userId }, "Failed to unlock advanced tips from command");
|
|
5836
|
+
return false;
|
|
5837
|
+
}
|
|
5838
|
+
}
|
|
5839
|
+
/** Post one fresh nonce-armed tip in a known General channel. */
|
|
5840
|
+
async promptTip(generalName, adapter, chatId, threadId) {
|
|
5841
|
+
const state = this.readTipState();
|
|
5842
|
+
if (!state)
|
|
5843
|
+
return "unavailable";
|
|
5844
|
+
if (!state.advancedUnlocked
|
|
5845
|
+
&& visibleTipLevels(true).has("advanced")
|
|
5846
|
+
&& canUnlockAdvancedTips(state.dismissed)) {
|
|
5847
|
+
return await this.promptAdvancedTipUnlock(generalName, adapter, chatId, threadId)
|
|
5848
|
+
? "posted"
|
|
5849
|
+
: "unavailable";
|
|
5850
|
+
}
|
|
5851
|
+
const tip = selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds());
|
|
5852
|
+
if (!tip)
|
|
5853
|
+
return this.scheduler ? "empty" : "unavailable";
|
|
5854
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5855
|
+
prefix: TIP_DISMISS_CALLBACK_PREFIX,
|
|
5856
|
+
alertType: "tip",
|
|
5857
|
+
instanceName: generalName,
|
|
5858
|
+
adapter,
|
|
5859
|
+
adapterId: adapter.id,
|
|
5860
|
+
chatId,
|
|
5861
|
+
threadId,
|
|
5862
|
+
message: this.formatTip(tip),
|
|
5863
|
+
choices: [
|
|
5864
|
+
{ action: "dismiss", label: t("tips.dismiss") },
|
|
5865
|
+
{ action: "confused", label: t("tips.confused") },
|
|
5866
|
+
],
|
|
5867
|
+
// Expiry removes only the stale button; the useful tip remains readable.
|
|
5868
|
+
expiredText: this.formatTip(tip),
|
|
5869
|
+
extra: { allowAnyUser: true, tipId: tip.id },
|
|
5870
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5871
|
+
});
|
|
5872
|
+
return nonce ? "posted" : "unavailable";
|
|
5873
|
+
}
|
|
5874
|
+
async promptAdvancedTipUnlock(generalName, adapter, chatId, threadId) {
|
|
5875
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5876
|
+
prefix: TIP_UNLOCK_CALLBACK_PREFIX,
|
|
5877
|
+
alertType: "tip",
|
|
5878
|
+
instanceName: generalName,
|
|
5879
|
+
adapter,
|
|
5880
|
+
adapterId: adapter.id,
|
|
5881
|
+
chatId,
|
|
5882
|
+
threadId,
|
|
5883
|
+
message: t("tips.advanced.unlock_prompt"),
|
|
5884
|
+
choices: [{ action: "unlock", label: t("tips.advanced.unlock") }],
|
|
5885
|
+
expiredText: t("tips.advanced.expired"),
|
|
5886
|
+
extra: { allowAnyUser: true },
|
|
5887
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5888
|
+
});
|
|
5889
|
+
return nonce !== null;
|
|
5890
|
+
}
|
|
5891
|
+
async sendTipToGeneral() {
|
|
5892
|
+
if (!this.tipsEnabled())
|
|
5893
|
+
return;
|
|
5894
|
+
const generalName = this.findGeneralInstance();
|
|
5895
|
+
if (!generalName)
|
|
5896
|
+
return;
|
|
5897
|
+
const adapter = this.getAdapterForInstance(generalName);
|
|
5898
|
+
const chatId = this.getGroupIdForInstance(generalName);
|
|
5899
|
+
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
5900
|
+
if (!adapter || !chatId)
|
|
5901
|
+
return;
|
|
5902
|
+
const result = await this.promptTip(generalName, adapter, chatId, topicId != null ? String(topicId) : undefined);
|
|
5903
|
+
if (result === "unavailable") {
|
|
5904
|
+
this.logger.warn({ generalName }, "Daily tip could not be posted");
|
|
5905
|
+
}
|
|
5906
|
+
}
|
|
5907
|
+
async handleTipDismiss(data, callbackAdapterId, receivingAdapter) {
|
|
5908
|
+
const claimed = this.consumeNonceCallback(TIP_DISMISS_CALLBACK_PREFIX, /^tip-dismiss:([0-9a-f]+):(dismiss|confused)$/, data, callbackAdapterId, receivingAdapter,
|
|
5909
|
+
// A restart orphans the expiry timer, so tip buttons can outlive their
|
|
5910
|
+
// nonce by days. Keep the tip text readable and always tell the clicker
|
|
5911
|
+
// what to do next (the keyboard edit itself can be refused: Telegram
|
|
5912
|
+
// rejects edits on messages older than 48h).
|
|
5913
|
+
{ keepText: true, notice: t("tips.expired_use_tips") });
|
|
5914
|
+
if (claimed === null)
|
|
5915
|
+
return false;
|
|
5916
|
+
if (claimed === "consumed")
|
|
5917
|
+
return true;
|
|
5918
|
+
const { entry: pending } = claimed;
|
|
5919
|
+
if (claimed.action === "confused") {
|
|
5920
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5921
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip feedback has incomplete persistence context");
|
|
5922
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5923
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId });
|
|
5924
|
+
return true;
|
|
5925
|
+
}
|
|
5926
|
+
try {
|
|
5927
|
+
this.scheduler.db.recordTipFeedback(data.userId, pending.tipId, "confused");
|
|
5928
|
+
}
|
|
5929
|
+
catch (err) {
|
|
5930
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip feedback");
|
|
5931
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5932
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId }).catch(() => { });
|
|
5933
|
+
return true;
|
|
5934
|
+
}
|
|
5935
|
+
this.logger.info({
|
|
5936
|
+
tipId: pending.tipId,
|
|
5937
|
+
userId: data.userId,
|
|
5938
|
+
feedbackType: "confused",
|
|
5939
|
+
}, "Tip feedback recorded");
|
|
5940
|
+
this.eventLog?.insert(pending.instanceName, "tip_feedback", {
|
|
5941
|
+
tipId: pending.tipId,
|
|
5942
|
+
userId: data.userId,
|
|
5943
|
+
feedbackType: "confused",
|
|
5944
|
+
});
|
|
5945
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5946
|
+
// Confusion is feedback, not a dismissal. Keep the Tip readable while
|
|
5947
|
+
// retiring this one-shot prompt; it remains eligible for future draws.
|
|
5948
|
+
pending.expiredText);
|
|
5949
|
+
await pending.adapter.sendText(pending.chatId, t("tips.feedback_recorded"), { threadId: pending.threadId }).catch(err => this.logger.warn({ err, tipId: pending.tipId }, "Failed to acknowledge tip feedback"));
|
|
5950
|
+
return true;
|
|
5951
|
+
}
|
|
5952
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5953
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip dismissal has incomplete persistence context");
|
|
5954
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5955
|
+
return true;
|
|
5956
|
+
}
|
|
5957
|
+
try {
|
|
5958
|
+
this.scheduler.db.dismissTip(data.userId, pending.tipId);
|
|
5959
|
+
}
|
|
5960
|
+
catch (err) {
|
|
5961
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip dismissal");
|
|
5962
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5963
|
+
return true;
|
|
5964
|
+
}
|
|
5965
|
+
this.eventLog?.insert(pending.instanceName, "tip_dismissed", {
|
|
5966
|
+
tipId: pending.tipId,
|
|
5967
|
+
userId: data.userId,
|
|
5968
|
+
});
|
|
5969
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5970
|
+
// Dismissal changes future selection only. Keep the useful tip text in
|
|
5971
|
+
// chat history and remove just the now-consumed button. Append a short
|
|
5972
|
+
// confirmation so the user sees feedback (unlike the pre-#605 version
|
|
5973
|
+
// that replaced the entire message with just the confirmation line).
|
|
5974
|
+
`${pending.expiredText}\n\n${t("tips.dismissed")}`);
|
|
5975
|
+
const state = this.readTipState();
|
|
5976
|
+
if (state && !state.advancedUnlocked && canUnlockAdvancedTips(state.dismissed)) {
|
|
5977
|
+
await this.promptAdvancedTipUnlock(pending.instanceName, pending.adapter, pending.chatId, pending.threadId);
|
|
5978
|
+
}
|
|
5979
|
+
return true;
|
|
5980
|
+
}
|
|
5981
|
+
async handleTipUnlock(data, callbackAdapterId, receivingAdapter) {
|
|
5982
|
+
const claimed = this.consumeNonceCallback(TIP_UNLOCK_CALLBACK_PREFIX, /^tip-unlock:([0-9a-f]+):(unlock)$/, data, callbackAdapterId, receivingAdapter, { keepText: true, notice: t("tips.expired_use_tips") });
|
|
5983
|
+
if (claimed === null)
|
|
5984
|
+
return false;
|
|
5985
|
+
if (claimed === "consumed")
|
|
5986
|
+
return true;
|
|
5987
|
+
const { entry: pending } = claimed;
|
|
5988
|
+
if (!data.userId || !this.scheduler) {
|
|
5989
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5990
|
+
return true;
|
|
5991
|
+
}
|
|
5992
|
+
try {
|
|
5993
|
+
this.scheduler.db.unlockAdvancedTips(data.userId);
|
|
5994
|
+
}
|
|
5995
|
+
catch (err) {
|
|
5996
|
+
this.logger.warn({ err, userId: data.userId }, "Failed to unlock advanced tips");
|
|
5997
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5998
|
+
return true;
|
|
5999
|
+
}
|
|
6000
|
+
this.eventLog?.insert(pending.instanceName, "tips_advanced_unlocked", { userId: data.userId });
|
|
6001
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.advanced.unlocked"));
|
|
6002
|
+
return true;
|
|
6003
|
+
}
|
|
4647
6004
|
/** Post the destructive `/clear` confirmation in the invoking channel. */
|
|
4648
6005
|
async promptClearConfirmation(instanceName, channelId, adapter, chatId, threadId) {
|
|
4649
6006
|
if (!this.topicCommands.supportsClear(instanceName))
|
|
4650
|
-
return
|
|
6007
|
+
return t("clear.unsupported");
|
|
4651
6008
|
const adapterId = adapter.id;
|
|
4652
6009
|
if (!adapterId)
|
|
4653
6010
|
return t("clear.prompt_unavailable");
|
|
@@ -4921,7 +6278,7 @@ export class FleetManager {
|
|
|
4921
6278
|
const force = target.kind === "classic" && !!data.options?.force;
|
|
4922
6279
|
const cmd = saveCommandForBackend(backend, filename, force);
|
|
4923
6280
|
if (!cmd) {
|
|
4924
|
-
await data.respond(
|
|
6281
|
+
await data.respond(t("save.unsupported"));
|
|
4925
6282
|
return;
|
|
4926
6283
|
}
|
|
4927
6284
|
if (target.kind === "classic") {
|
|
@@ -4974,7 +6331,7 @@ export class FleetManager {
|
|
|
4974
6331
|
}
|
|
4975
6332
|
return false;
|
|
4976
6333
|
}
|
|
4977
|
-
async sendCancelButton(instanceName, correlationId) {
|
|
6334
|
+
async sendCancelButton(instanceName, correlationId, preserveProgress = false) {
|
|
4978
6335
|
// Post first, retire after (see the tail of this method). Retiring up front
|
|
4979
6336
|
// meant that from the delete until the new message came back — a chat API
|
|
4980
6337
|
// round trip, and every reply goes through here — the instance had NO live
|
|
@@ -5035,6 +6392,10 @@ export class FleetManager {
|
|
|
5035
6392
|
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
5036
6393
|
// with nothing visibly changed.
|
|
5037
6394
|
lastProgressText: "👀 處理中…",
|
|
6395
|
+
// A reply can re-post the same in-flight bubble below the reply. Carry
|
|
6396
|
+
// its current list into that replacement; fresh inbound work must start
|
|
6397
|
+
// empty even if the daemon's reset broadcast is still in flight.
|
|
6398
|
+
toolProgress: preserveProgress ? this.instanceProgress.get(instanceName) : undefined,
|
|
5038
6399
|
};
|
|
5039
6400
|
this.startProgressTicker(entry);
|
|
5040
6401
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
@@ -5088,16 +6449,28 @@ export class FleetManager {
|
|
|
5088
6449
|
static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
|
|
5089
6450
|
if (elapsedMs < minElapsedMs)
|
|
5090
6451
|
return "👀 處理中…";
|
|
6452
|
+
const elapsed = FleetManager.formatProgressElapsed(elapsedMs);
|
|
6453
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
6454
|
+
return detail
|
|
6455
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
6456
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
6457
|
+
}
|
|
6458
|
+
/** Render elapsed time consistently in live and retained progress bubbles. */
|
|
6459
|
+
static formatProgressElapsed(elapsedMs) {
|
|
5091
6460
|
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
5092
6461
|
const minutes = Math.floor(totalSeconds / 60);
|
|
5093
6462
|
const seconds = totalSeconds % 60;
|
|
5094
|
-
|
|
6463
|
+
return minutes >= 60
|
|
5095
6464
|
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
5096
6465
|
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
5097
|
-
|
|
5098
|
-
|
|
5099
|
-
|
|
5100
|
-
|
|
6466
|
+
}
|
|
6467
|
+
/** Final read-only form of a bubble that contains semantic tool history.
|
|
6468
|
+
* Neutral wording is intentional: the same retirement primitive is used for
|
|
6469
|
+
* normal completion, a mid-turn bubble replacement, and user cancellation,
|
|
6470
|
+
* so claiming every retained checkpoint is "completed" would be false. */
|
|
6471
|
+
composeRetiredBubbleText(entry) {
|
|
6472
|
+
const elapsed = FleetManager.formatProgressElapsed(Date.now() - (entry.startedAt ?? Date.now()));
|
|
6473
|
+
return `🧾 工具歷程 (記錄至 ${elapsed})\n${entry.toolProgress}`;
|
|
5101
6474
|
}
|
|
5102
6475
|
/**
|
|
5103
6476
|
* Make a tool summary safe to paste into a channel message.
|
|
@@ -5130,6 +6503,88 @@ export class FleetManager {
|
|
|
5130
6503
|
else
|
|
5131
6504
|
this.instanceActivity.delete(name);
|
|
5132
6505
|
}
|
|
6506
|
+
/**
|
|
6507
|
+
* Cache the turn's tool-progress list and push it into the instance's live
|
|
6508
|
+
* bubble, coalesced so a burst of tool events cannot flood the Bot API
|
|
6509
|
+
* (Telegram's flood limit is per-chat across ALL forum topics, so edits are
|
|
6510
|
+
* rate-limited per bubble AND ride behind the daemon-side 3s coalescer).
|
|
6511
|
+
*/
|
|
6512
|
+
cacheInstanceProgress(name, progress) {
|
|
6513
|
+
if (progress)
|
|
6514
|
+
this.instanceProgress.set(name, progress);
|
|
6515
|
+
else
|
|
6516
|
+
this.instanceProgress.delete(name);
|
|
6517
|
+
for (const entry of this.cancelButtons.values()) {
|
|
6518
|
+
if (entry.instanceName !== name)
|
|
6519
|
+
continue;
|
|
6520
|
+
// Empty means the daemon completed/reset the turn. Clear the live cache
|
|
6521
|
+
// for the next turn, but retain this bubble's last non-empty list so its
|
|
6522
|
+
// retirement can preserve the history instead of deleting the message.
|
|
6523
|
+
if (progress)
|
|
6524
|
+
entry.toolProgress = progress;
|
|
6525
|
+
this.scheduleProgressEdit(entry);
|
|
6526
|
+
}
|
|
6527
|
+
}
|
|
6528
|
+
/** At most one progress-driven edit per bubble per TOOL_PROGRESS_EDIT_MIN_MS. */
|
|
6529
|
+
scheduleProgressEdit(entry) {
|
|
6530
|
+
if (entry.retiring)
|
|
6531
|
+
return;
|
|
6532
|
+
const since = Date.now() - (entry.lastProgressEditAt ?? 0);
|
|
6533
|
+
if (since >= TOOL_PROGRESS_EDIT_MIN_MS) {
|
|
6534
|
+
this.refreshBubble(entry);
|
|
6535
|
+
return;
|
|
6536
|
+
}
|
|
6537
|
+
if (entry.progressEditTimer)
|
|
6538
|
+
return; // trailing edit already scheduled
|
|
6539
|
+
entry.progressEditTimer = setTimeout(() => {
|
|
6540
|
+
entry.progressEditTimer = undefined;
|
|
6541
|
+
this.refreshBubble(entry);
|
|
6542
|
+
}, TOOL_PROGRESS_EDIT_MIN_MS - since);
|
|
6543
|
+
entry.progressEditTimer.unref?.();
|
|
6544
|
+
}
|
|
6545
|
+
/**
|
|
6546
|
+
* The ONE composer for the bubble text. Both writers — the elapsed-time
|
|
6547
|
+
* ticker and the tool-progress push — go through here; two independent
|
|
6548
|
+
* renderers editing the same message is how the progress list used to get
|
|
6549
|
+
* wiped by the next elapsed tick (#528 trap 2).
|
|
6550
|
+
*/
|
|
6551
|
+
composeBubbleText(entry) {
|
|
6552
|
+
return FleetManager.bubbleText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs(), entry.toolProgress);
|
|
6553
|
+
}
|
|
6554
|
+
/** Pure composition of header + tool list, exposed for tests. */
|
|
6555
|
+
static bubbleText(elapsedMs, activity, minElapsedMs, progress) {
|
|
6556
|
+
const header = FleetManager.progressText(elapsedMs,
|
|
6557
|
+
// The single-line activity detail is redundant once a tool list exists.
|
|
6558
|
+
progress ? undefined : activity, minElapsedMs);
|
|
6559
|
+
return progress ? `${header}\n${progress}` : header;
|
|
6560
|
+
}
|
|
6561
|
+
/** Recompose and edit the bubble in place; skips when nothing changed. */
|
|
6562
|
+
refreshBubble(entry) {
|
|
6563
|
+
if (entry.retiring || !this.cancelButtons.has(entry.messageId)) {
|
|
6564
|
+
clearInterval(entry.progressTimer);
|
|
6565
|
+
return;
|
|
6566
|
+
}
|
|
6567
|
+
const text = this.composeBubbleText(entry);
|
|
6568
|
+
if (text === entry.lastProgressText)
|
|
6569
|
+
return; // nothing changed — skip the API call
|
|
6570
|
+
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
6571
|
+
if (!adapter?.editAlert)
|
|
6572
|
+
return;
|
|
6573
|
+
entry.lastProgressText = text;
|
|
6574
|
+
entry.lastProgressEditAt = Date.now();
|
|
6575
|
+
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
6576
|
+
type: "cancel",
|
|
6577
|
+
instanceName: entry.instanceName,
|
|
6578
|
+
message: text,
|
|
6579
|
+
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
6580
|
+
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
6581
|
+
.catch(err => {
|
|
6582
|
+
// A failed progress edit must never escalate: the button still works and
|
|
6583
|
+
// the next tick retries. Common causes are a deleted message or a
|
|
6584
|
+
// rate limit.
|
|
6585
|
+
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
6586
|
+
});
|
|
6587
|
+
}
|
|
5133
6588
|
/**
|
|
5134
6589
|
* Refresh the button's text in place while the instance keeps working.
|
|
5135
6590
|
*
|
|
@@ -5146,32 +6601,8 @@ export class FleetManager {
|
|
|
5146
6601
|
}
|
|
5147
6602
|
return PROGRESS_MIN_ELAPSED_MS;
|
|
5148
6603
|
}
|
|
5149
|
-
startProgressTicker(entry) {
|
|
5150
|
-
const tick = () =>
|
|
5151
|
-
if (!this.cancelButtons.has(entry.messageId)) {
|
|
5152
|
-
clearInterval(entry.progressTimer);
|
|
5153
|
-
return;
|
|
5154
|
-
}
|
|
5155
|
-
const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
|
|
5156
|
-
if (text === entry.lastProgressText)
|
|
5157
|
-
return; // nothing changed — skip the API call
|
|
5158
|
-
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
5159
|
-
if (!adapter?.editAlert)
|
|
5160
|
-
return;
|
|
5161
|
-
entry.lastProgressText = text;
|
|
5162
|
-
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
5163
|
-
type: "cancel",
|
|
5164
|
-
instanceName: entry.instanceName,
|
|
5165
|
-
message: text,
|
|
5166
|
-
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
5167
|
-
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
5168
|
-
.catch(err => {
|
|
5169
|
-
// A failed progress edit must never escalate: the button still works and
|
|
5170
|
-
// the next tick retries. Common causes are a deleted message or a
|
|
5171
|
-
// rate limit.
|
|
5172
|
-
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
5173
|
-
});
|
|
5174
|
-
};
|
|
6604
|
+
startProgressTicker(entry) {
|
|
6605
|
+
const tick = () => this.refreshBubble(entry);
|
|
5175
6606
|
entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
|
|
5176
6607
|
entry.progressTimer.unref?.();
|
|
5177
6608
|
// One extra tick right when the threshold passes, so a 30s threshold shows
|
|
@@ -5185,8 +6616,11 @@ export class FleetManager {
|
|
|
5185
6616
|
}
|
|
5186
6617
|
/**
|
|
5187
6618
|
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
5188
|
-
* it has not, retire its button.
|
|
5189
|
-
*
|
|
6619
|
+
* it has not, retire its button. The daemon is asked for a fresh pane capture
|
|
6620
|
+
* before deciding. Reading only the transition cache stranded the first
|
|
6621
|
+
* post-restart bubble when its startup "working" report never got a matching
|
|
6622
|
+
* idle edge. Re-arming replaces the previous timer, so a burst of replies ends
|
|
6623
|
+
* with exactly one pending check.
|
|
5190
6624
|
*/
|
|
5191
6625
|
armReplyGrace(instanceName) {
|
|
5192
6626
|
for (const entry of this.cancelButtons.values()) {
|
|
@@ -5198,14 +6632,26 @@ export class FleetManager {
|
|
|
5198
6632
|
entry.replyGraceTimer = undefined;
|
|
5199
6633
|
if (!this.cancelButtons.has(entry.messageId))
|
|
5200
6634
|
return;
|
|
5201
|
-
|
|
5202
|
-
return; // resumed — a long run keeps its button
|
|
5203
|
-
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
5204
|
-
this.retireButton(entry);
|
|
6635
|
+
void this.finishReplyGrace(instanceName, entry);
|
|
5205
6636
|
}, REPLY_RETIRE_GRACE_MS);
|
|
5206
6637
|
entry.replyGraceTimer.unref?.();
|
|
5207
6638
|
}
|
|
5208
6639
|
}
|
|
6640
|
+
async finishReplyGrace(instanceName, entry) {
|
|
6641
|
+
const refreshed = await this.refreshInstanceExecutionState(instanceName, REPLY_STATE_REFRESH_TIMEOUT_MS);
|
|
6642
|
+
if (!this.cancelButtons.has(entry.messageId) || entry.retiring)
|
|
6643
|
+
return;
|
|
6644
|
+
if (!refreshed) {
|
|
6645
|
+
// Fail safe: without an authoritative answer, retain a potentially live
|
|
6646
|
+
// Cancel button. The 5-minute/30-minute/24-hour safety nets still apply.
|
|
6647
|
+
this.logger.debug({ instanceName, messageId: entry.messageId }, "Cancel reply-grace state refresh timed out");
|
|
6648
|
+
return;
|
|
6649
|
+
}
|
|
6650
|
+
if (!this.getInstanceIdle(instanceName))
|
|
6651
|
+
return; // genuine long run keeps its button
|
|
6652
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
6653
|
+
this.retireButton(entry);
|
|
6654
|
+
}
|
|
5209
6655
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
5210
6656
|
retireInstanceButtons(instanceName) {
|
|
5211
6657
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -5227,7 +6673,7 @@ export class FleetManager {
|
|
|
5227
6673
|
this.deleteButtonMessage(entry)
|
|
5228
6674
|
.then(() => {
|
|
5229
6675
|
this.discardButton(entry);
|
|
5230
|
-
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId }, "Cancel button
|
|
6676
|
+
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId, historyPreserved: Boolean(entry.toolProgress) }, "Cancel button retired");
|
|
5231
6677
|
})
|
|
5232
6678
|
.catch((err) => this.scheduleButtonRetry(entry, err));
|
|
5233
6679
|
}
|
|
@@ -5239,6 +6685,8 @@ export class FleetManager {
|
|
|
5239
6685
|
clearInterval(entry.idleCheckTimer);
|
|
5240
6686
|
if (entry.progressTimer)
|
|
5241
6687
|
clearInterval(entry.progressTimer);
|
|
6688
|
+
if (entry.progressEditTimer)
|
|
6689
|
+
clearTimeout(entry.progressEditTimer);
|
|
5242
6690
|
if (entry.replyGraceTimer)
|
|
5243
6691
|
clearTimeout(entry.replyGraceTimer);
|
|
5244
6692
|
this.cancelButtons.delete(entry.messageId);
|
|
@@ -5316,12 +6764,21 @@ export class FleetManager {
|
|
|
5316
6764
|
// Continue the same retire cycle (bypass the retiring-guard in retireButton).
|
|
5317
6765
|
entry.retryTimer = setTimeout(() => this.attemptButtonDelete(entry), CANCEL_BTN_RETRY_INTERVAL_MS);
|
|
5318
6766
|
}
|
|
5319
|
-
/**
|
|
6767
|
+
/** Retire one cancel bubble via its own adapter. A bubble that accumulated
|
|
6768
|
+
* tool progress is kept as a read-only history message; a plain bubble keeps
|
|
6769
|
+
* the legacy delete/edit-to-checkmark behavior. Resolves on success and
|
|
5320
6770
|
* rejects on failure so the caller can retry. */
|
|
5321
6771
|
deleteButtonMessage(e) {
|
|
5322
6772
|
const adapter = (e.adapterId ? this.worlds.get(e.adapterId)?.adapter : undefined) ?? this.adapter;
|
|
5323
6773
|
if (!adapter)
|
|
5324
6774
|
return Promise.reject(new Error("no adapter for cancel button"));
|
|
6775
|
+
if (e.toolProgress && adapter.editMessageRemoveButtons) {
|
|
6776
|
+
return adapter.editMessageRemoveButtons(e.chatId, e.messageId, this.composeRetiredBubbleText(e), e.threadId);
|
|
6777
|
+
}
|
|
6778
|
+
// All production adapters currently implement editMessageRemoveButtons.
|
|
6779
|
+
// A third-party adapter without it degrades to the legacy delete behavior:
|
|
6780
|
+
// removing a live Cancel control is safer than retaining an actionable,
|
|
6781
|
+
// untracked button forever.
|
|
5325
6782
|
if (adapter.deleteMessage)
|
|
5326
6783
|
return adapter.deleteMessage(e.chatId, e.messageId, e.threadId);
|
|
5327
6784
|
if (adapter.editMessageRemoveButtons)
|
|
@@ -5373,11 +6830,604 @@ export class FleetManager {
|
|
|
5373
6830
|
const daemon = this.daemons.get(instanceName);
|
|
5374
6831
|
if (!daemon)
|
|
5375
6832
|
return false;
|
|
6833
|
+
const deliveryEpoch = this.cancelPendingDeliveries(instanceName);
|
|
6834
|
+
daemon.clearPendingDeliveries?.(deliveryEpoch);
|
|
5376
6835
|
daemon.sendEscape().catch(e => this.logger.warn({ err: e, instanceName }, "sendEscape failed"));
|
|
5377
6836
|
this.lastInboundMsg.delete(instanceName);
|
|
5378
6837
|
this.clearCancelButton(instanceName);
|
|
5379
6838
|
return true;
|
|
5380
6839
|
}
|
|
6840
|
+
getDeliveryEpoch(instanceName) {
|
|
6841
|
+
return this.deliveryEpochs.get(instanceName) ?? 0;
|
|
6842
|
+
}
|
|
6843
|
+
isDeliveryEpochCurrent(instanceName, deliveryEpoch) {
|
|
6844
|
+
return deliveryEpoch === this.getDeliveryEpoch(instanceName);
|
|
6845
|
+
}
|
|
6846
|
+
/** Invalidate work queued before a user cancel and wake idle-gate waiters. */
|
|
6847
|
+
cancelPendingDeliveries(instanceName) {
|
|
6848
|
+
const next = this.getDeliveryEpoch(instanceName) + 1;
|
|
6849
|
+
this.deliveryEpochs.set(instanceName, next);
|
|
6850
|
+
for (const check of this.instanceIdleWaiters.get(instanceName) ?? [])
|
|
6851
|
+
check();
|
|
6852
|
+
return next;
|
|
6853
|
+
}
|
|
6854
|
+
// ── Remote CLI login (`/login`) ──────────────────────────────────────────
|
|
6855
|
+
//
|
|
6856
|
+
// One session at a time, in a dedicated tmux window — never an instance
|
|
6857
|
+
// pane. Credentials are per-backend (codex homes symlink auth.json to the
|
|
6858
|
+
// shared ~/.codex; the other CLIs use one real home), so a single sign-in
|
|
6859
|
+
// repairs every instance of that backend, and instance delivery, pane-state
|
|
6860
|
+
// detection, tool progress, and mcp_proxy_reply never observe login output.
|
|
6861
|
+
activeLogin = null;
|
|
6862
|
+
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
6863
|
+
async promptLoginBackends(chat) {
|
|
6864
|
+
const configured = new Set();
|
|
6865
|
+
for (const [, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
6866
|
+
configured.add(config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
6867
|
+
}
|
|
6868
|
+
const choices = Object.keys(LOGIN_FLOWS)
|
|
6869
|
+
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
6870
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6871
|
+
if (choices.length === 0) {
|
|
6872
|
+
await chat.adapter.sendText(chat.chatId, t("login.unsupported", [...configured].join(", ")), { threadId: chat.threadId });
|
|
6873
|
+
return;
|
|
6874
|
+
}
|
|
6875
|
+
await this.postNonceButtonPrompt({
|
|
6876
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6877
|
+
alertType: "login",
|
|
6878
|
+
instanceName: "login",
|
|
6879
|
+
adapter: chat.adapter,
|
|
6880
|
+
adapterId: chat.adapterId,
|
|
6881
|
+
chatId: chat.chatId,
|
|
6882
|
+
threadId: chat.threadId,
|
|
6883
|
+
message: t("login.choose_backend"),
|
|
6884
|
+
choices,
|
|
6885
|
+
expiredText: t("buttons.stale"),
|
|
6886
|
+
});
|
|
6887
|
+
}
|
|
6888
|
+
/**
|
|
6889
|
+
* Ask the General topic to approve a ClassicBot access request.
|
|
6890
|
+
*
|
|
6891
|
+
* Two affirmative buttons rather than one, when the trigger identified a
|
|
6892
|
+
* user. Allowing a group to talk and granting that person ClassicBot admin
|
|
6893
|
+
* (start/stop/model on classic channels — NOT fleet admin, which reads
|
|
6894
|
+
* fleet.yaml `channel.access.allowed_users`) have different blast radii, and
|
|
6895
|
+
* bundling them would force anyone who wants only the cheap one to accept the
|
|
6896
|
+
* expensive one or go edit YAML by hand.
|
|
6897
|
+
*
|
|
6898
|
+
* Built on postNonceButtonPrompt, so it inherits the canonical-address
|
|
6899
|
+
* binding (#682) that makes these buttons answer in a Telegram General topic.
|
|
6900
|
+
*/
|
|
6901
|
+
async promptClassicApproval(opts) {
|
|
6902
|
+
const adapter = this.getAdapterForInstance(opts.generalName);
|
|
6903
|
+
// An instance with no world binding yet (fresh restart, or a fleet whose
|
|
6904
|
+
// primary `channel:` block carries the config) still has a usable adapter —
|
|
6905
|
+
// fall back to that adapter's own id rather than dropping the buttons. The
|
|
6906
|
+
// id only has to match what the callback arrives with, which is this same
|
|
6907
|
+
// adapter.
|
|
6908
|
+
const adapterId = this.getInstanceAdapterId(opts.generalName) ?? adapter?.id;
|
|
6909
|
+
const chatId = this.getGroupIdForInstance(opts.generalName);
|
|
6910
|
+
const topicId = this.fleetConfig?.instances[opts.generalName]?.topic_id;
|
|
6911
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6912
|
+
// Never lose the notification because the buttons could not be addressed.
|
|
6913
|
+
this.notifyInstanceTopic(opts.generalName, opts.message);
|
|
6914
|
+
return;
|
|
6915
|
+
}
|
|
6916
|
+
const choices = [{ action: "allow", label: t("classic.approve_group") }];
|
|
6917
|
+
if (opts.userId)
|
|
6918
|
+
choices.push({ action: "allow-admin", label: t("classic.approve_group_admin") });
|
|
6919
|
+
choices.push({ action: "ignore", label: t("classic.approve_ignore") });
|
|
6920
|
+
await this.postNonceButtonPrompt({
|
|
6921
|
+
prefix: CLASSIC_APPROVE_CALLBACK_PREFIX,
|
|
6922
|
+
alertType: "classic_approve",
|
|
6923
|
+
instanceName: opts.generalName,
|
|
6924
|
+
adapter,
|
|
6925
|
+
adapterId,
|
|
6926
|
+
chatId,
|
|
6927
|
+
threadId: topicId != null ? String(topicId) : undefined,
|
|
6928
|
+
message: opts.message,
|
|
6929
|
+
choices,
|
|
6930
|
+
expiredText: t("buttons.stale"),
|
|
6931
|
+
extra: { classicGroupId: opts.groupId, classicUserId: opts.userId, classicScope: opts.scope },
|
|
6932
|
+
});
|
|
6933
|
+
}
|
|
6934
|
+
/**
|
|
6935
|
+
* Apply an approved ClassicBot access request.
|
|
6936
|
+
*
|
|
6937
|
+
* Writes through ClassicChannelManager's typed mutators rather than handing
|
|
6938
|
+
* General a YAML edit: the id is stored via String() so a Discord snowflake
|
|
6939
|
+
* cannot land as a YAML integer and lose precision, the existing save() path
|
|
6940
|
+
* is reused, in-memory state is correct immediately (no waiting on the 30s
|
|
6941
|
+
* reload poll), and a click cannot silently do nothing because General
|
|
6942
|
+
* happened to be paused. General is still told the outcome.
|
|
6943
|
+
*/
|
|
6944
|
+
async applyClassicApproval(entry, groupId, adminUserId) {
|
|
6945
|
+
const classic = this.classicChannels;
|
|
6946
|
+
if (!classic) {
|
|
6947
|
+
this.notifyInstanceTopic(entry.instanceName, t("classic.approve_failed", groupId));
|
|
6948
|
+
return;
|
|
6949
|
+
}
|
|
6950
|
+
const scope = entry.classicScope ?? "guild";
|
|
6951
|
+
const access = scope === "group" ? classic.allowGroup(groupId) : classic.allowGuild(groupId);
|
|
6952
|
+
const admin = adminUserId ? classic.addAdminUser(adminUserId) : null;
|
|
6953
|
+
const lines = [t(access === "added" ? "classic.approve_done_group"
|
|
6954
|
+
: access === "already" ? "classic.approve_done_group_already"
|
|
6955
|
+
: "classic.approve_done_group_open", groupId)];
|
|
6956
|
+
if (admin) {
|
|
6957
|
+
lines.push(t(admin === "added" ? "classic.approve_done_admin" : "classic.approve_done_admin_already", adminUserId ?? ""));
|
|
6958
|
+
}
|
|
6959
|
+
this.notifyInstanceTopic(entry.instanceName, lines.join("\n"));
|
|
6960
|
+
}
|
|
6961
|
+
/** Consume a ClassicBot approval button. */
|
|
6962
|
+
async handleClassicApproval(data, callbackAdapterId, receivingAdapter) {
|
|
6963
|
+
const claimed = this.consumeNonceCallback(CLASSIC_APPROVE_CALLBACK_PREFIX, /^classic-approve:([0-9a-f]+):(allow|allow-admin|ignore)$/, data, callbackAdapterId, receivingAdapter);
|
|
6964
|
+
if (claimed === null)
|
|
6965
|
+
return false;
|
|
6966
|
+
if (claimed === "consumed")
|
|
6967
|
+
return true;
|
|
6968
|
+
const { entry, action } = claimed;
|
|
6969
|
+
const groupId = entry.classicGroupId ?? "";
|
|
6970
|
+
const userId = entry.classicUserId;
|
|
6971
|
+
if (action === "ignore") {
|
|
6972
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("classic.approve_ignored", groupId));
|
|
6973
|
+
return true;
|
|
6974
|
+
}
|
|
6975
|
+
const grantAdmin = action === "allow-admin" && !!userId;
|
|
6976
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, grantAdmin ? t("classic.approve_applying_admin", groupId, userId ?? "")
|
|
6977
|
+
: t("classic.approve_applying", groupId));
|
|
6978
|
+
await this.applyClassicApproval(entry, groupId, grantAdmin ? userId : undefined);
|
|
6979
|
+
return true;
|
|
6980
|
+
}
|
|
6981
|
+
/**
|
|
6982
|
+
* Backend chooser for a bare `/install-cli`, mirroring promptLoginBackends so
|
|
6983
|
+
* both commands feel the same. Built on postNonceButtonPrompt rather than the
|
|
6984
|
+
* `/model` selection coordinator: that is the mechanism `/login` already uses,
|
|
6985
|
+
* and the one whose canonical-address binding (#682) makes the buttons answer
|
|
6986
|
+
* in a Telegram General topic.
|
|
6987
|
+
*
|
|
6988
|
+
* Unlike the login chooser this does NOT filter to backends the fleet already
|
|
6989
|
+
* runs. Installing is how you get a backend you do not have yet, so filtering
|
|
6990
|
+
* by configured backends would hide the only entry the admin came for.
|
|
6991
|
+
*
|
|
6992
|
+
* gemini-cli is omitted: it is deprecated (see backend/factory.ts). Typing
|
|
6993
|
+
* `/install-cli gemini-cli` still works — this only stops recommending it.
|
|
6994
|
+
*/
|
|
6995
|
+
async promptInstallBackends(chat) {
|
|
6996
|
+
const choices = Object.keys(BACKEND_INSTALLATION_INFO)
|
|
6997
|
+
.filter(backend => backend !== "gemini-cli")
|
|
6998
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6999
|
+
await this.postNonceButtonPrompt({
|
|
7000
|
+
prefix: INSTALL_CALLBACK_PREFIX,
|
|
7001
|
+
alertType: "install",
|
|
7002
|
+
instanceName: "install",
|
|
7003
|
+
adapter: chat.adapter,
|
|
7004
|
+
adapterId: chat.adapterId,
|
|
7005
|
+
chatId: chat.chatId,
|
|
7006
|
+
threadId: chat.threadId,
|
|
7007
|
+
message: t("install.choose_backend"),
|
|
7008
|
+
choices,
|
|
7009
|
+
expiredText: t("buttons.stale"),
|
|
7010
|
+
});
|
|
7011
|
+
}
|
|
7012
|
+
/** Backend chooser button → start that backend's install session. */
|
|
7013
|
+
async handleInstallBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7014
|
+
const claimed = this.consumeNonceCallback(INSTALL_CALLBACK_PREFIX, /^install-select:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
7015
|
+
if (claimed === null)
|
|
7016
|
+
return false;
|
|
7017
|
+
if (claimed === "consumed")
|
|
7018
|
+
return true;
|
|
7019
|
+
const { entry, action: backend } = claimed;
|
|
7020
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.starting_backend", backend));
|
|
7021
|
+
const text = await this.startInstallSession(backend, {
|
|
7022
|
+
adapter: entry.adapter,
|
|
7023
|
+
adapterId: entry.adapterId,
|
|
7024
|
+
chatId: entry.chatId,
|
|
7025
|
+
threadId: entry.threadId,
|
|
7026
|
+
});
|
|
7027
|
+
if (text)
|
|
7028
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7029
|
+
return true;
|
|
7030
|
+
}
|
|
7031
|
+
/**
|
|
7032
|
+
* Start a login session for one backend. Caller enforces admin.
|
|
7033
|
+
* Returns a status line to post, or null when a confirmation prompt was
|
|
7034
|
+
* posted instead (auth still valid — see the pre-check below).
|
|
7035
|
+
*/
|
|
7036
|
+
async startLoginSession(backendArg, chat, opts = {}) {
|
|
7037
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7038
|
+
const flow = LOGIN_FLOWS[backend];
|
|
7039
|
+
if (!flow)
|
|
7040
|
+
return t("login.unsupported", backendArg);
|
|
7041
|
+
if (this.activeLogin)
|
|
7042
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7043
|
+
// Install sessions share the window namespace — never run both at once.
|
|
7044
|
+
if (this.activeInstall)
|
|
7045
|
+
return t("install.busy");
|
|
7046
|
+
// Token-free pre-check (5s cap): re-login while auth still works is
|
|
7047
|
+
// usually a mistake, so it needs a confirmed click. An invalid OR
|
|
7048
|
+
// uncertain result (timeout, missing binary) proceeds straight to login —
|
|
7049
|
+
// an unreliable probe must never block the re-login the admin asked for.
|
|
7050
|
+
if (!opts.skipAuthCheck && flow.authCheck) {
|
|
7051
|
+
const status = await checkAuthStatus(flow.authCheck);
|
|
7052
|
+
if (status === "valid") {
|
|
7053
|
+
await this.postNonceButtonPrompt({
|
|
7054
|
+
prefix: LOGIN_CONFIRM_CALLBACK_PREFIX,
|
|
7055
|
+
alertType: "login",
|
|
7056
|
+
instanceName: backend,
|
|
7057
|
+
adapter: chat.adapter,
|
|
7058
|
+
adapterId: chat.adapterId,
|
|
7059
|
+
chatId: chat.chatId,
|
|
7060
|
+
threadId: chat.threadId,
|
|
7061
|
+
message: t("login.still_valid", backend),
|
|
7062
|
+
choices: [
|
|
7063
|
+
{ action: "go", label: t("login.relogin_go") },
|
|
7064
|
+
{ action: "cancel", label: t("login.relogin_cancel") },
|
|
7065
|
+
],
|
|
7066
|
+
expiredText: t("buttons.stale"),
|
|
7067
|
+
});
|
|
7068
|
+
return null;
|
|
7069
|
+
}
|
|
7070
|
+
}
|
|
7071
|
+
return this.launchLoginSession(flow, backend, chat);
|
|
7072
|
+
}
|
|
7073
|
+
/** Create the login window and session (pre-check already settled). */
|
|
7074
|
+
async launchLoginSession(flow, backend, chat) {
|
|
7075
|
+
if (this.activeLogin)
|
|
7076
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7077
|
+
// Install sessions share the window namespace — never run both at once.
|
|
7078
|
+
if (this.activeInstall)
|
|
7079
|
+
return t("install.busy");
|
|
7080
|
+
const sessionName = getTmuxSession();
|
|
7081
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7082
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
7083
|
+
const session = new LoginSession(flow, tmux, {
|
|
7084
|
+
onMenu: async (options) => {
|
|
7085
|
+
await this.postNonceButtonPrompt({
|
|
7086
|
+
prefix: LOGIN_MENU_CALLBACK_PREFIX,
|
|
7087
|
+
alertType: "login",
|
|
7088
|
+
instanceName: backend,
|
|
7089
|
+
adapter: chat.adapter,
|
|
7090
|
+
adapterId: chat.adapterId,
|
|
7091
|
+
chatId: chat.chatId,
|
|
7092
|
+
threadId: chat.threadId,
|
|
7093
|
+
message: t("login.choose_provider"),
|
|
7094
|
+
choices: options.map((label, index) => ({ action: String(index), label })),
|
|
7095
|
+
expiredText: t("buttons.stale"),
|
|
7096
|
+
});
|
|
7097
|
+
},
|
|
7098
|
+
onAuthHint: async (url, code) => {
|
|
7099
|
+
await this.sendLoginSecret(chat, backend, url, code);
|
|
7100
|
+
},
|
|
7101
|
+
onNeedInput: async (promptExcerpt) => {
|
|
7102
|
+
await chat.adapter.sendText(chat.chatId, t("login.need_input", backend, promptExcerpt), { threadId: chat.threadId }).catch(() => { });
|
|
7103
|
+
},
|
|
7104
|
+
onDone: async ({ ok, detail }) => {
|
|
7105
|
+
this.activeLogin = null;
|
|
7106
|
+
let text;
|
|
7107
|
+
if (ok) {
|
|
7108
|
+
const { woken, restarted } = await this.recoverBackendInstances(backend);
|
|
7109
|
+
const none = t("login.none");
|
|
7110
|
+
text = t("login.success", backend, woken.length ? woken.join(", ") : none, restarted.length ? restarted.join(", ") : none);
|
|
7111
|
+
}
|
|
7112
|
+
else if (detail === "cancelled") {
|
|
7113
|
+
// The cancel command's own reply already announced this — a second
|
|
7114
|
+
// message here was a duplicate.
|
|
7115
|
+
return;
|
|
7116
|
+
}
|
|
7117
|
+
else {
|
|
7118
|
+
text = t("login.failed", backend, detail);
|
|
7119
|
+
}
|
|
7120
|
+
await chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
7121
|
+
},
|
|
7122
|
+
}, this.logger);
|
|
7123
|
+
// Claim the slot before the first await so two admins racing /login cannot
|
|
7124
|
+
// both create windows; release on startup failure.
|
|
7125
|
+
this.activeLogin = { session, backend, chat };
|
|
7126
|
+
try {
|
|
7127
|
+
await session.start();
|
|
7128
|
+
}
|
|
7129
|
+
catch (err) {
|
|
7130
|
+
this.activeLogin = null;
|
|
7131
|
+
return t("login.failed", backend, err.message);
|
|
7132
|
+
}
|
|
7133
|
+
return t("login.started", backend);
|
|
7134
|
+
}
|
|
7135
|
+
/** `/login code <text>` — paste admin-supplied text into the login window. */
|
|
7136
|
+
async loginSubmitInput(text) {
|
|
7137
|
+
if (!this.activeLogin)
|
|
7138
|
+
return t("login.no_session");
|
|
7139
|
+
const ok = await this.activeLogin.session.submitInput(text);
|
|
7140
|
+
return ok ? t("login.input_sent") : t("login.input_failed");
|
|
7141
|
+
}
|
|
7142
|
+
/** `/login cancel` — abort the active session and remove its window. */
|
|
7143
|
+
async cancelLoginSession() {
|
|
7144
|
+
if (!this.activeLogin)
|
|
7145
|
+
return t("login.no_session");
|
|
7146
|
+
const backend = this.activeLogin.backend;
|
|
7147
|
+
await this.activeLogin.session.cancel();
|
|
7148
|
+
return t("login.cancelled", backend);
|
|
7149
|
+
}
|
|
7150
|
+
/**
|
|
7151
|
+
* The URL (+ code) is a live credential: whoever completes it binds THEIR
|
|
7152
|
+
* account to this fleet's CLI. Telegram gets an HTML spoiler (same treatment
|
|
7153
|
+
* as the dashboard token); other adapters get plain text with the warning.
|
|
7154
|
+
*/
|
|
7155
|
+
async sendLoginSecret(chat, backend, url, code) {
|
|
7156
|
+
const codeLine = code ? `\n${t("login.auth_code", code)}` : "";
|
|
7157
|
+
try {
|
|
7158
|
+
if (chat.adapter.type === "telegram") {
|
|
7159
|
+
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
7160
|
+
await chat.adapter.sendText(chat.chatId, `${esc(t("login.auth_hint", backend))}\n<tg-spoiler>${esc(url)}${esc(codeLine)}</tg-spoiler>`, { threadId: chat.threadId, format: "html" });
|
|
7161
|
+
}
|
|
7162
|
+
else {
|
|
7163
|
+
await chat.adapter.sendText(chat.chatId, `${t("login.auth_hint", backend)}\n${url}${codeLine}`, { threadId: chat.threadId });
|
|
7164
|
+
}
|
|
7165
|
+
}
|
|
7166
|
+
catch (err) {
|
|
7167
|
+
this.logger.warn({ err: err.message, backend }, "Failed to deliver login URL");
|
|
7168
|
+
}
|
|
7169
|
+
}
|
|
7170
|
+
/**
|
|
7171
|
+
* After a successful login: wake the paused instances of that backend AND
|
|
7172
|
+
* restart the running ones — a live CLI holds the old token in memory and
|
|
7173
|
+
* only re-reads credentials on process start (a paused instance's CLI is
|
|
7174
|
+
* already dead, so waking it respawns with the new token for free).
|
|
7175
|
+
*/
|
|
7176
|
+
async recoverBackendInstances(backend) {
|
|
7177
|
+
const woken = [];
|
|
7178
|
+
const restarted = [];
|
|
7179
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
7180
|
+
const effective = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
7181
|
+
if (effective !== backend)
|
|
7182
|
+
continue;
|
|
7183
|
+
const status = this.getInstanceStatus(name);
|
|
7184
|
+
try {
|
|
7185
|
+
if (status === "paused") {
|
|
7186
|
+
if (this.daemons.has(name))
|
|
7187
|
+
await this.lifecycle.wake(name, 30_000);
|
|
7188
|
+
else
|
|
7189
|
+
await this.startPersistedPausedInstance(name);
|
|
7190
|
+
woken.push(name);
|
|
7191
|
+
}
|
|
7192
|
+
else if (status === "running") {
|
|
7193
|
+
await this.restartSingleInstance(name);
|
|
7194
|
+
restarted.push(name);
|
|
7195
|
+
}
|
|
7196
|
+
}
|
|
7197
|
+
catch (err) {
|
|
7198
|
+
this.logger.warn({ err: err.message, name, status }, "Post-login recovery failed");
|
|
7199
|
+
}
|
|
7200
|
+
}
|
|
7201
|
+
return { woken, restarted };
|
|
7202
|
+
}
|
|
7203
|
+
/** Backend chooser button → start that backend's login session. */
|
|
7204
|
+
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7205
|
+
const claimed = this.consumeNonceCallback(LOGIN_CALLBACK_PREFIX, /^login:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
7206
|
+
if (claimed === null)
|
|
7207
|
+
return false;
|
|
7208
|
+
if (claimed === "consumed")
|
|
7209
|
+
return true;
|
|
7210
|
+
const { entry, action: backend } = claimed;
|
|
7211
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7212
|
+
const text = await this.startLoginSession(backend, {
|
|
7213
|
+
adapter: entry.adapter,
|
|
7214
|
+
adapterId: entry.adapterId,
|
|
7215
|
+
chatId: entry.chatId,
|
|
7216
|
+
threadId: entry.threadId,
|
|
7217
|
+
});
|
|
7218
|
+
if (text)
|
|
7219
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7220
|
+
return true;
|
|
7221
|
+
}
|
|
7222
|
+
/** Re-login confirmation (auth pre-check said credentials still work). */
|
|
7223
|
+
async handleLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7224
|
+
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
7225
|
+
if (claimed === null)
|
|
7226
|
+
return false;
|
|
7227
|
+
if (claimed === "consumed")
|
|
7228
|
+
return true;
|
|
7229
|
+
const { entry, action } = claimed;
|
|
7230
|
+
const backend = entry.instanceName;
|
|
7231
|
+
if (action === "cancel") {
|
|
7232
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.cancelled", backend));
|
|
7233
|
+
return true;
|
|
7234
|
+
}
|
|
7235
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7236
|
+
const text = await this.startLoginSession(backend, {
|
|
7237
|
+
adapter: entry.adapter,
|
|
7238
|
+
adapterId: entry.adapterId,
|
|
7239
|
+
chatId: entry.chatId,
|
|
7240
|
+
threadId: entry.threadId,
|
|
7241
|
+
}, { skipAuthCheck: true });
|
|
7242
|
+
if (text)
|
|
7243
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7244
|
+
return true;
|
|
7245
|
+
}
|
|
7246
|
+
/** Kiro provider button → drive the CLI's arrow-key selector. */
|
|
7247
|
+
async handleLoginMenuSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7248
|
+
const claimed = this.consumeNonceCallback(LOGIN_MENU_CALLBACK_PREFIX, /^login-menu:([0-9a-f]+):(\d)$/, data, callbackAdapterId, receivingAdapter);
|
|
7249
|
+
if (claimed === null)
|
|
7250
|
+
return false;
|
|
7251
|
+
if (claimed === "consumed")
|
|
7252
|
+
return true;
|
|
7253
|
+
const { entry, action } = claimed;
|
|
7254
|
+
const index = Number(action);
|
|
7255
|
+
const label = this.activeLogin?.session.flow.menu?.options[index] ?? action;
|
|
7256
|
+
const ok = await this.activeLogin?.session.selectMenuOption(index) ?? false;
|
|
7257
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, ok ? t("login.provider_selected", label) : t("login.no_session"));
|
|
7258
|
+
return true;
|
|
7259
|
+
}
|
|
7260
|
+
// ── Remote CLI install (`/install-cli`) ──────────────────────────────────
|
|
7261
|
+
//
|
|
7262
|
+
// Same dedicated-window model as /login (and the same LoginSession state
|
|
7263
|
+
// machine — an install is a login flow with no auth hints): run the
|
|
7264
|
+
// installer, judge by exit code, verify the binary on a fresh login shell,
|
|
7265
|
+
// then offer to chain straight into /login.
|
|
7266
|
+
activeInstall = null;
|
|
7267
|
+
/** Start a CLI install session. Caller enforces admin. */
|
|
7268
|
+
async startInstallSession(backendArg, chat) {
|
|
7269
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7270
|
+
const info = BACKEND_INSTALLATION_INFO[backend];
|
|
7271
|
+
if (!info)
|
|
7272
|
+
return t("install.unsupported", backendArg);
|
|
7273
|
+
if (checkBinaryInstalled(info.binary))
|
|
7274
|
+
return t("install.already", backend, info.binary);
|
|
7275
|
+
if (this.activeInstall)
|
|
7276
|
+
return t("install.busy");
|
|
7277
|
+
if (this.activeLogin)
|
|
7278
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7279
|
+
const sessionName = getTmuxSession();
|
|
7280
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7281
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
7282
|
+
// A synthetic login flow: success is decided by the installer's exit code
|
|
7283
|
+
// (LoginSession treats a clean exit as success), never by pane text — and
|
|
7284
|
+
// installer output that happens to contain a URL must not be forwarded as
|
|
7285
|
+
// an auth hint, hence the no-op events below.
|
|
7286
|
+
const flow = {
|
|
7287
|
+
backend,
|
|
7288
|
+
command: info.install,
|
|
7289
|
+
successPattern: /$^/,
|
|
7290
|
+
timeoutMs: 10 * 60 * 1000,
|
|
7291
|
+
};
|
|
7292
|
+
const session = new LoginSession(flow, tmux, {
|
|
7293
|
+
onMenu: () => { },
|
|
7294
|
+
onAuthHint: () => { },
|
|
7295
|
+
onNeedInput: () => { },
|
|
7296
|
+
onDone: async ({ ok, detail }) => {
|
|
7297
|
+
this.activeInstall = null;
|
|
7298
|
+
if (!ok) {
|
|
7299
|
+
// A cancel is user-initiated — the cancel command's own reply already
|
|
7300
|
+
// said so; a second message here would be a duplicate.
|
|
7301
|
+
if (detail !== "cancelled") {
|
|
7302
|
+
await chat.adapter.sendText(chat.chatId, t("install.failed", backend, detail), { threadId: chat.threadId }).catch(() => { });
|
|
7303
|
+
}
|
|
7304
|
+
return;
|
|
7305
|
+
}
|
|
7306
|
+
// The installer may only have added the binary to a profile PATH; a
|
|
7307
|
+
// fresh login shell sees that, the fleet process's PATH may not.
|
|
7308
|
+
if (!this.verifyBinaryOnLoginShell(info.binary)) {
|
|
7309
|
+
await chat.adapter.sendText(chat.chatId, t("install.verify_failed", backend, info.binary), { threadId: chat.threadId }).catch(() => { });
|
|
7310
|
+
return;
|
|
7311
|
+
}
|
|
7312
|
+
if (!LOGIN_FLOWS[backend]) {
|
|
7313
|
+
await chat.adapter.sendText(chat.chatId, t("install.success_no_login", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7314
|
+
return;
|
|
7315
|
+
}
|
|
7316
|
+
await this.postNonceButtonPrompt({
|
|
7317
|
+
prefix: INSTALL_LOGIN_CALLBACK_PREFIX,
|
|
7318
|
+
alertType: "login",
|
|
7319
|
+
instanceName: backend,
|
|
7320
|
+
adapter: chat.adapter,
|
|
7321
|
+
adapterId: chat.adapterId,
|
|
7322
|
+
chatId: chat.chatId,
|
|
7323
|
+
threadId: chat.threadId,
|
|
7324
|
+
message: t("install.login_prompt", backend),
|
|
7325
|
+
choices: [
|
|
7326
|
+
{ action: "go", label: t("install.login_now") },
|
|
7327
|
+
{ action: "later", label: t("install.later") },
|
|
7328
|
+
],
|
|
7329
|
+
expiredText: t("install.later_ack", backend),
|
|
7330
|
+
});
|
|
7331
|
+
},
|
|
7332
|
+
}, this.logger);
|
|
7333
|
+
this.activeInstall = { session, backend, chat };
|
|
7334
|
+
try {
|
|
7335
|
+
await session.start();
|
|
7336
|
+
}
|
|
7337
|
+
catch (err) {
|
|
7338
|
+
this.activeInstall = null;
|
|
7339
|
+
return t("install.failed", backend, err.message);
|
|
7340
|
+
}
|
|
7341
|
+
return t("install.started", backend);
|
|
7342
|
+
}
|
|
7343
|
+
/** `/install-cli cancel` — abort the active install and remove its window. */
|
|
7344
|
+
async cancelInstallSession() {
|
|
7345
|
+
if (!this.activeInstall)
|
|
7346
|
+
return t("install.no_session");
|
|
7347
|
+
const backend = this.activeInstall.backend;
|
|
7348
|
+
await this.activeInstall.session.cancel();
|
|
7349
|
+
return t("install.cancelled", backend);
|
|
7350
|
+
}
|
|
7351
|
+
/** `command -v` on a login shell, so PATH additions from rc files count. */
|
|
7352
|
+
verifyBinaryOnLoginShell(binary) {
|
|
7353
|
+
try {
|
|
7354
|
+
const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe" });
|
|
7355
|
+
return result.status === 0;
|
|
7356
|
+
}
|
|
7357
|
+
catch {
|
|
7358
|
+
return false;
|
|
7359
|
+
}
|
|
7360
|
+
}
|
|
7361
|
+
/** "Sign in now?" button after a successful install. */
|
|
7362
|
+
async handleInstallLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7363
|
+
const claimed = this.consumeNonceCallback(INSTALL_LOGIN_CALLBACK_PREFIX, /^install-login:([0-9a-f]+):(go|later)$/, data, callbackAdapterId, receivingAdapter);
|
|
7364
|
+
if (claimed === null)
|
|
7365
|
+
return false;
|
|
7366
|
+
if (claimed === "consumed")
|
|
7367
|
+
return true;
|
|
7368
|
+
const { entry, action } = claimed;
|
|
7369
|
+
const backend = entry.instanceName;
|
|
7370
|
+
if (action === "later") {
|
|
7371
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.later_ack", backend));
|
|
7372
|
+
return true;
|
|
7373
|
+
}
|
|
7374
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7375
|
+
const text = await this.startLoginSession(backend, {
|
|
7376
|
+
adapter: entry.adapter,
|
|
7377
|
+
adapterId: entry.adapterId,
|
|
7378
|
+
chatId: entry.chatId,
|
|
7379
|
+
threadId: entry.threadId,
|
|
7380
|
+
});
|
|
7381
|
+
if (text)
|
|
7382
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7383
|
+
return true;
|
|
7384
|
+
}
|
|
7385
|
+
/** Discord native `/login` slash — shared by every adapter dispatch block. */
|
|
7386
|
+
async handleLoginSlash(data, adapterId, adapter) {
|
|
7387
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7388
|
+
await data.respond(t("permission.denied"));
|
|
7389
|
+
return;
|
|
7390
|
+
}
|
|
7391
|
+
const chat = { adapter, adapterId, chatId: data.channelId };
|
|
7392
|
+
if (data.options?.cancel === true) {
|
|
7393
|
+
await data.respond(await this.cancelLoginSession());
|
|
7394
|
+
return;
|
|
7395
|
+
}
|
|
7396
|
+
const code = String(data.options?.code ?? "").trim();
|
|
7397
|
+
if (code) {
|
|
7398
|
+
await data.respond(await this.loginSubmitInput(code));
|
|
7399
|
+
return;
|
|
7400
|
+
}
|
|
7401
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7402
|
+
if (backend) {
|
|
7403
|
+
const text = await this.startLoginSession(backend, chat);
|
|
7404
|
+
await data.respond(text ?? t("login.confirm_posted"));
|
|
7405
|
+
return;
|
|
7406
|
+
}
|
|
7407
|
+
await this.promptLoginBackends(chat);
|
|
7408
|
+
await data.respond(t("login.chooser_posted"));
|
|
7409
|
+
}
|
|
7410
|
+
/** Discord native `/install-cli` slash — shared by every dispatch block. */
|
|
7411
|
+
async handleInstallCliSlash(data, adapterId, adapter) {
|
|
7412
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7413
|
+
await data.respond(t("permission.denied"));
|
|
7414
|
+
return;
|
|
7415
|
+
}
|
|
7416
|
+
if (data.options?.cancel === true) {
|
|
7417
|
+
await data.respond(await this.cancelInstallSession());
|
|
7418
|
+
return;
|
|
7419
|
+
}
|
|
7420
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7421
|
+
if (!backend) {
|
|
7422
|
+
// Bare call: offer the backends instead of printing a usage line the user
|
|
7423
|
+
// then has to retype. A Discord slash that DID pick the native `backend`
|
|
7424
|
+
// choice never lands here, so the two paths cannot both fire.
|
|
7425
|
+
await this.promptInstallBackends({ adapter, adapterId, chatId: data.channelId });
|
|
7426
|
+
await data.respond(t("install.chooser_posted"));
|
|
7427
|
+
return;
|
|
7428
|
+
}
|
|
7429
|
+
await data.respond(await this.startInstallSession(backend, { adapter, adapterId, chatId: data.channelId }));
|
|
7430
|
+
}
|
|
5381
7431
|
queueMirrorMessage(text) {
|
|
5382
7432
|
const mirrorTopicId = this.fleetConfig?.channel?.mirror_topic_id;
|
|
5383
7433
|
if (mirrorTopicId == null || !this.adapter)
|
|
@@ -5549,9 +7599,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
5549
7599
|
}
|
|
5550
7600
|
/** Resolve the workspace path a backend actually uses before publishing knowledge. */
|
|
5551
7601
|
resolveKnowledgeWorkDir(workDir, backend, instanceName) {
|
|
5552
|
-
//
|
|
5553
|
-
//
|
|
5554
|
-
// CLI actually runs, not where fleet.yaml points.
|
|
7602
|
+
// Backend resolution may create/normalize the real cwd. Instructions and
|
|
7603
|
+
// skills must land where the CLI actually runs, not an assumed path.
|
|
5555
7604
|
try {
|
|
5556
7605
|
const resolved = createBackend(backend, join(getAgendHome(), "cli-env"))
|
|
5557
7606
|
.resolveWorkingDirectory?.(workDir, instanceName);
|
|
@@ -6207,6 +8256,24 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6207
8256
|
return null;
|
|
6208
8257
|
const probed = await be.probeCLIEnv({ workingDirectory: "", instanceDir: join(getAgendHome(), "cli-env"), instanceName: `probe-${backend}`, mcpServers: {} });
|
|
6209
8258
|
const env = { backend, probedAt: Date.now(), ...probed };
|
|
8259
|
+
// An empty result must never overwrite a catalog we already have. Some
|
|
8260
|
+
// probes hit the network (`agy models` fetches, 5s cap), so a slow moment
|
|
8261
|
+
// returns [] — and writing that would blank the list for the whole 24h
|
|
8262
|
+
// TTL, long after the CLI recovered. Observed live: a good 11-model
|
|
8263
|
+
// antigravity cache replaced by an empty one. Keep the known models and
|
|
8264
|
+
// let the fresher currentModel/version through.
|
|
8265
|
+
if (!env.models?.length) {
|
|
8266
|
+
const previous = this.readCliEnv(backend);
|
|
8267
|
+
if (previous?.models?.length)
|
|
8268
|
+
env.models = previous.models;
|
|
8269
|
+
}
|
|
8270
|
+
// Same protection for the extended catalog: one offline moment must not
|
|
8271
|
+
// blank a good account list for the whole cache TTL.
|
|
8272
|
+
if (!env.apiModels?.length) {
|
|
8273
|
+
const previous = this.readCliEnv(backend);
|
|
8274
|
+
if (previous?.apiModels?.length)
|
|
8275
|
+
env.apiModels = previous.apiModels;
|
|
8276
|
+
}
|
|
6210
8277
|
const path = this.cliEnvPath(backend);
|
|
6211
8278
|
mkdirSync(dirname(path), { recursive: true });
|
|
6212
8279
|
writeFileSync(path, JSON.stringify(env, null, 2));
|
|
@@ -6245,6 +8312,108 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6245
8312
|
const env = await this.probeBackend(backendName);
|
|
6246
8313
|
return env?.models ?? [];
|
|
6247
8314
|
}
|
|
8315
|
+
/**
|
|
8316
|
+
* Model catalog behind the `list_models` tool.
|
|
8317
|
+
*
|
|
8318
|
+
* The two scopes are not cosmetic. "global" is the account/CLI catalog served
|
|
8319
|
+
* from the startup probe cache; "instance" is resolved through that instance's
|
|
8320
|
+
* OWN backend config, and for a Codex instance on a custom provider that is a
|
|
8321
|
+
* different catalog entirely — `listModels()` reads models_cache.json out of
|
|
8322
|
+
* the instance's private CODEX_HOME. Answering such an instance with the
|
|
8323
|
+
* account list would name models its CLI rejects, which is exactly the
|
|
8324
|
+
* mistake this tool exists to prevent.
|
|
8325
|
+
*
|
|
8326
|
+
* `scope` always describes where the returned LIST came from, not what was
|
|
8327
|
+
* asked for: an instance query that falls back to the account catalog reports
|
|
8328
|
+
* scope "global" and says so in `note`, rather than implying instance-level
|
|
8329
|
+
* accuracy it does not have.
|
|
8330
|
+
*
|
|
8331
|
+
* Never throws — a model listing is an aid, and failing it must not fail a turn.
|
|
8332
|
+
*/
|
|
8333
|
+
async listModelCatalog(opts = {}) {
|
|
8334
|
+
const { instanceName } = opts;
|
|
8335
|
+
if (instanceName) {
|
|
8336
|
+
const backend = this.backendNameForInstance(instanceName);
|
|
8337
|
+
const resolved = this.resolveInstanceModel(instanceName);
|
|
8338
|
+
const currentModel = resolved.source === "unresolved" ? null : resolved.model;
|
|
8339
|
+
const provider = this.customProviderFor(instanceName, backend);
|
|
8340
|
+
const scoped = await this.instanceScopedModels(instanceName, backend);
|
|
8341
|
+
if (scoped.length) {
|
|
8342
|
+
return {
|
|
8343
|
+
backend, scope: "instance", instance: instanceName,
|
|
8344
|
+
current_model: currentModel, models: scoped, source: "live",
|
|
8345
|
+
...(provider ? { note: `Catalog read through this instance's ${backend} provider "${provider}" — it may differ from the account catalog.` } : {}),
|
|
8346
|
+
};
|
|
8347
|
+
}
|
|
8348
|
+
// No instance-local catalog (never launched, or the backend has no
|
|
8349
|
+
// per-instance list). The account catalog is the best available answer,
|
|
8350
|
+
// but it is labelled honestly rather than dressed up as instance scope.
|
|
8351
|
+
const global = await this.globalModelCatalog(backend);
|
|
8352
|
+
return {
|
|
8353
|
+
...global, instance: instanceName, current_model: currentModel,
|
|
8354
|
+
note: provider
|
|
8355
|
+
? `No instance-local catalog yet; showing the account catalog, which may NOT match this instance's ${backend} provider "${provider}".`
|
|
8356
|
+
: "No instance-local catalog yet; showing the account catalog.",
|
|
8357
|
+
};
|
|
8358
|
+
}
|
|
8359
|
+
return this.globalModelCatalog(opts.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
8360
|
+
}
|
|
8361
|
+
/** The custom provider an instance overrides its backend with, if any. */
|
|
8362
|
+
customProviderFor(instanceName, backend) {
|
|
8363
|
+
const opts = this.fleetConfig?.instances?.[instanceName]?.backend_options?.[backend]
|
|
8364
|
+
?? this.fleetConfig?.defaults?.backend_options?.[backend];
|
|
8365
|
+
const provider = opts?.provider;
|
|
8366
|
+
return typeof provider === "string" && provider.trim() ? provider.trim() : null;
|
|
8367
|
+
}
|
|
8368
|
+
/** Ask a backend for its catalog using ONE instance's real config. Never throws. */
|
|
8369
|
+
async instanceScopedModels(instanceName, backend) {
|
|
8370
|
+
try {
|
|
8371
|
+
const inst = this.fleetConfig?.instances?.[instanceName];
|
|
8372
|
+
const instanceDir = this.getInstanceDir(instanceName);
|
|
8373
|
+
const be = createBackend(backend, instanceDir);
|
|
8374
|
+
if (!be.listModels)
|
|
8375
|
+
return [];
|
|
8376
|
+
return await be.listModels({
|
|
8377
|
+
workingDirectory: inst?.working_directory ?? "",
|
|
8378
|
+
instanceDir,
|
|
8379
|
+
instanceName,
|
|
8380
|
+
mcpServers: {},
|
|
8381
|
+
model: inst?.model,
|
|
8382
|
+
backendOptions: inst?.backend_options?.[backend] ?? this.fleetConfig?.defaults?.backend_options?.[backend],
|
|
8383
|
+
}) ?? [];
|
|
8384
|
+
}
|
|
8385
|
+
catch {
|
|
8386
|
+
// listModels is documented never to throw, but a backend constructor can
|
|
8387
|
+
// (missing binary). A catalog is an aid; degrade to the account list.
|
|
8388
|
+
return [];
|
|
8389
|
+
}
|
|
8390
|
+
}
|
|
8391
|
+
/** Account-wide catalog: probe cache first, live probe on miss. */
|
|
8392
|
+
async globalModelCatalog(backend) {
|
|
8393
|
+
const cached = this.readCliEnv(backend);
|
|
8394
|
+
if (cached?.models?.length) {
|
|
8395
|
+
return {
|
|
8396
|
+
backend, scope: "global", current_model: cached.currentModel ?? null,
|
|
8397
|
+
models: cached.models, source: "cache",
|
|
8398
|
+
probed_at: new Date(cached.probedAt).toISOString(),
|
|
8399
|
+
};
|
|
8400
|
+
}
|
|
8401
|
+
const env = await this.probeBackend(backend);
|
|
8402
|
+
if (env?.models?.length) {
|
|
8403
|
+
return {
|
|
8404
|
+
backend, scope: "global", current_model: env.currentModel ?? null,
|
|
8405
|
+
models: env.models, source: "live",
|
|
8406
|
+
probed_at: new Date(env.probedAt).toISOString(),
|
|
8407
|
+
};
|
|
8408
|
+
}
|
|
8409
|
+
// Reported rather than thrown: "we could not enumerate" is a useful answer,
|
|
8410
|
+
// and the caller can still set a model by name (AgEnD passes it through).
|
|
8411
|
+
return {
|
|
8412
|
+
backend, scope: "global", current_model: env?.currentModel ?? null,
|
|
8413
|
+
models: [], source: "fallback",
|
|
8414
|
+
note: `Could not enumerate models for ${backend} (CLI missing, not logged in, or it offers no list). Model names are passed through to the CLI, so a known-good name still works.`,
|
|
8415
|
+
};
|
|
8416
|
+
}
|
|
6248
8417
|
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
6249
8418
|
/** Label an effort choice, marking the one currently configured. */
|
|
6250
8419
|
effortChoiceLabel(level, current) {
|
|
@@ -6253,10 +8422,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6253
8422
|
effortMenuHeader(instanceName) {
|
|
6254
8423
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6255
8424
|
if (!effort)
|
|
6256
|
-
return "
|
|
8425
|
+
return t("effort.current_default");
|
|
6257
8426
|
return source === "fleet-default"
|
|
6258
|
-
?
|
|
6259
|
-
:
|
|
8427
|
+
? t("effort.current_fleet", effort)
|
|
8428
|
+
: t("effort.current", effort);
|
|
6260
8429
|
}
|
|
6261
8430
|
/** `/effort` — DC Select Menu, or apply directly when a level is given. */
|
|
6262
8431
|
async handleEffortSlash(data, adapterId) {
|
|
@@ -6277,11 +8446,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6277
8446
|
}
|
|
6278
8447
|
const levels = this.effortLevelsFor(name);
|
|
6279
8448
|
if (levels.length === 0) {
|
|
6280
|
-
await data.respond(
|
|
8449
|
+
await data.respond(t("effort.unsupported", this.backendNameForInstance(name)));
|
|
6281
8450
|
return;
|
|
6282
8451
|
}
|
|
6283
8452
|
if (!data.respondChoices) {
|
|
6284
|
-
await data.respond(
|
|
8453
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6285
8454
|
return;
|
|
6286
8455
|
}
|
|
6287
8456
|
const current = this.resolveInstanceEffort(name).effort;
|
|
@@ -6294,20 +8463,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6294
8463
|
timer.unref?.();
|
|
6295
8464
|
this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
6296
8465
|
try {
|
|
6297
|
-
await data.respondChoices(
|
|
8466
|
+
await data.respondChoices(t("effort.menu", this.effortMenuHeader(name)), choices);
|
|
6298
8467
|
}
|
|
6299
8468
|
catch (err) {
|
|
6300
8469
|
this.pendingEffortSelects.delete(nonce);
|
|
6301
8470
|
clearTimeout(timer);
|
|
6302
8471
|
this.logger.warn({ err, instanceName: name }, "effort menu failed");
|
|
6303
|
-
await data.respond(
|
|
8472
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6304
8473
|
}
|
|
6305
8474
|
}
|
|
6306
8475
|
/** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
|
|
6307
8476
|
async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6308
8477
|
const levels = this.effortLevelsFor(instanceName);
|
|
6309
8478
|
if (levels.length === 0) {
|
|
6310
|
-
return
|
|
8479
|
+
return t("effort.unsupported", this.backendNameForInstance(instanceName));
|
|
6311
8480
|
}
|
|
6312
8481
|
const current = this.resolveInstanceEffort(instanceName).effort;
|
|
6313
8482
|
const nonce = randomBytes(6).toString("hex");
|
|
@@ -6323,13 +8492,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6323
8492
|
const p = this.pendingEffortSelects.get(nonce);
|
|
6324
8493
|
if (p) {
|
|
6325
8494
|
this.pendingEffortSelects.delete(nonce);
|
|
6326
|
-
p.respond("
|
|
8495
|
+
p.respond(t("effort.selection_expired")).catch(() => { });
|
|
6327
8496
|
}
|
|
6328
8497
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6329
8498
|
timer.unref?.();
|
|
6330
8499
|
this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6331
8500
|
try {
|
|
6332
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8501
|
+
const menuMessageId = await adapter.promptUser(chatId, t("effort.menu", this.effortMenuHeader(instanceName)), choices, { threadId });
|
|
6333
8502
|
const pending = this.pendingEffortSelects.get(nonce);
|
|
6334
8503
|
if (pending)
|
|
6335
8504
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6339,7 +8508,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6339
8508
|
this.pendingEffortSelects.delete(nonce);
|
|
6340
8509
|
clearTimeout(timer);
|
|
6341
8510
|
this.logger.warn({ err, instanceName }, "TG effort menu failed");
|
|
6342
|
-
return
|
|
8511
|
+
return t("effort.usage", levels.join("|"));
|
|
6343
8512
|
}
|
|
6344
8513
|
}
|
|
6345
8514
|
/** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
|
|
@@ -6360,7 +8529,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6360
8529
|
this.pendingEffortSelects.delete(match[1]);
|
|
6361
8530
|
clearTimeout(pending.timer);
|
|
6362
8531
|
const level = match[2];
|
|
6363
|
-
const progressText =
|
|
8532
|
+
const progressText = t("effort.setting", pending.instanceName, level);
|
|
6364
8533
|
let progressMsgId;
|
|
6365
8534
|
if (pending.adapter && pending.adapterChatId) {
|
|
6366
8535
|
const menuMessageId = pending.menuMessageId ?? data.messageId;
|
|
@@ -6392,7 +8561,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6392
8561
|
}
|
|
6393
8562
|
catch (err) {
|
|
6394
8563
|
this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
|
|
6395
|
-
result =
|
|
8564
|
+
result = t("effort.switch_failed", level, err instanceof Error ? err.message : String(err));
|
|
6396
8565
|
}
|
|
6397
8566
|
if (pending.adapter && pending.adapterChatId) {
|
|
6398
8567
|
if (progressMsgId) {
|
|
@@ -6429,32 +8598,36 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6429
8598
|
}
|
|
6430
8599
|
// No arg (or --refresh) → menu. Menu is DC-only this round (respondChoices); TG uses `/model <name>`.
|
|
6431
8600
|
if (!data.respondChoices) {
|
|
6432
|
-
await data.respond("
|
|
8601
|
+
await data.respond(t("model.usage"));
|
|
6433
8602
|
return;
|
|
6434
8603
|
}
|
|
6435
8604
|
const options = await this.getModelOptions(name, isRefresh);
|
|
6436
8605
|
if (options.length === 0) {
|
|
6437
|
-
await data.respond(
|
|
8606
|
+
await data.respond(t("model.list_unavailable", name));
|
|
6438
8607
|
return;
|
|
6439
8608
|
}
|
|
6440
8609
|
// Raw id for ✓-matching options; display resolves an inherited CLI default.
|
|
6441
8610
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(name);
|
|
6442
8611
|
const nonce = randomBytes(6).toString("hex");
|
|
6443
|
-
const
|
|
8612
|
+
const isClaude = this.backendNameForInstance(name) === "claude-code";
|
|
8613
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6444
8614
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6445
8615
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6446
8616
|
}));
|
|
8617
|
+
if (isClaude) {
|
|
8618
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8619
|
+
}
|
|
6447
8620
|
const timer = setTimeout(() => this.pendingModelSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6448
8621
|
timer.unref?.();
|
|
6449
|
-
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
8622
|
+
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond, respondChoices: data.respondChoices });
|
|
6450
8623
|
try {
|
|
6451
|
-
await data.respondChoices(
|
|
8624
|
+
await data.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
6452
8625
|
}
|
|
6453
8626
|
catch (err) {
|
|
6454
8627
|
this.pendingModelSelects.delete(nonce);
|
|
6455
8628
|
clearTimeout(timer);
|
|
6456
8629
|
this.logger.warn({ err, instanceName: name }, "model menu failed");
|
|
6457
|
-
await data.respond("
|
|
8630
|
+
await data.respond(t("model.usage"));
|
|
6458
8631
|
}
|
|
6459
8632
|
}
|
|
6460
8633
|
/**
|
|
@@ -6465,14 +8638,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6465
8638
|
async promptModelMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6466
8639
|
const options = await this.getModelOptions(instanceName);
|
|
6467
8640
|
if (options.length === 0) {
|
|
6468
|
-
return
|
|
8641
|
+
return t("model.list_unavailable", instanceName);
|
|
6469
8642
|
}
|
|
6470
8643
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(instanceName);
|
|
6471
8644
|
const nonce = randomBytes(6).toString("hex");
|
|
6472
|
-
const
|
|
8645
|
+
const isClaude = this.backendNameForInstance(instanceName) === "claude-code";
|
|
8646
|
+
// Keep the more-models entry inside Discord's 25-option select cap.
|
|
8647
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6473
8648
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6474
8649
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6475
8650
|
}));
|
|
8651
|
+
if (isClaude) {
|
|
8652
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8653
|
+
}
|
|
6476
8654
|
const respond = async (text) => {
|
|
6477
8655
|
await adapter.sendText(chatId, text, { threadId });
|
|
6478
8656
|
return undefined;
|
|
@@ -6481,13 +8659,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6481
8659
|
const p = this.pendingModelSelects.get(nonce);
|
|
6482
8660
|
if (p) {
|
|
6483
8661
|
this.pendingModelSelects.delete(nonce);
|
|
6484
|
-
p.respond("
|
|
8662
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
6485
8663
|
}
|
|
6486
8664
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6487
8665
|
timer.unref?.();
|
|
6488
8666
|
this.pendingModelSelects.set(nonce, { instanceName, model: "", userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6489
8667
|
try {
|
|
6490
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8668
|
+
const menuMessageId = await adapter.promptUser(chatId, t("model.menu", currentDisplay), choices, { threadId });
|
|
6491
8669
|
const pending = this.pendingModelSelects.get(nonce);
|
|
6492
8670
|
if (pending)
|
|
6493
8671
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6497,7 +8675,63 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6497
8675
|
this.pendingModelSelects.delete(nonce);
|
|
6498
8676
|
clearTimeout(timer);
|
|
6499
8677
|
this.logger.warn({ err, instanceName }, "TG model menu failed");
|
|
6500
|
-
return
|
|
8678
|
+
return t("model.usage");
|
|
8679
|
+
}
|
|
8680
|
+
}
|
|
8681
|
+
/** Cached-or-live account catalog behind "/model → more models" (claude). */
|
|
8682
|
+
async claudeApiModelOptions() {
|
|
8683
|
+
const cached = this.readCliEnv("claude-code");
|
|
8684
|
+
if (cached?.apiModels?.length)
|
|
8685
|
+
return cached.apiModels;
|
|
8686
|
+
const env = await this.probeBackend("claude-code");
|
|
8687
|
+
return env?.apiModels ?? [];
|
|
8688
|
+
}
|
|
8689
|
+
/** Replace a consumed "/model" menu with the full account catalog tier. */
|
|
8690
|
+
async expandClaudeModelMenu(pending) {
|
|
8691
|
+
const expanded = await this.claudeApiModelOptions();
|
|
8692
|
+
if (!expanded.length) {
|
|
8693
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8694
|
+
return;
|
|
8695
|
+
}
|
|
8696
|
+
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(pending.instanceName);
|
|
8697
|
+
const nonce = randomBytes(6).toString("hex");
|
|
8698
|
+
const choices = expanded.slice(0, 25).map(o => ({
|
|
8699
|
+
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
8700
|
+
label: this.modelChoiceLabel(o, currentModel),
|
|
8701
|
+
}));
|
|
8702
|
+
const timer = setTimeout(() => {
|
|
8703
|
+
const p = this.pendingModelSelects.get(nonce);
|
|
8704
|
+
if (p) {
|
|
8705
|
+
this.pendingModelSelects.delete(nonce);
|
|
8706
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
8707
|
+
}
|
|
8708
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
8709
|
+
timer.unref?.();
|
|
8710
|
+
this.pendingModelSelects.set(nonce, { ...pending, model: "", timer });
|
|
8711
|
+
try {
|
|
8712
|
+
if (pending.respondChoices) {
|
|
8713
|
+
// Discord select menu: edit the same interaction reply in place.
|
|
8714
|
+
await pending.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
8715
|
+
return;
|
|
8716
|
+
}
|
|
8717
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
8718
|
+
// Telegram: retire the tier-1 keyboard, then post the expanded menu.
|
|
8719
|
+
if (pending.menuMessageId && pending.adapter.editMessageRemoveButtons) {
|
|
8720
|
+
await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, pending.menuMessageId, t("model.more"), pending.adapterThreadId).catch(() => { });
|
|
8721
|
+
}
|
|
8722
|
+
const menuMessageId = await pending.adapter.promptUser(pending.adapterChatId, t("model.menu", currentDisplay), choices, { threadId: pending.adapterThreadId });
|
|
8723
|
+
const fresh = this.pendingModelSelects.get(nonce);
|
|
8724
|
+
if (fresh)
|
|
8725
|
+
fresh.menuMessageId = menuMessageId;
|
|
8726
|
+
return;
|
|
8727
|
+
}
|
|
8728
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8729
|
+
}
|
|
8730
|
+
catch (err) {
|
|
8731
|
+
this.pendingModelSelects.delete(nonce);
|
|
8732
|
+
clearTimeout(timer);
|
|
8733
|
+
this.logger.warn({ err, instanceName: pending.instanceName }, "Expanded model menu failed");
|
|
8734
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
6501
8735
|
}
|
|
6502
8736
|
}
|
|
6503
8737
|
/** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
|
|
@@ -6519,8 +8753,14 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6519
8753
|
this.pendingModelSelects.delete(match[1]);
|
|
6520
8754
|
clearTimeout(pending.timer);
|
|
6521
8755
|
const model = match[2];
|
|
8756
|
+
// "More models…" is a navigation choice, not a model: swap the menu for the
|
|
8757
|
+
// full account catalog (live /v1/models with [1m] variants).
|
|
8758
|
+
if (model === "__more__") {
|
|
8759
|
+
await this.expandClaudeModelMenu(pending);
|
|
8760
|
+
return true;
|
|
8761
|
+
}
|
|
6522
8762
|
// Send immediate "⏳ Switching..." feedback, then apply in background.
|
|
6523
|
-
const progressText =
|
|
8763
|
+
const progressText = t("model.switching", pending.instanceName, model);
|
|
6524
8764
|
let progressMsgId;
|
|
6525
8765
|
if (pending.adapter && pending.adapterChatId) {
|
|
6526
8766
|
// TG path: turn the original menu into the progress message. This both
|
|
@@ -6557,7 +8797,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6557
8797
|
}
|
|
6558
8798
|
catch (err) {
|
|
6559
8799
|
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
6560
|
-
result =
|
|
8800
|
+
result = t("model.switch_failed", model, err instanceof Error ? err.message : String(err));
|
|
6561
8801
|
}
|
|
6562
8802
|
if (pending.adapter && pending.adapterChatId) {
|
|
6563
8803
|
if (progressMsgId) {
|
|
@@ -6654,17 +8894,17 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6654
8894
|
}
|
|
6655
8895
|
catch { /* treated as unsupported below */ }
|
|
6656
8896
|
if (strategy === "unsupported" || supported.length === 0) {
|
|
6657
|
-
return
|
|
8897
|
+
return t("effort.unsupported", backendName);
|
|
6658
8898
|
}
|
|
6659
8899
|
if (!FleetManager.EFFORT_LEVELS.includes(level)) {
|
|
6660
|
-
return
|
|
8900
|
+
return t("effort.unknown", level, FleetManager.EFFORT_LEVELS.join(", "));
|
|
6661
8901
|
}
|
|
6662
8902
|
const applied = FleetManager.clampEffort(level, supported);
|
|
6663
8903
|
if (!applied)
|
|
6664
|
-
return
|
|
8904
|
+
return t("effort.no_canonical", backendName);
|
|
6665
8905
|
const warn = applied === level
|
|
6666
8906
|
? ""
|
|
6667
|
-
:
|
|
8907
|
+
: t("effort.clamped", applied, level, backendName);
|
|
6668
8908
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6669
8909
|
// or the instance silently reverts on restart.
|
|
6670
8910
|
if (this.fleetConfig?.instances[instanceName]) {
|
|
@@ -6673,12 +8913,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6673
8913
|
}
|
|
6674
8914
|
if (strategy === "runtime") {
|
|
6675
8915
|
if (!this.instanceIpcClients.get(instanceName))
|
|
6676
|
-
return `${warn}
|
|
8916
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6677
8917
|
this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
|
|
6678
|
-
return `${warn}
|
|
8918
|
+
return `${warn}${t("effort.runtime_success", instanceName, applied)}`;
|
|
6679
8919
|
}
|
|
6680
8920
|
await this.restartSingleInstance(instanceName);
|
|
6681
|
-
return `${warn}
|
|
8921
|
+
return `${warn}${t("effort.restart_success", instanceName, applied)}`;
|
|
6682
8922
|
}
|
|
6683
8923
|
async applyModel(instanceName, model) {
|
|
6684
8924
|
const backendName = this.backendNameForInstance(instanceName);
|
|
@@ -6687,9 +8927,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6687
8927
|
strategy = createBackend(backendName, this.getInstanceDir(instanceName)).getModelSwitchStrategy?.(model) ?? "restart";
|
|
6688
8928
|
}
|
|
6689
8929
|
catch { /* default restart */ }
|
|
6690
|
-
const warn = isModelCompatible(backendName, model) ? "" :
|
|
8930
|
+
const warn = isModelCompatible(backendName, model) ? "" : t("model.pattern_warning", model, backendName);
|
|
6691
8931
|
if (strategy === "runtime" && !this.instanceIpcClients.get(instanceName)) {
|
|
6692
|
-
return `${warn}
|
|
8932
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6693
8933
|
}
|
|
6694
8934
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6695
8935
|
// or the instance silently reverts to the CLI default after a fleet restart.
|
|
@@ -6703,13 +8943,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6703
8943
|
persisted = true;
|
|
6704
8944
|
}
|
|
6705
8945
|
if (!persisted)
|
|
6706
|
-
return `${warn}
|
|
8946
|
+
return `${warn}${t("model.persist_failed", instanceName)}`;
|
|
6707
8947
|
if (strategy === "runtime") {
|
|
6708
8948
|
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
6709
|
-
return `${warn}
|
|
8949
|
+
return `${warn}${t("model.runtime_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6710
8950
|
}
|
|
6711
8951
|
await this.restartSingleInstance(instanceName);
|
|
6712
|
-
return `${warn}
|
|
8952
|
+
return `${warn}${t("model.restart_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6713
8953
|
}
|
|
6714
8954
|
/**
|
|
6715
8955
|
* The trailing "Current effort: …" line for a /model reply.
|
|
@@ -6723,10 +8963,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6723
8963
|
return "";
|
|
6724
8964
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6725
8965
|
if (!effort)
|
|
6726
|
-
return "
|
|
8966
|
+
return `\n${t("effort.current_default")}`;
|
|
6727
8967
|
return source === "fleet-default"
|
|
6728
|
-
? `\
|
|
6729
|
-
: `\
|
|
8968
|
+
? `\n${t("effort.current_fleet", effort)}`
|
|
8969
|
+
: `\n${t("effort.current", effort)}`;
|
|
6730
8970
|
}
|
|
6731
8971
|
/** Read recent chat log for agent context */
|
|
6732
8972
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
@@ -6761,11 +9001,21 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6761
9001
|
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
6762
9002
|
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
6763
9003
|
if (!this.classicChannels)
|
|
6764
|
-
return "
|
|
9004
|
+
return t("classic.manager_unavailable");
|
|
6765
9005
|
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
6766
9006
|
const generalId = this.findGeneralInstance(adapterId);
|
|
6767
|
-
if (generalId)
|
|
6768
|
-
|
|
9007
|
+
if (generalId) {
|
|
9008
|
+
// Fire-and-forget, exactly as the notifyInstanceTopic it replaces: this
|
|
9009
|
+
// function's return value is the rejection shown to the user, and it
|
|
9010
|
+
// must not wait on posting buttons into the General topic.
|
|
9011
|
+
void this.promptClassicApproval({
|
|
9012
|
+
generalName: generalId,
|
|
9013
|
+
message: t("alert.unauth_guild", guildId, userId),
|
|
9014
|
+
groupId: String(guildId),
|
|
9015
|
+
scope: "guild",
|
|
9016
|
+
userId: userId ? String(userId) : undefined,
|
|
9017
|
+
}).catch(err => this.logger.warn({ err, guildId }, "Classic approval prompt failed"));
|
|
9018
|
+
}
|
|
6769
9019
|
return t("classic.not_authorized_guild");
|
|
6770
9020
|
}
|
|
6771
9021
|
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
@@ -6908,6 +9158,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6908
9158
|
return;
|
|
6909
9159
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
6910
9160
|
ensureWorkspaceGit(workDir);
|
|
9161
|
+
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
6911
9162
|
const config = {
|
|
6912
9163
|
...DEFAULT_INSTANCE_CONFIG,
|
|
6913
9164
|
...this.fleetConfig?.defaults,
|
|
@@ -6915,6 +9166,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6915
9166
|
lightweight: true,
|
|
6916
9167
|
...(backend ? { backend } : {}),
|
|
6917
9168
|
...(model ? { model } : {}),
|
|
9169
|
+
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
9170
|
+
...(classicIdentity?.description ? { description: classicIdentity.description } : {}),
|
|
6918
9171
|
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
6919
9172
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
6920
9173
|
};
|
|
@@ -6928,7 +9181,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6928
9181
|
return blocker;
|
|
6929
9182
|
const classicChannels = this.classicChannels;
|
|
6930
9183
|
if (!classicChannels)
|
|
6931
|
-
return "
|
|
9184
|
+
return t("classic.manager_unavailable");
|
|
6932
9185
|
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
6933
9186
|
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
6934
9187
|
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
@@ -6950,7 +9203,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6950
9203
|
/** Handle /stop slash command — unregister classic channel */
|
|
6951
9204
|
async handleClassicStop(channelId, adapterId) {
|
|
6952
9205
|
if (!this.classicChannels)
|
|
6953
|
-
return "
|
|
9206
|
+
return t("classic.manager_unavailable");
|
|
6954
9207
|
const ch = this.classicChannels.unregister(channelId, adapterId);
|
|
6955
9208
|
if (!ch)
|
|
6956
9209
|
return t("classic.no_agent");
|
|
@@ -6982,6 +9235,21 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6982
9235
|
// await in the same tick as the signal handler, so no event can slip in.
|
|
6983
9236
|
this.shuttingDown = true;
|
|
6984
9237
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
9238
|
+
// Release held delivery promises before awaiting daemon shutdown, then
|
|
9239
|
+
// reject spawn work which has not started. Otherwise a storm backoff could
|
|
9240
|
+
// make `agend stop` wait forever for its own queue.
|
|
9241
|
+
this.stormWindow.shutdown();
|
|
9242
|
+
this.spawnGate.shutdown();
|
|
9243
|
+
for (const pending of this.startupRetries.values())
|
|
9244
|
+
clearTimeout(pending.timer);
|
|
9245
|
+
this.startupRetries.clear();
|
|
9246
|
+
for (const pending of this.startupRetryNotices.values())
|
|
9247
|
+
clearTimeout(pending.timer);
|
|
9248
|
+
this.startupRetryNotices.clear();
|
|
9249
|
+
if (this.stormOpenNotifyTimer) {
|
|
9250
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
9251
|
+
this.stormOpenNotifyTimer = null;
|
|
9252
|
+
}
|
|
6985
9253
|
sdNotifyBlocking("STOPPING=1");
|
|
6986
9254
|
if (this.watchdogTimer) {
|
|
6987
9255
|
clearInterval(this.watchdogTimer);
|
|
@@ -6997,6 +9265,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6997
9265
|
this.clearStatuslineWatchers();
|
|
6998
9266
|
this.costGuard?.stop();
|
|
6999
9267
|
this.dailySummary?.stop();
|
|
9268
|
+
this.dailyTipScheduler?.stop();
|
|
9269
|
+
this.dailyTipScheduler = null;
|
|
7000
9270
|
if (this.updateCheckTimer) {
|
|
7001
9271
|
clearTimeout(this.updateCheckTimer);
|
|
7002
9272
|
clearInterval(this.updateCheckTimer);
|
|
@@ -7214,7 +9484,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7214
9484
|
/**
|
|
7215
9485
|
* Hot-reload: re-read fleet.yaml and reconcile running instances.
|
|
7216
9486
|
* Starts new, stops removed, restarts modified instances.
|
|
7217
|
-
*
|
|
9487
|
+
* Whitelisted runtime fields are pushed into live daemons; all other instance
|
|
9488
|
+
* fields, plus cold fleet-level settings, retain restart semantics.
|
|
7218
9489
|
*/
|
|
7219
9490
|
async reconcileInstances() {
|
|
7220
9491
|
if (!this.configPath)
|
|
@@ -7256,11 +9527,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7256
9527
|
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
7257
9528
|
// with no explanation anywhere they are looking.
|
|
7258
9529
|
const why = !validation.valid
|
|
7259
|
-
?
|
|
9530
|
+
? t("fleet.reload_validation", validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n"))
|
|
7260
9531
|
: unsafeEmpty
|
|
7261
|
-
?
|
|
7262
|
-
:
|
|
7263
|
-
this.notifyFleetError(
|
|
9532
|
+
? t("fleet.reload_removed_all", oldCount)
|
|
9533
|
+
: t("fleet.reload_removed_half", oldCount, newCount);
|
|
9534
|
+
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
7264
9535
|
return;
|
|
7265
9536
|
}
|
|
7266
9537
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -7268,9 +9539,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7268
9539
|
this.scheduler?.reload();
|
|
7269
9540
|
const newInstances = this.fleetConfig.instances;
|
|
7270
9541
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
7271
|
-
// Detect fleet-level
|
|
7272
|
-
|
|
7273
|
-
const
|
|
9542
|
+
// Detect fleet-level changes which still need a restart. Hot defaults are
|
|
9543
|
+
// reconciled below and must not produce a misleading restart warning.
|
|
9544
|
+
const oldDefaultColdWithFleetFields = oldConfig?.defaults
|
|
9545
|
+
? splitHotColdConfig(oldConfig.defaults).cold
|
|
9546
|
+
: {};
|
|
9547
|
+
const newDefaultColdWithFleetFields = this.fleetConfig?.defaults
|
|
9548
|
+
? splitHotColdConfig(this.fleetConfig.defaults).cold
|
|
9549
|
+
: {};
|
|
9550
|
+
// `tips` is a fleet-owned hot switch; it is read at post time and never
|
|
9551
|
+
// belongs in a daemon config or a "restart required" warning.
|
|
9552
|
+
const { tips: _oldTips, ...oldDefaultCold } = oldDefaultColdWithFleetFields;
|
|
9553
|
+
const { tips: _newTips, ...newDefaultCold } = newDefaultColdWithFleetFields;
|
|
9554
|
+
const oldFleetLevel = JSON.stringify({ channel: oldConfig?.channel, defaults: oldDefaultCold });
|
|
9555
|
+
const newFleetLevel = JSON.stringify({ channel: this.fleetConfig?.channel, defaults: newDefaultCold });
|
|
7274
9556
|
if (oldFleetLevel !== newFleetLevel) {
|
|
7275
9557
|
this.logger.warn("Fleet-level config changed (channel/defaults) — use /restart for full effect");
|
|
7276
9558
|
}
|
|
@@ -7282,22 +9564,43 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7282
9564
|
await this.stopInstance(name).catch(err => this.logger.error({ err, name }, "Failed to stop removed instance"));
|
|
7283
9565
|
}
|
|
7284
9566
|
}
|
|
7285
|
-
// Start new +
|
|
9567
|
+
// Start new + reconcile modified instances. Hot values are always sent as a
|
|
9568
|
+
// complete snapshot: Settings mutates FleetManager's config before SIGHUP,
|
|
9569
|
+
// so an old/new diff alone can miss the live daemon's stale value.
|
|
7286
9570
|
for (const [name, config] of Object.entries(newInstances)) {
|
|
7287
9571
|
if (!this.daemons.has(name)) {
|
|
7288
9572
|
// New instance — startInstance already calls connectIpcToInstance
|
|
7289
9573
|
this.logger.info({ name }, "New instance in config — starting");
|
|
7290
|
-
await this.
|
|
9574
|
+
await this.startInstanceUnattended(name, config, topicMode, "new instance");
|
|
7291
9575
|
}
|
|
7292
9576
|
else if (oldConfig?.instances[name]) {
|
|
7293
|
-
|
|
7294
|
-
|
|
9577
|
+
const daemon = this.daemons.get(name);
|
|
9578
|
+
const runtimeConfig = daemon.getConfigSnapshot?.() ?? oldConfig.instances[name];
|
|
9579
|
+
const oldParts = splitHotColdConfig(runtimeConfig);
|
|
9580
|
+
const newParts = splitHotColdConfig(config);
|
|
9581
|
+
// Every field not explicitly classified hot is cold by default.
|
|
9582
|
+
if (!isDeepStrictEqual(oldParts.cold, newParts.cold)) {
|
|
7295
9583
|
this.logger.info({ name }, "Instance config changed — restarting");
|
|
7296
9584
|
await this.stopInstance(name).catch(() => { });
|
|
7297
|
-
await this.
|
|
9585
|
+
await this.startInstanceUnattended(name, config, topicMode, "modified instance");
|
|
9586
|
+
}
|
|
9587
|
+
else if (!isDeepStrictEqual(oldParts.hot, newParts.hot)) {
|
|
9588
|
+
const update = hotConfigUpdate(config);
|
|
9589
|
+
const ipc = this.instanceIpcClients.get(name);
|
|
9590
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
9591
|
+
if (!sent) {
|
|
9592
|
+
// Daemon is in-process, so a reconnect gap must not leave runtime
|
|
9593
|
+
// state stale. Normal operation still uses the explicit IPC contract.
|
|
9594
|
+
daemon.applyConfigUpdate(update);
|
|
9595
|
+
this.logger.warn({ name }, "Config-update IPC unavailable — applied hot config in-process");
|
|
9596
|
+
}
|
|
9597
|
+
this.logger.info({ name, fields: [...HOT_INSTANCE_CONFIG_KEYS] }, "Instance hot config reloaded");
|
|
7298
9598
|
}
|
|
7299
9599
|
}
|
|
7300
9600
|
}
|
|
9601
|
+
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
9602
|
+
// currently idle instances instead of waiting for a future state edge.
|
|
9603
|
+
this.enforceWarmCap();
|
|
7301
9604
|
this.logger.info({ running: this.daemons.size, configured: Object.keys(newInstances).length }, "Reconcile complete");
|
|
7302
9605
|
}
|
|
7303
9606
|
async restartInstances() {
|
|
@@ -7378,14 +9681,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7378
9681
|
const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
7379
9682
|
const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
7380
9683
|
for (const [name, cfg] of restartGenerals) {
|
|
7381
|
-
|
|
7382
|
-
|
|
7383
|
-
if (this.daemons.has(name))
|
|
7384
|
-
restartProgress.markReady();
|
|
7385
|
-
}
|
|
7386
|
-
catch (err) {
|
|
7387
|
-
this.logger.error({ err, name }, "Failed to start general instance");
|
|
7388
|
-
}
|
|
9684
|
+
if (await this.startInstanceUnattended(name, cfg, topicMode, "general instance"))
|
|
9685
|
+
restartProgress.markReady();
|
|
7389
9686
|
}
|
|
7390
9687
|
// General is ready again; now its topic can own the live progress message.
|
|
7391
9688
|
await restartProgress.start(progressTarget);
|
|
@@ -7406,14 +9703,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7406
9703
|
while (idx < channels.length) {
|
|
7407
9704
|
const batch = channels.slice(idx, idx + concurrency);
|
|
7408
9705
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
if (this.daemons.has(ch.instanceName))
|
|
7412
|
-
restartProgress.markReady();
|
|
7413
|
-
}
|
|
7414
|
-
catch (err) {
|
|
7415
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
7416
|
-
}
|
|
9706
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
9707
|
+
restartProgress.markReady();
|
|
7417
9708
|
}));
|
|
7418
9709
|
idx += concurrency;
|
|
7419
9710
|
}
|
|
@@ -7537,7 +9828,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7537
9828
|
if (generalId) {
|
|
7538
9829
|
// No release URL — Discord's SuppressEmbeds proved unreliable and the
|
|
7539
9830
|
// link preview looked bad. Version + /update instruction is enough.
|
|
7540
|
-
this.notifyInstanceTopic(generalId, t("update.
|
|
9831
|
+
this.notifyInstanceTopic(generalId, t("update.available_current", `v${target}`, `v${currentVersion}`));
|
|
7541
9832
|
}
|
|
7542
9833
|
}
|
|
7543
9834
|
}
|
|
@@ -7671,63 +9962,81 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7671
9962
|
}
|
|
7672
9963
|
// Fleet API (enriched for agent board)
|
|
7673
9964
|
if (req.method === "GET" && req.url === "/api/fleet") {
|
|
7674
|
-
|
|
7675
|
-
|
|
7676
|
-
|
|
7677
|
-
|
|
7678
|
-
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
|
|
7682
|
-
|
|
7683
|
-
|
|
7684
|
-
|
|
7685
|
-
|
|
7686
|
-
|
|
7687
|
-
|
|
7688
|
-
|
|
7689
|
-
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
const
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
|
|
7697
|
-
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
|
|
7705
|
-
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
|
|
9965
|
+
try {
|
|
9966
|
+
const sysInfo = this.getSysInfo();
|
|
9967
|
+
const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
|
|
9968
|
+
const fleetNames = new Set(fleetInstances.map(inst => inst.name));
|
|
9969
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
9970
|
+
.filter(channel => !fleetNames.has(channel.instanceName))
|
|
9971
|
+
.map(channel => ({
|
|
9972
|
+
name: channel.instanceName,
|
|
9973
|
+
status: this.getInstanceStatus(channel.instanceName),
|
|
9974
|
+
state: this.getInstanceExecutionState(channel.instanceName),
|
|
9975
|
+
ipc: this.instanceIpcClients.has(channel.instanceName),
|
|
9976
|
+
costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
|
|
9977
|
+
rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
|
|
9978
|
+
classic: true,
|
|
9979
|
+
classicName: channel.name,
|
|
9980
|
+
channelId: channel.channelId,
|
|
9981
|
+
adapterId: channel.adapterId ?? null,
|
|
9982
|
+
}));
|
|
9983
|
+
const enriched = [...fleetInstances, ...classicInstances].map(inst => {
|
|
9984
|
+
const config = this.fleetConfig?.instances[inst.name];
|
|
9985
|
+
const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
|
|
9986
|
+
const lastActivity = inst.classic
|
|
9987
|
+
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
9988
|
+
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
9989
|
+
const backend = this.backendNameForInstance(inst.name);
|
|
9990
|
+
const resolvedModel = this.resolveInstanceModel(inst.name);
|
|
9991
|
+
const effortStrategy = this.effortStrategyFor(inst.name);
|
|
9992
|
+
const resolvedEffort = this.resolveInstanceEffort(inst.name);
|
|
9993
|
+
// Find claimed tasks for this instance
|
|
9994
|
+
let currentTask = null;
|
|
9995
|
+
try {
|
|
9996
|
+
const tasks = this.scheduler?.db.listTasks({ assignee: inst.name, status: "claimed" });
|
|
9997
|
+
if (tasks?.length)
|
|
9998
|
+
currentTask = tasks[0].title;
|
|
9999
|
+
}
|
|
10000
|
+
catch (err) {
|
|
10001
|
+
this.logger.debug({ err, name: inst.name }, "Scheduler listTasks failed (/api/fleet)");
|
|
10002
|
+
}
|
|
10003
|
+
return {
|
|
10004
|
+
...inst,
|
|
10005
|
+
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
10006
|
+
backend,
|
|
10007
|
+
// Settings renders these runtime-effective values rather than the
|
|
10008
|
+
// sparse user-authored YAML. `auto` means the supported CLI is
|
|
10009
|
+
// using its own effort default; null is reserved for unsupported.
|
|
10010
|
+
model: resolvedModel.model,
|
|
10011
|
+
model_display: resolvedModel.display,
|
|
10012
|
+
model_source: resolvedModel.source,
|
|
10013
|
+
effort: effortStrategy === "unsupported" ? null : (resolvedEffort.effort ?? "auto"),
|
|
10014
|
+
effort_supported: effortStrategy !== "unsupported",
|
|
10015
|
+
tool_set: config?.tool_set ?? "full",
|
|
10016
|
+
general_topic: config?.general_topic ?? false,
|
|
10017
|
+
// User activity is persisted by the daemon, so both the board and
|
|
10018
|
+
// auto-pause retain an accurate age across fleet restarts.
|
|
10019
|
+
lastActivity,
|
|
10020
|
+
currentTask,
|
|
10021
|
+
idle: this.getInstanceIdle(inst.name),
|
|
10022
|
+
state: this.getInstanceExecutionState(inst.name),
|
|
10023
|
+
};
|
|
10024
|
+
});
|
|
10025
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
10026
|
+
res.writeHead(200);
|
|
10027
|
+
res.end(JSON.stringify({
|
|
10028
|
+
...sysInfo,
|
|
10029
|
+
version: this.currentVersion,
|
|
10030
|
+
instances: enriched,
|
|
10031
|
+
}));
|
|
10032
|
+
}
|
|
10033
|
+
catch (err) {
|
|
10034
|
+
this.logger.error({ err }, "/api/fleet failed");
|
|
10035
|
+
if (!res.headersSent) {
|
|
10036
|
+
res.writeHead(500);
|
|
10037
|
+
res.end("Internal Server Error");
|
|
7709
10038
|
}
|
|
7710
|
-
|
|
7711
|
-
...inst,
|
|
7712
|
-
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
7713
|
-
backend,
|
|
7714
|
-
tool_set: config?.tool_set ?? "full",
|
|
7715
|
-
general_topic: config?.general_topic ?? false,
|
|
7716
|
-
// User activity is persisted by the daemon, so both the board and
|
|
7717
|
-
// auto-pause retain an accurate age across fleet restarts.
|
|
7718
|
-
lastActivity,
|
|
7719
|
-
currentTask,
|
|
7720
|
-
idle: this.getInstanceIdle(inst.name),
|
|
7721
|
-
state: this.getInstanceExecutionState(inst.name),
|
|
7722
|
-
};
|
|
7723
|
-
});
|
|
7724
|
-
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
7725
|
-
res.writeHead(200);
|
|
7726
|
-
res.end(JSON.stringify({
|
|
7727
|
-
...sysInfo,
|
|
7728
|
-
version: this.currentVersion,
|
|
7729
|
-
instances: enriched,
|
|
7730
|
-
}));
|
|
10039
|
+
}
|
|
7731
10040
|
return;
|
|
7732
10041
|
}
|
|
7733
10042
|
// Activity API
|
|
@@ -7857,7 +10166,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7857
10166
|
if (err.code === "EADDRINUSE") {
|
|
7858
10167
|
if (this.healthPortRetried) {
|
|
7859
10168
|
this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
|
|
7860
|
-
this.notifyFleetError(
|
|
10169
|
+
this.notifyFleetError(t("dashboard.port_in_use", port));
|
|
7861
10170
|
return;
|
|
7862
10171
|
}
|
|
7863
10172
|
this.healthPortRetried = true;
|
|
@@ -7883,7 +10192,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7883
10192
|
return;
|
|
7884
10193
|
}
|
|
7885
10194
|
this.logger.error({ err, port }, "Health server error");
|
|
7886
|
-
this.notifyFleetError(
|
|
10195
|
+
this.notifyFleetError(t("dashboard.server_failed", err.message));
|
|
7887
10196
|
});
|
|
7888
10197
|
this.healthServer.listen(port, "127.0.0.1", () => markListening());
|
|
7889
10198
|
}
|