@songsid/agend 2.1.4-beta.9 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-cli-instructions.md +2 -2
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +12 -8
- package/dist/backend/antigravity.js +189 -42
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +87 -23
- package/dist/backend/claude-code.js +361 -46
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +139 -5
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +52 -2
- package/dist/backend/kiro.js +267 -12
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.js +17 -4
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +122 -3
- package/dist/backend/types.js +5 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/backend-outage.d.ts +61 -0
- package/dist/backend-outage.js +71 -0
- package/dist/backend-outage.js.map +1 -0
- package/dist/channel/adapters/discord.d.ts +44 -2
- package/dist/channel/adapters/discord.js +523 -90
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +30 -0
- package/dist/channel/adapters/telegram.js +221 -19
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/ipc-bridge.d.ts +2 -1
- package/dist/channel/ipc-bridge.js +33 -4
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/markdown-chunk.d.ts +42 -0
- package/dist/channel/markdown-chunk.js +205 -0
- package/dist/channel/markdown-chunk.js.map +1 -0
- package/dist/channel/mcp-server.js +34 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +5 -3
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.js +64 -8
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/types.d.ts +31 -1
- package/dist/classic-channel-manager.d.ts +65 -0
- package/dist/classic-channel-manager.js +172 -3
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +124 -83
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +13 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +12 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +6 -0
- package/dist/cross-instance-envelope.js +30 -0
- package/dist/cross-instance-envelope.js.map +1 -0
- package/dist/daemon.d.ts +491 -14
- package/dist/daemon.js +2121 -276
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +41 -0
- package/dist/doctor.js +267 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +63 -0
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +402 -11
- package/dist/fleet-manager.js +2594 -422
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +63 -16
- package/dist/general-knowledge/skills/session-management/SKILL.md +171 -8
- package/dist/instance-lifecycle.d.ts +114 -1
- package/dist/instance-lifecycle.js +404 -14
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +42 -4
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +1024 -320
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +91 -0
- package/dist/login-flows.js +170 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/network-family.d.ts +18 -0
- package/dist/network-family.js +20 -0
- package/dist/network-family.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -1
- package/dist/outbound-handlers.js +174 -40
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +10 -3
- package/dist/outbound-schemas.js +10 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +52 -0
- package/dist/pane-input-residue.js +107 -0
- package/dist/pane-input-residue.js.map +1 -0
- package/dist/process-memory.d.ts +6 -0
- package/dist/process-memory.js +45 -0
- package/dist/process-memory.js.map +1 -1
- package/dist/reply-dedup.d.ts +7 -8
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/spawn-gate.d.ts +30 -0
- package/dist/spawn-gate.js +79 -0
- package/dist/spawn-gate.js.map +1 -0
- package/dist/steer-capability.d.ts +1 -0
- package/dist/steer-capability.js +6 -0
- package/dist/steer-capability.js.map +1 -0
- package/dist/storm-window.d.ts +83 -0
- package/dist/storm-window.js +251 -0
- package/dist/storm-window.js.map +1 -0
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +16 -0
- package/dist/tmux-manager.js +110 -25
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +31 -7
- package/dist/topic-commands.js +342 -70
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-sources.d.ts +35 -10
- package/dist/transcript-sources.js +190 -38
- package/dist/transcript-sources.js.map +1 -1
- package/dist/types.d.ts +6 -1
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +200 -60
- package/dist/ui/view.html +147 -30
- package/dist/usage/format-rich.d.ts +1 -1
- package/dist/usage/format-rich.js +28 -24
- package/dist/usage/format-rich.js.map +1 -1
- package/dist/usage/i18n-keys.d.ts +7 -0
- package/dist/usage/i18n-keys.js +34 -0
- package/dist/usage/i18n-keys.js.map +1 -0
- package/dist/usage/i18n.d.ts +5 -0
- package/dist/usage/i18n.js +27 -0
- package/dist/usage/i18n.js.map +1 -0
- package/dist/usage/providers.d.ts +21 -0
- package/dist/usage/providers.js +153 -75
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +11 -3
- package/dist/usage/usage-api.js +61 -24
- package/dist/usage/usage-api.js.map +1 -1
- package/package.json +2 -2
package/dist/fleet-manager.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { freemem, totalmem } from "node:os";
|
|
3
5
|
import { createServer } from "node:http";
|
|
4
6
|
import { join, dirname, basename } from "node:path";
|
|
5
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
10
|
import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
|
|
8
11
|
import { formatUpdateProgress } from "./update-progress.js";
|
|
9
12
|
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
10
13
|
import { readFleetMemory } from "./process-memory.js";
|
|
11
14
|
import { ReplyDeduper } from "./reply-dedup.js";
|
|
12
|
-
import { isScalar, parseDocument } from "yaml";
|
|
15
|
+
import { isMap, isScalar, parseDocument } from "yaml";
|
|
13
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
14
17
|
const __dirname = dirname(__filename);
|
|
15
18
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
@@ -30,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
30
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
31
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
32
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
33
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE,
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
34
37
|
import { DailySummary } from "./daily-summary.js";
|
|
35
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
36
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -42,9 +45,11 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
42
45
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
43
46
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
44
47
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
45
|
-
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
48
|
+
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
|
+
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
|
+
import { LoginSession } from "./login-manager.js";
|
|
46
51
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
47
|
-
import { setLocale, detectLocale, t } from "./locale.js";
|
|
52
|
+
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
48
53
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
49
54
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
50
55
|
import { validateFleetConfig } from "./config-validator.js";
|
|
@@ -54,6 +59,11 @@ import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
|
54
59
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
55
60
|
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
56
61
|
import { RestartProgress } from "./restart-progress.js";
|
|
62
|
+
import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
|
|
63
|
+
import { StormWindow } from "./storm-window.js";
|
|
64
|
+
import { SpawnGate } from "./spawn-gate.js";
|
|
65
|
+
import { BackendOutageTracker } from "./backend-outage.js";
|
|
66
|
+
import { canUnlockAdvancedTips, DailyTipScheduler, selectTip, visibleTipLevels, } from "./tips.js";
|
|
57
67
|
import { getTmuxSession } from "./config.js";
|
|
58
68
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
59
69
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -106,6 +116,8 @@ const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
|
|
|
106
116
|
* button alone (the idle edge retires it when the run really ends).
|
|
107
117
|
*/
|
|
108
118
|
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
119
|
+
/** Bound for the daemon to capture the pane and answer a post-reply state query. */
|
|
120
|
+
const REPLY_STATE_REFRESH_TIMEOUT_MS = 2_000;
|
|
109
121
|
/**
|
|
110
122
|
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
111
123
|
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
@@ -162,6 +174,30 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
162
174
|
* emoji never changes the documented delivery-state protocol.
|
|
163
175
|
*/
|
|
164
176
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
177
|
+
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
178
|
+
"tool_progress",
|
|
179
|
+
"mcp_proxy_reply",
|
|
180
|
+
"auto_pause_after",
|
|
181
|
+
"warm_cap",
|
|
182
|
+
"display_name",
|
|
183
|
+
"description",
|
|
184
|
+
"tags",
|
|
185
|
+
"log_level",
|
|
186
|
+
]);
|
|
187
|
+
function splitHotColdConfig(config) {
|
|
188
|
+
const hot = {};
|
|
189
|
+
const cold = {};
|
|
190
|
+
for (const [key, value] of Object.entries(config)) {
|
|
191
|
+
(HOT_INSTANCE_CONFIG_KEYS.has(key) ? hot : cold)[key] = value;
|
|
192
|
+
}
|
|
193
|
+
return { hot, cold };
|
|
194
|
+
}
|
|
195
|
+
function hotConfigUpdate(config) {
|
|
196
|
+
const update = {};
|
|
197
|
+
for (const key of HOT_INSTANCE_CONFIG_KEYS)
|
|
198
|
+
update[key] = config[key] ?? null;
|
|
199
|
+
return update;
|
|
200
|
+
}
|
|
165
201
|
/**
|
|
166
202
|
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
167
203
|
*
|
|
@@ -178,9 +214,18 @@ const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
|
|
|
178
214
|
const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
|
|
179
215
|
const HANG_CALLBACK_PREFIX = "hang:";
|
|
180
216
|
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
217
|
+
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
218
|
+
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
219
|
+
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
220
|
+
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
221
|
+
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
222
|
+
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
223
|
+
const LOGIN_CONFIRM_CALLBACK_PREFIX = "login-confirm:";
|
|
224
|
+
const INSTALL_LOGIN_CALLBACK_PREFIX = "install-login:";
|
|
181
225
|
const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
|
|
182
226
|
/** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
|
|
183
227
|
const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
|
|
228
|
+
const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
|
|
184
229
|
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
185
230
|
export class FleetManager {
|
|
186
231
|
dataDir;
|
|
@@ -188,6 +233,10 @@ export class FleetManager {
|
|
|
188
233
|
static sighupHandlerInstalled = false;
|
|
189
234
|
children = new Map();
|
|
190
235
|
lifecycle;
|
|
236
|
+
stormWindow;
|
|
237
|
+
spawnGate;
|
|
238
|
+
/** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
|
|
239
|
+
backendOutage = new BackendOutageTracker();
|
|
191
240
|
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
192
241
|
get daemons() { return this.lifecycle.daemons; }
|
|
193
242
|
fleetConfig = null;
|
|
@@ -227,6 +276,7 @@ export class FleetManager {
|
|
|
227
276
|
costGuard = null;
|
|
228
277
|
statuslineWatcher;
|
|
229
278
|
dailySummary = null;
|
|
279
|
+
dailyTipScheduler = null;
|
|
230
280
|
webhookEmitter = null;
|
|
231
281
|
// Topic icon + auto-archive state
|
|
232
282
|
topicIcons = {};
|
|
@@ -239,6 +289,13 @@ export class FleetManager {
|
|
|
239
289
|
warmCapEvicting = new Set();
|
|
240
290
|
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
241
291
|
idleGatedDeliveryTails = new Map();
|
|
292
|
+
/**
|
|
293
|
+
* Per-instance cancellation epoch for deliveries which have not reached the
|
|
294
|
+
* daemon yet. A cancel advances the epoch; queued work captures the old value
|
|
295
|
+
* and becomes a no-op, while messages arriving after the click capture the new
|
|
296
|
+
* value and remain deliverable.
|
|
297
|
+
*/
|
|
298
|
+
deliveryEpochs = new Map();
|
|
242
299
|
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
243
300
|
lastDeliveryAt = new Map();
|
|
244
301
|
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
@@ -261,6 +318,28 @@ export class FleetManager {
|
|
|
261
318
|
ipcWaitTails = new Map();
|
|
262
319
|
/** instanceName → restart currently executing; concurrent callers join it. */
|
|
263
320
|
restartsInFlight = new Map();
|
|
321
|
+
/**
|
|
322
|
+
* Delayed automatic retries for instances whose startup failed. Before this a
|
|
323
|
+
* failed start was logged once and the instance stayed `stopped` until an
|
|
324
|
+
* operator noticed (2026-09-03: 4 kiro instances, all victims of the same
|
|
325
|
+
* backend outage during a post-update herd). instanceName → pending retry.
|
|
326
|
+
*/
|
|
327
|
+
startupRetries = new Map();
|
|
328
|
+
/** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
|
|
329
|
+
explicitStopGeneration = new Map();
|
|
330
|
+
/** instanceName → outage hand-off currently executing; a repeat joins it. */
|
|
331
|
+
handOffsInFlight = new Map();
|
|
332
|
+
/** instanceName → explicit stop currently executing; the outage hand-off waits for it. */
|
|
333
|
+
stopsInFlight = new Map();
|
|
334
|
+
/** Aggregation window for the "N instances failed to start" notice. */
|
|
335
|
+
startupRetryNotices = new Map();
|
|
336
|
+
/** Backoff between automatic startup retries; the last step repeats while the backend is down. */
|
|
337
|
+
static STARTUP_RETRY_BACKOFF_MS = [60_000, 5 * 60_000, 15 * 60_000];
|
|
338
|
+
/** Hard cap on automatic startup retries (3 backoff steps + up to 3 more during a backend outage). */
|
|
339
|
+
static STARTUP_RETRY_MAX_ATTEMPTS = 6;
|
|
340
|
+
/** Re-check interval when a retry is due but the tmux storm window still blocks spawns. */
|
|
341
|
+
static STARTUP_RETRY_STORM_DEFER_MS = 60_000;
|
|
342
|
+
static STARTUP_RETRY_NOTICE_AGGREGATE_MS = 1_000;
|
|
264
343
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
265
344
|
lastInboundMsg = new Map();
|
|
266
345
|
topicArchiver;
|
|
@@ -297,12 +376,14 @@ export class FleetManager {
|
|
|
297
376
|
updateProgressTimer = null;
|
|
298
377
|
updateProgressEditRunning = false;
|
|
299
378
|
lastUpdateProgressText = null;
|
|
379
|
+
updateCompletionTipText = null;
|
|
300
380
|
eventLogPruneTimer = null;
|
|
301
381
|
logRotateTimer = null;
|
|
302
382
|
/** Days of event/activity history to keep. */
|
|
303
383
|
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
304
384
|
watchdogTimer = null;
|
|
305
385
|
startedAt = 0;
|
|
386
|
+
stormOpenNotifyTimer = null;
|
|
306
387
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
307
388
|
mirrorBuffer = [];
|
|
308
389
|
mirrorTimer = null;
|
|
@@ -318,11 +399,63 @@ export class FleetManager {
|
|
|
318
399
|
process.on("SIGHUP", () => FleetManager.signalTarget?.handleSighup());
|
|
319
400
|
FleetManager.sighupHandlerInstalled = true;
|
|
320
401
|
}
|
|
402
|
+
this.stormWindow = new StormWindow();
|
|
403
|
+
this.spawnGate = new SpawnGate({
|
|
404
|
+
storm: this.stormWindow,
|
|
405
|
+
concurrency: () => this.spawnConcurrency(),
|
|
406
|
+
staggerMs: () => this.fleetConfig?.defaults?.startup?.stagger_delay_ms ?? 500,
|
|
407
|
+
});
|
|
408
|
+
this.bindStormWindowEvents();
|
|
321
409
|
this.lifecycle = new InstanceLifecycle(this);
|
|
322
410
|
this.topicCommands = new TopicCommands(this);
|
|
323
411
|
this.topicArchiver = new TopicArchiver(this);
|
|
324
412
|
this.statuslineWatcher = new StatuslineWatcher(this);
|
|
325
413
|
}
|
|
414
|
+
spawnConcurrency() {
|
|
415
|
+
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
416
|
+
if (explicit != null)
|
|
417
|
+
return Math.max(1, Math.min(20, explicit));
|
|
418
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
419
|
+
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
420
|
+
}
|
|
421
|
+
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
422
|
+
bindStormWindowEvents() {
|
|
423
|
+
this.stormWindow.on("opened", (snapshot) => {
|
|
424
|
+
for (const [name, daemon] of this.daemons) {
|
|
425
|
+
if (!daemon.isPaused)
|
|
426
|
+
this.stormWindow.addAffected(name);
|
|
427
|
+
}
|
|
428
|
+
this.logger.error({ ...snapshot }, "tmux server storm opened — holding respawns and delivery");
|
|
429
|
+
if (this.stormOpenNotifyTimer)
|
|
430
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
431
|
+
this.stormOpenNotifyTimer = setTimeout(() => {
|
|
432
|
+
this.stormOpenNotifyTimer = null;
|
|
433
|
+
const current = this.stormWindow.snapshot();
|
|
434
|
+
this.notifyFleetError(t("storm.opened", current.affected.length, this.formatStormDelay(current.backoffMs)));
|
|
435
|
+
}, 1_000);
|
|
436
|
+
this.stormOpenNotifyTimer.unref?.();
|
|
437
|
+
});
|
|
438
|
+
this.stormWindow.on("extended", (snapshot) => {
|
|
439
|
+
this.logger.error({ ...snapshot }, "tmux server storm repeated — backoff extended");
|
|
440
|
+
this.notifyFleetError(t("storm.extended", snapshot.crashCount, this.formatStormDelay(snapshot.backoffMs)));
|
|
441
|
+
});
|
|
442
|
+
this.stormWindow.on("recovery_due", (snapshot) => {
|
|
443
|
+
this.logger.warn({ ...snapshot }, "tmux storm backoff elapsed — rolling recovery started");
|
|
444
|
+
});
|
|
445
|
+
this.stormWindow.on("closed", (snapshot, reason) => {
|
|
446
|
+
const unresolved = snapshot.affected.filter(name => !snapshot.recovered.includes(name));
|
|
447
|
+
this.logger.info({ ...snapshot, reason, unresolved }, "tmux server storm closed");
|
|
448
|
+
this.notifyFleetError(t(reason === "timeout" ? "storm.timed_out" : "storm.recovered", snapshot.recovered.length, snapshot.affected.length, unresolved.length > 0 ? unresolved.join(", ") : t("storm.none")));
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
formatStormDelay(ms) {
|
|
452
|
+
if (ms >= 60_000)
|
|
453
|
+
return `${Math.round(ms / 60_000)}m`;
|
|
454
|
+
return `${Math.round(ms / 1_000)}s`;
|
|
455
|
+
}
|
|
456
|
+
stormSuppressed(kind) {
|
|
457
|
+
return this.stormWindow.shouldSuppress(kind);
|
|
458
|
+
}
|
|
326
459
|
handleSighup() {
|
|
327
460
|
this.logger.info("Received SIGHUP, hot-reloading config...");
|
|
328
461
|
if (!this.startupComplete) {
|
|
@@ -345,7 +478,7 @@ export class FleetManager {
|
|
|
345
478
|
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
346
479
|
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
347
480
|
const message = err instanceof Error ? err.message : String(err);
|
|
348
|
-
this.notifyFleetError(
|
|
481
|
+
this.notifyFleetError(t("fleet.reload_failed", message));
|
|
349
482
|
})
|
|
350
483
|
.finally(() => {
|
|
351
484
|
this.reconcileInFlight = null;
|
|
@@ -434,7 +567,8 @@ export class FleetManager {
|
|
|
434
567
|
getSysInfo() {
|
|
435
568
|
const mem = process.memoryUsage();
|
|
436
569
|
const toMB = (b) => Math.round(b / 1024 / 1024 * 10) / 10;
|
|
437
|
-
|
|
570
|
+
// Fleet instances (fleet.yaml)
|
|
571
|
+
const fleetInstances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
438
572
|
name,
|
|
439
573
|
status: this.getInstanceStatus(name),
|
|
440
574
|
state: this.getInstanceExecutionState(name),
|
|
@@ -442,12 +576,46 @@ export class FleetManager {
|
|
|
442
576
|
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
443
577
|
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
444
578
|
}));
|
|
579
|
+
// Classic instances (classicBot.yaml) — dedupe against fleet
|
|
580
|
+
const fleetNames = new Set(fleetInstances.map(i => i.name));
|
|
581
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
582
|
+
.filter(ch => !fleetNames.has(ch.instanceName))
|
|
583
|
+
.map(ch => ({
|
|
584
|
+
name: ch.instanceName,
|
|
585
|
+
status: this.getInstanceStatus(ch.instanceName),
|
|
586
|
+
state: this.getInstanceExecutionState(ch.instanceName),
|
|
587
|
+
ipc: this.instanceIpcClients.has(ch.instanceName),
|
|
588
|
+
costCents: this.costGuard?.getDailyCostCents(ch.instanceName) ?? 0,
|
|
589
|
+
rateLimits: this.statuslineWatcher.getRateLimits(ch.instanceName) ?? null,
|
|
590
|
+
}));
|
|
591
|
+
// Combined roster (matches /api/fleet and agend ls)
|
|
592
|
+
const allInstances = [...fleetInstances, ...classicInstances];
|
|
593
|
+
// Fleet summary counts (fleet + Classic combined)
|
|
594
|
+
const running_count = allInstances.filter(i => i.status === "running").length;
|
|
595
|
+
const paused_count = allInstances.filter(i => i.status === "paused").length;
|
|
596
|
+
// System memory (GB, 1 decimal)
|
|
597
|
+
const totalGB = totalmem() / (1024 ** 3);
|
|
598
|
+
const usedGB = (totalmem() - freemem()) / (1024 ** 3);
|
|
599
|
+
const system_mem_gb = {
|
|
600
|
+
used: Math.round(usedGB * 10) / 10,
|
|
601
|
+
total: Math.round(totalGB * 10) / 10,
|
|
602
|
+
};
|
|
603
|
+
// Fleet memory: O(1) cgroup read (includes entire service tree: fleet + CLIs + MCP servers)
|
|
604
|
+
// This avoids blocking the event loop with per-instance tree scans.
|
|
605
|
+
const fleetMem = readFleetMemory();
|
|
606
|
+
const fleet_mem_mb = fleetMem.cgroupAnonBytes != null
|
|
607
|
+
? Math.round(fleetMem.cgroupAnonBytes / (1024 * 1024) * 10) / 10
|
|
608
|
+
: null;
|
|
445
609
|
return {
|
|
446
610
|
uptime_seconds: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
447
611
|
memory_mb: { rss: toMB(mem.rss), heapUsed: toMB(mem.heapUsed), heapTotal: toMB(mem.heapTotal) },
|
|
448
|
-
instances,
|
|
612
|
+
instances: fleetInstances, // SysInfo.instances is fleet-only; /api/fleet enriches with Classic
|
|
449
613
|
fleet_cost_cents: this.costGuard?.getFleetTotalCents() ?? 0,
|
|
450
614
|
fleet_cost_limit_cents: this.costGuard?.getLimitCents() ?? 0,
|
|
615
|
+
running_count,
|
|
616
|
+
paused_count,
|
|
617
|
+
fleet_mem_mb,
|
|
618
|
+
system_mem_gb,
|
|
451
619
|
};
|
|
452
620
|
}
|
|
453
621
|
/** Load fleet.yaml and build routing table */
|
|
@@ -595,6 +763,57 @@ export class FleetManager {
|
|
|
595
763
|
child.once("error", err => this.failUpdateProgress(err.message));
|
|
596
764
|
child.unref();
|
|
597
765
|
}
|
|
766
|
+
async handleTipsSlash(data, adapterId) {
|
|
767
|
+
if (!this.fleetConfig)
|
|
768
|
+
return;
|
|
769
|
+
const mode = typeof data.options?.mode === "string" ? data.options.mode : "";
|
|
770
|
+
if (mode === "advanced on") {
|
|
771
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
772
|
+
await data.respond(t("permission.denied"));
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
await data.respond(this.unlockAdvancedTips(data.userId)
|
|
776
|
+
? t("tips.advanced.unlocked")
|
|
777
|
+
: t("tips.unavailable"));
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
if (mode === "on" || mode === "off") {
|
|
781
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
782
|
+
await data.respond(t("permission.denied"));
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
this.fleetConfig.defaults.tips = mode === "on";
|
|
786
|
+
this.saveFleetConfig();
|
|
787
|
+
await data.respond(t(mode === "on" ? "tips.enabled" : "tips.disabled"));
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
791
|
+
if (!adapter) {
|
|
792
|
+
await data.respond(t("tips.unavailable"));
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const targetName = this.resolveSlashTarget(data.channelId, adapterId)
|
|
796
|
+
?? this.findGeneralInstance(adapterId)
|
|
797
|
+
?? "general";
|
|
798
|
+
// Discord slash interactions are deferred ephemerally, but the Tip itself
|
|
799
|
+
// belongs in the channel where /tips was invoked. Post there directly,
|
|
800
|
+
// then delete the empty acknowledgement instead of leaving a slash corpse.
|
|
801
|
+
//
|
|
802
|
+
// The callback binding uses (chatId, threadId) that match what the adapter
|
|
803
|
+
// emits on callback_query. Discord emits (guildId, channelId); Telegram
|
|
804
|
+
// emits (supergroup chatId, topic threadId). For /tips invoked in an
|
|
805
|
+
// arbitrary channel, pass the canonical group id as chatId and the
|
|
806
|
+
// invocation channel as threadId, exactly as sendTipToGeneral() does.
|
|
807
|
+
const chatId = this.getGroupIdForInstance(targetName) || data.channelId;
|
|
808
|
+
const result = await this.promptTip(targetName, adapter, chatId, data.channelId);
|
|
809
|
+
if (result === "posted" && data.dismissResponse) {
|
|
810
|
+
await data.dismissResponse();
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
await data.respond(t(result === "posted"
|
|
814
|
+
? "tips.posted"
|
|
815
|
+
: result === "empty" ? "tips.empty" : "tips.unavailable"));
|
|
816
|
+
}
|
|
598
817
|
/** Admin-only full conversation reset for fleet-topic and Classic instances. */
|
|
599
818
|
async handleClearSlash(data, adapterId) {
|
|
600
819
|
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
@@ -747,6 +966,39 @@ export class FleetManager {
|
|
|
747
966
|
return null;
|
|
748
967
|
return this.instanceStateCache.get(name)?.state ?? null;
|
|
749
968
|
}
|
|
969
|
+
/**
|
|
970
|
+
* Subscription providers currently relevant to this fleet. Stopped/crashed
|
|
971
|
+
* rows do not keep a usage card alive, while persisted paused instances do.
|
|
972
|
+
* Classic channels use their effective backend merge chain, not the raw row.
|
|
973
|
+
*/
|
|
974
|
+
getActiveUsageProviderIds() {
|
|
975
|
+
const providers = new Set();
|
|
976
|
+
for (const backend of this.getActiveBackendIds()) {
|
|
977
|
+
const provider = usageProviderIdForBackend(backend);
|
|
978
|
+
if (provider)
|
|
979
|
+
providers.add(provider);
|
|
980
|
+
}
|
|
981
|
+
return providers;
|
|
982
|
+
}
|
|
983
|
+
/** Effective backends with a running or persisted-paused fleet/Classic instance. */
|
|
984
|
+
getActiveBackendIds() {
|
|
985
|
+
const backends = new Set();
|
|
986
|
+
const add = (name, backend) => {
|
|
987
|
+
const status = this.getInstanceStatus(name);
|
|
988
|
+
if (status !== "running" && status !== "paused")
|
|
989
|
+
return;
|
|
990
|
+
if (backend)
|
|
991
|
+
backends.add(backend);
|
|
992
|
+
};
|
|
993
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
994
|
+
// loadFleetConfig() has already merged the fleet default into each row.
|
|
995
|
+
add(name, config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
996
|
+
}
|
|
997
|
+
for (const channel of this.classicChannels?.getAll() ?? []) {
|
|
998
|
+
add(channel.instanceName, this.classicChannels?.getBackendByInstance(channel.instanceName, this.fleetConfig?.defaults?.backend));
|
|
999
|
+
}
|
|
1000
|
+
return backends;
|
|
1001
|
+
}
|
|
750
1002
|
isClassicInstance(name) {
|
|
751
1003
|
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
752
1004
|
}
|
|
@@ -859,7 +1111,7 @@ export class FleetManager {
|
|
|
859
1111
|
.finally(() => this.warmCapEvicting.delete(victim));
|
|
860
1112
|
}
|
|
861
1113
|
}
|
|
862
|
-
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
1114
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0, cancelled) {
|
|
863
1115
|
const isReady = () => {
|
|
864
1116
|
const snapshot = this.instanceStateCache.get(instanceName);
|
|
865
1117
|
return snapshot?.state === "idle"
|
|
@@ -881,8 +1133,12 @@ export class FleetManager {
|
|
|
881
1133
|
this.instanceIdleWaiters.delete(instanceName);
|
|
882
1134
|
resolve(idle);
|
|
883
1135
|
};
|
|
884
|
-
const check = () => {
|
|
885
|
-
|
|
1136
|
+
const check = () => {
|
|
1137
|
+
if (cancelled?.())
|
|
1138
|
+
finish(false);
|
|
1139
|
+
else if (isReady())
|
|
1140
|
+
finish(true);
|
|
1141
|
+
};
|
|
886
1142
|
const query = () => {
|
|
887
1143
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
888
1144
|
if (ipc?.connected) {
|
|
@@ -898,24 +1154,90 @@ export class FleetManager {
|
|
|
898
1154
|
query();
|
|
899
1155
|
});
|
|
900
1156
|
}
|
|
901
|
-
|
|
1157
|
+
/**
|
|
1158
|
+
* Ask the daemon to capture the pane before answering, then wait for any state
|
|
1159
|
+
* report produced after this request. The ordinary query is intentionally
|
|
1160
|
+
* cache-only (idle gates call it frequently); this opt-in refresh is reserved
|
|
1161
|
+
* for lifecycle decisions where a stale state would strand UI.
|
|
1162
|
+
*/
|
|
1163
|
+
refreshInstanceExecutionState(instanceName, timeoutMs) {
|
|
1164
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1165
|
+
if (!ipc?.connected)
|
|
1166
|
+
return Promise.resolve(false);
|
|
1167
|
+
const previous = this.instanceStateCache.get(instanceName);
|
|
1168
|
+
return new Promise(resolve => {
|
|
1169
|
+
let settled = false;
|
|
1170
|
+
const finish = (refreshed) => {
|
|
1171
|
+
if (settled)
|
|
1172
|
+
return;
|
|
1173
|
+
settled = true;
|
|
1174
|
+
clearTimeout(timeout);
|
|
1175
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
1176
|
+
waiters?.delete(check);
|
|
1177
|
+
if (waiters?.size === 0)
|
|
1178
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
1179
|
+
resolve(refreshed);
|
|
1180
|
+
};
|
|
1181
|
+
const check = () => {
|
|
1182
|
+
if (this.instanceStateCache.get(instanceName) !== previous)
|
|
1183
|
+
finish(true);
|
|
1184
|
+
};
|
|
1185
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
1186
|
+
waiters.add(check);
|
|
1187
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
1188
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
1189
|
+
timeout.unref?.();
|
|
1190
|
+
const sent = ipc.send({
|
|
1191
|
+
type: "query_instance_state",
|
|
1192
|
+
requestId: `reply-grace-${Date.now()}`,
|
|
1193
|
+
refresh: true,
|
|
1194
|
+
});
|
|
1195
|
+
if (!sent)
|
|
1196
|
+
finish(false);
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
|
|
1200
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1201
|
+
return;
|
|
1202
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1203
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1204
|
+
return;
|
|
902
1205
|
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
903
1206
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
904
1207
|
const wakeStartedAt = Date.now();
|
|
905
1208
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1209
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1210
|
+
return;
|
|
906
1211
|
// Waking added one to the warm count — make room by evicting a different
|
|
907
1212
|
// LRU idle instance (never this one; it's about to work).
|
|
908
1213
|
this.enforceWarmCap(instanceName);
|
|
909
1214
|
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
910
1215
|
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
911
1216
|
}
|
|
912
|
-
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
1217
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
|
|
1218
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1219
|
+
this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
// A server crash can land while waitForInstanceIdle is pending. Re-check
|
|
1223
|
+
// immediately before the old timeout path would force text into a boot UI.
|
|
1224
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1225
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1226
|
+
return;
|
|
913
1227
|
if (!idle) {
|
|
914
1228
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
915
1229
|
}
|
|
916
|
-
await this.sendWhenConnected(instanceName, payload);
|
|
1230
|
+
await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
|
|
1231
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1232
|
+
return;
|
|
917
1233
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
918
1234
|
}
|
|
1235
|
+
async holdDeliveryForStorm(instanceName, deliveryEpoch) {
|
|
1236
|
+
if (!this.stormWindow.isDeliveryHeld(instanceName))
|
|
1237
|
+
return;
|
|
1238
|
+
this.logger.warn({ instanceName, deliveryEpoch }, "Delivery held during tmux server recovery");
|
|
1239
|
+
await this.stormWindow.waitForDeliveryAllowed(instanceName);
|
|
1240
|
+
}
|
|
919
1241
|
/**
|
|
920
1242
|
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
921
1243
|
*
|
|
@@ -932,7 +1254,9 @@ export class FleetManager {
|
|
|
932
1254
|
* instance, *including* when the socket happens to be up: otherwise a message
|
|
933
1255
|
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
934
1256
|
*/
|
|
935
|
-
async sendWhenConnected(instanceName, payload) {
|
|
1257
|
+
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
1258
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1259
|
+
return;
|
|
936
1260
|
const queued = this.ipcWaitTails.get(instanceName);
|
|
937
1261
|
if (!queued) {
|
|
938
1262
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -941,7 +1265,11 @@ export class FleetManager {
|
|
|
941
1265
|
}
|
|
942
1266
|
const attempt = (queued ?? Promise.resolve())
|
|
943
1267
|
.catch(() => { })
|
|
944
|
-
.then(() =>
|
|
1268
|
+
.then(() => {
|
|
1269
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1270
|
+
return;
|
|
1271
|
+
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
|
|
1272
|
+
});
|
|
945
1273
|
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
946
1274
|
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
947
1275
|
const tail = attempt.catch(() => { });
|
|
@@ -958,10 +1286,12 @@ export class FleetManager {
|
|
|
958
1286
|
}
|
|
959
1287
|
}
|
|
960
1288
|
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
961
|
-
async sendAfterIpcReturns(instanceName, payload) {
|
|
1289
|
+
async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
962
1290
|
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
963
1291
|
let warned = false;
|
|
964
1292
|
for (;;) {
|
|
1293
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1294
|
+
return;
|
|
965
1295
|
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
966
1296
|
// so a cached reference would stay dead forever.
|
|
967
1297
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -977,8 +1307,14 @@ export class FleetManager {
|
|
|
977
1307
|
await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
|
|
978
1308
|
}
|
|
979
1309
|
}
|
|
1310
|
+
/** Whether this target already has an ordinary non-user delivery in its FIFO. */
|
|
1311
|
+
hasPendingIdleGatedDelivery(instanceName) {
|
|
1312
|
+
return this.idleGatedDeliveryTails.has(instanceName);
|
|
1313
|
+
}
|
|
980
1314
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
981
1315
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
1316
|
+
const deliveryEpoch = this.getDeliveryEpoch(instanceName);
|
|
1317
|
+
const deliveryPayload = { ...payload, delivery_epoch: deliveryEpoch };
|
|
982
1318
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
983
1319
|
? payload.meta
|
|
984
1320
|
: undefined;
|
|
@@ -990,16 +1326,20 @@ export class FleetManager {
|
|
|
990
1326
|
if (!waitForIdle) {
|
|
991
1327
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
992
1328
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1329
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1330
|
+
return;
|
|
993
1331
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
994
1332
|
}
|
|
995
|
-
await this.sendWhenConnected(instanceName,
|
|
1333
|
+
await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
|
|
1334
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1335
|
+
return;
|
|
996
1336
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
997
1337
|
// working must not trust the stale idle snapshot from before the send.
|
|
998
1338
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
999
1339
|
return;
|
|
1000
1340
|
}
|
|
1001
1341
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
1002
|
-
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName,
|
|
1342
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
|
|
1003
1343
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
1004
1344
|
try {
|
|
1005
1345
|
await delivery;
|
|
@@ -1033,6 +1373,7 @@ export class FleetManager {
|
|
|
1033
1373
|
throw new Error("Classic channel manager not initialized");
|
|
1034
1374
|
const wasRunning = this.daemons.has(instanceName);
|
|
1035
1375
|
this.classicChannels.reloadFromDisk();
|
|
1376
|
+
this.reportClassicUnrecoverableIds();
|
|
1036
1377
|
this.reregisterClassicChannels();
|
|
1037
1378
|
const channel = this.classicChannels.getAll().find(item => item.instanceName === instanceName);
|
|
1038
1379
|
if (!channel)
|
|
@@ -1050,6 +1391,9 @@ export class FleetManager {
|
|
|
1050
1391
|
* across a fleet restart.
|
|
1051
1392
|
*/
|
|
1052
1393
|
resumePaused = false) {
|
|
1394
|
+
// Any start supersedes a pending automatic retry (it would otherwise fire
|
|
1395
|
+
// into a running instance — harmless, but noisy — or race this start).
|
|
1396
|
+
this.cancelStartupRetry(name);
|
|
1053
1397
|
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
1054
1398
|
await this.lifecycle.wake(name, 30_000);
|
|
1055
1399
|
// A successful wake clears the persisted pause marker and produces a
|
|
@@ -1080,12 +1424,6 @@ export class FleetManager {
|
|
|
1080
1424
|
}
|
|
1081
1425
|
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
1082
1426
|
if (config.general_topic) {
|
|
1083
|
-
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
1084
|
-
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
1085
|
-
if (backend === "antigravity") {
|
|
1086
|
-
this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
|
|
1087
|
-
this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
|
|
1088
|
-
}
|
|
1089
1427
|
this.ensureGeneralInstructions(config.working_directory, backend, name);
|
|
1090
1428
|
}
|
|
1091
1429
|
else if (kind === "fleet-topic") {
|
|
@@ -1151,86 +1489,243 @@ export class FleetManager {
|
|
|
1151
1489
|
}
|
|
1152
1490
|
if (runnableEntries.length === 0)
|
|
1153
1491
|
return;
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1492
|
+
if (this.fleetConfig?.defaults?.startup?.concurrency == null) {
|
|
1493
|
+
this.logger.info({
|
|
1494
|
+
concurrency: this.spawnConcurrency(),
|
|
1495
|
+
freeMemMB: Math.round(freemem() / (1024 * 1024)),
|
|
1496
|
+
totalInstances: runnableEntries.length,
|
|
1497
|
+
}, "Adaptive startup concurrency");
|
|
1498
|
+
}
|
|
1499
|
+
await Promise.all(runnableEntries.map(([name, config]) => this.spawnGate.run({
|
|
1500
|
+
instanceName: name,
|
|
1501
|
+
workingDirectory: config.working_directory,
|
|
1502
|
+
reason: "startup",
|
|
1503
|
+
}, async () => {
|
|
1504
|
+
if (await this.startInstanceUnattended(name, config, topicMode, "instance"))
|
|
1505
|
+
onReady?.(name);
|
|
1506
|
+
})));
|
|
1507
|
+
}
|
|
1508
|
+
/**
|
|
1509
|
+
* Start from an UNATTENDED path (fleet startup, full restart, config
|
|
1510
|
+
* reconcile): nobody is watching the result, so a failure is logged and
|
|
1511
|
+
* handed to the delayed automatic retry instead of leaving the instance
|
|
1512
|
+
* `stopped` forever. Explicit operator/API starts call startInstance directly
|
|
1513
|
+
* and keep their synchronous error. Returns whether the instance is up.
|
|
1514
|
+
*/
|
|
1515
|
+
async startInstanceUnattended(name, config, topicMode, what) {
|
|
1516
|
+
try {
|
|
1517
|
+
await this.startInstance(name, config, topicMode);
|
|
1518
|
+
return this.daemons.has(name);
|
|
1164
1519
|
}
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1520
|
+
catch (err) {
|
|
1521
|
+
this.logger.error({ err, name }, `Failed to start ${what}`);
|
|
1522
|
+
this.scheduleStartupRetry(name, 0);
|
|
1523
|
+
return false;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
// ── Delayed automatic startup retries ──────────────────────────────────
|
|
1527
|
+
/**
|
|
1528
|
+
* Schedule attempt `attempt` (0-based) of the automatic startup retry for an
|
|
1529
|
+
* instance whose start just failed. Backoff 1 → 5 → 15 min; while the
|
|
1530
|
+
* instance's backend is known to be unreachable the 15-min step repeats up to
|
|
1531
|
+
* STARTUP_RETRY_MAX_ATTEMPTS, after which we give up with one notice. Each
|
|
1532
|
+
* retry re-checks the world (still configured, not running, not paused, no
|
|
1533
|
+
* tmux storm) and runs through the SpawnGate as "recovery" — so a herd of
|
|
1534
|
+
* failed instances comes back at the gate's concurrency, never all at once.
|
|
1535
|
+
*/
|
|
1536
|
+
scheduleStartupRetry(name, attempt) {
|
|
1537
|
+
if (this.shuttingDown)
|
|
1538
|
+
return;
|
|
1539
|
+
if (this.startupRetries.has(name))
|
|
1540
|
+
return;
|
|
1541
|
+
const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
|
|
1542
|
+
const outage = this.backendOutage.isActive(this.backendNameOf(name));
|
|
1543
|
+
const exhausted = attempt >= backoff.length && !(outage && attempt < FleetManager.STARTUP_RETRY_MAX_ATTEMPTS);
|
|
1544
|
+
if (attempt >= FleetManager.STARTUP_RETRY_MAX_ATTEMPTS || exhausted) {
|
|
1545
|
+
this.logger.error({ name, attempts: attempt }, "Giving up automatic startup retries");
|
|
1546
|
+
this.queueStartupRetryNotice("gave_up", name, 0);
|
|
1547
|
+
return;
|
|
1548
|
+
}
|
|
1549
|
+
const delayMs = backoff[Math.min(attempt, backoff.length - 1)];
|
|
1550
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, delayMs);
|
|
1551
|
+
timer.unref?.();
|
|
1552
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1553
|
+
this.logger.warn({ name, attempt: attempt + 1, delayMs, backendOutage: outage }, "Startup failed — automatic retry scheduled");
|
|
1554
|
+
if (attempt === 0)
|
|
1555
|
+
this.queueStartupRetryNotice("scheduled", name, delayMs);
|
|
1556
|
+
}
|
|
1557
|
+
/** Drop a pending automatic retry (an operator start/stop/restart supersedes it). */
|
|
1558
|
+
cancelStartupRetry(name) {
|
|
1559
|
+
const pending = this.startupRetries.get(name);
|
|
1560
|
+
if (!pending)
|
|
1561
|
+
return;
|
|
1562
|
+
clearTimeout(pending.timer);
|
|
1563
|
+
this.startupRetries.delete(name);
|
|
1564
|
+
this.logger.info({ name }, "Pending automatic startup retry cancelled");
|
|
1565
|
+
}
|
|
1566
|
+
/** Pending automatic retry, if any (status display / tests). */
|
|
1567
|
+
pendingStartupRetry(name) {
|
|
1568
|
+
const pending = this.startupRetries.get(name);
|
|
1569
|
+
return pending ? { attempt: pending.attempt } : null;
|
|
1570
|
+
}
|
|
1571
|
+
/**
|
|
1572
|
+
* A daemon's crash-respawn hit the backend outage (startup_backend_unreachable):
|
|
1573
|
+
* stop THAT daemon and schedule the delayed retry. Serialized against the
|
|
1574
|
+
* operator paths: an in-flight restart is awaited first (it replaces the
|
|
1575
|
+
* daemon itself), the stop is identity-checked so a fresh daemon registered
|
|
1576
|
+
* meanwhile is never deleted, and an explicit stop/restart that began during
|
|
1577
|
+
* the hand-off (generation bump) owns the outcome — no retry is scheduled
|
|
1578
|
+
* behind an operator's back.
|
|
1579
|
+
*/
|
|
1580
|
+
async handOffToStartupRetry(name, daemon) {
|
|
1581
|
+
const joined = this.handOffsInFlight.get(name);
|
|
1582
|
+
if (joined)
|
|
1583
|
+
return joined;
|
|
1584
|
+
const run = (async () => {
|
|
1585
|
+
// Let any operator restart/stop that is already executing finish first:
|
|
1586
|
+
// a restart replaces the daemon itself, a stop removes it — either way
|
|
1587
|
+
// the identity check below then sees "not ours any more" and we do
|
|
1588
|
+
// nothing. Loop: one operation can be chained behind another.
|
|
1589
|
+
for (;;) {
|
|
1590
|
+
const inFlight = this.restartsInFlight.get(name) ?? this.stopsInFlight.get(name);
|
|
1591
|
+
if (!inFlight)
|
|
1592
|
+
break;
|
|
1593
|
+
await inFlight.catch(() => { });
|
|
1594
|
+
}
|
|
1595
|
+
const stopGen = this.explicitStopGeneration.get(name) ?? 0;
|
|
1596
|
+
if (this.lifecycle.daemons.get(name) !== daemon)
|
|
1597
|
+
return; // already replaced or stopped
|
|
1598
|
+
await this.lifecycle.stopIfCurrent(name, daemon);
|
|
1599
|
+
if (this.shuttingDown)
|
|
1600
|
+
return;
|
|
1601
|
+
if ((this.explicitStopGeneration.get(name) ?? 0) !== stopGen) {
|
|
1602
|
+
this.logger.info({ name }, "Outage hand-off superseded by an explicit stop/restart — no automatic retry");
|
|
1185
1603
|
return;
|
|
1186
1604
|
}
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1605
|
+
if (this.lifecycle.daemons.has(name) || this.lifecycle.isPaused(name))
|
|
1606
|
+
return;
|
|
1607
|
+
this.scheduleStartupRetry(name, 0);
|
|
1608
|
+
})().finally(() => this.handOffsInFlight.delete(name));
|
|
1609
|
+
this.handOffsInFlight.set(name, run);
|
|
1610
|
+
return run;
|
|
1611
|
+
}
|
|
1612
|
+
async runStartupRetry(name, attempt) {
|
|
1613
|
+
this.startupRetries.delete(name);
|
|
1614
|
+
if (this.shuttingDown)
|
|
1615
|
+
return;
|
|
1616
|
+
if (!this.isConfiguredInstance(name))
|
|
1617
|
+
return; // removed from fleet.yaml / classic channel meanwhile
|
|
1618
|
+
if (this.lifecycle.isPaused(name) || this.daemons.has(name))
|
|
1619
|
+
return; // operator acted meanwhile
|
|
1620
|
+
if (this.stormWindow.isSpawnBlocked()) {
|
|
1621
|
+
// The tmux server is being restarted; joining that herd is what we are
|
|
1622
|
+
// trying to avoid. Same attempt again after the storm's own recovery.
|
|
1623
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, FleetManager.STARTUP_RETRY_STORM_DEFER_MS);
|
|
1624
|
+
timer.unref?.();
|
|
1625
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1626
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Startup retry deferred — tmux storm window is blocking spawns");
|
|
1627
|
+
return;
|
|
1628
|
+
}
|
|
1629
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
1630
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
1631
|
+
try {
|
|
1632
|
+
await this.spawnGate.run({
|
|
1633
|
+
instanceName: name,
|
|
1634
|
+
workingDirectory: this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name),
|
|
1635
|
+
reason: "recovery",
|
|
1636
|
+
}, () => this.startConfiguredInstance(name, topicMode));
|
|
1637
|
+
if (this.daemons.has(name)) {
|
|
1638
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Automatic startup retry succeeded");
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
catch (err) {
|
|
1642
|
+
this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
|
|
1643
|
+
this.scheduleStartupRetry(name, attempt + 1);
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
/** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
|
|
1647
|
+
isConfiguredInstance(name) {
|
|
1648
|
+
if (this.fleetConfig?.instances[name])
|
|
1649
|
+
return true;
|
|
1650
|
+
return !!this.classicChannels?.getAll().some(channel => channel.instanceName === name);
|
|
1651
|
+
}
|
|
1652
|
+
/**
|
|
1653
|
+
* Kind-aware start for the automatic retry: fleet-topic instances come from
|
|
1654
|
+
* fleet.yaml, ClassicBot instances exist only in the classic channel manager
|
|
1655
|
+
* and must be rebuilt through startClassicInstance (a fleet.yaml lookup alone
|
|
1656
|
+
* silently dropped them — the retry timer fired and nothing happened).
|
|
1657
|
+
*/
|
|
1658
|
+
async startConfiguredInstance(name, topicMode) {
|
|
1659
|
+
const config = this.fleetConfig?.instances[name];
|
|
1660
|
+
if (config) {
|
|
1661
|
+
await this.startInstance(name, config, topicMode);
|
|
1662
|
+
return;
|
|
1663
|
+
}
|
|
1664
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
1665
|
+
if (!channel || !this.classicChannels)
|
|
1666
|
+
throw new Error(`Instance '${name}' is no longer configured`);
|
|
1667
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1668
|
+
}
|
|
1669
|
+
/**
|
|
1670
|
+
* Unattended ClassicBot start (fleet startup batch, full-restart batch, the
|
|
1671
|
+
* classicBot.yaml reconcile): same contract as startInstanceUnattended —
|
|
1672
|
+
* failures are logged and handed to the delayed automatic retry, whose
|
|
1673
|
+
* kind-aware startConfiguredInstance rebuilds the Classic instance. Returns
|
|
1674
|
+
* whether the instance is up.
|
|
1675
|
+
*/
|
|
1676
|
+
async startClassicInstanceUnattended(ch, what) {
|
|
1677
|
+
try {
|
|
1678
|
+
await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1679
|
+
return this.daemons.has(ch.instanceName);
|
|
1680
|
+
}
|
|
1681
|
+
catch (err) {
|
|
1682
|
+
this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
|
|
1683
|
+
this.scheduleStartupRetry(ch.instanceName, 0);
|
|
1684
|
+
return false;
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
backendNameOf(name) {
|
|
1688
|
+
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1689
|
+
const configured = this.fleetConfig?.instances[name]?.backend;
|
|
1690
|
+
if (configured)
|
|
1691
|
+
return configured;
|
|
1692
|
+
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1693
|
+
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1694
|
+
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1695
|
+
}
|
|
1696
|
+
return fleetDefault ?? "claude-code";
|
|
1697
|
+
}
|
|
1698
|
+
/**
|
|
1699
|
+
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1700
|
+
* fails many instances within the same second. Two notices per incident at
|
|
1701
|
+
* most — "N failed, retrying in X" and, if it comes to that, "gave up on N".
|
|
1702
|
+
*/
|
|
1703
|
+
queueStartupRetryNotice(kind, name, delayMs) {
|
|
1704
|
+
const pending = this.startupRetryNotices.get(kind);
|
|
1705
|
+
if (pending) {
|
|
1706
|
+
if (!pending.names.includes(name))
|
|
1707
|
+
pending.names.push(name);
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
const timer = setTimeout(() => {
|
|
1711
|
+
const entry = this.startupRetryNotices.get(kind);
|
|
1712
|
+
this.startupRetryNotices.delete(kind);
|
|
1713
|
+
if (!entry)
|
|
1714
|
+
return;
|
|
1715
|
+
const list = entry.names.join(", ");
|
|
1716
|
+
if (kind === "scheduled") {
|
|
1717
|
+
let text = t("fleet.startup_retry_scheduled", entry.names.length, list, this.formatStormDelay(entry.delayMs));
|
|
1718
|
+
const downBackends = [...new Set(entry.names.map(n => this.backendNameOf(n)))].filter(b => this.backendOutage.isActive(b));
|
|
1719
|
+
if (downBackends.length)
|
|
1720
|
+
text += `\n${t("fleet.startup_retry_outage", downBackends.join(", "))}`;
|
|
1721
|
+
this.notifyFleetError(text);
|
|
1722
|
+
}
|
|
1723
|
+
else {
|
|
1724
|
+
this.notifyFleetError(t("fleet.startup_retry_gave_up", entry.names.length, list));
|
|
1725
|
+
}
|
|
1726
|
+
}, FleetManager.STARTUP_RETRY_NOTICE_AGGREGATE_MS);
|
|
1727
|
+
timer.unref?.();
|
|
1728
|
+
this.startupRetryNotices.set(kind, { names: [name], delayMs, timer });
|
|
1234
1729
|
}
|
|
1235
1730
|
runnableStartupCount(fleet, includeClassic) {
|
|
1236
1731
|
const names = this.configuredStartupInstanceNames(fleet, includeClassic);
|
|
@@ -1265,6 +1760,8 @@ export class FleetManager {
|
|
|
1265
1760
|
};
|
|
1266
1761
|
}
|
|
1267
1762
|
async stopInstance(name) {
|
|
1763
|
+
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
1764
|
+
this.cancelStartupRetry(name);
|
|
1268
1765
|
this.failoverActive.delete(name);
|
|
1269
1766
|
this.cancelIdleButtonRetirement(name);
|
|
1270
1767
|
this.instanceStateCache.delete(name);
|
|
@@ -1276,12 +1773,20 @@ export class FleetManager {
|
|
|
1276
1773
|
// interactive-prompt / clean-exit events whose handlers post fresh prompts
|
|
1277
1774
|
// while the stop is still awaiting (the TOCTOU sol's review called out).
|
|
1278
1775
|
this.clearNoncePromptsForInstance(name);
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1776
|
+
// Published so the outage hand-off can wait for an explicit stop that began
|
|
1777
|
+
// BEFORE it (the generation fence alone only catches stops that begin after
|
|
1778
|
+
// the hand-off snapshotted it).
|
|
1779
|
+
const run = (async () => {
|
|
1780
|
+
try {
|
|
1781
|
+
await this.lifecycle.stop(name);
|
|
1782
|
+
}
|
|
1783
|
+
finally {
|
|
1784
|
+
this.clearNoncePromptsForInstance(name);
|
|
1785
|
+
}
|
|
1786
|
+
})().finally(() => { if (this.stopsInFlight.get(name) === run)
|
|
1787
|
+
this.stopsInFlight.delete(name); });
|
|
1788
|
+
this.stopsInFlight.set(name, run);
|
|
1789
|
+
return run;
|
|
1285
1790
|
}
|
|
1286
1791
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
1287
1792
|
async restartSingleInstance(name, opts) {
|
|
@@ -1294,7 +1799,12 @@ export class FleetManager {
|
|
|
1294
1799
|
this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
|
|
1295
1800
|
return inFlight;
|
|
1296
1801
|
}
|
|
1297
|
-
const
|
|
1802
|
+
const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
|
|
1803
|
+
const run = this.spawnGate.run({
|
|
1804
|
+
instanceName: name,
|
|
1805
|
+
workingDirectory,
|
|
1806
|
+
reason: "restart",
|
|
1807
|
+
}, () => this.doRestartSingleInstance(name, opts))
|
|
1298
1808
|
.finally(() => this.restartsInFlight.delete(name));
|
|
1299
1809
|
this.restartsInFlight.set(name, run);
|
|
1300
1810
|
return run;
|
|
@@ -1312,6 +1822,7 @@ export class FleetManager {
|
|
|
1312
1822
|
this.writeFreshStartMarker(name);
|
|
1313
1823
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
1314
1824
|
await this.startInstance(name, config, topicMode ?? false);
|
|
1825
|
+
this.stormWindow.markRecovered(name);
|
|
1315
1826
|
return;
|
|
1316
1827
|
}
|
|
1317
1828
|
// Classic instance fallback
|
|
@@ -1324,6 +1835,7 @@ export class FleetManager {
|
|
|
1324
1835
|
if (opts?.freshStart)
|
|
1325
1836
|
this.writeFreshStartMarker(name);
|
|
1326
1837
|
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1838
|
+
this.stormWindow.markRecovered(name);
|
|
1327
1839
|
return;
|
|
1328
1840
|
}
|
|
1329
1841
|
throw new Error(`Instance not found: ${name}`);
|
|
@@ -1389,6 +1901,7 @@ export class FleetManager {
|
|
|
1389
1901
|
messageId,
|
|
1390
1902
|
});
|
|
1391
1903
|
this.lastUpdateProgressText = null;
|
|
1904
|
+
this.updateCompletionTipText = null;
|
|
1392
1905
|
this.startUpdateProgressMonitor(adapter);
|
|
1393
1906
|
}
|
|
1394
1907
|
failUpdateProgress(message) {
|
|
@@ -1412,7 +1925,17 @@ export class FleetManager {
|
|
|
1412
1925
|
const adapter = this.adapters.get(target.adapterId) ?? (initialAdapter?.id === target.adapterId ? initialAdapter : undefined);
|
|
1413
1926
|
if (!adapter)
|
|
1414
1927
|
return;
|
|
1415
|
-
|
|
1928
|
+
let text = formatUpdateProgress(marker);
|
|
1929
|
+
// No-restart updates complete in the old fleet process, so they never
|
|
1930
|
+
// reach RestartProgress.finish(). Append the same optional tip here.
|
|
1931
|
+
if (marker.progress.stage === "complete" && this.tipsEnabled()) {
|
|
1932
|
+
if (this.updateCompletionTipText === null) {
|
|
1933
|
+
const tip = this.pickAvailableTip();
|
|
1934
|
+
this.updateCompletionTipText = tip ? this.formatTip(tip) : "";
|
|
1935
|
+
}
|
|
1936
|
+
if (this.updateCompletionTipText)
|
|
1937
|
+
text += `\n\n${this.updateCompletionTipText}`;
|
|
1938
|
+
}
|
|
1416
1939
|
if (text === this.lastUpdateProgressText)
|
|
1417
1940
|
return;
|
|
1418
1941
|
this.updateProgressEditRunning = true;
|
|
@@ -1453,6 +1976,7 @@ export class FleetManager {
|
|
|
1453
1976
|
// Rotate fleet.log if oversized (before any logging)
|
|
1454
1977
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1455
1978
|
const fleet = this.loadConfig(configPath);
|
|
1979
|
+
this.slimFleetConfigAtStartup();
|
|
1456
1980
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
1457
1981
|
const savedUpdateProgress = readUpdateProgress(this.dataDir);
|
|
1458
1982
|
const pendingUpdateProgress = savedUpdateProgress
|
|
@@ -1469,6 +1993,10 @@ export class FleetManager {
|
|
|
1469
1993
|
const { getTmuxSocketName: getSocket } = await import("./paths.js");
|
|
1470
1994
|
TmuxManager.setSocketName(getSocket());
|
|
1471
1995
|
await TmuxManager.ensureSession(getTmuxSession());
|
|
1996
|
+
// Pre-flight (advisory, fire-and-forget): warm the per-backend auth cache
|
|
1997
|
+
// before instances spawn, so a CLI that comes up on its sign-in screen gets
|
|
1998
|
+
// an instant 🔑 verdict instead of decaying into crash/MCP-died noise.
|
|
1999
|
+
this.lifecycle.primeAuthVerification(Object.values(fleet.instances).map(config => config.backend ?? fleet.defaults?.backend ?? "claude-code"));
|
|
1472
2000
|
// Start tmux control mode client for idle detection
|
|
1473
2001
|
if (!this.controlClient) {
|
|
1474
2002
|
this.controlClient = new TmuxControlClient(getTmuxSession(), 2000, this.logger);
|
|
@@ -1512,6 +2040,11 @@ export class FleetManager {
|
|
|
1512
2040
|
this.classicChannels = new ClassicChannelManager(this.dataDir, this.logger);
|
|
1513
2041
|
const classicAdapters = fleet.channels?.length ? fleet.channels : (fleet.channel ? [fleet.channel] : []);
|
|
1514
2042
|
this.classicChannels.configureAdapters(classicAdapters);
|
|
2043
|
+
// The unrecoverable-id report is deliberately NOT sent here: no adapter
|
|
2044
|
+
// exists yet at this point in startAll(), so notifyFleetError would find
|
|
2045
|
+
// nothing to deliver through, drop the message silently, AND burn its
|
|
2046
|
+
// 10-minute throttle key on the way out. It is sent once the shared adapter
|
|
2047
|
+
// is up — see the adapterStartup continuation below.
|
|
1515
2048
|
// Restore the persisted bot binding so replies/cancel go through the right
|
|
1516
2049
|
// bot after a restart (before this, inbound would re-bind lazily).
|
|
1517
2050
|
for (const ch of this.classicChannels.getAll()) {
|
|
@@ -1535,6 +2068,9 @@ export class FleetManager {
|
|
|
1535
2068
|
}
|
|
1536
2069
|
if (!this.classicChannels.checkReload())
|
|
1537
2070
|
return;
|
|
2071
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2072
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
2073
|
+
this.reportClassicUnrecoverableIds();
|
|
1538
2074
|
this.reregisterClassicChannels();
|
|
1539
2075
|
for (const ch of this.classicChannels.getAll()) {
|
|
1540
2076
|
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
@@ -1548,7 +2084,10 @@ export class FleetManager {
|
|
|
1548
2084
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1549
2085
|
// Small delay to let tmux window clean up
|
|
1550
2086
|
await new Promise(r => setTimeout(r, 2000));
|
|
1551
|
-
|
|
2087
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
2088
|
+
// unattended helper reads them from it and schedules the delayed
|
|
2089
|
+
// retry on failure like every other unattended start.
|
|
2090
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1552
2091
|
}
|
|
1553
2092
|
}
|
|
1554
2093
|
}
|
|
@@ -1699,6 +2238,13 @@ export class FleetManager {
|
|
|
1699
2238
|
this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule) => this.handleScheduleTrigger(schedule), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
|
|
1700
2239
|
this.scheduler.init();
|
|
1701
2240
|
this.logger.info("Scheduler initialized");
|
|
2241
|
+
// Tips share the daily-report clock but remain an independent internal
|
|
2242
|
+
// job: disabling daily_summary must not disable tips (and vice versa).
|
|
2243
|
+
// The callback checks defaults.tips at fire time, so /tips on|off is hot.
|
|
2244
|
+
this.dailyTipScheduler = new DailyTipScheduler(summaryConfig, costGuardConfig.timezone, () => this.sendTipToGeneral().catch(err => {
|
|
2245
|
+
this.logger.warn({ err }, "Failed to send daily tip");
|
|
2246
|
+
}));
|
|
2247
|
+
this.dailyTipScheduler.start();
|
|
1702
2248
|
// Inject active decisions as env var for MCP instructions.
|
|
1703
2249
|
// Snapshotted at startup — new decisions via post_decision are available
|
|
1704
2250
|
// through list_decisions tool but not auto-injected until restart.
|
|
@@ -1728,6 +2274,9 @@ export class FleetManager {
|
|
|
1728
2274
|
}
|
|
1729
2275
|
catch (err) {
|
|
1730
2276
|
this.logger.error({ err, name }, "Failed to start general instance");
|
|
2277
|
+
// General is the most important instance to bring back: it also gets
|
|
2278
|
+
// the delayed automatic retry (the topic notice below still goes out).
|
|
2279
|
+
this.scheduleStartupRetry(name, 0);
|
|
1731
2280
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1732
2281
|
const topicId = cfg.topic_id ? String(cfg.topic_id) : undefined;
|
|
1733
2282
|
if (this.adapter && topicId) {
|
|
@@ -1756,6 +2305,10 @@ export class FleetManager {
|
|
|
1756
2305
|
catch (err) {
|
|
1757
2306
|
this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
|
|
1758
2307
|
}
|
|
2308
|
+
// Now that there is somewhere to deliver: a classicBot.yaml that already
|
|
2309
|
+
// holds an unmatchable id at boot is the COMMON case, and reporting it
|
|
2310
|
+
// before the adapter existed meant the operator heard nothing at all.
|
|
2311
|
+
this.reportClassicUnrecoverableIds();
|
|
1759
2312
|
})();
|
|
1760
2313
|
progressStart = adapterStartup.then(() => {
|
|
1761
2314
|
if (pendingUpdateProgress) {
|
|
@@ -1847,14 +2400,8 @@ export class FleetManager {
|
|
|
1847
2400
|
while (idx < channels.length) {
|
|
1848
2401
|
const batch = channels.slice(idx, idx + concurrency);
|
|
1849
2402
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
if (this.daemons.has(ch.instanceName))
|
|
1853
|
-
startupProgress.markReady();
|
|
1854
|
-
}
|
|
1855
|
-
catch (err) {
|
|
1856
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
1857
|
-
}
|
|
2403
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
2404
|
+
startupProgress.markReady();
|
|
1858
2405
|
}));
|
|
1859
2406
|
idx += concurrency;
|
|
1860
2407
|
}
|
|
@@ -1881,6 +2428,12 @@ export class FleetManager {
|
|
|
1881
2428
|
version: agendVersion,
|
|
1882
2429
|
pausedNames,
|
|
1883
2430
|
failedNames,
|
|
2431
|
+
tipText: pendingUpdateProgress && this.tipsEnabled()
|
|
2432
|
+
? (() => {
|
|
2433
|
+
const tip = this.pickAvailableTip();
|
|
2434
|
+
return tip ? this.formatTip(tip) : undefined;
|
|
2435
|
+
})()
|
|
2436
|
+
: undefined,
|
|
1884
2437
|
});
|
|
1885
2438
|
if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
|
|
1886
2439
|
let text;
|
|
@@ -2042,20 +2595,33 @@ export class FleetManager {
|
|
|
2042
2595
|
notifyAdapterFailure(adapterId, error) {
|
|
2043
2596
|
const generalId = this.findGeneralInstance();
|
|
2044
2597
|
if (generalId) {
|
|
2045
|
-
this.notifyInstanceTopic(generalId,
|
|
2598
|
+
this.notifyInstanceTopic(generalId, t("adapter.start_failed", adapterId, error));
|
|
2046
2599
|
}
|
|
2047
2600
|
}
|
|
2048
2601
|
/** Notify admin that a retried adapter reconnected. */
|
|
2049
2602
|
notifyAdapterRecovery(adapterId, attempts) {
|
|
2050
2603
|
const generalId = this.findGeneralInstance();
|
|
2051
2604
|
if (generalId) {
|
|
2052
|
-
this.notifyInstanceTopic(generalId,
|
|
2605
|
+
this.notifyInstanceTopic(generalId, t("adapter.reconnected", adapterId, attempts));
|
|
2053
2606
|
}
|
|
2054
2607
|
}
|
|
2055
2608
|
/** Get adapter states for /status visibility. */
|
|
2056
2609
|
getAdapterStates() {
|
|
2057
2610
|
return this.adapterState;
|
|
2058
2611
|
}
|
|
2612
|
+
bindAdapterHealth(adapter, adapterId) {
|
|
2613
|
+
adapter.on("gateway_health", (snapshot) => {
|
|
2614
|
+
const previous = this.adapterState.get(adapterId);
|
|
2615
|
+
const status = snapshot.status === "connected" ? "connected"
|
|
2616
|
+
: snapshot.status === "stopped" ? "failed"
|
|
2617
|
+
: "retrying";
|
|
2618
|
+
this.adapterState.set(adapterId, {
|
|
2619
|
+
status,
|
|
2620
|
+
retryCount: previous?.retryCount ?? 0,
|
|
2621
|
+
lastError: status === "connected" ? undefined : snapshot.lastReconnectReason ?? previous?.lastError,
|
|
2622
|
+
});
|
|
2623
|
+
});
|
|
2624
|
+
}
|
|
2059
2625
|
/**
|
|
2060
2626
|
* Real, checkable fleet health for `/health` and the operator.
|
|
2061
2627
|
*
|
|
@@ -2085,10 +2651,17 @@ export class FleetManager {
|
|
|
2085
2651
|
counts.stopped++;
|
|
2086
2652
|
}
|
|
2087
2653
|
const states = {};
|
|
2654
|
+
const details = {};
|
|
2088
2655
|
let connected = 0;
|
|
2089
2656
|
for (const [id, state] of this.adapterState) {
|
|
2090
|
-
|
|
2091
|
-
if (
|
|
2657
|
+
const snapshot = this.adapters.get(id)?.getHealthSnapshot?.();
|
|
2658
|
+
if (snapshot)
|
|
2659
|
+
details[id] = snapshot;
|
|
2660
|
+
const effectiveStatus = snapshot?.status === "connected" ? "connected"
|
|
2661
|
+
: snapshot && snapshot.status !== "stopped" ? "retrying"
|
|
2662
|
+
: state.status;
|
|
2663
|
+
states[id] = effectiveStatus;
|
|
2664
|
+
if (effectiveStatus === "connected")
|
|
2092
2665
|
connected++;
|
|
2093
2666
|
}
|
|
2094
2667
|
const problems = [];
|
|
@@ -2096,9 +2669,9 @@ export class FleetManager {
|
|
|
2096
2669
|
problems.push("no channel adapter is connected");
|
|
2097
2670
|
if (counts.crashed > 0)
|
|
2098
2671
|
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
2099
|
-
for (const [id, state] of
|
|
2100
|
-
if (state
|
|
2101
|
-
problems.push(`adapter ${id} is ${state
|
|
2672
|
+
for (const [id, state] of Object.entries(states)) {
|
|
2673
|
+
if (state !== "connected")
|
|
2674
|
+
problems.push(`adapter ${id} is ${state}`);
|
|
2102
2675
|
}
|
|
2103
2676
|
if (!this.startupComplete)
|
|
2104
2677
|
problems.push("startup has not completed");
|
|
@@ -2111,7 +2684,7 @@ export class FleetManager {
|
|
|
2111
2684
|
status,
|
|
2112
2685
|
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
2113
2686
|
instances: counts,
|
|
2114
|
-
adapters: { total: this.adapterState.size, connected, states },
|
|
2687
|
+
adapters: { total: this.adapterState.size, connected, states, details },
|
|
2115
2688
|
startupComplete: this.startupComplete,
|
|
2116
2689
|
memory: readFleetMemory(),
|
|
2117
2690
|
problems,
|
|
@@ -2140,6 +2713,7 @@ export class FleetManager {
|
|
|
2140
2713
|
const world = new AdapterWorld(adapterId, this.adapter, accessManager, channelConfig);
|
|
2141
2714
|
this.worlds.set(adapterId, world);
|
|
2142
2715
|
this.adapters.set(adapterId, this.adapter);
|
|
2716
|
+
this.bindAdapterHealth(this.adapter, adapterId);
|
|
2143
2717
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2144
2718
|
await this.handleInboundMessage(msg);
|
|
2145
2719
|
}, this.logger, "adapter.message"));
|
|
@@ -2147,6 +2721,22 @@ export class FleetManager {
|
|
|
2147
2721
|
await this.handleInboundReaction(r);
|
|
2148
2722
|
}, this.logger, "adapter.reaction"));
|
|
2149
2723
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
2724
|
+
if (await this.handleTipDismiss(data, adapterId, this.adapter ?? undefined))
|
|
2725
|
+
return;
|
|
2726
|
+
if (await this.handleTipUnlock(data, adapterId, this.adapter ?? undefined))
|
|
2727
|
+
return;
|
|
2728
|
+
if (await this.handleLoginBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2729
|
+
return;
|
|
2730
|
+
if (await this.handleInstallBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2731
|
+
return;
|
|
2732
|
+
if (await this.handleClassicApproval(data, adapterId, this.adapter ?? undefined))
|
|
2733
|
+
return;
|
|
2734
|
+
if (await this.handleLoginMenuSelect(data, adapterId, this.adapter ?? undefined))
|
|
2735
|
+
return;
|
|
2736
|
+
if (await this.handleLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2737
|
+
return;
|
|
2738
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2739
|
+
return;
|
|
2150
2740
|
if (await this.handleClearConfirmation(data, adapterId, this.adapter ?? undefined))
|
|
2151
2741
|
return;
|
|
2152
2742
|
if (await this.handleExitRestartPrompt(data, adapterId, this.adapter ?? undefined))
|
|
@@ -2259,6 +2849,29 @@ export class FleetManager {
|
|
|
2259
2849
|
});
|
|
2260
2850
|
await data.respond(result);
|
|
2261
2851
|
}
|
|
2852
|
+
else if (data.command === "btw") {
|
|
2853
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2854
|
+
if (!name) {
|
|
2855
|
+
await data.respond(t("classic.no_agent"));
|
|
2856
|
+
return;
|
|
2857
|
+
}
|
|
2858
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
2859
|
+
if (!btwText) {
|
|
2860
|
+
await data.respond(t("btw.usage"));
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
2864
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2865
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2866
|
+
});
|
|
2867
|
+
await data.respond(result);
|
|
2868
|
+
}
|
|
2869
|
+
else if (data.command === "login") {
|
|
2870
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
2871
|
+
}
|
|
2872
|
+
else if (data.command === "install-cli") {
|
|
2873
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
2874
|
+
}
|
|
2262
2875
|
else if (data.command === "clear") {
|
|
2263
2876
|
await this.handleClearSlash(data, adapterId);
|
|
2264
2877
|
}
|
|
@@ -2332,12 +2945,15 @@ export class FleetManager {
|
|
|
2332
2945
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2333
2946
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2334
2947
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2335
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
2948
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2336
2949
|
}
|
|
2337
2950
|
catch (err) {
|
|
2338
|
-
await data.respond(
|
|
2951
|
+
await data.respond(t("usage.failed", err.message));
|
|
2339
2952
|
}
|
|
2340
2953
|
}
|
|
2954
|
+
else if (data.command === "tips") {
|
|
2955
|
+
await this.handleTipsSlash(data, adapterId);
|
|
2956
|
+
}
|
|
2341
2957
|
else if (data.command === "status") {
|
|
2342
2958
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2343
2959
|
// instance's cost and IPC health.
|
|
@@ -2349,7 +2965,8 @@ export class FleetManager {
|
|
|
2349
2965
|
await data.respond(text);
|
|
2350
2966
|
}
|
|
2351
2967
|
else if (data.command === "sysinfo") {
|
|
2352
|
-
|
|
2968
|
+
// Slash commands are Discord-only; use plain lines (no markdown table)
|
|
2969
|
+
await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
|
|
2353
2970
|
}
|
|
2354
2971
|
else if (data.command === "dashboard") {
|
|
2355
2972
|
// Reply is ephemeral (adapter defers non-chat commands ephemerally), so
|
|
@@ -2403,6 +3020,29 @@ export class FleetManager {
|
|
|
2403
3020
|
});
|
|
2404
3021
|
await data.respond(result);
|
|
2405
3022
|
}
|
|
3023
|
+
else if (data.command === "btw") {
|
|
3024
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3025
|
+
if (!name) {
|
|
3026
|
+
await data.respond(t("classic.no_agent"));
|
|
3027
|
+
return;
|
|
3028
|
+
}
|
|
3029
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
3030
|
+
if (!btwText) {
|
|
3031
|
+
await data.respond(t("btw.usage"));
|
|
3032
|
+
return;
|
|
3033
|
+
}
|
|
3034
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
3035
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3036
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3037
|
+
});
|
|
3038
|
+
await data.respond(result);
|
|
3039
|
+
}
|
|
3040
|
+
else if (data.command === "login") {
|
|
3041
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
3042
|
+
}
|
|
3043
|
+
else if (data.command === "install-cli") {
|
|
3044
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
3045
|
+
}
|
|
2406
3046
|
}, this.logger, "adapter.slash_command"));
|
|
2407
3047
|
await this.topicCommands.registerBotCommands().catch(e => this.logger.warn({ err: e }, "registerBotCommands failed (non-fatal)"));
|
|
2408
3048
|
// Background-probe each backend's CLI env (version/models) → cli-env cache.
|
|
@@ -2429,13 +3069,14 @@ export class FleetManager {
|
|
|
2429
3069
|
}, this.logger, "adapter.handler_error"));
|
|
2430
3070
|
this.adapter.on("error", (err) => {
|
|
2431
3071
|
this.logger.error({ err }, "Primary adapter fatal error");
|
|
2432
|
-
this.restartAdapter(this.adapter,
|
|
3072
|
+
this.restartAdapter(this.adapter, adapterId).catch(() => { });
|
|
2433
3073
|
});
|
|
2434
|
-
this.adapter.on("new_group_detected", safeHandler((data) => {
|
|
3074
|
+
this.adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2435
3075
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2436
3076
|
const generalId = this.findGeneralInstance();
|
|
3077
|
+
// No user to promote: the bot was just added, nobody has run /start yet.
|
|
2437
3078
|
if (generalId)
|
|
2438
|
-
this.
|
|
3079
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2439
3080
|
}, this.logger, "adapter.new_group_detected"));
|
|
2440
3081
|
// Start adapter AFTER all event listeners are registered (started event sets botUsername)
|
|
2441
3082
|
await this.adapter.start();
|
|
@@ -2471,6 +3112,7 @@ export class FleetManager {
|
|
|
2471
3112
|
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2472
3113
|
this.worlds.set(adapterId, world);
|
|
2473
3114
|
this.adapters.set(adapterId, adapter);
|
|
3115
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2474
3116
|
// Wire up event handlers (same as primary, routes through shared handleInboundMessage)
|
|
2475
3117
|
adapter.on("message", safeHandler(async (msg) => {
|
|
2476
3118
|
await this.handleInboundMessage(msg);
|
|
@@ -2479,6 +3121,22 @@ export class FleetManager {
|
|
|
2479
3121
|
await this.handleInboundReaction(r);
|
|
2480
3122
|
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
2481
3123
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
3124
|
+
if (await this.handleTipDismiss(data, adapterId, adapter))
|
|
3125
|
+
return;
|
|
3126
|
+
if (await this.handleTipUnlock(data, adapterId, adapter))
|
|
3127
|
+
return;
|
|
3128
|
+
if (await this.handleLoginBackendSelect(data, adapterId, adapter))
|
|
3129
|
+
return;
|
|
3130
|
+
if (await this.handleInstallBackendSelect(data, adapterId, adapter))
|
|
3131
|
+
return;
|
|
3132
|
+
if (await this.handleClassicApproval(data, adapterId, adapter))
|
|
3133
|
+
return;
|
|
3134
|
+
if (await this.handleLoginMenuSelect(data, adapterId, adapter))
|
|
3135
|
+
return;
|
|
3136
|
+
if (await this.handleLoginConfirm(data, adapterId, adapter))
|
|
3137
|
+
return;
|
|
3138
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, adapter))
|
|
3139
|
+
return;
|
|
2482
3140
|
if (await this.handleClearConfirmation(data, adapterId, adapter))
|
|
2483
3141
|
return;
|
|
2484
3142
|
if (await this.handleExitRestartPrompt(data, adapterId, adapter))
|
|
@@ -2634,12 +3292,15 @@ export class FleetManager {
|
|
|
2634
3292
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2635
3293
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2636
3294
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2637
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
3295
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2638
3296
|
}
|
|
2639
3297
|
catch (err) {
|
|
2640
|
-
await data.respond(
|
|
3298
|
+
await data.respond(t("usage.failed", err.message));
|
|
2641
3299
|
}
|
|
2642
3300
|
}
|
|
3301
|
+
else if (data.command === "tips") {
|
|
3302
|
+
await this.handleTipsSlash(data, adapterId);
|
|
3303
|
+
}
|
|
2643
3304
|
else if (data.command === "status") {
|
|
2644
3305
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2645
3306
|
// instance's cost and IPC health.
|
|
@@ -2651,7 +3312,8 @@ export class FleetManager {
|
|
|
2651
3312
|
await data.respond(text);
|
|
2652
3313
|
}
|
|
2653
3314
|
else if (data.command === "sysinfo") {
|
|
2654
|
-
|
|
3315
|
+
// Slash commands are Discord-only; use plain lines (no markdown table)
|
|
3316
|
+
await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
|
|
2655
3317
|
}
|
|
2656
3318
|
else if (data.command === "dashboard") {
|
|
2657
3319
|
// Reply is ephemeral (adapter defers non-chat commands ephemerally), so
|
|
@@ -2705,11 +3367,30 @@ export class FleetManager {
|
|
|
2705
3367
|
});
|
|
2706
3368
|
await data.respond(result);
|
|
2707
3369
|
}
|
|
3370
|
+
else if (data.command === "btw") {
|
|
3371
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3372
|
+
if (!name) {
|
|
3373
|
+
await data.respond(t("classic.no_agent"));
|
|
3374
|
+
return;
|
|
3375
|
+
}
|
|
3376
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
3377
|
+
if (!btwText) {
|
|
3378
|
+
await data.respond(t("btw.usage"));
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
3381
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
3382
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3383
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3384
|
+
});
|
|
3385
|
+
await data.respond(result);
|
|
3386
|
+
}
|
|
3387
|
+
else if (data.command === "login") {
|
|
3388
|
+
await this.handleLoginSlash(data, adapterId, adapter);
|
|
3389
|
+
}
|
|
3390
|
+
else if (data.command === "install-cli") {
|
|
3391
|
+
await this.handleInstallCliSlash(data, adapterId, adapter);
|
|
3392
|
+
}
|
|
2708
3393
|
}, this.logger, `adapter[${adapterId}].slash_command`));
|
|
2709
|
-
await adapter.start();
|
|
2710
|
-
if (channelConfig.group_id) {
|
|
2711
|
-
adapter.setChatId(String(channelConfig.group_id));
|
|
2712
|
-
}
|
|
2713
3394
|
adapter.on("started", safeHandler((username, userId) => {
|
|
2714
3395
|
this.logger.info(`[${adapterId}] Bot @${username} polling started.`);
|
|
2715
3396
|
const world = this.worlds.get(adapterId);
|
|
@@ -2719,16 +3400,21 @@ export class FleetManager {
|
|
|
2719
3400
|
world.botUserId = userId;
|
|
2720
3401
|
}
|
|
2721
3402
|
}, this.logger, `adapter[${adapterId}].started`));
|
|
2722
|
-
adapter.on("new_group_detected", safeHandler((data) => {
|
|
3403
|
+
adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2723
3404
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2724
3405
|
const generalId = this.findGeneralInstance(adapterId);
|
|
2725
3406
|
if (generalId)
|
|
2726
|
-
this.
|
|
3407
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2727
3408
|
}, this.logger, `adapter[${adapterId}].new_group_detected`));
|
|
2728
3409
|
adapter.on("error", (err) => {
|
|
2729
3410
|
this.logger.error({ err, adapterId }, "Additional adapter fatal error");
|
|
2730
3411
|
this.restartAdapter(adapter, adapterId).catch(() => { });
|
|
2731
3412
|
});
|
|
3413
|
+
// Register lifecycle listeners before login; a fast ready/error must not be lost.
|
|
3414
|
+
await adapter.start();
|
|
3415
|
+
if (channelConfig.group_id) {
|
|
3416
|
+
adapter.setChatId(String(channelConfig.group_id));
|
|
3417
|
+
}
|
|
2732
3418
|
this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
|
|
2733
3419
|
}
|
|
2734
3420
|
/** Connect IPC to a single instance with all handlers */
|
|
@@ -2825,6 +3511,39 @@ export class FleetManager {
|
|
|
2825
3511
|
else if (msg.type === "instance_progress") {
|
|
2826
3512
|
this.cacheInstanceProgress(name, msg.progress || null);
|
|
2827
3513
|
}
|
|
3514
|
+
else if (msg.type === "cross_instance_delivery_failed") {
|
|
3515
|
+
const senderSession = String(msg.senderSession ?? "");
|
|
3516
|
+
const targetInstance = String(msg.targetInstance ?? name);
|
|
3517
|
+
const correlationId = String(msg.correlationId ?? "unknown");
|
|
3518
|
+
const error = String(msg.error ?? "unknown delivery failure");
|
|
3519
|
+
const senderInstance = this.instanceIpcClients.has(senderSession)
|
|
3520
|
+
? senderSession
|
|
3521
|
+
: this.sessionRegistry.get(senderSession);
|
|
3522
|
+
this.logger.error({ senderSession, targetInstance, correlationId, error }, "Cross-instance pane delivery failed after queue acceptance");
|
|
3523
|
+
this.eventLog?.insert(targetInstance, "cross_instance_delivery_failed", {
|
|
3524
|
+
from: senderSession, correlation_id: correlationId, error,
|
|
3525
|
+
});
|
|
3526
|
+
if (senderInstance) {
|
|
3527
|
+
this.notifyInstanceTopic(senderInstance, t("cross_instance.delivery_failed_notice", targetInstance, error, correlationId));
|
|
3528
|
+
void this.deliverToInstance(senderInstance, {
|
|
3529
|
+
type: "fleet_inbound",
|
|
3530
|
+
targetSession: senderSession,
|
|
3531
|
+
content: t("cross_instance.delivery_failed_agent", targetInstance, error, correlationId),
|
|
3532
|
+
meta: {
|
|
3533
|
+
chat_id: "",
|
|
3534
|
+
message_id: `delivery-failed-${Date.now()}`,
|
|
3535
|
+
user: "AgEnD",
|
|
3536
|
+
user_id: "system:agend",
|
|
3537
|
+
ts: new Date().toISOString(),
|
|
3538
|
+
thread_id: "",
|
|
3539
|
+
request_kind: "update",
|
|
3540
|
+
correlation_id: correlationId,
|
|
3541
|
+
},
|
|
3542
|
+
}, { waitForIdle: true }).catch(deliveryError => {
|
|
3543
|
+
this.logger.error({ err: deliveryError, senderSession, correlationId }, "Could not deliver cross-instance failure status back to sender");
|
|
3544
|
+
});
|
|
3545
|
+
}
|
|
3546
|
+
}
|
|
2828
3547
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2829
3548
|
this.cacheInstanceExecutionState(name, msg);
|
|
2830
3549
|
if (msg.type === "instance_state_response") {
|
|
@@ -2938,6 +3657,26 @@ export class FleetManager {
|
|
|
2938
3657
|
const previous = this.adapterState.get(id);
|
|
2939
3658
|
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2940
3659
|
try {
|
|
3660
|
+
if (adapter.reconnectGateway) {
|
|
3661
|
+
try {
|
|
3662
|
+
// Discord requires a fresh Client after destroy(). Its adapter owns the
|
|
3663
|
+
// single-flight, generation fence and bounded IDENTIFY backoff, so all
|
|
3664
|
+
// watchdog/manual/error triggers must converge here instead of stop/start.
|
|
3665
|
+
await adapter.reconnectGateway(previous?.lastError ?? "fleet adapter restart");
|
|
3666
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
3667
|
+
this.logger.info({ id }, "Adapter gateway rebuilt successfully");
|
|
3668
|
+
}
|
|
3669
|
+
catch (err) {
|
|
3670
|
+
if (!this.ipcStoppingInstances.has("__fleet_stopping__")) {
|
|
3671
|
+
this.adapterState.set(id, {
|
|
3672
|
+
status: "failed",
|
|
3673
|
+
retryCount: previous?.retryCount ?? 0,
|
|
3674
|
+
lastError: err?.message ?? String(err),
|
|
3675
|
+
});
|
|
3676
|
+
}
|
|
3677
|
+
}
|
|
3678
|
+
return;
|
|
3679
|
+
}
|
|
2941
3680
|
for (let attempt = 1;; attempt++) {
|
|
2942
3681
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
2943
3682
|
return;
|
|
@@ -3041,9 +3780,148 @@ export class FleetManager {
|
|
|
3041
3780
|
consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
|
|
3042
3781
|
};
|
|
3043
3782
|
}
|
|
3783
|
+
/**
|
|
3784
|
+
* Which adapter's access policy governs an inbound message.
|
|
3785
|
+
*
|
|
3786
|
+
* `authoritative` means the thread resolved to an owning instance, so that
|
|
3787
|
+
* adapter's policy is the only one entitled to decide. A resolved owner whose
|
|
3788
|
+
* policy cannot be read is NOT the same as having no owner: the owner's world
|
|
3789
|
+
* may not exist yet (an adapter that fails its token check returns before its
|
|
3790
|
+
* world and AccessManager are created) while sibling bots are already live and
|
|
3791
|
+
* receiving copies. Falling back to a sibling's policy — or to the fleet-wide
|
|
3792
|
+
* one — would let the owner's rules be decided by another adapter, so callers
|
|
3793
|
+
* must refuse instead.
|
|
3794
|
+
*
|
|
3795
|
+
* Without an owner (classic channels, the no-thread Telegram path, unrouted
|
|
3796
|
+
* threads, no adapter identity) the receiving adapter's policy applies, as
|
|
3797
|
+
* before.
|
|
3798
|
+
*/
|
|
3799
|
+
governingAccess(msg, threadId) {
|
|
3800
|
+
if (threadId && !this.classicChannels?.hasChannel(threadId)) {
|
|
3801
|
+
const target = this.routing.resolve(threadId);
|
|
3802
|
+
if (target) {
|
|
3803
|
+
const owner = this.getInstanceAdapterId(target.name);
|
|
3804
|
+
if (owner)
|
|
3805
|
+
return { adapterId: owner, authoritative: true };
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
return { adapterId: msg.adapterId, authoritative: false };
|
|
3809
|
+
}
|
|
3810
|
+
/**
|
|
3811
|
+
* Why this adapter should leave an inbound message to another adapter, or null
|
|
3812
|
+
* to handle it.
|
|
3813
|
+
*
|
|
3814
|
+
* A fleet topic is served by exactly one instance, and that instance is
|
|
3815
|
+
* answered by exactly one adapter. When several bots share a guild they each
|
|
3816
|
+
* receive their own copy of every message, so the copy that is acted on must
|
|
3817
|
+
* be chosen by ownership rather than by which inbound happened to fire first.
|
|
3818
|
+
* Ownership is per-instance — the adapter the instance is bound to, falling
|
|
3819
|
+
* back to channels[0] when it is not bound — so an instance answered by a
|
|
3820
|
+
* non-default bot still receives its own traffic, and the access policy that
|
|
3821
|
+
* applies is that instance's own adapter's.
|
|
3822
|
+
*
|
|
3823
|
+
* Callers must ask before claiming the shared dedup key: a copy that is going
|
|
3824
|
+
* to be left alone must not consume the key, or the owner's copy would be
|
|
3825
|
+
* discarded as a duplicate and the message lost.
|
|
3826
|
+
*
|
|
3827
|
+
* Exempt: classic channels (each bot owns its own agent there and handles its
|
|
3828
|
+
* own copy, which is why their dedup key is adapter-scoped), the no-thread
|
|
3829
|
+
* Telegram path (no thread to resolve an instance from), and messages with no
|
|
3830
|
+
* adapter id (single-adapter fleets).
|
|
3831
|
+
*/
|
|
3832
|
+
topicOwnerDropReason(msg, threadId) {
|
|
3833
|
+
if (!threadId || !msg.adapterId)
|
|
3834
|
+
return null;
|
|
3835
|
+
if (this.classicChannels?.hasChannel(threadId))
|
|
3836
|
+
return null;
|
|
3837
|
+
const target = this.routing.resolve(threadId);
|
|
3838
|
+
if (!target)
|
|
3839
|
+
return null;
|
|
3840
|
+
const ownerAdapterId = this.getInstanceAdapterId(target.name);
|
|
3841
|
+
if (ownerAdapterId && msg.adapterId !== ownerAdapterId) {
|
|
3842
|
+
return `not the adapter bound to ${target.name} (that is ${ownerAdapterId})`;
|
|
3843
|
+
}
|
|
3844
|
+
return null;
|
|
3845
|
+
}
|
|
3846
|
+
/**
|
|
3847
|
+
* Why this adapter must ignore a bot/webhook message, or null to accept it.
|
|
3848
|
+
* Pure: callers rely on being able to ask before claiming the dedup key.
|
|
3849
|
+
*/
|
|
3850
|
+
botMessageDropReason(msg, threadId) {
|
|
3851
|
+
if (!threadId) {
|
|
3852
|
+
// TG classic: allow if the bot @mentions our bot or access mode is open
|
|
3853
|
+
const world = this.worlds.get(msg.adapterId ?? "");
|
|
3854
|
+
const botUser = world?.botUsername;
|
|
3855
|
+
const isOpen = this.getChannelConfig(msg.adapterId)?.access?.mode === "open";
|
|
3856
|
+
const mentionsUs = !!(botUser && msg.text?.toLowerCase().includes(`@${botUser.toLowerCase()}`));
|
|
3857
|
+
return (!isOpen && !mentionsUs) ? "tg-classic: not open and does not mention us" : null;
|
|
3858
|
+
}
|
|
3859
|
+
if (this.classicChannels?.hasChannel(threadId)) {
|
|
3860
|
+
// Classic channel (per-bot): bot messages only when THIS bot owns an
|
|
3861
|
+
// agent here and collab is on for it.
|
|
3862
|
+
if (!this.classicChannels.getInstanceByChannel(threadId, msg.adapterId)) {
|
|
3863
|
+
return "classic: this bot owns no agent in the channel";
|
|
3864
|
+
}
|
|
3865
|
+
if (!this.classicChannels.isCollab(threadId, msg.adapterId))
|
|
3866
|
+
return "classic: collab off";
|
|
3867
|
+
return null;
|
|
3868
|
+
}
|
|
3869
|
+
const target = this.routing.resolve(threadId);
|
|
3870
|
+
if (!target)
|
|
3871
|
+
return "fleet topic: no instance routed for this thread";
|
|
3872
|
+
// Fleet topic: allow if collab enabled OR access mode is open
|
|
3873
|
+
const isOpen = this.getChannelConfig(msg.adapterId)?.access?.mode === "open";
|
|
3874
|
+
if (!isOpen && !this.collabInstances.has(target.name)) {
|
|
3875
|
+
return `fleet topic: adapter not open and collab off for ${target.name}`;
|
|
3876
|
+
}
|
|
3877
|
+
return null;
|
|
3878
|
+
}
|
|
3044
3879
|
async handleInboundMessage(msg) {
|
|
3045
3880
|
const threadId = msg.threadId || undefined;
|
|
3046
3881
|
this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
|
|
3882
|
+
// Ownership and per-adapter filtering both run before the dedup claim below.
|
|
3883
|
+
// When several bots share a guild they all receive the same message; a copy
|
|
3884
|
+
// this adapter is going to leave alone must not consume the shared dedup key,
|
|
3885
|
+
// or the owning adapter's copy would be discarded as a duplicate and the
|
|
3886
|
+
// message lost entirely. Claiming the key only on surviving copies also keeps
|
|
3887
|
+
// delivery single when more than one adapter would otherwise accept it.
|
|
3888
|
+
// Only bot messages are settled by ownership: a bot answering in a fleet
|
|
3889
|
+
// topic would speak as the wrong identity, whereas a human message is just
|
|
3890
|
+
// input for the instance and its reply is canonicalized to the owning
|
|
3891
|
+
// adapter downstream — so a topic only a non-owner bot can see still works.
|
|
3892
|
+
const drop = msg.isBotMessage
|
|
3893
|
+
? (this.topicOwnerDropReason(msg, threadId) ?? this.botMessageDropReason(msg, threadId))
|
|
3894
|
+
: null;
|
|
3895
|
+
if (drop) {
|
|
3896
|
+
this.logger.debug({ adapterId: msg.adapterId, threadId: threadId ?? null, messageId: msg.messageId, reason: drop }, "Inbound message dropped before the dedup claim — key not consumed");
|
|
3897
|
+
return;
|
|
3898
|
+
}
|
|
3899
|
+
// Access control — classic channels are open to all, others require an allowed
|
|
3900
|
+
// user. This runs before the dedup claim, and is judged by the adapter that
|
|
3901
|
+
// owns the topic rather than the one that happened to receive the message:
|
|
3902
|
+
// otherwise a sibling bot could settle the message under its own policy —
|
|
3903
|
+
// claiming the shared key and then refusing it, or admitting a user the
|
|
3904
|
+
// owning adapter does not allow — purely by arriving first.
|
|
3905
|
+
const governing = this.governingAccess(msg, threadId);
|
|
3906
|
+
const ownerWorld = governing.adapterId ? this.worlds.get(governing.adapterId) : undefined;
|
|
3907
|
+
if (governing.authoritative && !ownerWorld) {
|
|
3908
|
+
this.logger.warn({ adapterId: msg.adapterId, owner: governing.adapterId, threadId, messageId: msg.messageId }, "Refusing inbound: the owning adapter is not running, so its access policy cannot be applied");
|
|
3909
|
+
return;
|
|
3910
|
+
}
|
|
3911
|
+
const am = governing.authoritative
|
|
3912
|
+
? ownerWorld?.accessManager
|
|
3913
|
+
: (ownerWorld?.accessManager ?? this.accessManager);
|
|
3914
|
+
if (am && !am.isAllowed(msg.userId)) {
|
|
3915
|
+
const adapterGroupId = String(this.getChannelConfig(msg.adapterId)?.group_id ?? "");
|
|
3916
|
+
const isTelegramClassicCandidate = msg.source === "telegram" && msg.chatId !== adapterGroupId && !threadId;
|
|
3917
|
+
if (!isTelegramClassicCandidate) {
|
|
3918
|
+
// Classic channels are open to all; check per-bot ownership (or fleet topic).
|
|
3919
|
+
const isClassic = !!(threadId && this.classicChannels?.hasChannel(threadId));
|
|
3920
|
+
this.logger.info({ userId: msg.userId, threadId, isClassic, owner: governing.adapterId }, "Access DENIED for non-allowed user");
|
|
3921
|
+
if (!isClassic)
|
|
3922
|
+
return;
|
|
3923
|
+
}
|
|
3924
|
+
}
|
|
3047
3925
|
// Multi-adapter dedup: when several bots share a guild, each adapter fires
|
|
3048
3926
|
// its own "message" event for the same underlying message. Process it once.
|
|
3049
3927
|
// Routing (by topic/channel) and reply-adapter selection (by channel_id
|
|
@@ -3072,55 +3950,6 @@ export class FleetManager {
|
|
|
3072
3950
|
this.recentMessageIds.delete(oldest);
|
|
3073
3951
|
}
|
|
3074
3952
|
}
|
|
3075
|
-
// Bot messages: only allow in collab channels or TG classic with @mention
|
|
3076
|
-
if (msg.isBotMessage) {
|
|
3077
|
-
if (!threadId) {
|
|
3078
|
-
// TG classic: allow if bot @mentions our bot or access mode is open
|
|
3079
|
-
const world = this.worlds.get(msg.adapterId ?? "");
|
|
3080
|
-
const botUser = world?.botUsername;
|
|
3081
|
-
const channelCfg = this.getChannelConfig(msg.adapterId);
|
|
3082
|
-
const isOpen = channelCfg?.access?.mode === "open";
|
|
3083
|
-
const mentionsUs = !!(botUser && msg.text?.toLowerCase().includes(`@${botUser.toLowerCase()}`));
|
|
3084
|
-
this.logger.debug({ botUser, mentionsUs, isOpen, isBotMessage: true, threadId: null }, "Bot message filter (no threadId path)");
|
|
3085
|
-
if (!isOpen && !mentionsUs)
|
|
3086
|
-
return;
|
|
3087
|
-
// Fall through to TG classic handling below
|
|
3088
|
-
}
|
|
3089
|
-
else if (this.classicChannels?.hasChannel(threadId)) {
|
|
3090
|
-
// Classic channel (per-bot): bot messages only when THIS bot owns an
|
|
3091
|
-
// agent here and collab is on for it.
|
|
3092
|
-
const classicName = this.classicChannels.getInstanceByChannel(threadId, msg.adapterId);
|
|
3093
|
-
if (!classicName)
|
|
3094
|
-
return;
|
|
3095
|
-
if (!this.classicChannels.isCollab(threadId, msg.adapterId))
|
|
3096
|
-
return;
|
|
3097
|
-
// Fall through to channel handling
|
|
3098
|
-
}
|
|
3099
|
-
else {
|
|
3100
|
-
const target = this.routing.resolve(threadId);
|
|
3101
|
-
if (!target)
|
|
3102
|
-
return;
|
|
3103
|
-
// Fleet topic: allow if collab enabled OR access mode is open
|
|
3104
|
-
const channelCfg = this.getChannelConfig(msg.adapterId);
|
|
3105
|
-
const isOpen = channelCfg?.access?.mode === "open";
|
|
3106
|
-
if (!isOpen && !this.collabInstances.has(target.name))
|
|
3107
|
-
return;
|
|
3108
|
-
// Fall through to channel handling
|
|
3109
|
-
}
|
|
3110
|
-
}
|
|
3111
|
-
// Access control — classic channels are open to all, others require allowed user
|
|
3112
|
-
const am = (msg.adapterId ? this.worlds.get(msg.adapterId)?.accessManager : undefined) ?? this.accessManager;
|
|
3113
|
-
if (am && !am.isAllowed(msg.userId)) {
|
|
3114
|
-
const adapterGroupId = String(this.getChannelConfig(msg.adapterId)?.group_id ?? "");
|
|
3115
|
-
const isTelegramClassicCandidate = msg.source === "telegram" && msg.chatId !== adapterGroupId && !threadId;
|
|
3116
|
-
if (!isTelegramClassicCandidate) {
|
|
3117
|
-
// Classic channels are open to all; check per-bot ownership (or fleet topic).
|
|
3118
|
-
const isClassic = !!(threadId && this.classicChannels?.hasChannel(threadId));
|
|
3119
|
-
this.logger.info({ userId: msg.userId, threadId, isClassic }, "Access DENIED for non-allowed user");
|
|
3120
|
-
if (!isClassic)
|
|
3121
|
-
return;
|
|
3122
|
-
}
|
|
3123
|
-
}
|
|
3124
3953
|
if (threadId == null) {
|
|
3125
3954
|
// ── Telegram Classic Mode ──
|
|
3126
3955
|
// Messages from chats other than the primary forum group are classic mode candidates.
|
|
@@ -3175,7 +4004,13 @@ export class FleetManager {
|
|
|
3175
4004
|
const adminMsg = t("alert.new_group", groupTitle, chatId, msg.username, msg.userId, msg.source);
|
|
3176
4005
|
const generalId = this.findGeneralInstance(msg.adapterId);
|
|
3177
4006
|
if (generalId) {
|
|
3178
|
-
this.
|
|
4007
|
+
await this.promptClassicApproval({
|
|
4008
|
+
// Guarded by isGroupAllowed → this belongs in allowed_groups,
|
|
4009
|
+
// NOT allowed_guilds; writing the latter would change the file
|
|
4010
|
+
// without unblocking the group.
|
|
4011
|
+
generalName: generalId, message: adminMsg, groupId: String(chatId),
|
|
4012
|
+
scope: "group", userId: msg.userId,
|
|
4013
|
+
});
|
|
3179
4014
|
}
|
|
3180
4015
|
await msgAdapter?.sendText(chatId, t("classic.access_requested"));
|
|
3181
4016
|
return;
|
|
@@ -3290,6 +4125,24 @@ export class FleetManager {
|
|
|
3290
4125
|
await msgAdapter?.sendText(chatId, result);
|
|
3291
4126
|
return;
|
|
3292
4127
|
}
|
|
4128
|
+
// /btw — Claude Code's native side-thread command. Like /steer it is
|
|
4129
|
+
// not admin-gated, but it remains a distinct raw CLI command so Claude
|
|
4130
|
+
// does not fold the question into the active task.
|
|
4131
|
+
if (text === "/btw" || text.startsWith("/btw ") || text.startsWith("/btw@")) {
|
|
4132
|
+
const btwName = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
4133
|
+
if (!btwName) {
|
|
4134
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
4135
|
+
return;
|
|
4136
|
+
}
|
|
4137
|
+
const btwContent = text.replace(/^\/btw(@\S+)?/, "").trim();
|
|
4138
|
+
if (!btwContent) {
|
|
4139
|
+
await msgAdapter?.sendText(chatId, t("btw.usage"));
|
|
4140
|
+
return;
|
|
4141
|
+
}
|
|
4142
|
+
const result = this.topicCommands.sendBtw(btwName, btwContent, msg);
|
|
4143
|
+
await msgAdapter?.sendText(chatId, result);
|
|
4144
|
+
return;
|
|
4145
|
+
}
|
|
3293
4146
|
// Handle /clear command (admin only) — unlike /compact this starts a
|
|
3294
4147
|
// fresh conversation and intentionally discards the current history.
|
|
3295
4148
|
if (text === "/clear" || text.startsWith("/clear@")) {
|
|
@@ -3317,7 +4170,7 @@ export class FleetManager {
|
|
|
3317
4170
|
return;
|
|
3318
4171
|
}
|
|
3319
4172
|
const ok = this.cancelInstance(cancelName);
|
|
3320
|
-
await msgAdapter?.sendText(chatId, ok ?
|
|
4173
|
+
await msgAdapter?.sendText(chatId, ok ? t("cancel.sent", cancelName) : t("cancel.not_running", cancelName));
|
|
3321
4174
|
return;
|
|
3322
4175
|
}
|
|
3323
4176
|
// Handle /ctx command
|
|
@@ -3354,7 +4207,7 @@ export class FleetManager {
|
|
|
3354
4207
|
const backend = this.classicChannels.getBackendByInstance(saveName, this.fleetConfig?.defaults?.backend);
|
|
3355
4208
|
const cmd = saveCommandForBackend(backend, filename);
|
|
3356
4209
|
if (!cmd) {
|
|
3357
|
-
await msgAdapter?.sendText(chatId,
|
|
4210
|
+
await msgAdapter?.sendText(chatId, t("save.unsupported"));
|
|
3358
4211
|
return;
|
|
3359
4212
|
}
|
|
3360
4213
|
this.pasteRawToClassicInstance(saveName, cmd);
|
|
@@ -3572,10 +4425,10 @@ export class FleetManager {
|
|
|
3572
4425
|
return;
|
|
3573
4426
|
let warning = "";
|
|
3574
4427
|
if (rl.five_hour_pct >= 95) {
|
|
3575
|
-
warning =
|
|
4428
|
+
warning = t("rate_limit.five_hour", instanceName, Math.round(rl.five_hour_pct));
|
|
3576
4429
|
}
|
|
3577
4430
|
else if (rl.seven_day_pct >= 95) {
|
|
3578
|
-
warning =
|
|
4431
|
+
warning = t("rate_limit.weekly", instanceName, Math.round(rl.seven_day_pct));
|
|
3579
4432
|
}
|
|
3580
4433
|
if (!warning)
|
|
3581
4434
|
return;
|
|
@@ -3590,8 +4443,6 @@ export class FleetManager {
|
|
|
3590
4443
|
}
|
|
3591
4444
|
/** Handle outbound tool calls from a daemon instance */
|
|
3592
4445
|
async handleOutboundFromInstance(instanceName, msg) {
|
|
3593
|
-
if (this.worlds.size === 0)
|
|
3594
|
-
return;
|
|
3595
4446
|
this.touchActivity(instanceName);
|
|
3596
4447
|
this.setTopicIcon(instanceName, "green");
|
|
3597
4448
|
const tool = msg.tool;
|
|
@@ -3601,13 +4452,21 @@ export class FleetManager {
|
|
|
3601
4452
|
const senderSessionName = msg.senderSessionName;
|
|
3602
4453
|
const respond = (result, error) => {
|
|
3603
4454
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
4455
|
+
let sent = false;
|
|
3604
4456
|
if (fleetRequestId) {
|
|
3605
|
-
ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error });
|
|
4457
|
+
sent = ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error }) ?? false;
|
|
3606
4458
|
}
|
|
3607
4459
|
else {
|
|
3608
|
-
ipc?.send({ type: "fleet_outbound_response", requestId, result, error });
|
|
4460
|
+
sent = ipc?.send({ type: "fleet_outbound_response", requestId, result, error }) ?? false;
|
|
4461
|
+
}
|
|
4462
|
+
if (!sent) {
|
|
4463
|
+
this.logger.warn({ instanceName, tool, requestId, fleetRequestId, error }, "Fleet outbound result could not be returned — instance IPC is disconnected");
|
|
3609
4464
|
}
|
|
3610
4465
|
};
|
|
4466
|
+
if (this.worlds.size === 0) {
|
|
4467
|
+
respond(null, "Channel adapters are not ready — retry shortly");
|
|
4468
|
+
return;
|
|
4469
|
+
}
|
|
3611
4470
|
// Resolve threadId: use sender's topic_id if sender is a known fleet instance,
|
|
3612
4471
|
// fall back to general topic if sender is unknown, or IPC owner if no sender.
|
|
3613
4472
|
const senderInstanceName = senderSessionName && this.fleetConfig?.instances[senderSessionName]
|
|
@@ -3652,17 +4511,30 @@ export class FleetManager {
|
|
|
3652
4511
|
if (tool === "reply") {
|
|
3653
4512
|
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3654
4513
|
if (ticket.duplicate) {
|
|
3655
|
-
this.logger.info({ instanceName }, "
|
|
4514
|
+
this.logger.info({ instanceName }, "Concurrent duplicate reply joined — awaiting the original send's outcome");
|
|
3656
4515
|
ticket.subscribe(respond);
|
|
3657
4516
|
return;
|
|
3658
4517
|
}
|
|
3659
4518
|
const original = respond;
|
|
3660
4519
|
const respondAndRecord = (result, error) => {
|
|
3661
4520
|
ticket.complete(result, error);
|
|
4521
|
+
// Return the platform outcome first. Bookkeeping below must never turn
|
|
4522
|
+
// a confirmed Discord/Telegram POST into a tool error if a secondary
|
|
4523
|
+
// log/button side effect happens to throw.
|
|
3662
4524
|
original(result, error);
|
|
4525
|
+
// The adapter resolves only after the platform POST returns. Keep
|
|
4526
|
+
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4527
|
+
// a routed-but-failed reply is not a delivered reply.
|
|
4528
|
+
if (!error && result != null) {
|
|
4529
|
+
try {
|
|
4530
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4531
|
+
}
|
|
4532
|
+
catch (err) {
|
|
4533
|
+
this.logger.warn({ err, instanceName }, "Reply delivered but post-delivery bookkeeping failed");
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
3663
4536
|
};
|
|
3664
4537
|
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3665
|
-
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3666
4538
|
return;
|
|
3667
4539
|
}
|
|
3668
4540
|
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
@@ -3699,7 +4571,7 @@ export class FleetManager {
|
|
|
3699
4571
|
this.clearCancelButton(instanceName);
|
|
3700
4572
|
}
|
|
3701
4573
|
else {
|
|
3702
|
-
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
4574
|
+
void this.sendCancelButton(instanceName, undefined, true).then(() => this.armReplyGrace(instanceName));
|
|
3703
4575
|
}
|
|
3704
4576
|
this.reactDone(instanceName);
|
|
3705
4577
|
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
@@ -3736,7 +4608,11 @@ export class FleetManager {
|
|
|
3736
4608
|
if (silent) {
|
|
3737
4609
|
const ipc = this.instanceIpcClients.get(target);
|
|
3738
4610
|
if (ipc) {
|
|
3739
|
-
ipc.send({
|
|
4611
|
+
ipc.send({
|
|
4612
|
+
type: "raw_paste",
|
|
4613
|
+
content: message,
|
|
4614
|
+
delivery_epoch: this.getDeliveryEpoch(target),
|
|
4615
|
+
});
|
|
3740
4616
|
this.scheduler.recordRun(id, "delivered");
|
|
3741
4617
|
this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
|
|
3742
4618
|
}
|
|
@@ -3751,10 +4627,19 @@ export class FleetManager {
|
|
|
3751
4627
|
const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
|
|
3752
4628
|
const deliver = async () => {
|
|
3753
4629
|
try {
|
|
4630
|
+
// A schedule has no live inbound adapter context. Seed the daemon with
|
|
4631
|
+
// the target instance's configured world so replies use its persona
|
|
4632
|
+
// after a fresh start instead of falling back to channels[0].
|
|
4633
|
+
const adapterId = this.getInstanceAdapterId(target);
|
|
3754
4634
|
await this.deliverToInstance(target, {
|
|
3755
4635
|
type: "fleet_schedule_trigger",
|
|
3756
4636
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
3757
|
-
meta: {
|
|
4637
|
+
meta: {
|
|
4638
|
+
chat_id: reply_chat_id,
|
|
4639
|
+
thread_id: reply_thread_id,
|
|
4640
|
+
user: "scheduler",
|
|
4641
|
+
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4642
|
+
},
|
|
3758
4643
|
}, { waitForIdle: true });
|
|
3759
4644
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
3760
4645
|
void this.sendCancelButton(target);
|
|
@@ -3902,7 +4787,29 @@ export class FleetManager {
|
|
|
3902
4787
|
}
|
|
3903
4788
|
/** Resolve display name for an instance, fallback to instance name. */
|
|
3904
4789
|
resolveDisplayName(instanceName) {
|
|
3905
|
-
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4790
|
+
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4791
|
+
?? this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName)?.displayName
|
|
4792
|
+
?? instanceName;
|
|
4793
|
+
}
|
|
4794
|
+
/** Persist identity to the instance's actual config store. Classic instances
|
|
4795
|
+
* are registry rows in classicBot.yaml, not fleet.yaml instance entries. */
|
|
4796
|
+
setInstanceDisplayName(instanceName, displayName) {
|
|
4797
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4798
|
+
if (fleetInstance) {
|
|
4799
|
+
fleetInstance.display_name = displayName;
|
|
4800
|
+
this.saveFleetConfig();
|
|
4801
|
+
return true;
|
|
4802
|
+
}
|
|
4803
|
+
return this.classicChannels?.setDisplayNameByInstance(instanceName, displayName) ?? false;
|
|
4804
|
+
}
|
|
4805
|
+
setInstanceDescription(instanceName, description) {
|
|
4806
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4807
|
+
if (fleetInstance) {
|
|
4808
|
+
fleetInstance.description = description;
|
|
4809
|
+
this.saveFleetConfig();
|
|
4810
|
+
return true;
|
|
4811
|
+
}
|
|
4812
|
+
return this.classicChannels?.setDescriptionByInstance(instanceName, description) ?? false;
|
|
3906
4813
|
}
|
|
3907
4814
|
handleSetDisplayName(instanceName, msg) {
|
|
3908
4815
|
const fleetRequestId = msg.fleetRequestId;
|
|
@@ -3915,8 +4822,10 @@ export class FleetManager {
|
|
|
3915
4822
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: "Name must be 1-30 characters" });
|
|
3916
4823
|
return;
|
|
3917
4824
|
}
|
|
3918
|
-
this.
|
|
3919
|
-
|
|
4825
|
+
if (!this.setInstanceDisplayName(instanceName, displayName)) {
|
|
4826
|
+
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4827
|
+
return;
|
|
4828
|
+
}
|
|
3920
4829
|
this.logger.info({ instanceName, displayName }, "Display name set");
|
|
3921
4830
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, result: { display_name: displayName } });
|
|
3922
4831
|
}
|
|
@@ -3931,8 +4840,10 @@ export class FleetManager {
|
|
|
3931
4840
|
ipc.send({ type: "fleet_description_response", fleetRequestId, error: "Description cannot be empty" });
|
|
3932
4841
|
return;
|
|
3933
4842
|
}
|
|
3934
|
-
this.
|
|
3935
|
-
|
|
4843
|
+
if (!this.setInstanceDescription(instanceName, description)) {
|
|
4844
|
+
ipc.send({ type: "fleet_description_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4845
|
+
return;
|
|
4846
|
+
}
|
|
3936
4847
|
this.logger.info({ instanceName, description: description.slice(0, 80) }, "Description set");
|
|
3937
4848
|
ipc.send({ type: "fleet_description_response", fleetRequestId, result: { description } });
|
|
3938
4849
|
}
|
|
@@ -4061,8 +4972,8 @@ export class FleetManager {
|
|
|
4061
4972
|
return { error: "Fleet config not available" };
|
|
4062
4973
|
if (!name || name.length > 30)
|
|
4063
4974
|
return { error: "Name must be 1-30 characters" };
|
|
4064
|
-
this.
|
|
4065
|
-
|
|
4975
|
+
if (!this.setInstanceDisplayName(instance, name))
|
|
4976
|
+
return { error: `Instance '${instance}' not found` };
|
|
4066
4977
|
return { display_name: name };
|
|
4067
4978
|
}
|
|
4068
4979
|
async handleSetDescriptionHttp(instance, description) {
|
|
@@ -4070,8 +4981,8 @@ export class FleetManager {
|
|
|
4070
4981
|
return { error: "Fleet config not available" };
|
|
4071
4982
|
if (!description)
|
|
4072
4983
|
return { error: "Description cannot be empty" };
|
|
4073
|
-
this.
|
|
4074
|
-
|
|
4984
|
+
if (!this.setInstanceDescription(instance, description))
|
|
4985
|
+
return { error: `Instance '${instance}' not found` };
|
|
4075
4986
|
return { description };
|
|
4076
4987
|
}
|
|
4077
4988
|
summarizeToolCall(tool, args) {
|
|
@@ -4203,7 +5114,8 @@ export class FleetManager {
|
|
|
4203
5114
|
}
|
|
4204
5115
|
/**
|
|
4205
5116
|
* Patch only values changed in the effective config into the original YAML
|
|
4206
|
-
* document. Unknown keys
|
|
5117
|
+
* document. Unknown keys and comments remain untouched; redundant
|
|
5118
|
+
* non-identity instance leaves are canonicalized to inheritance afterward.
|
|
4207
5119
|
*/
|
|
4208
5120
|
saveFleetConfig(explicitPatches = []) {
|
|
4209
5121
|
if (!this.fleetConfig || !this.configPath)
|
|
@@ -4219,9 +5131,9 @@ export class FleetManager {
|
|
|
4219
5131
|
}
|
|
4220
5132
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4221
5133
|
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
4222
|
-
// Settings edits are expressed against the raw config
|
|
4223
|
-
//
|
|
4224
|
-
//
|
|
5134
|
+
// Settings edits are expressed against the raw config, so apply them before
|
|
5135
|
+
// canonicalization. A non-identity value equal to its inherited default is
|
|
5136
|
+
// intentionally stored as inheritance rather than an explicit duplicate.
|
|
4225
5137
|
for (const patch of explicitPatches) {
|
|
4226
5138
|
if (patch.remove) {
|
|
4227
5139
|
// YAML's deleteIn throws when an inherited nested key has no raw parent
|
|
@@ -4236,7 +5148,24 @@ export class FleetManager {
|
|
|
4236
5148
|
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
4237
5149
|
}
|
|
4238
5150
|
}
|
|
5151
|
+
const rawAfterPatches = this.rawFleetDocument.toJS();
|
|
5152
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(rawAfterPatches);
|
|
5153
|
+
for (const path of redundantPaths) {
|
|
5154
|
+
if (this.rawFleetDocument.hasIn(path))
|
|
5155
|
+
this.rawFleetDocument.deleteIn(path);
|
|
5156
|
+
// Avoid leaving empty operational maps such as `terminal: {}` while
|
|
5157
|
+
// preserving the instance mapping itself and all surrounding comments.
|
|
5158
|
+
for (let depth = path.length - 1; depth > 2; depth--) {
|
|
5159
|
+
const parentPath = path.slice(0, depth);
|
|
5160
|
+
const parent = this.rawFleetDocument.getIn(parentPath, true);
|
|
5161
|
+
if (!isMap(parent) || parent.items.length > 0)
|
|
5162
|
+
break;
|
|
5163
|
+
this.rawFleetDocument.deleteIn(parentPath);
|
|
5164
|
+
}
|
|
5165
|
+
}
|
|
4239
5166
|
const output = String(this.rawFleetDocument);
|
|
5167
|
+
if (redundantPaths.length > 0)
|
|
5168
|
+
this.writeFleetConfigBackup(source);
|
|
4240
5169
|
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
4241
5170
|
writeFileSync(tempPath, output, "utf-8");
|
|
4242
5171
|
if (existsSync(this.configPath))
|
|
@@ -4244,7 +5173,36 @@ export class FleetManager {
|
|
|
4244
5173
|
renameSync(tempPath, this.configPath);
|
|
4245
5174
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4246
5175
|
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
4247
|
-
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
5176
|
+
this.logger.info({ path: this.configPath, strippedDefaults: redundantPaths.length }, "Saved fleet config (lossless patch)");
|
|
5177
|
+
}
|
|
5178
|
+
/** One-time upgrade migration; invalid YAML is never rewritten. */
|
|
5179
|
+
slimFleetConfigAtStartup() {
|
|
5180
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(this.rawFleetConfig);
|
|
5181
|
+
if (redundantPaths.length === 0)
|
|
5182
|
+
return;
|
|
5183
|
+
const validation = validateFleetConfig(this.rawFleetConfig);
|
|
5184
|
+
if (!validation.valid) {
|
|
5185
|
+
this.logger.warn({ errors: validation.errors, redundantDefaults: redundantPaths.length }, "Skipping fleet.yaml default slimming because the raw config is invalid");
|
|
5186
|
+
return;
|
|
5187
|
+
}
|
|
5188
|
+
try {
|
|
5189
|
+
this.saveFleetConfig();
|
|
5190
|
+
this.logger.info({ strippedDefaults: redundantPaths.length, backup: `${this.configPath}.bak` }, "Slimmed redundant instance defaults in fleet.yaml");
|
|
5191
|
+
}
|
|
5192
|
+
catch (err) {
|
|
5193
|
+
// A migration must not turn a previously bootable fleet into an outage.
|
|
5194
|
+
this.logger.warn({ err }, "Could not slim fleet.yaml; continuing with the original config");
|
|
5195
|
+
}
|
|
5196
|
+
}
|
|
5197
|
+
writeFleetConfigBackup(source) {
|
|
5198
|
+
if (!this.configPath)
|
|
5199
|
+
return;
|
|
5200
|
+
const backupPath = `${this.configPath}.bak`;
|
|
5201
|
+
const tempPath = `${backupPath}.tmp-${process.pid}`;
|
|
5202
|
+
writeFileSync(tempPath, source, "utf-8");
|
|
5203
|
+
if (existsSync(this.configPath))
|
|
5204
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
5205
|
+
renameSync(tempPath, backupPath);
|
|
4248
5206
|
}
|
|
4249
5207
|
patchFleetDocument(document, path, before, after) {
|
|
4250
5208
|
if (Object.is(before, after))
|
|
@@ -4296,7 +5254,12 @@ export class FleetManager {
|
|
|
4296
5254
|
currentNode.value = after;
|
|
4297
5255
|
}
|
|
4298
5256
|
else {
|
|
4299
|
-
|
|
5257
|
+
// Use a YAML collection node for newly-added objects. Passing the raw
|
|
5258
|
+
// object creates a scalar wrapper: it serializes, but nested hasIn /
|
|
5259
|
+
// deleteIn cannot traverse it (notably when slimming create_instance).
|
|
5260
|
+
document.setIn(path, after !== null && typeof after === "object"
|
|
5261
|
+
? document.createNode(after)
|
|
5262
|
+
: after);
|
|
4300
5263
|
}
|
|
4301
5264
|
}
|
|
4302
5265
|
}
|
|
@@ -4387,7 +5350,7 @@ export class FleetManager {
|
|
|
4387
5350
|
from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct,
|
|
4388
5351
|
});
|
|
4389
5352
|
this.webhookEmitter?.emit("model_failover", name, { from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct });
|
|
4390
|
-
this.notifyInstanceTopic(name,
|
|
5353
|
+
this.notifyInstanceTopic(name, t("failover.triggered", fiveHourPct, fallbackModel, primaryModel));
|
|
4391
5354
|
}
|
|
4392
5355
|
else if (fiveHourPct < FleetManager.FAILOVER_RECOVER_PCT && currentFailover) {
|
|
4393
5356
|
// Recover: switch back to primary
|
|
@@ -4398,7 +5361,7 @@ export class FleetManager {
|
|
|
4398
5361
|
restored: primaryModel, five_hour_pct: fiveHourPct,
|
|
4399
5362
|
});
|
|
4400
5363
|
this.webhookEmitter?.emit("model_recovered", name, { restored: primaryModel, five_hour_pct: fiveHourPct });
|
|
4401
|
-
this.notifyInstanceTopic(name,
|
|
5364
|
+
this.notifyInstanceTopic(name, t("failover.recovered", fiveHourPct, primaryModel));
|
|
4402
5365
|
}
|
|
4403
5366
|
}
|
|
4404
5367
|
toggleFleetCollab(instanceName) {
|
|
@@ -4452,9 +5415,9 @@ export class FleetManager {
|
|
|
4452
5415
|
if (output)
|
|
4453
5416
|
return output;
|
|
4454
5417
|
if (e.code === "ENOENT") {
|
|
4455
|
-
return "
|
|
5418
|
+
return t("doctor.cli_missing");
|
|
4456
5419
|
}
|
|
4457
|
-
return stripAnsi(e.message ?? "").trim() || "
|
|
5420
|
+
return stripAnsi(e.message ?? "").trim() || t("doctor.failed");
|
|
4458
5421
|
}
|
|
4459
5422
|
}
|
|
4460
5423
|
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
@@ -4538,6 +5501,31 @@ export class FleetManager {
|
|
|
4538
5501
|
* The log line is written by the caller regardless: if every adapter is down,
|
|
4539
5502
|
* the only notification path is the one that is broken.
|
|
4540
5503
|
*/
|
|
5504
|
+
/**
|
|
5505
|
+
* Surface ids that can never match into the operator's General topic.
|
|
5506
|
+
*
|
|
5507
|
+
* The load-time log is still silence for anyone not reading daemon.log, and a
|
|
5508
|
+
* chat that never gets in with nothing said anywhere is exactly the failure
|
|
5509
|
+
* this line of work exists to remove. notifyFleetError throttles by message
|
|
5510
|
+
* text for 10 minutes and resolves its target from config, so the 30s reload
|
|
5511
|
+
* poll cannot flood the topic and a down General does not swallow it.
|
|
5512
|
+
*/
|
|
5513
|
+
reportClassicUnrecoverableIds() {
|
|
5514
|
+
const bad = this.classicChannels?.getUnrecoverableIds?.() ?? [];
|
|
5515
|
+
if (bad.length === 0)
|
|
5516
|
+
return;
|
|
5517
|
+
// notifyFleetError claims its throttle key BEFORE attempting delivery, so
|
|
5518
|
+
// calling it with no adapter would silence this message for ten minutes
|
|
5519
|
+
// without anyone having seen it. Log and leave the key unspent; a later
|
|
5520
|
+
// call, once an adapter exists, still gets through.
|
|
5521
|
+
if (!this.adapter && this.adapters.size === 0) {
|
|
5522
|
+
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match — deferring the operator notice until an adapter is up");
|
|
5523
|
+
return;
|
|
5524
|
+
}
|
|
5525
|
+
const list = bad.map(e => `${e.field}: ${e.value}`).join(", ");
|
|
5526
|
+
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5527
|
+
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5528
|
+
}
|
|
4541
5529
|
notifyFleetError(text) {
|
|
4542
5530
|
const now = Date.now();
|
|
4543
5531
|
const key = text.slice(0, 200);
|
|
@@ -4547,16 +5535,8 @@ export class FleetManager {
|
|
|
4547
5535
|
return;
|
|
4548
5536
|
}
|
|
4549
5537
|
const suppressed = seen?.suppressed ?? 0;
|
|
4550
|
-
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
4551
|
-
// Bound the map: it is keyed by message text, and a message with a varying
|
|
4552
|
-
// suffix (a path, an id) would otherwise grow it without limit.
|
|
4553
|
-
if (this.fleetErrorNotices.size > 100) {
|
|
4554
|
-
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
4555
|
-
if (oldest !== undefined)
|
|
4556
|
-
this.fleetErrorNotices.delete(oldest);
|
|
4557
|
-
}
|
|
4558
5538
|
const body = suppressed > 0
|
|
4559
|
-
? `${text}\n(
|
|
5539
|
+
? `${text}\n${t("fleet.error_suppressed", suppressed, Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000))}`
|
|
4560
5540
|
: text;
|
|
4561
5541
|
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4562
5542
|
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
@@ -4564,26 +5544,60 @@ export class FleetManager {
|
|
|
4564
5544
|
// topic_id to post into it.
|
|
4565
5545
|
const general = Object.entries(this.fleetConfig?.instances ?? {})
|
|
4566
5546
|
.find(([, config]) => config.general_topic === true)?.[0];
|
|
5547
|
+
// "dispatched", not "delivered": see notifyInstanceTopic. A platform that
|
|
5548
|
+
// rejects the message afterwards still consumes the throttle window, which
|
|
5549
|
+
// is the pre-existing semantic and not something this change alters.
|
|
5550
|
+
let dispatched = false;
|
|
4567
5551
|
if (general) {
|
|
4568
|
-
this.notifyInstanceTopic(general, body);
|
|
4569
|
-
return;
|
|
5552
|
+
dispatched = this.notifyInstanceTopic(general, body);
|
|
4570
5553
|
}
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
5554
|
+
else {
|
|
5555
|
+
// No General instance — fall back to the primary channel's group.
|
|
5556
|
+
const groupId = this.getChannelConfig()?.group_id;
|
|
5557
|
+
if (this.adapter && groupId) {
|
|
5558
|
+
this.adapter.sendText(String(groupId), body)
|
|
5559
|
+
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5560
|
+
dispatched = true;
|
|
5561
|
+
}
|
|
5562
|
+
}
|
|
5563
|
+
if (!dispatched) {
|
|
5564
|
+
// Do NOT record the throttle key. It used to be claimed before delivery
|
|
5565
|
+
// was attempted, so a message nobody could receive — every fleet error
|
|
5566
|
+
// raised before adapters exist, which is when startup faults happen —
|
|
5567
|
+
// also suppressed the next ten minutes of identical ones. Leaving the key
|
|
5568
|
+
// unspent lets the next caller, once an adapter is up, actually get
|
|
5569
|
+
// through. The counter is not reset either: nothing was shown.
|
|
5570
|
+
this.logger.warn({ text: body }, "Fleet error could not be delivered (no adapter or no target yet) — not consuming the throttle window");
|
|
4577
5571
|
return;
|
|
4578
5572
|
}
|
|
4579
|
-
this.
|
|
5573
|
+
this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
|
|
5574
|
+
// Bound the map: it is keyed by message text, and a message with a varying
|
|
5575
|
+
// suffix (a path, an id) would otherwise grow it without limit.
|
|
5576
|
+
if (this.fleetErrorNotices.size > 100) {
|
|
5577
|
+
const oldest = this.fleetErrorNotices.keys().next().value;
|
|
5578
|
+
if (oldest !== undefined)
|
|
5579
|
+
this.fleetErrorNotices.delete(oldest);
|
|
5580
|
+
}
|
|
4580
5581
|
}
|
|
4581
5582
|
static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
|
|
4582
5583
|
fleetErrorNotices = new Map();
|
|
5584
|
+
/**
|
|
5585
|
+
* Post into an instance's topic. Returns whether a send was DISPATCHED — a
|
|
5586
|
+
* target was resolved and sendText was called — so a caller that must not
|
|
5587
|
+
* lose the message (notifyFleetError, which spends a throttle key) can tell
|
|
5588
|
+
* that from silence. Existing callers ignore the result and are unaffected.
|
|
5589
|
+
*
|
|
5590
|
+
* Not a delivery guarantee: sendText is fire-and-forget, and a platform-side
|
|
5591
|
+
* rejection surfaces only as a warn in its .catch. True delivery confirmation
|
|
5592
|
+
* would have to make this async and change every caller.
|
|
5593
|
+
*/
|
|
4583
5594
|
notifyInstanceTopic(instanceName, text, extraOpts) {
|
|
4584
5595
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
4585
|
-
if (!adapter)
|
|
4586
|
-
|
|
5596
|
+
if (!adapter) {
|
|
5597
|
+
// Early startup: adapters are created well after the fleet object exists.
|
|
5598
|
+
this.logger.warn({ instanceName }, "No adapter yet — instance topic notification not sent");
|
|
5599
|
+
return false;
|
|
5600
|
+
}
|
|
4587
5601
|
const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
|
|
4588
5602
|
const groupId = channelCfg?.group_id;
|
|
4589
5603
|
// Fleet topic instance
|
|
@@ -4591,26 +5605,30 @@ export class FleetManager {
|
|
|
4591
5605
|
if (threadId != null && groupId) {
|
|
4592
5606
|
adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
|
|
4593
5607
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
|
|
4594
|
-
return;
|
|
5608
|
+
return true;
|
|
4595
5609
|
}
|
|
4596
5610
|
// Classic instance: find its channelId from the classic manager
|
|
4597
5611
|
const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
|
|
4598
5612
|
if (classicChatId) {
|
|
4599
5613
|
adapter.sendText(classicChatId, text, extraOpts)
|
|
4600
5614
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
4601
|
-
return;
|
|
5615
|
+
return true;
|
|
4602
5616
|
}
|
|
4603
5617
|
// Fallback: send to group without threadId
|
|
4604
5618
|
if (groupId) {
|
|
4605
5619
|
adapter.sendText(String(groupId), text, extraOpts)
|
|
4606
5620
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5621
|
+
return true;
|
|
4607
5622
|
}
|
|
5623
|
+
this.logger.warn({ instanceName }, "No group id — instance topic notification not sent");
|
|
5624
|
+
return false;
|
|
4608
5625
|
}
|
|
4609
5626
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
4610
5627
|
//
|
|
4611
5628
|
// One shared lifecycle for every "notification with decision buttons":
|
|
4612
5629
|
// post with a 128-bit nonce, arm a bounded expiry, bind the click to the
|
|
4613
|
-
// exact adapter+chat+thread+message that created it, require fleet admin
|
|
5630
|
+
// exact adapter+chat+thread+message that created it, require fleet admin for
|
|
5631
|
+
// actions that mutate runtime state (Tips are the harmless exception),
|
|
4614
5632
|
// consume exactly once. The features differ only in what they post
|
|
4615
5633
|
// and what a consumed click does.
|
|
4616
5634
|
/**
|
|
@@ -4621,8 +5639,14 @@ export class FleetManager {
|
|
|
4621
5639
|
*/
|
|
4622
5640
|
async postNonceButtonPrompt(opts) {
|
|
4623
5641
|
// 16 bytes = the 128-bit capability the design claims. Telegram's 64-byte
|
|
4624
|
-
// callback_data cap still holds
|
|
4625
|
-
// + 32 hex + ":confirm" (8) = 59
|
|
5642
|
+
// callback_data cap still holds, with two prefixes tied at the longest:
|
|
5643
|
+
// "interactive-assist:" (19) + 32 hex + ":confirm" (8) = 59, and
|
|
5644
|
+
// "install-select:" (15) + 32 hex + ":" + the longest backend name
|
|
5645
|
+
// ("claude-code"/"antigravity", 11) = 59. The longest overall is
|
|
5646
|
+
// "classic-approve:" (16) + 32 hex + ":allow-admin" (12) = 60, leaving 4
|
|
5647
|
+
// bytes. A longer prefix or action would be silently rejected by Telegram —
|
|
5648
|
+
// both sets are pinned by callback_data assertions in
|
|
5649
|
+
// install-backend-menu.test.ts and classic-approve-buttons.test.ts.
|
|
4626
5650
|
const nonce = randomBytes(16).toString("hex");
|
|
4627
5651
|
const entry = {
|
|
4628
5652
|
prefix: opts.prefix,
|
|
@@ -4652,6 +5676,17 @@ export class FleetManager {
|
|
|
4652
5676
|
message: opts.message,
|
|
4653
5677
|
choices: opts.choices.map(c => ({ id: `${opts.prefix}${nonce}:${c.action}`, label: c.label })),
|
|
4654
5678
|
}, opts.threadId ? { threadId: opts.threadId } : undefined);
|
|
5679
|
+
// Bind the nonce to the provider's canonical delivery address, not the
|
|
5680
|
+
// logical routing input. Telegram, for example, represents General as
|
|
5681
|
+
// AgEnD topic "1" on input but omits message_thread_id on the wire and in
|
|
5682
|
+
// callback queries. Exact callback matching below remains fail-closed.
|
|
5683
|
+
entry.chatId = sent.chatId;
|
|
5684
|
+
// Preserve compatibility with adapters that predate SentMessage.threadId
|
|
5685
|
+
// and omit the optional property altogether. An explicit undefined is a
|
|
5686
|
+
// canonical flat/root delivery context (notably Telegram General).
|
|
5687
|
+
if (Object.prototype.hasOwnProperty.call(sent, "threadId")) {
|
|
5688
|
+
entry.threadId = sent.threadId;
|
|
5689
|
+
}
|
|
4655
5690
|
entry.messageId = sent.messageId;
|
|
4656
5691
|
return nonce;
|
|
4657
5692
|
}
|
|
@@ -4664,19 +5699,18 @@ export class FleetManager {
|
|
|
4664
5699
|
}
|
|
4665
5700
|
}
|
|
4666
5701
|
/**
|
|
4667
|
-
*
|
|
5702
|
+
* Validate a nonce-armed callback and claim it exactly once.
|
|
4668
5703
|
*
|
|
4669
5704
|
* Returns:
|
|
4670
5705
|
* - null — the callback is not for this prefix; try the next handler
|
|
4671
5706
|
* - "consumed" — for this prefix but stale/denied/malformed; stop dispatch
|
|
4672
|
-
* - the entry+action — the click is authorized and
|
|
4673
|
-
* from the map before any await, so double clicks cannot act twice)
|
|
5707
|
+
* - the entry+action — the click is authorized and claimed before any await
|
|
4674
5708
|
*
|
|
4675
5709
|
* A stale click (expired nonce, or a pre-upgrade button whose payload no
|
|
4676
5710
|
* longer parses) collapses the clicked message so the dead button stops
|
|
4677
5711
|
* inviting clicks — the same courtesy the cancel button extends.
|
|
4678
5712
|
*/
|
|
4679
|
-
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter) {
|
|
5713
|
+
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter, staleHandling) {
|
|
4680
5714
|
if (!data.callbackData.startsWith(prefix))
|
|
4681
5715
|
return null;
|
|
4682
5716
|
const match = data.callbackData.match(actionRe);
|
|
@@ -4688,27 +5722,52 @@ export class FleetManager {
|
|
|
4688
5722
|
pending = undefined;
|
|
4689
5723
|
if (!match || !pending) {
|
|
4690
5724
|
const adapter = receivingAdapter ?? this.adapter;
|
|
4691
|
-
|
|
5725
|
+
if (staleHandling?.keepText && adapter?.removeMessageButtons) {
|
|
5726
|
+
adapter.removeMessageButtons(data.chatId, data.messageId, data.threadId)
|
|
5727
|
+
.catch(() => { });
|
|
5728
|
+
}
|
|
5729
|
+
else {
|
|
5730
|
+
adapter?.editMessageRemoveButtons?.(data.chatId, data.messageId, t("buttons.stale"), data.threadId).catch(() => { });
|
|
5731
|
+
}
|
|
5732
|
+
if (staleHandling?.notice) {
|
|
5733
|
+
adapter?.sendText(data.chatId, staleHandling.notice, { threadId: data.threadId })
|
|
5734
|
+
.catch(() => { });
|
|
5735
|
+
}
|
|
4692
5736
|
return "consumed";
|
|
4693
5737
|
}
|
|
4694
5738
|
// Bind the capability to the exact message/world that created it. Telegram
|
|
4695
|
-
// keyboards are visible to everyone, so
|
|
5739
|
+
// keyboards are visible to everyone, so mutating actions require fleet admin;
|
|
5740
|
+
// a Tip acknowledgement only records that the shared content was read.
|
|
4696
5741
|
const isAuthorized = data.userId
|
|
4697
|
-
? pending.
|
|
4698
|
-
?
|
|
4699
|
-
:
|
|
5742
|
+
? pending.allowAnyUser
|
|
5743
|
+
? true
|
|
5744
|
+
: pending.authChannelId
|
|
5745
|
+
? this.isModelAdmin(data.userId, pending.authChannelId, callbackAdapterId)
|
|
5746
|
+
: this.isFleetAdmin(data.userId, callbackAdapterId)
|
|
4700
5747
|
: false;
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
5748
|
+
const mismatchedFields = [];
|
|
5749
|
+
if (pending.adapterId !== callbackAdapterId)
|
|
5750
|
+
mismatchedFields.push("adapterId");
|
|
5751
|
+
if (data.chatId !== pending.chatId)
|
|
5752
|
+
mismatchedFields.push("chatId");
|
|
5753
|
+
if (pending.threadId != null && data.threadId !== pending.threadId)
|
|
5754
|
+
mismatchedFields.push("threadId");
|
|
5755
|
+
if (pending.messageId != null && data.messageId !== pending.messageId)
|
|
5756
|
+
mismatchedFields.push("messageId");
|
|
5757
|
+
if (!isAuthorized)
|
|
5758
|
+
mismatchedFields.push("authorization");
|
|
5759
|
+
if (mismatchedFields.length > 0) {
|
|
4706
5760
|
// Deliberately does NOT consume the nonce: the real admin can still click.
|
|
4707
|
-
this.logger.warn({
|
|
5761
|
+
this.logger.warn({
|
|
5762
|
+
instanceName: pending.instanceName,
|
|
5763
|
+
prefix,
|
|
5764
|
+
userId: data.userId,
|
|
5765
|
+
mismatchedFields: mismatchedFields.join(","),
|
|
5766
|
+
}, "Rejected unauthorized or mismatched button callback");
|
|
4708
5767
|
return "consumed";
|
|
4709
5768
|
}
|
|
4710
|
-
// Claim before any await: double clicks and duplicate callback delivery
|
|
4711
|
-
// never act twice.
|
|
5769
|
+
// Claim before any await: double clicks and duplicate callback delivery
|
|
5770
|
+
// can never act twice for state-changing actions.
|
|
4712
5771
|
this.pendingNonceButtons.delete(match[1]);
|
|
4713
5772
|
if (pending.timer)
|
|
4714
5773
|
clearTimeout(pending.timer);
|
|
@@ -4729,10 +5788,223 @@ export class FleetManager {
|
|
|
4729
5788
|
this.logger.warn({ err, instanceName: pending.instanceName }, "Failed to retire prompt buttons");
|
|
4730
5789
|
}
|
|
4731
5790
|
}
|
|
5791
|
+
tipsEnabled() {
|
|
5792
|
+
return this.fleetConfig?.defaults.tips !== false;
|
|
5793
|
+
}
|
|
5794
|
+
readTipState() {
|
|
5795
|
+
if (!this.scheduler)
|
|
5796
|
+
return null;
|
|
5797
|
+
try {
|
|
5798
|
+
return {
|
|
5799
|
+
dismissed: this.scheduler.db.listDismissedTipIds(),
|
|
5800
|
+
advancedUnlocked: this.scheduler.db.isAdvancedTipsUnlocked(),
|
|
5801
|
+
};
|
|
5802
|
+
}
|
|
5803
|
+
catch (err) {
|
|
5804
|
+
// Tips are additive. A damaged/locked scheduler DB must never block fleet
|
|
5805
|
+
// startup, update completion, or a General command.
|
|
5806
|
+
this.logger.warn({ err }, "Failed to read tip state");
|
|
5807
|
+
return null;
|
|
5808
|
+
}
|
|
5809
|
+
}
|
|
5810
|
+
pickAvailableTip() {
|
|
5811
|
+
const state = this.readTipState();
|
|
5812
|
+
return state
|
|
5813
|
+
? selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds())
|
|
5814
|
+
: null;
|
|
5815
|
+
}
|
|
5816
|
+
tipText(tip) {
|
|
5817
|
+
return getLocale() === "zh-TW" ? tip.text_zh : tip.text_en;
|
|
5818
|
+
}
|
|
5819
|
+
formatTip(tip) {
|
|
5820
|
+
return `💡 ${t("tips.label")}: ${this.tipText(tip)}`;
|
|
5821
|
+
}
|
|
5822
|
+
/** Persistently enable advanced tips through the admin-only slash command. */
|
|
5823
|
+
unlockAdvancedTips(userId) {
|
|
5824
|
+
if (!this.scheduler)
|
|
5825
|
+
return false;
|
|
5826
|
+
try {
|
|
5827
|
+
this.scheduler.db.unlockAdvancedTips(userId);
|
|
5828
|
+
this.eventLog?.insert(this.findGeneralInstance() ?? "general", "tips_advanced_unlocked", {
|
|
5829
|
+
userId,
|
|
5830
|
+
source: "command",
|
|
5831
|
+
});
|
|
5832
|
+
return true;
|
|
5833
|
+
}
|
|
5834
|
+
catch (err) {
|
|
5835
|
+
this.logger.warn({ err, userId }, "Failed to unlock advanced tips from command");
|
|
5836
|
+
return false;
|
|
5837
|
+
}
|
|
5838
|
+
}
|
|
5839
|
+
/** Post one fresh nonce-armed tip in a known General channel. */
|
|
5840
|
+
async promptTip(generalName, adapter, chatId, threadId) {
|
|
5841
|
+
const state = this.readTipState();
|
|
5842
|
+
if (!state)
|
|
5843
|
+
return "unavailable";
|
|
5844
|
+
if (!state.advancedUnlocked
|
|
5845
|
+
&& visibleTipLevels(true).has("advanced")
|
|
5846
|
+
&& canUnlockAdvancedTips(state.dismissed)) {
|
|
5847
|
+
return await this.promptAdvancedTipUnlock(generalName, adapter, chatId, threadId)
|
|
5848
|
+
? "posted"
|
|
5849
|
+
: "unavailable";
|
|
5850
|
+
}
|
|
5851
|
+
const tip = selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds());
|
|
5852
|
+
if (!tip)
|
|
5853
|
+
return this.scheduler ? "empty" : "unavailable";
|
|
5854
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5855
|
+
prefix: TIP_DISMISS_CALLBACK_PREFIX,
|
|
5856
|
+
alertType: "tip",
|
|
5857
|
+
instanceName: generalName,
|
|
5858
|
+
adapter,
|
|
5859
|
+
adapterId: adapter.id,
|
|
5860
|
+
chatId,
|
|
5861
|
+
threadId,
|
|
5862
|
+
message: this.formatTip(tip),
|
|
5863
|
+
choices: [
|
|
5864
|
+
{ action: "dismiss", label: t("tips.dismiss") },
|
|
5865
|
+
{ action: "confused", label: t("tips.confused") },
|
|
5866
|
+
],
|
|
5867
|
+
// Expiry removes only the stale button; the useful tip remains readable.
|
|
5868
|
+
expiredText: this.formatTip(tip),
|
|
5869
|
+
extra: { allowAnyUser: true, tipId: tip.id },
|
|
5870
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5871
|
+
});
|
|
5872
|
+
return nonce ? "posted" : "unavailable";
|
|
5873
|
+
}
|
|
5874
|
+
async promptAdvancedTipUnlock(generalName, adapter, chatId, threadId) {
|
|
5875
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5876
|
+
prefix: TIP_UNLOCK_CALLBACK_PREFIX,
|
|
5877
|
+
alertType: "tip",
|
|
5878
|
+
instanceName: generalName,
|
|
5879
|
+
adapter,
|
|
5880
|
+
adapterId: adapter.id,
|
|
5881
|
+
chatId,
|
|
5882
|
+
threadId,
|
|
5883
|
+
message: t("tips.advanced.unlock_prompt"),
|
|
5884
|
+
choices: [{ action: "unlock", label: t("tips.advanced.unlock") }],
|
|
5885
|
+
expiredText: t("tips.advanced.expired"),
|
|
5886
|
+
extra: { allowAnyUser: true },
|
|
5887
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5888
|
+
});
|
|
5889
|
+
return nonce !== null;
|
|
5890
|
+
}
|
|
5891
|
+
async sendTipToGeneral() {
|
|
5892
|
+
if (!this.tipsEnabled())
|
|
5893
|
+
return;
|
|
5894
|
+
const generalName = this.findGeneralInstance();
|
|
5895
|
+
if (!generalName)
|
|
5896
|
+
return;
|
|
5897
|
+
const adapter = this.getAdapterForInstance(generalName);
|
|
5898
|
+
const chatId = this.getGroupIdForInstance(generalName);
|
|
5899
|
+
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
5900
|
+
if (!adapter || !chatId)
|
|
5901
|
+
return;
|
|
5902
|
+
const result = await this.promptTip(generalName, adapter, chatId, topicId != null ? String(topicId) : undefined);
|
|
5903
|
+
if (result === "unavailable") {
|
|
5904
|
+
this.logger.warn({ generalName }, "Daily tip could not be posted");
|
|
5905
|
+
}
|
|
5906
|
+
}
|
|
5907
|
+
async handleTipDismiss(data, callbackAdapterId, receivingAdapter) {
|
|
5908
|
+
const claimed = this.consumeNonceCallback(TIP_DISMISS_CALLBACK_PREFIX, /^tip-dismiss:([0-9a-f]+):(dismiss|confused)$/, data, callbackAdapterId, receivingAdapter,
|
|
5909
|
+
// A restart orphans the expiry timer, so tip buttons can outlive their
|
|
5910
|
+
// nonce by days. Keep the tip text readable and always tell the clicker
|
|
5911
|
+
// what to do next (the keyboard edit itself can be refused: Telegram
|
|
5912
|
+
// rejects edits on messages older than 48h).
|
|
5913
|
+
{ keepText: true, notice: t("tips.expired_use_tips") });
|
|
5914
|
+
if (claimed === null)
|
|
5915
|
+
return false;
|
|
5916
|
+
if (claimed === "consumed")
|
|
5917
|
+
return true;
|
|
5918
|
+
const { entry: pending } = claimed;
|
|
5919
|
+
if (claimed.action === "confused") {
|
|
5920
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5921
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip feedback has incomplete persistence context");
|
|
5922
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5923
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId });
|
|
5924
|
+
return true;
|
|
5925
|
+
}
|
|
5926
|
+
try {
|
|
5927
|
+
this.scheduler.db.recordTipFeedback(data.userId, pending.tipId, "confused");
|
|
5928
|
+
}
|
|
5929
|
+
catch (err) {
|
|
5930
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip feedback");
|
|
5931
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5932
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId }).catch(() => { });
|
|
5933
|
+
return true;
|
|
5934
|
+
}
|
|
5935
|
+
this.logger.info({
|
|
5936
|
+
tipId: pending.tipId,
|
|
5937
|
+
userId: data.userId,
|
|
5938
|
+
feedbackType: "confused",
|
|
5939
|
+
}, "Tip feedback recorded");
|
|
5940
|
+
this.eventLog?.insert(pending.instanceName, "tip_feedback", {
|
|
5941
|
+
tipId: pending.tipId,
|
|
5942
|
+
userId: data.userId,
|
|
5943
|
+
feedbackType: "confused",
|
|
5944
|
+
});
|
|
5945
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5946
|
+
// Confusion is feedback, not a dismissal. Keep the Tip readable while
|
|
5947
|
+
// retiring this one-shot prompt; it remains eligible for future draws.
|
|
5948
|
+
pending.expiredText);
|
|
5949
|
+
await pending.adapter.sendText(pending.chatId, t("tips.feedback_recorded"), { threadId: pending.threadId }).catch(err => this.logger.warn({ err, tipId: pending.tipId }, "Failed to acknowledge tip feedback"));
|
|
5950
|
+
return true;
|
|
5951
|
+
}
|
|
5952
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5953
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip dismissal has incomplete persistence context");
|
|
5954
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5955
|
+
return true;
|
|
5956
|
+
}
|
|
5957
|
+
try {
|
|
5958
|
+
this.scheduler.db.dismissTip(data.userId, pending.tipId);
|
|
5959
|
+
}
|
|
5960
|
+
catch (err) {
|
|
5961
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip dismissal");
|
|
5962
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5963
|
+
return true;
|
|
5964
|
+
}
|
|
5965
|
+
this.eventLog?.insert(pending.instanceName, "tip_dismissed", {
|
|
5966
|
+
tipId: pending.tipId,
|
|
5967
|
+
userId: data.userId,
|
|
5968
|
+
});
|
|
5969
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5970
|
+
// Dismissal changes future selection only. Keep the useful tip text in
|
|
5971
|
+
// chat history and remove just the now-consumed button. Append a short
|
|
5972
|
+
// confirmation so the user sees feedback (unlike the pre-#605 version
|
|
5973
|
+
// that replaced the entire message with just the confirmation line).
|
|
5974
|
+
`${pending.expiredText}\n\n${t("tips.dismissed")}`);
|
|
5975
|
+
const state = this.readTipState();
|
|
5976
|
+
if (state && !state.advancedUnlocked && canUnlockAdvancedTips(state.dismissed)) {
|
|
5977
|
+
await this.promptAdvancedTipUnlock(pending.instanceName, pending.adapter, pending.chatId, pending.threadId);
|
|
5978
|
+
}
|
|
5979
|
+
return true;
|
|
5980
|
+
}
|
|
5981
|
+
async handleTipUnlock(data, callbackAdapterId, receivingAdapter) {
|
|
5982
|
+
const claimed = this.consumeNonceCallback(TIP_UNLOCK_CALLBACK_PREFIX, /^tip-unlock:([0-9a-f]+):(unlock)$/, data, callbackAdapterId, receivingAdapter, { keepText: true, notice: t("tips.expired_use_tips") });
|
|
5983
|
+
if (claimed === null)
|
|
5984
|
+
return false;
|
|
5985
|
+
if (claimed === "consumed")
|
|
5986
|
+
return true;
|
|
5987
|
+
const { entry: pending } = claimed;
|
|
5988
|
+
if (!data.userId || !this.scheduler) {
|
|
5989
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5990
|
+
return true;
|
|
5991
|
+
}
|
|
5992
|
+
try {
|
|
5993
|
+
this.scheduler.db.unlockAdvancedTips(data.userId);
|
|
5994
|
+
}
|
|
5995
|
+
catch (err) {
|
|
5996
|
+
this.logger.warn({ err, userId: data.userId }, "Failed to unlock advanced tips");
|
|
5997
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5998
|
+
return true;
|
|
5999
|
+
}
|
|
6000
|
+
this.eventLog?.insert(pending.instanceName, "tips_advanced_unlocked", { userId: data.userId });
|
|
6001
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.advanced.unlocked"));
|
|
6002
|
+
return true;
|
|
6003
|
+
}
|
|
4732
6004
|
/** Post the destructive `/clear` confirmation in the invoking channel. */
|
|
4733
6005
|
async promptClearConfirmation(instanceName, channelId, adapter, chatId, threadId) {
|
|
4734
6006
|
if (!this.topicCommands.supportsClear(instanceName))
|
|
4735
|
-
return
|
|
6007
|
+
return t("clear.unsupported");
|
|
4736
6008
|
const adapterId = adapter.id;
|
|
4737
6009
|
if (!adapterId)
|
|
4738
6010
|
return t("clear.prompt_unavailable");
|
|
@@ -5006,7 +6278,7 @@ export class FleetManager {
|
|
|
5006
6278
|
const force = target.kind === "classic" && !!data.options?.force;
|
|
5007
6279
|
const cmd = saveCommandForBackend(backend, filename, force);
|
|
5008
6280
|
if (!cmd) {
|
|
5009
|
-
await data.respond(
|
|
6281
|
+
await data.respond(t("save.unsupported"));
|
|
5010
6282
|
return;
|
|
5011
6283
|
}
|
|
5012
6284
|
if (target.kind === "classic") {
|
|
@@ -5059,7 +6331,7 @@ export class FleetManager {
|
|
|
5059
6331
|
}
|
|
5060
6332
|
return false;
|
|
5061
6333
|
}
|
|
5062
|
-
async sendCancelButton(instanceName, correlationId) {
|
|
6334
|
+
async sendCancelButton(instanceName, correlationId, preserveProgress = false) {
|
|
5063
6335
|
// Post first, retire after (see the tail of this method). Retiring up front
|
|
5064
6336
|
// meant that from the delete until the new message came back — a chat API
|
|
5065
6337
|
// round trip, and every reply goes through here — the instance had NO live
|
|
@@ -5120,6 +6392,10 @@ export class FleetManager {
|
|
|
5120
6392
|
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
5121
6393
|
// with nothing visibly changed.
|
|
5122
6394
|
lastProgressText: "👀 處理中…",
|
|
6395
|
+
// A reply can re-post the same in-flight bubble below the reply. Carry
|
|
6396
|
+
// its current list into that replacement; fresh inbound work must start
|
|
6397
|
+
// empty even if the daemon's reset broadcast is still in flight.
|
|
6398
|
+
toolProgress: preserveProgress ? this.instanceProgress.get(instanceName) : undefined,
|
|
5123
6399
|
};
|
|
5124
6400
|
this.startProgressTicker(entry);
|
|
5125
6401
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
@@ -5173,16 +6449,28 @@ export class FleetManager {
|
|
|
5173
6449
|
static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
|
|
5174
6450
|
if (elapsedMs < minElapsedMs)
|
|
5175
6451
|
return "👀 處理中…";
|
|
6452
|
+
const elapsed = FleetManager.formatProgressElapsed(elapsedMs);
|
|
6453
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
6454
|
+
return detail
|
|
6455
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
6456
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
6457
|
+
}
|
|
6458
|
+
/** Render elapsed time consistently in live and retained progress bubbles. */
|
|
6459
|
+
static formatProgressElapsed(elapsedMs) {
|
|
5176
6460
|
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
5177
6461
|
const minutes = Math.floor(totalSeconds / 60);
|
|
5178
6462
|
const seconds = totalSeconds % 60;
|
|
5179
|
-
|
|
6463
|
+
return minutes >= 60
|
|
5180
6464
|
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
5181
6465
|
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
6466
|
+
}
|
|
6467
|
+
/** Final read-only form of a bubble that contains semantic tool history.
|
|
6468
|
+
* Neutral wording is intentional: the same retirement primitive is used for
|
|
6469
|
+
* normal completion, a mid-turn bubble replacement, and user cancellation,
|
|
6470
|
+
* so claiming every retained checkpoint is "completed" would be false. */
|
|
6471
|
+
composeRetiredBubbleText(entry) {
|
|
6472
|
+
const elapsed = FleetManager.formatProgressElapsed(Date.now() - (entry.startedAt ?? Date.now()));
|
|
6473
|
+
return `🧾 工具歷程 (記錄至 ${elapsed})\n${entry.toolProgress}`;
|
|
5186
6474
|
}
|
|
5187
6475
|
/**
|
|
5188
6476
|
* Make a tool summary safe to paste into a channel message.
|
|
@@ -5227,12 +6515,20 @@ export class FleetManager {
|
|
|
5227
6515
|
else
|
|
5228
6516
|
this.instanceProgress.delete(name);
|
|
5229
6517
|
for (const entry of this.cancelButtons.values()) {
|
|
5230
|
-
if (entry.instanceName
|
|
5231
|
-
|
|
6518
|
+
if (entry.instanceName !== name)
|
|
6519
|
+
continue;
|
|
6520
|
+
// Empty means the daemon completed/reset the turn. Clear the live cache
|
|
6521
|
+
// for the next turn, but retain this bubble's last non-empty list so its
|
|
6522
|
+
// retirement can preserve the history instead of deleting the message.
|
|
6523
|
+
if (progress)
|
|
6524
|
+
entry.toolProgress = progress;
|
|
6525
|
+
this.scheduleProgressEdit(entry);
|
|
5232
6526
|
}
|
|
5233
6527
|
}
|
|
5234
6528
|
/** At most one progress-driven edit per bubble per TOOL_PROGRESS_EDIT_MIN_MS. */
|
|
5235
6529
|
scheduleProgressEdit(entry) {
|
|
6530
|
+
if (entry.retiring)
|
|
6531
|
+
return;
|
|
5236
6532
|
const since = Date.now() - (entry.lastProgressEditAt ?? 0);
|
|
5237
6533
|
if (since >= TOOL_PROGRESS_EDIT_MIN_MS) {
|
|
5238
6534
|
this.refreshBubble(entry);
|
|
@@ -5253,7 +6549,7 @@ export class FleetManager {
|
|
|
5253
6549
|
* wiped by the next elapsed tick (#528 trap 2).
|
|
5254
6550
|
*/
|
|
5255
6551
|
composeBubbleText(entry) {
|
|
5256
|
-
return FleetManager.bubbleText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs(),
|
|
6552
|
+
return FleetManager.bubbleText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs(), entry.toolProgress);
|
|
5257
6553
|
}
|
|
5258
6554
|
/** Pure composition of header + tool list, exposed for tests. */
|
|
5259
6555
|
static bubbleText(elapsedMs, activity, minElapsedMs, progress) {
|
|
@@ -5264,7 +6560,7 @@ export class FleetManager {
|
|
|
5264
6560
|
}
|
|
5265
6561
|
/** Recompose and edit the bubble in place; skips when nothing changed. */
|
|
5266
6562
|
refreshBubble(entry) {
|
|
5267
|
-
if (!this.cancelButtons.has(entry.messageId)) {
|
|
6563
|
+
if (entry.retiring || !this.cancelButtons.has(entry.messageId)) {
|
|
5268
6564
|
clearInterval(entry.progressTimer);
|
|
5269
6565
|
return;
|
|
5270
6566
|
}
|
|
@@ -5320,8 +6616,11 @@ export class FleetManager {
|
|
|
5320
6616
|
}
|
|
5321
6617
|
/**
|
|
5322
6618
|
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
5323
|
-
* it has not, retire its button.
|
|
5324
|
-
*
|
|
6619
|
+
* it has not, retire its button. The daemon is asked for a fresh pane capture
|
|
6620
|
+
* before deciding. Reading only the transition cache stranded the first
|
|
6621
|
+
* post-restart bubble when its startup "working" report never got a matching
|
|
6622
|
+
* idle edge. Re-arming replaces the previous timer, so a burst of replies ends
|
|
6623
|
+
* with exactly one pending check.
|
|
5325
6624
|
*/
|
|
5326
6625
|
armReplyGrace(instanceName) {
|
|
5327
6626
|
for (const entry of this.cancelButtons.values()) {
|
|
@@ -5333,14 +6632,26 @@ export class FleetManager {
|
|
|
5333
6632
|
entry.replyGraceTimer = undefined;
|
|
5334
6633
|
if (!this.cancelButtons.has(entry.messageId))
|
|
5335
6634
|
return;
|
|
5336
|
-
|
|
5337
|
-
return; // resumed — a long run keeps its button
|
|
5338
|
-
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
5339
|
-
this.retireButton(entry);
|
|
6635
|
+
void this.finishReplyGrace(instanceName, entry);
|
|
5340
6636
|
}, REPLY_RETIRE_GRACE_MS);
|
|
5341
6637
|
entry.replyGraceTimer.unref?.();
|
|
5342
6638
|
}
|
|
5343
6639
|
}
|
|
6640
|
+
async finishReplyGrace(instanceName, entry) {
|
|
6641
|
+
const refreshed = await this.refreshInstanceExecutionState(instanceName, REPLY_STATE_REFRESH_TIMEOUT_MS);
|
|
6642
|
+
if (!this.cancelButtons.has(entry.messageId) || entry.retiring)
|
|
6643
|
+
return;
|
|
6644
|
+
if (!refreshed) {
|
|
6645
|
+
// Fail safe: without an authoritative answer, retain a potentially live
|
|
6646
|
+
// Cancel button. The 5-minute/30-minute/24-hour safety nets still apply.
|
|
6647
|
+
this.logger.debug({ instanceName, messageId: entry.messageId }, "Cancel reply-grace state refresh timed out");
|
|
6648
|
+
return;
|
|
6649
|
+
}
|
|
6650
|
+
if (!this.getInstanceIdle(instanceName))
|
|
6651
|
+
return; // genuine long run keeps its button
|
|
6652
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
6653
|
+
this.retireButton(entry);
|
|
6654
|
+
}
|
|
5344
6655
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
5345
6656
|
retireInstanceButtons(instanceName) {
|
|
5346
6657
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -5362,7 +6673,7 @@ export class FleetManager {
|
|
|
5362
6673
|
this.deleteButtonMessage(entry)
|
|
5363
6674
|
.then(() => {
|
|
5364
6675
|
this.discardButton(entry);
|
|
5365
|
-
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId }, "Cancel button
|
|
6676
|
+
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId, historyPreserved: Boolean(entry.toolProgress) }, "Cancel button retired");
|
|
5366
6677
|
})
|
|
5367
6678
|
.catch((err) => this.scheduleButtonRetry(entry, err));
|
|
5368
6679
|
}
|
|
@@ -5453,12 +6764,21 @@ export class FleetManager {
|
|
|
5453
6764
|
// Continue the same retire cycle (bypass the retiring-guard in retireButton).
|
|
5454
6765
|
entry.retryTimer = setTimeout(() => this.attemptButtonDelete(entry), CANCEL_BTN_RETRY_INTERVAL_MS);
|
|
5455
6766
|
}
|
|
5456
|
-
/**
|
|
6767
|
+
/** Retire one cancel bubble via its own adapter. A bubble that accumulated
|
|
6768
|
+
* tool progress is kept as a read-only history message; a plain bubble keeps
|
|
6769
|
+
* the legacy delete/edit-to-checkmark behavior. Resolves on success and
|
|
5457
6770
|
* rejects on failure so the caller can retry. */
|
|
5458
6771
|
deleteButtonMessage(e) {
|
|
5459
6772
|
const adapter = (e.adapterId ? this.worlds.get(e.adapterId)?.adapter : undefined) ?? this.adapter;
|
|
5460
6773
|
if (!adapter)
|
|
5461
6774
|
return Promise.reject(new Error("no adapter for cancel button"));
|
|
6775
|
+
if (e.toolProgress && adapter.editMessageRemoveButtons) {
|
|
6776
|
+
return adapter.editMessageRemoveButtons(e.chatId, e.messageId, this.composeRetiredBubbleText(e), e.threadId);
|
|
6777
|
+
}
|
|
6778
|
+
// All production adapters currently implement editMessageRemoveButtons.
|
|
6779
|
+
// A third-party adapter without it degrades to the legacy delete behavior:
|
|
6780
|
+
// removing a live Cancel control is safer than retaining an actionable,
|
|
6781
|
+
// untracked button forever.
|
|
5462
6782
|
if (adapter.deleteMessage)
|
|
5463
6783
|
return adapter.deleteMessage(e.chatId, e.messageId, e.threadId);
|
|
5464
6784
|
if (adapter.editMessageRemoveButtons)
|
|
@@ -5510,11 +6830,604 @@ export class FleetManager {
|
|
|
5510
6830
|
const daemon = this.daemons.get(instanceName);
|
|
5511
6831
|
if (!daemon)
|
|
5512
6832
|
return false;
|
|
6833
|
+
const deliveryEpoch = this.cancelPendingDeliveries(instanceName);
|
|
6834
|
+
daemon.clearPendingDeliveries?.(deliveryEpoch);
|
|
5513
6835
|
daemon.sendEscape().catch(e => this.logger.warn({ err: e, instanceName }, "sendEscape failed"));
|
|
5514
6836
|
this.lastInboundMsg.delete(instanceName);
|
|
5515
6837
|
this.clearCancelButton(instanceName);
|
|
5516
6838
|
return true;
|
|
5517
6839
|
}
|
|
6840
|
+
getDeliveryEpoch(instanceName) {
|
|
6841
|
+
return this.deliveryEpochs.get(instanceName) ?? 0;
|
|
6842
|
+
}
|
|
6843
|
+
isDeliveryEpochCurrent(instanceName, deliveryEpoch) {
|
|
6844
|
+
return deliveryEpoch === this.getDeliveryEpoch(instanceName);
|
|
6845
|
+
}
|
|
6846
|
+
/** Invalidate work queued before a user cancel and wake idle-gate waiters. */
|
|
6847
|
+
cancelPendingDeliveries(instanceName) {
|
|
6848
|
+
const next = this.getDeliveryEpoch(instanceName) + 1;
|
|
6849
|
+
this.deliveryEpochs.set(instanceName, next);
|
|
6850
|
+
for (const check of this.instanceIdleWaiters.get(instanceName) ?? [])
|
|
6851
|
+
check();
|
|
6852
|
+
return next;
|
|
6853
|
+
}
|
|
6854
|
+
// ── Remote CLI login (`/login`) ──────────────────────────────────────────
|
|
6855
|
+
//
|
|
6856
|
+
// One session at a time, in a dedicated tmux window — never an instance
|
|
6857
|
+
// pane. Credentials are per-backend (codex homes symlink auth.json to the
|
|
6858
|
+
// shared ~/.codex; the other CLIs use one real home), so a single sign-in
|
|
6859
|
+
// repairs every instance of that backend, and instance delivery, pane-state
|
|
6860
|
+
// detection, tool progress, and mcp_proxy_reply never observe login output.
|
|
6861
|
+
activeLogin = null;
|
|
6862
|
+
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
6863
|
+
async promptLoginBackends(chat) {
|
|
6864
|
+
const configured = new Set();
|
|
6865
|
+
for (const [, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
6866
|
+
configured.add(config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
6867
|
+
}
|
|
6868
|
+
const choices = Object.keys(LOGIN_FLOWS)
|
|
6869
|
+
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
6870
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6871
|
+
if (choices.length === 0) {
|
|
6872
|
+
await chat.adapter.sendText(chat.chatId, t("login.unsupported", [...configured].join(", ")), { threadId: chat.threadId });
|
|
6873
|
+
return;
|
|
6874
|
+
}
|
|
6875
|
+
await this.postNonceButtonPrompt({
|
|
6876
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6877
|
+
alertType: "login",
|
|
6878
|
+
instanceName: "login",
|
|
6879
|
+
adapter: chat.adapter,
|
|
6880
|
+
adapterId: chat.adapterId,
|
|
6881
|
+
chatId: chat.chatId,
|
|
6882
|
+
threadId: chat.threadId,
|
|
6883
|
+
message: t("login.choose_backend"),
|
|
6884
|
+
choices,
|
|
6885
|
+
expiredText: t("buttons.stale"),
|
|
6886
|
+
});
|
|
6887
|
+
}
|
|
6888
|
+
/**
|
|
6889
|
+
* Ask the General topic to approve a ClassicBot access request.
|
|
6890
|
+
*
|
|
6891
|
+
* Two affirmative buttons rather than one, when the trigger identified a
|
|
6892
|
+
* user. Allowing a group to talk and granting that person ClassicBot admin
|
|
6893
|
+
* (start/stop/model on classic channels — NOT fleet admin, which reads
|
|
6894
|
+
* fleet.yaml `channel.access.allowed_users`) have different blast radii, and
|
|
6895
|
+
* bundling them would force anyone who wants only the cheap one to accept the
|
|
6896
|
+
* expensive one or go edit YAML by hand.
|
|
6897
|
+
*
|
|
6898
|
+
* Built on postNonceButtonPrompt, so it inherits the canonical-address
|
|
6899
|
+
* binding (#682) that makes these buttons answer in a Telegram General topic.
|
|
6900
|
+
*/
|
|
6901
|
+
async promptClassicApproval(opts) {
|
|
6902
|
+
const adapter = this.getAdapterForInstance(opts.generalName);
|
|
6903
|
+
// An instance with no world binding yet (fresh restart, or a fleet whose
|
|
6904
|
+
// primary `channel:` block carries the config) still has a usable adapter —
|
|
6905
|
+
// fall back to that adapter's own id rather than dropping the buttons. The
|
|
6906
|
+
// id only has to match what the callback arrives with, which is this same
|
|
6907
|
+
// adapter.
|
|
6908
|
+
const adapterId = this.getInstanceAdapterId(opts.generalName) ?? adapter?.id;
|
|
6909
|
+
const chatId = this.getGroupIdForInstance(opts.generalName);
|
|
6910
|
+
const topicId = this.fleetConfig?.instances[opts.generalName]?.topic_id;
|
|
6911
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6912
|
+
// Never lose the notification because the buttons could not be addressed.
|
|
6913
|
+
this.notifyInstanceTopic(opts.generalName, opts.message);
|
|
6914
|
+
return;
|
|
6915
|
+
}
|
|
6916
|
+
const choices = [{ action: "allow", label: t("classic.approve_group") }];
|
|
6917
|
+
if (opts.userId)
|
|
6918
|
+
choices.push({ action: "allow-admin", label: t("classic.approve_group_admin") });
|
|
6919
|
+
choices.push({ action: "ignore", label: t("classic.approve_ignore") });
|
|
6920
|
+
await this.postNonceButtonPrompt({
|
|
6921
|
+
prefix: CLASSIC_APPROVE_CALLBACK_PREFIX,
|
|
6922
|
+
alertType: "classic_approve",
|
|
6923
|
+
instanceName: opts.generalName,
|
|
6924
|
+
adapter,
|
|
6925
|
+
adapterId,
|
|
6926
|
+
chatId,
|
|
6927
|
+
threadId: topicId != null ? String(topicId) : undefined,
|
|
6928
|
+
message: opts.message,
|
|
6929
|
+
choices,
|
|
6930
|
+
expiredText: t("buttons.stale"),
|
|
6931
|
+
extra: { classicGroupId: opts.groupId, classicUserId: opts.userId, classicScope: opts.scope },
|
|
6932
|
+
});
|
|
6933
|
+
}
|
|
6934
|
+
/**
|
|
6935
|
+
* Apply an approved ClassicBot access request.
|
|
6936
|
+
*
|
|
6937
|
+
* Writes through ClassicChannelManager's typed mutators rather than handing
|
|
6938
|
+
* General a YAML edit: the id is stored via String() so a Discord snowflake
|
|
6939
|
+
* cannot land as a YAML integer and lose precision, the existing save() path
|
|
6940
|
+
* is reused, in-memory state is correct immediately (no waiting on the 30s
|
|
6941
|
+
* reload poll), and a click cannot silently do nothing because General
|
|
6942
|
+
* happened to be paused. General is still told the outcome.
|
|
6943
|
+
*/
|
|
6944
|
+
async applyClassicApproval(entry, groupId, adminUserId) {
|
|
6945
|
+
const classic = this.classicChannels;
|
|
6946
|
+
if (!classic) {
|
|
6947
|
+
this.notifyInstanceTopic(entry.instanceName, t("classic.approve_failed", groupId));
|
|
6948
|
+
return;
|
|
6949
|
+
}
|
|
6950
|
+
const scope = entry.classicScope ?? "guild";
|
|
6951
|
+
const access = scope === "group" ? classic.allowGroup(groupId) : classic.allowGuild(groupId);
|
|
6952
|
+
const admin = adminUserId ? classic.addAdminUser(adminUserId) : null;
|
|
6953
|
+
const lines = [t(access === "added" ? "classic.approve_done_group"
|
|
6954
|
+
: access === "already" ? "classic.approve_done_group_already"
|
|
6955
|
+
: "classic.approve_done_group_open", groupId)];
|
|
6956
|
+
if (admin) {
|
|
6957
|
+
lines.push(t(admin === "added" ? "classic.approve_done_admin" : "classic.approve_done_admin_already", adminUserId ?? ""));
|
|
6958
|
+
}
|
|
6959
|
+
this.notifyInstanceTopic(entry.instanceName, lines.join("\n"));
|
|
6960
|
+
}
|
|
6961
|
+
/** Consume a ClassicBot approval button. */
|
|
6962
|
+
async handleClassicApproval(data, callbackAdapterId, receivingAdapter) {
|
|
6963
|
+
const claimed = this.consumeNonceCallback(CLASSIC_APPROVE_CALLBACK_PREFIX, /^classic-approve:([0-9a-f]+):(allow|allow-admin|ignore)$/, data, callbackAdapterId, receivingAdapter);
|
|
6964
|
+
if (claimed === null)
|
|
6965
|
+
return false;
|
|
6966
|
+
if (claimed === "consumed")
|
|
6967
|
+
return true;
|
|
6968
|
+
const { entry, action } = claimed;
|
|
6969
|
+
const groupId = entry.classicGroupId ?? "";
|
|
6970
|
+
const userId = entry.classicUserId;
|
|
6971
|
+
if (action === "ignore") {
|
|
6972
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("classic.approve_ignored", groupId));
|
|
6973
|
+
return true;
|
|
6974
|
+
}
|
|
6975
|
+
const grantAdmin = action === "allow-admin" && !!userId;
|
|
6976
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, grantAdmin ? t("classic.approve_applying_admin", groupId, userId ?? "")
|
|
6977
|
+
: t("classic.approve_applying", groupId));
|
|
6978
|
+
await this.applyClassicApproval(entry, groupId, grantAdmin ? userId : undefined);
|
|
6979
|
+
return true;
|
|
6980
|
+
}
|
|
6981
|
+
/**
|
|
6982
|
+
* Backend chooser for a bare `/install-cli`, mirroring promptLoginBackends so
|
|
6983
|
+
* both commands feel the same. Built on postNonceButtonPrompt rather than the
|
|
6984
|
+
* `/model` selection coordinator: that is the mechanism `/login` already uses,
|
|
6985
|
+
* and the one whose canonical-address binding (#682) makes the buttons answer
|
|
6986
|
+
* in a Telegram General topic.
|
|
6987
|
+
*
|
|
6988
|
+
* Unlike the login chooser this does NOT filter to backends the fleet already
|
|
6989
|
+
* runs. Installing is how you get a backend you do not have yet, so filtering
|
|
6990
|
+
* by configured backends would hide the only entry the admin came for.
|
|
6991
|
+
*
|
|
6992
|
+
* gemini-cli is omitted: it is deprecated (see backend/factory.ts). Typing
|
|
6993
|
+
* `/install-cli gemini-cli` still works — this only stops recommending it.
|
|
6994
|
+
*/
|
|
6995
|
+
async promptInstallBackends(chat) {
|
|
6996
|
+
const choices = Object.keys(BACKEND_INSTALLATION_INFO)
|
|
6997
|
+
.filter(backend => backend !== "gemini-cli")
|
|
6998
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6999
|
+
await this.postNonceButtonPrompt({
|
|
7000
|
+
prefix: INSTALL_CALLBACK_PREFIX,
|
|
7001
|
+
alertType: "install",
|
|
7002
|
+
instanceName: "install",
|
|
7003
|
+
adapter: chat.adapter,
|
|
7004
|
+
adapterId: chat.adapterId,
|
|
7005
|
+
chatId: chat.chatId,
|
|
7006
|
+
threadId: chat.threadId,
|
|
7007
|
+
message: t("install.choose_backend"),
|
|
7008
|
+
choices,
|
|
7009
|
+
expiredText: t("buttons.stale"),
|
|
7010
|
+
});
|
|
7011
|
+
}
|
|
7012
|
+
/** Backend chooser button → start that backend's install session. */
|
|
7013
|
+
async handleInstallBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7014
|
+
const claimed = this.consumeNonceCallback(INSTALL_CALLBACK_PREFIX, /^install-select:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
7015
|
+
if (claimed === null)
|
|
7016
|
+
return false;
|
|
7017
|
+
if (claimed === "consumed")
|
|
7018
|
+
return true;
|
|
7019
|
+
const { entry, action: backend } = claimed;
|
|
7020
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.starting_backend", backend));
|
|
7021
|
+
const text = await this.startInstallSession(backend, {
|
|
7022
|
+
adapter: entry.adapter,
|
|
7023
|
+
adapterId: entry.adapterId,
|
|
7024
|
+
chatId: entry.chatId,
|
|
7025
|
+
threadId: entry.threadId,
|
|
7026
|
+
});
|
|
7027
|
+
if (text)
|
|
7028
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7029
|
+
return true;
|
|
7030
|
+
}
|
|
7031
|
+
/**
|
|
7032
|
+
* Start a login session for one backend. Caller enforces admin.
|
|
7033
|
+
* Returns a status line to post, or null when a confirmation prompt was
|
|
7034
|
+
* posted instead (auth still valid — see the pre-check below).
|
|
7035
|
+
*/
|
|
7036
|
+
async startLoginSession(backendArg, chat, opts = {}) {
|
|
7037
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7038
|
+
const flow = LOGIN_FLOWS[backend];
|
|
7039
|
+
if (!flow)
|
|
7040
|
+
return t("login.unsupported", backendArg);
|
|
7041
|
+
if (this.activeLogin)
|
|
7042
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7043
|
+
// Install sessions share the window namespace — never run both at once.
|
|
7044
|
+
if (this.activeInstall)
|
|
7045
|
+
return t("install.busy");
|
|
7046
|
+
// Token-free pre-check (5s cap): re-login while auth still works is
|
|
7047
|
+
// usually a mistake, so it needs a confirmed click. An invalid OR
|
|
7048
|
+
// uncertain result (timeout, missing binary) proceeds straight to login —
|
|
7049
|
+
// an unreliable probe must never block the re-login the admin asked for.
|
|
7050
|
+
if (!opts.skipAuthCheck && flow.authCheck) {
|
|
7051
|
+
const status = await checkAuthStatus(flow.authCheck);
|
|
7052
|
+
if (status === "valid") {
|
|
7053
|
+
await this.postNonceButtonPrompt({
|
|
7054
|
+
prefix: LOGIN_CONFIRM_CALLBACK_PREFIX,
|
|
7055
|
+
alertType: "login",
|
|
7056
|
+
instanceName: backend,
|
|
7057
|
+
adapter: chat.adapter,
|
|
7058
|
+
adapterId: chat.adapterId,
|
|
7059
|
+
chatId: chat.chatId,
|
|
7060
|
+
threadId: chat.threadId,
|
|
7061
|
+
message: t("login.still_valid", backend),
|
|
7062
|
+
choices: [
|
|
7063
|
+
{ action: "go", label: t("login.relogin_go") },
|
|
7064
|
+
{ action: "cancel", label: t("login.relogin_cancel") },
|
|
7065
|
+
],
|
|
7066
|
+
expiredText: t("buttons.stale"),
|
|
7067
|
+
});
|
|
7068
|
+
return null;
|
|
7069
|
+
}
|
|
7070
|
+
}
|
|
7071
|
+
return this.launchLoginSession(flow, backend, chat);
|
|
7072
|
+
}
|
|
7073
|
+
/** Create the login window and session (pre-check already settled). */
|
|
7074
|
+
async launchLoginSession(flow, backend, chat) {
|
|
7075
|
+
if (this.activeLogin)
|
|
7076
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7077
|
+
// Install sessions share the window namespace — never run both at once.
|
|
7078
|
+
if (this.activeInstall)
|
|
7079
|
+
return t("install.busy");
|
|
7080
|
+
const sessionName = getTmuxSession();
|
|
7081
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7082
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
7083
|
+
const session = new LoginSession(flow, tmux, {
|
|
7084
|
+
onMenu: async (options) => {
|
|
7085
|
+
await this.postNonceButtonPrompt({
|
|
7086
|
+
prefix: LOGIN_MENU_CALLBACK_PREFIX,
|
|
7087
|
+
alertType: "login",
|
|
7088
|
+
instanceName: backend,
|
|
7089
|
+
adapter: chat.adapter,
|
|
7090
|
+
adapterId: chat.adapterId,
|
|
7091
|
+
chatId: chat.chatId,
|
|
7092
|
+
threadId: chat.threadId,
|
|
7093
|
+
message: t("login.choose_provider"),
|
|
7094
|
+
choices: options.map((label, index) => ({ action: String(index), label })),
|
|
7095
|
+
expiredText: t("buttons.stale"),
|
|
7096
|
+
});
|
|
7097
|
+
},
|
|
7098
|
+
onAuthHint: async (url, code) => {
|
|
7099
|
+
await this.sendLoginSecret(chat, backend, url, code);
|
|
7100
|
+
},
|
|
7101
|
+
onNeedInput: async (promptExcerpt) => {
|
|
7102
|
+
await chat.adapter.sendText(chat.chatId, t("login.need_input", backend, promptExcerpt), { threadId: chat.threadId }).catch(() => { });
|
|
7103
|
+
},
|
|
7104
|
+
onDone: async ({ ok, detail }) => {
|
|
7105
|
+
this.activeLogin = null;
|
|
7106
|
+
let text;
|
|
7107
|
+
if (ok) {
|
|
7108
|
+
const { woken, restarted } = await this.recoverBackendInstances(backend);
|
|
7109
|
+
const none = t("login.none");
|
|
7110
|
+
text = t("login.success", backend, woken.length ? woken.join(", ") : none, restarted.length ? restarted.join(", ") : none);
|
|
7111
|
+
}
|
|
7112
|
+
else if (detail === "cancelled") {
|
|
7113
|
+
// The cancel command's own reply already announced this — a second
|
|
7114
|
+
// message here was a duplicate.
|
|
7115
|
+
return;
|
|
7116
|
+
}
|
|
7117
|
+
else {
|
|
7118
|
+
text = t("login.failed", backend, detail);
|
|
7119
|
+
}
|
|
7120
|
+
await chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
7121
|
+
},
|
|
7122
|
+
}, this.logger);
|
|
7123
|
+
// Claim the slot before the first await so two admins racing /login cannot
|
|
7124
|
+
// both create windows; release on startup failure.
|
|
7125
|
+
this.activeLogin = { session, backend, chat };
|
|
7126
|
+
try {
|
|
7127
|
+
await session.start();
|
|
7128
|
+
}
|
|
7129
|
+
catch (err) {
|
|
7130
|
+
this.activeLogin = null;
|
|
7131
|
+
return t("login.failed", backend, err.message);
|
|
7132
|
+
}
|
|
7133
|
+
return t("login.started", backend);
|
|
7134
|
+
}
|
|
7135
|
+
/** `/login code <text>` — paste admin-supplied text into the login window. */
|
|
7136
|
+
async loginSubmitInput(text) {
|
|
7137
|
+
if (!this.activeLogin)
|
|
7138
|
+
return t("login.no_session");
|
|
7139
|
+
const ok = await this.activeLogin.session.submitInput(text);
|
|
7140
|
+
return ok ? t("login.input_sent") : t("login.input_failed");
|
|
7141
|
+
}
|
|
7142
|
+
/** `/login cancel` — abort the active session and remove its window. */
|
|
7143
|
+
async cancelLoginSession() {
|
|
7144
|
+
if (!this.activeLogin)
|
|
7145
|
+
return t("login.no_session");
|
|
7146
|
+
const backend = this.activeLogin.backend;
|
|
7147
|
+
await this.activeLogin.session.cancel();
|
|
7148
|
+
return t("login.cancelled", backend);
|
|
7149
|
+
}
|
|
7150
|
+
/**
|
|
7151
|
+
* The URL (+ code) is a live credential: whoever completes it binds THEIR
|
|
7152
|
+
* account to this fleet's CLI. Telegram gets an HTML spoiler (same treatment
|
|
7153
|
+
* as the dashboard token); other adapters get plain text with the warning.
|
|
7154
|
+
*/
|
|
7155
|
+
async sendLoginSecret(chat, backend, url, code) {
|
|
7156
|
+
const codeLine = code ? `\n${t("login.auth_code", code)}` : "";
|
|
7157
|
+
try {
|
|
7158
|
+
if (chat.adapter.type === "telegram") {
|
|
7159
|
+
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
7160
|
+
await chat.adapter.sendText(chat.chatId, `${esc(t("login.auth_hint", backend))}\n<tg-spoiler>${esc(url)}${esc(codeLine)}</tg-spoiler>`, { threadId: chat.threadId, format: "html" });
|
|
7161
|
+
}
|
|
7162
|
+
else {
|
|
7163
|
+
await chat.adapter.sendText(chat.chatId, `${t("login.auth_hint", backend)}\n${url}${codeLine}`, { threadId: chat.threadId });
|
|
7164
|
+
}
|
|
7165
|
+
}
|
|
7166
|
+
catch (err) {
|
|
7167
|
+
this.logger.warn({ err: err.message, backend }, "Failed to deliver login URL");
|
|
7168
|
+
}
|
|
7169
|
+
}
|
|
7170
|
+
/**
|
|
7171
|
+
* After a successful login: wake the paused instances of that backend AND
|
|
7172
|
+
* restart the running ones — a live CLI holds the old token in memory and
|
|
7173
|
+
* only re-reads credentials on process start (a paused instance's CLI is
|
|
7174
|
+
* already dead, so waking it respawns with the new token for free).
|
|
7175
|
+
*/
|
|
7176
|
+
async recoverBackendInstances(backend) {
|
|
7177
|
+
const woken = [];
|
|
7178
|
+
const restarted = [];
|
|
7179
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
7180
|
+
const effective = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
7181
|
+
if (effective !== backend)
|
|
7182
|
+
continue;
|
|
7183
|
+
const status = this.getInstanceStatus(name);
|
|
7184
|
+
try {
|
|
7185
|
+
if (status === "paused") {
|
|
7186
|
+
if (this.daemons.has(name))
|
|
7187
|
+
await this.lifecycle.wake(name, 30_000);
|
|
7188
|
+
else
|
|
7189
|
+
await this.startPersistedPausedInstance(name);
|
|
7190
|
+
woken.push(name);
|
|
7191
|
+
}
|
|
7192
|
+
else if (status === "running") {
|
|
7193
|
+
await this.restartSingleInstance(name);
|
|
7194
|
+
restarted.push(name);
|
|
7195
|
+
}
|
|
7196
|
+
}
|
|
7197
|
+
catch (err) {
|
|
7198
|
+
this.logger.warn({ err: err.message, name, status }, "Post-login recovery failed");
|
|
7199
|
+
}
|
|
7200
|
+
}
|
|
7201
|
+
return { woken, restarted };
|
|
7202
|
+
}
|
|
7203
|
+
/** Backend chooser button → start that backend's login session. */
|
|
7204
|
+
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7205
|
+
const claimed = this.consumeNonceCallback(LOGIN_CALLBACK_PREFIX, /^login:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
7206
|
+
if (claimed === null)
|
|
7207
|
+
return false;
|
|
7208
|
+
if (claimed === "consumed")
|
|
7209
|
+
return true;
|
|
7210
|
+
const { entry, action: backend } = claimed;
|
|
7211
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7212
|
+
const text = await this.startLoginSession(backend, {
|
|
7213
|
+
adapter: entry.adapter,
|
|
7214
|
+
adapterId: entry.adapterId,
|
|
7215
|
+
chatId: entry.chatId,
|
|
7216
|
+
threadId: entry.threadId,
|
|
7217
|
+
});
|
|
7218
|
+
if (text)
|
|
7219
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7220
|
+
return true;
|
|
7221
|
+
}
|
|
7222
|
+
/** Re-login confirmation (auth pre-check said credentials still work). */
|
|
7223
|
+
async handleLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7224
|
+
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
7225
|
+
if (claimed === null)
|
|
7226
|
+
return false;
|
|
7227
|
+
if (claimed === "consumed")
|
|
7228
|
+
return true;
|
|
7229
|
+
const { entry, action } = claimed;
|
|
7230
|
+
const backend = entry.instanceName;
|
|
7231
|
+
if (action === "cancel") {
|
|
7232
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.cancelled", backend));
|
|
7233
|
+
return true;
|
|
7234
|
+
}
|
|
7235
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7236
|
+
const text = await this.startLoginSession(backend, {
|
|
7237
|
+
adapter: entry.adapter,
|
|
7238
|
+
adapterId: entry.adapterId,
|
|
7239
|
+
chatId: entry.chatId,
|
|
7240
|
+
threadId: entry.threadId,
|
|
7241
|
+
}, { skipAuthCheck: true });
|
|
7242
|
+
if (text)
|
|
7243
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7244
|
+
return true;
|
|
7245
|
+
}
|
|
7246
|
+
/** Kiro provider button → drive the CLI's arrow-key selector. */
|
|
7247
|
+
async handleLoginMenuSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7248
|
+
const claimed = this.consumeNonceCallback(LOGIN_MENU_CALLBACK_PREFIX, /^login-menu:([0-9a-f]+):(\d)$/, data, callbackAdapterId, receivingAdapter);
|
|
7249
|
+
if (claimed === null)
|
|
7250
|
+
return false;
|
|
7251
|
+
if (claimed === "consumed")
|
|
7252
|
+
return true;
|
|
7253
|
+
const { entry, action } = claimed;
|
|
7254
|
+
const index = Number(action);
|
|
7255
|
+
const label = this.activeLogin?.session.flow.menu?.options[index] ?? action;
|
|
7256
|
+
const ok = await this.activeLogin?.session.selectMenuOption(index) ?? false;
|
|
7257
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, ok ? t("login.provider_selected", label) : t("login.no_session"));
|
|
7258
|
+
return true;
|
|
7259
|
+
}
|
|
7260
|
+
// ── Remote CLI install (`/install-cli`) ──────────────────────────────────
|
|
7261
|
+
//
|
|
7262
|
+
// Same dedicated-window model as /login (and the same LoginSession state
|
|
7263
|
+
// machine — an install is a login flow with no auth hints): run the
|
|
7264
|
+
// installer, judge by exit code, verify the binary on a fresh login shell,
|
|
7265
|
+
// then offer to chain straight into /login.
|
|
7266
|
+
activeInstall = null;
|
|
7267
|
+
/** Start a CLI install session. Caller enforces admin. */
|
|
7268
|
+
async startInstallSession(backendArg, chat) {
|
|
7269
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7270
|
+
const info = BACKEND_INSTALLATION_INFO[backend];
|
|
7271
|
+
if (!info)
|
|
7272
|
+
return t("install.unsupported", backendArg);
|
|
7273
|
+
if (checkBinaryInstalled(info.binary))
|
|
7274
|
+
return t("install.already", backend, info.binary);
|
|
7275
|
+
if (this.activeInstall)
|
|
7276
|
+
return t("install.busy");
|
|
7277
|
+
if (this.activeLogin)
|
|
7278
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7279
|
+
const sessionName = getTmuxSession();
|
|
7280
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7281
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
7282
|
+
// A synthetic login flow: success is decided by the installer's exit code
|
|
7283
|
+
// (LoginSession treats a clean exit as success), never by pane text — and
|
|
7284
|
+
// installer output that happens to contain a URL must not be forwarded as
|
|
7285
|
+
// an auth hint, hence the no-op events below.
|
|
7286
|
+
const flow = {
|
|
7287
|
+
backend,
|
|
7288
|
+
command: info.install,
|
|
7289
|
+
successPattern: /$^/,
|
|
7290
|
+
timeoutMs: 10 * 60 * 1000,
|
|
7291
|
+
};
|
|
7292
|
+
const session = new LoginSession(flow, tmux, {
|
|
7293
|
+
onMenu: () => { },
|
|
7294
|
+
onAuthHint: () => { },
|
|
7295
|
+
onNeedInput: () => { },
|
|
7296
|
+
onDone: async ({ ok, detail }) => {
|
|
7297
|
+
this.activeInstall = null;
|
|
7298
|
+
if (!ok) {
|
|
7299
|
+
// A cancel is user-initiated — the cancel command's own reply already
|
|
7300
|
+
// said so; a second message here would be a duplicate.
|
|
7301
|
+
if (detail !== "cancelled") {
|
|
7302
|
+
await chat.adapter.sendText(chat.chatId, t("install.failed", backend, detail), { threadId: chat.threadId }).catch(() => { });
|
|
7303
|
+
}
|
|
7304
|
+
return;
|
|
7305
|
+
}
|
|
7306
|
+
// The installer may only have added the binary to a profile PATH; a
|
|
7307
|
+
// fresh login shell sees that, the fleet process's PATH may not.
|
|
7308
|
+
if (!this.verifyBinaryOnLoginShell(info.binary)) {
|
|
7309
|
+
await chat.adapter.sendText(chat.chatId, t("install.verify_failed", backend, info.binary), { threadId: chat.threadId }).catch(() => { });
|
|
7310
|
+
return;
|
|
7311
|
+
}
|
|
7312
|
+
if (!LOGIN_FLOWS[backend]) {
|
|
7313
|
+
await chat.adapter.sendText(chat.chatId, t("install.success_no_login", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7314
|
+
return;
|
|
7315
|
+
}
|
|
7316
|
+
await this.postNonceButtonPrompt({
|
|
7317
|
+
prefix: INSTALL_LOGIN_CALLBACK_PREFIX,
|
|
7318
|
+
alertType: "login",
|
|
7319
|
+
instanceName: backend,
|
|
7320
|
+
adapter: chat.adapter,
|
|
7321
|
+
adapterId: chat.adapterId,
|
|
7322
|
+
chatId: chat.chatId,
|
|
7323
|
+
threadId: chat.threadId,
|
|
7324
|
+
message: t("install.login_prompt", backend),
|
|
7325
|
+
choices: [
|
|
7326
|
+
{ action: "go", label: t("install.login_now") },
|
|
7327
|
+
{ action: "later", label: t("install.later") },
|
|
7328
|
+
],
|
|
7329
|
+
expiredText: t("install.later_ack", backend),
|
|
7330
|
+
});
|
|
7331
|
+
},
|
|
7332
|
+
}, this.logger);
|
|
7333
|
+
this.activeInstall = { session, backend, chat };
|
|
7334
|
+
try {
|
|
7335
|
+
await session.start();
|
|
7336
|
+
}
|
|
7337
|
+
catch (err) {
|
|
7338
|
+
this.activeInstall = null;
|
|
7339
|
+
return t("install.failed", backend, err.message);
|
|
7340
|
+
}
|
|
7341
|
+
return t("install.started", backend);
|
|
7342
|
+
}
|
|
7343
|
+
/** `/install-cli cancel` — abort the active install and remove its window. */
|
|
7344
|
+
async cancelInstallSession() {
|
|
7345
|
+
if (!this.activeInstall)
|
|
7346
|
+
return t("install.no_session");
|
|
7347
|
+
const backend = this.activeInstall.backend;
|
|
7348
|
+
await this.activeInstall.session.cancel();
|
|
7349
|
+
return t("install.cancelled", backend);
|
|
7350
|
+
}
|
|
7351
|
+
/** `command -v` on a login shell, so PATH additions from rc files count. */
|
|
7352
|
+
verifyBinaryOnLoginShell(binary) {
|
|
7353
|
+
try {
|
|
7354
|
+
const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe" });
|
|
7355
|
+
return result.status === 0;
|
|
7356
|
+
}
|
|
7357
|
+
catch {
|
|
7358
|
+
return false;
|
|
7359
|
+
}
|
|
7360
|
+
}
|
|
7361
|
+
/** "Sign in now?" button after a successful install. */
|
|
7362
|
+
async handleInstallLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7363
|
+
const claimed = this.consumeNonceCallback(INSTALL_LOGIN_CALLBACK_PREFIX, /^install-login:([0-9a-f]+):(go|later)$/, data, callbackAdapterId, receivingAdapter);
|
|
7364
|
+
if (claimed === null)
|
|
7365
|
+
return false;
|
|
7366
|
+
if (claimed === "consumed")
|
|
7367
|
+
return true;
|
|
7368
|
+
const { entry, action } = claimed;
|
|
7369
|
+
const backend = entry.instanceName;
|
|
7370
|
+
if (action === "later") {
|
|
7371
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.later_ack", backend));
|
|
7372
|
+
return true;
|
|
7373
|
+
}
|
|
7374
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7375
|
+
const text = await this.startLoginSession(backend, {
|
|
7376
|
+
adapter: entry.adapter,
|
|
7377
|
+
adapterId: entry.adapterId,
|
|
7378
|
+
chatId: entry.chatId,
|
|
7379
|
+
threadId: entry.threadId,
|
|
7380
|
+
});
|
|
7381
|
+
if (text)
|
|
7382
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7383
|
+
return true;
|
|
7384
|
+
}
|
|
7385
|
+
/** Discord native `/login` slash — shared by every adapter dispatch block. */
|
|
7386
|
+
async handleLoginSlash(data, adapterId, adapter) {
|
|
7387
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7388
|
+
await data.respond(t("permission.denied"));
|
|
7389
|
+
return;
|
|
7390
|
+
}
|
|
7391
|
+
const chat = { adapter, adapterId, chatId: data.channelId };
|
|
7392
|
+
if (data.options?.cancel === true) {
|
|
7393
|
+
await data.respond(await this.cancelLoginSession());
|
|
7394
|
+
return;
|
|
7395
|
+
}
|
|
7396
|
+
const code = String(data.options?.code ?? "").trim();
|
|
7397
|
+
if (code) {
|
|
7398
|
+
await data.respond(await this.loginSubmitInput(code));
|
|
7399
|
+
return;
|
|
7400
|
+
}
|
|
7401
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7402
|
+
if (backend) {
|
|
7403
|
+
const text = await this.startLoginSession(backend, chat);
|
|
7404
|
+
await data.respond(text ?? t("login.confirm_posted"));
|
|
7405
|
+
return;
|
|
7406
|
+
}
|
|
7407
|
+
await this.promptLoginBackends(chat);
|
|
7408
|
+
await data.respond(t("login.chooser_posted"));
|
|
7409
|
+
}
|
|
7410
|
+
/** Discord native `/install-cli` slash — shared by every dispatch block. */
|
|
7411
|
+
async handleInstallCliSlash(data, adapterId, adapter) {
|
|
7412
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7413
|
+
await data.respond(t("permission.denied"));
|
|
7414
|
+
return;
|
|
7415
|
+
}
|
|
7416
|
+
if (data.options?.cancel === true) {
|
|
7417
|
+
await data.respond(await this.cancelInstallSession());
|
|
7418
|
+
return;
|
|
7419
|
+
}
|
|
7420
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7421
|
+
if (!backend) {
|
|
7422
|
+
// Bare call: offer the backends instead of printing a usage line the user
|
|
7423
|
+
// then has to retype. A Discord slash that DID pick the native `backend`
|
|
7424
|
+
// choice never lands here, so the two paths cannot both fire.
|
|
7425
|
+
await this.promptInstallBackends({ adapter, adapterId, chatId: data.channelId });
|
|
7426
|
+
await data.respond(t("install.chooser_posted"));
|
|
7427
|
+
return;
|
|
7428
|
+
}
|
|
7429
|
+
await data.respond(await this.startInstallSession(backend, { adapter, adapterId, chatId: data.channelId }));
|
|
7430
|
+
}
|
|
5518
7431
|
queueMirrorMessage(text) {
|
|
5519
7432
|
const mirrorTopicId = this.fleetConfig?.channel?.mirror_topic_id;
|
|
5520
7433
|
if (mirrorTopicId == null || !this.adapter)
|
|
@@ -5686,9 +7599,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
5686
7599
|
}
|
|
5687
7600
|
/** Resolve the workspace path a backend actually uses before publishing knowledge. */
|
|
5688
7601
|
resolveKnowledgeWorkDir(workDir, backend, instanceName) {
|
|
5689
|
-
//
|
|
5690
|
-
//
|
|
5691
|
-
// CLI actually runs, not where fleet.yaml points.
|
|
7602
|
+
// Backend resolution may create/normalize the real cwd. Instructions and
|
|
7603
|
+
// skills must land where the CLI actually runs, not an assumed path.
|
|
5692
7604
|
try {
|
|
5693
7605
|
const resolved = createBackend(backend, join(getAgendHome(), "cli-env"))
|
|
5694
7606
|
.resolveWorkingDirectory?.(workDir, instanceName);
|
|
@@ -6344,6 +8256,24 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6344
8256
|
return null;
|
|
6345
8257
|
const probed = await be.probeCLIEnv({ workingDirectory: "", instanceDir: join(getAgendHome(), "cli-env"), instanceName: `probe-${backend}`, mcpServers: {} });
|
|
6346
8258
|
const env = { backend, probedAt: Date.now(), ...probed };
|
|
8259
|
+
// An empty result must never overwrite a catalog we already have. Some
|
|
8260
|
+
// probes hit the network (`agy models` fetches, 5s cap), so a slow moment
|
|
8261
|
+
// returns [] — and writing that would blank the list for the whole 24h
|
|
8262
|
+
// TTL, long after the CLI recovered. Observed live: a good 11-model
|
|
8263
|
+
// antigravity cache replaced by an empty one. Keep the known models and
|
|
8264
|
+
// let the fresher currentModel/version through.
|
|
8265
|
+
if (!env.models?.length) {
|
|
8266
|
+
const previous = this.readCliEnv(backend);
|
|
8267
|
+
if (previous?.models?.length)
|
|
8268
|
+
env.models = previous.models;
|
|
8269
|
+
}
|
|
8270
|
+
// Same protection for the extended catalog: one offline moment must not
|
|
8271
|
+
// blank a good account list for the whole cache TTL.
|
|
8272
|
+
if (!env.apiModels?.length) {
|
|
8273
|
+
const previous = this.readCliEnv(backend);
|
|
8274
|
+
if (previous?.apiModels?.length)
|
|
8275
|
+
env.apiModels = previous.apiModels;
|
|
8276
|
+
}
|
|
6347
8277
|
const path = this.cliEnvPath(backend);
|
|
6348
8278
|
mkdirSync(dirname(path), { recursive: true });
|
|
6349
8279
|
writeFileSync(path, JSON.stringify(env, null, 2));
|
|
@@ -6382,6 +8312,108 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6382
8312
|
const env = await this.probeBackend(backendName);
|
|
6383
8313
|
return env?.models ?? [];
|
|
6384
8314
|
}
|
|
8315
|
+
/**
|
|
8316
|
+
* Model catalog behind the `list_models` tool.
|
|
8317
|
+
*
|
|
8318
|
+
* The two scopes are not cosmetic. "global" is the account/CLI catalog served
|
|
8319
|
+
* from the startup probe cache; "instance" is resolved through that instance's
|
|
8320
|
+
* OWN backend config, and for a Codex instance on a custom provider that is a
|
|
8321
|
+
* different catalog entirely — `listModels()` reads models_cache.json out of
|
|
8322
|
+
* the instance's private CODEX_HOME. Answering such an instance with the
|
|
8323
|
+
* account list would name models its CLI rejects, which is exactly the
|
|
8324
|
+
* mistake this tool exists to prevent.
|
|
8325
|
+
*
|
|
8326
|
+
* `scope` always describes where the returned LIST came from, not what was
|
|
8327
|
+
* asked for: an instance query that falls back to the account catalog reports
|
|
8328
|
+
* scope "global" and says so in `note`, rather than implying instance-level
|
|
8329
|
+
* accuracy it does not have.
|
|
8330
|
+
*
|
|
8331
|
+
* Never throws — a model listing is an aid, and failing it must not fail a turn.
|
|
8332
|
+
*/
|
|
8333
|
+
async listModelCatalog(opts = {}) {
|
|
8334
|
+
const { instanceName } = opts;
|
|
8335
|
+
if (instanceName) {
|
|
8336
|
+
const backend = this.backendNameForInstance(instanceName);
|
|
8337
|
+
const resolved = this.resolveInstanceModel(instanceName);
|
|
8338
|
+
const currentModel = resolved.source === "unresolved" ? null : resolved.model;
|
|
8339
|
+
const provider = this.customProviderFor(instanceName, backend);
|
|
8340
|
+
const scoped = await this.instanceScopedModels(instanceName, backend);
|
|
8341
|
+
if (scoped.length) {
|
|
8342
|
+
return {
|
|
8343
|
+
backend, scope: "instance", instance: instanceName,
|
|
8344
|
+
current_model: currentModel, models: scoped, source: "live",
|
|
8345
|
+
...(provider ? { note: `Catalog read through this instance's ${backend} provider "${provider}" — it may differ from the account catalog.` } : {}),
|
|
8346
|
+
};
|
|
8347
|
+
}
|
|
8348
|
+
// No instance-local catalog (never launched, or the backend has no
|
|
8349
|
+
// per-instance list). The account catalog is the best available answer,
|
|
8350
|
+
// but it is labelled honestly rather than dressed up as instance scope.
|
|
8351
|
+
const global = await this.globalModelCatalog(backend);
|
|
8352
|
+
return {
|
|
8353
|
+
...global, instance: instanceName, current_model: currentModel,
|
|
8354
|
+
note: provider
|
|
8355
|
+
? `No instance-local catalog yet; showing the account catalog, which may NOT match this instance's ${backend} provider "${provider}".`
|
|
8356
|
+
: "No instance-local catalog yet; showing the account catalog.",
|
|
8357
|
+
};
|
|
8358
|
+
}
|
|
8359
|
+
return this.globalModelCatalog(opts.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
8360
|
+
}
|
|
8361
|
+
/** The custom provider an instance overrides its backend with, if any. */
|
|
8362
|
+
customProviderFor(instanceName, backend) {
|
|
8363
|
+
const opts = this.fleetConfig?.instances?.[instanceName]?.backend_options?.[backend]
|
|
8364
|
+
?? this.fleetConfig?.defaults?.backend_options?.[backend];
|
|
8365
|
+
const provider = opts?.provider;
|
|
8366
|
+
return typeof provider === "string" && provider.trim() ? provider.trim() : null;
|
|
8367
|
+
}
|
|
8368
|
+
/** Ask a backend for its catalog using ONE instance's real config. Never throws. */
|
|
8369
|
+
async instanceScopedModels(instanceName, backend) {
|
|
8370
|
+
try {
|
|
8371
|
+
const inst = this.fleetConfig?.instances?.[instanceName];
|
|
8372
|
+
const instanceDir = this.getInstanceDir(instanceName);
|
|
8373
|
+
const be = createBackend(backend, instanceDir);
|
|
8374
|
+
if (!be.listModels)
|
|
8375
|
+
return [];
|
|
8376
|
+
return await be.listModels({
|
|
8377
|
+
workingDirectory: inst?.working_directory ?? "",
|
|
8378
|
+
instanceDir,
|
|
8379
|
+
instanceName,
|
|
8380
|
+
mcpServers: {},
|
|
8381
|
+
model: inst?.model,
|
|
8382
|
+
backendOptions: inst?.backend_options?.[backend] ?? this.fleetConfig?.defaults?.backend_options?.[backend],
|
|
8383
|
+
}) ?? [];
|
|
8384
|
+
}
|
|
8385
|
+
catch {
|
|
8386
|
+
// listModels is documented never to throw, but a backend constructor can
|
|
8387
|
+
// (missing binary). A catalog is an aid; degrade to the account list.
|
|
8388
|
+
return [];
|
|
8389
|
+
}
|
|
8390
|
+
}
|
|
8391
|
+
/** Account-wide catalog: probe cache first, live probe on miss. */
|
|
8392
|
+
async globalModelCatalog(backend) {
|
|
8393
|
+
const cached = this.readCliEnv(backend);
|
|
8394
|
+
if (cached?.models?.length) {
|
|
8395
|
+
return {
|
|
8396
|
+
backend, scope: "global", current_model: cached.currentModel ?? null,
|
|
8397
|
+
models: cached.models, source: "cache",
|
|
8398
|
+
probed_at: new Date(cached.probedAt).toISOString(),
|
|
8399
|
+
};
|
|
8400
|
+
}
|
|
8401
|
+
const env = await this.probeBackend(backend);
|
|
8402
|
+
if (env?.models?.length) {
|
|
8403
|
+
return {
|
|
8404
|
+
backend, scope: "global", current_model: env.currentModel ?? null,
|
|
8405
|
+
models: env.models, source: "live",
|
|
8406
|
+
probed_at: new Date(env.probedAt).toISOString(),
|
|
8407
|
+
};
|
|
8408
|
+
}
|
|
8409
|
+
// Reported rather than thrown: "we could not enumerate" is a useful answer,
|
|
8410
|
+
// and the caller can still set a model by name (AgEnD passes it through).
|
|
8411
|
+
return {
|
|
8412
|
+
backend, scope: "global", current_model: env?.currentModel ?? null,
|
|
8413
|
+
models: [], source: "fallback",
|
|
8414
|
+
note: `Could not enumerate models for ${backend} (CLI missing, not logged in, or it offers no list). Model names are passed through to the CLI, so a known-good name still works.`,
|
|
8415
|
+
};
|
|
8416
|
+
}
|
|
6385
8417
|
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
6386
8418
|
/** Label an effort choice, marking the one currently configured. */
|
|
6387
8419
|
effortChoiceLabel(level, current) {
|
|
@@ -6390,10 +8422,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6390
8422
|
effortMenuHeader(instanceName) {
|
|
6391
8423
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6392
8424
|
if (!effort)
|
|
6393
|
-
return "
|
|
8425
|
+
return t("effort.current_default");
|
|
6394
8426
|
return source === "fleet-default"
|
|
6395
|
-
?
|
|
6396
|
-
:
|
|
8427
|
+
? t("effort.current_fleet", effort)
|
|
8428
|
+
: t("effort.current", effort);
|
|
6397
8429
|
}
|
|
6398
8430
|
/** `/effort` — DC Select Menu, or apply directly when a level is given. */
|
|
6399
8431
|
async handleEffortSlash(data, adapterId) {
|
|
@@ -6414,11 +8446,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6414
8446
|
}
|
|
6415
8447
|
const levels = this.effortLevelsFor(name);
|
|
6416
8448
|
if (levels.length === 0) {
|
|
6417
|
-
await data.respond(
|
|
8449
|
+
await data.respond(t("effort.unsupported", this.backendNameForInstance(name)));
|
|
6418
8450
|
return;
|
|
6419
8451
|
}
|
|
6420
8452
|
if (!data.respondChoices) {
|
|
6421
|
-
await data.respond(
|
|
8453
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6422
8454
|
return;
|
|
6423
8455
|
}
|
|
6424
8456
|
const current = this.resolveInstanceEffort(name).effort;
|
|
@@ -6431,20 +8463,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6431
8463
|
timer.unref?.();
|
|
6432
8464
|
this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
6433
8465
|
try {
|
|
6434
|
-
await data.respondChoices(
|
|
8466
|
+
await data.respondChoices(t("effort.menu", this.effortMenuHeader(name)), choices);
|
|
6435
8467
|
}
|
|
6436
8468
|
catch (err) {
|
|
6437
8469
|
this.pendingEffortSelects.delete(nonce);
|
|
6438
8470
|
clearTimeout(timer);
|
|
6439
8471
|
this.logger.warn({ err, instanceName: name }, "effort menu failed");
|
|
6440
|
-
await data.respond(
|
|
8472
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6441
8473
|
}
|
|
6442
8474
|
}
|
|
6443
8475
|
/** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
|
|
6444
8476
|
async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6445
8477
|
const levels = this.effortLevelsFor(instanceName);
|
|
6446
8478
|
if (levels.length === 0) {
|
|
6447
|
-
return
|
|
8479
|
+
return t("effort.unsupported", this.backendNameForInstance(instanceName));
|
|
6448
8480
|
}
|
|
6449
8481
|
const current = this.resolveInstanceEffort(instanceName).effort;
|
|
6450
8482
|
const nonce = randomBytes(6).toString("hex");
|
|
@@ -6460,13 +8492,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6460
8492
|
const p = this.pendingEffortSelects.get(nonce);
|
|
6461
8493
|
if (p) {
|
|
6462
8494
|
this.pendingEffortSelects.delete(nonce);
|
|
6463
|
-
p.respond("
|
|
8495
|
+
p.respond(t("effort.selection_expired")).catch(() => { });
|
|
6464
8496
|
}
|
|
6465
8497
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6466
8498
|
timer.unref?.();
|
|
6467
8499
|
this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6468
8500
|
try {
|
|
6469
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8501
|
+
const menuMessageId = await adapter.promptUser(chatId, t("effort.menu", this.effortMenuHeader(instanceName)), choices, { threadId });
|
|
6470
8502
|
const pending = this.pendingEffortSelects.get(nonce);
|
|
6471
8503
|
if (pending)
|
|
6472
8504
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6476,7 +8508,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6476
8508
|
this.pendingEffortSelects.delete(nonce);
|
|
6477
8509
|
clearTimeout(timer);
|
|
6478
8510
|
this.logger.warn({ err, instanceName }, "TG effort menu failed");
|
|
6479
|
-
return
|
|
8511
|
+
return t("effort.usage", levels.join("|"));
|
|
6480
8512
|
}
|
|
6481
8513
|
}
|
|
6482
8514
|
/** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
|
|
@@ -6497,7 +8529,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6497
8529
|
this.pendingEffortSelects.delete(match[1]);
|
|
6498
8530
|
clearTimeout(pending.timer);
|
|
6499
8531
|
const level = match[2];
|
|
6500
|
-
const progressText =
|
|
8532
|
+
const progressText = t("effort.setting", pending.instanceName, level);
|
|
6501
8533
|
let progressMsgId;
|
|
6502
8534
|
if (pending.adapter && pending.adapterChatId) {
|
|
6503
8535
|
const menuMessageId = pending.menuMessageId ?? data.messageId;
|
|
@@ -6529,7 +8561,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6529
8561
|
}
|
|
6530
8562
|
catch (err) {
|
|
6531
8563
|
this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
|
|
6532
|
-
result =
|
|
8564
|
+
result = t("effort.switch_failed", level, err instanceof Error ? err.message : String(err));
|
|
6533
8565
|
}
|
|
6534
8566
|
if (pending.adapter && pending.adapterChatId) {
|
|
6535
8567
|
if (progressMsgId) {
|
|
@@ -6566,32 +8598,36 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6566
8598
|
}
|
|
6567
8599
|
// No arg (or --refresh) → menu. Menu is DC-only this round (respondChoices); TG uses `/model <name>`.
|
|
6568
8600
|
if (!data.respondChoices) {
|
|
6569
|
-
await data.respond("
|
|
8601
|
+
await data.respond(t("model.usage"));
|
|
6570
8602
|
return;
|
|
6571
8603
|
}
|
|
6572
8604
|
const options = await this.getModelOptions(name, isRefresh);
|
|
6573
8605
|
if (options.length === 0) {
|
|
6574
|
-
await data.respond(
|
|
8606
|
+
await data.respond(t("model.list_unavailable", name));
|
|
6575
8607
|
return;
|
|
6576
8608
|
}
|
|
6577
8609
|
// Raw id for ✓-matching options; display resolves an inherited CLI default.
|
|
6578
8610
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(name);
|
|
6579
8611
|
const nonce = randomBytes(6).toString("hex");
|
|
6580
|
-
const
|
|
8612
|
+
const isClaude = this.backendNameForInstance(name) === "claude-code";
|
|
8613
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6581
8614
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6582
8615
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6583
8616
|
}));
|
|
8617
|
+
if (isClaude) {
|
|
8618
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8619
|
+
}
|
|
6584
8620
|
const timer = setTimeout(() => this.pendingModelSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6585
8621
|
timer.unref?.();
|
|
6586
|
-
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
8622
|
+
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond, respondChoices: data.respondChoices });
|
|
6587
8623
|
try {
|
|
6588
|
-
await data.respondChoices(
|
|
8624
|
+
await data.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
6589
8625
|
}
|
|
6590
8626
|
catch (err) {
|
|
6591
8627
|
this.pendingModelSelects.delete(nonce);
|
|
6592
8628
|
clearTimeout(timer);
|
|
6593
8629
|
this.logger.warn({ err, instanceName: name }, "model menu failed");
|
|
6594
|
-
await data.respond("
|
|
8630
|
+
await data.respond(t("model.usage"));
|
|
6595
8631
|
}
|
|
6596
8632
|
}
|
|
6597
8633
|
/**
|
|
@@ -6602,14 +8638,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6602
8638
|
async promptModelMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6603
8639
|
const options = await this.getModelOptions(instanceName);
|
|
6604
8640
|
if (options.length === 0) {
|
|
6605
|
-
return
|
|
8641
|
+
return t("model.list_unavailable", instanceName);
|
|
6606
8642
|
}
|
|
6607
8643
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(instanceName);
|
|
6608
8644
|
const nonce = randomBytes(6).toString("hex");
|
|
6609
|
-
const
|
|
8645
|
+
const isClaude = this.backendNameForInstance(instanceName) === "claude-code";
|
|
8646
|
+
// Keep the more-models entry inside Discord's 25-option select cap.
|
|
8647
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6610
8648
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6611
8649
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6612
8650
|
}));
|
|
8651
|
+
if (isClaude) {
|
|
8652
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8653
|
+
}
|
|
6613
8654
|
const respond = async (text) => {
|
|
6614
8655
|
await adapter.sendText(chatId, text, { threadId });
|
|
6615
8656
|
return undefined;
|
|
@@ -6618,13 +8659,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6618
8659
|
const p = this.pendingModelSelects.get(nonce);
|
|
6619
8660
|
if (p) {
|
|
6620
8661
|
this.pendingModelSelects.delete(nonce);
|
|
6621
|
-
p.respond("
|
|
8662
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
6622
8663
|
}
|
|
6623
8664
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6624
8665
|
timer.unref?.();
|
|
6625
8666
|
this.pendingModelSelects.set(nonce, { instanceName, model: "", userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6626
8667
|
try {
|
|
6627
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8668
|
+
const menuMessageId = await adapter.promptUser(chatId, t("model.menu", currentDisplay), choices, { threadId });
|
|
6628
8669
|
const pending = this.pendingModelSelects.get(nonce);
|
|
6629
8670
|
if (pending)
|
|
6630
8671
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6634,7 +8675,63 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6634
8675
|
this.pendingModelSelects.delete(nonce);
|
|
6635
8676
|
clearTimeout(timer);
|
|
6636
8677
|
this.logger.warn({ err, instanceName }, "TG model menu failed");
|
|
6637
|
-
return
|
|
8678
|
+
return t("model.usage");
|
|
8679
|
+
}
|
|
8680
|
+
}
|
|
8681
|
+
/** Cached-or-live account catalog behind "/model → more models" (claude). */
|
|
8682
|
+
async claudeApiModelOptions() {
|
|
8683
|
+
const cached = this.readCliEnv("claude-code");
|
|
8684
|
+
if (cached?.apiModels?.length)
|
|
8685
|
+
return cached.apiModels;
|
|
8686
|
+
const env = await this.probeBackend("claude-code");
|
|
8687
|
+
return env?.apiModels ?? [];
|
|
8688
|
+
}
|
|
8689
|
+
/** Replace a consumed "/model" menu with the full account catalog tier. */
|
|
8690
|
+
async expandClaudeModelMenu(pending) {
|
|
8691
|
+
const expanded = await this.claudeApiModelOptions();
|
|
8692
|
+
if (!expanded.length) {
|
|
8693
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8694
|
+
return;
|
|
8695
|
+
}
|
|
8696
|
+
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(pending.instanceName);
|
|
8697
|
+
const nonce = randomBytes(6).toString("hex");
|
|
8698
|
+
const choices = expanded.slice(0, 25).map(o => ({
|
|
8699
|
+
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
8700
|
+
label: this.modelChoiceLabel(o, currentModel),
|
|
8701
|
+
}));
|
|
8702
|
+
const timer = setTimeout(() => {
|
|
8703
|
+
const p = this.pendingModelSelects.get(nonce);
|
|
8704
|
+
if (p) {
|
|
8705
|
+
this.pendingModelSelects.delete(nonce);
|
|
8706
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
8707
|
+
}
|
|
8708
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
8709
|
+
timer.unref?.();
|
|
8710
|
+
this.pendingModelSelects.set(nonce, { ...pending, model: "", timer });
|
|
8711
|
+
try {
|
|
8712
|
+
if (pending.respondChoices) {
|
|
8713
|
+
// Discord select menu: edit the same interaction reply in place.
|
|
8714
|
+
await pending.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
8715
|
+
return;
|
|
8716
|
+
}
|
|
8717
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
8718
|
+
// Telegram: retire the tier-1 keyboard, then post the expanded menu.
|
|
8719
|
+
if (pending.menuMessageId && pending.adapter.editMessageRemoveButtons) {
|
|
8720
|
+
await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, pending.menuMessageId, t("model.more"), pending.adapterThreadId).catch(() => { });
|
|
8721
|
+
}
|
|
8722
|
+
const menuMessageId = await pending.adapter.promptUser(pending.adapterChatId, t("model.menu", currentDisplay), choices, { threadId: pending.adapterThreadId });
|
|
8723
|
+
const fresh = this.pendingModelSelects.get(nonce);
|
|
8724
|
+
if (fresh)
|
|
8725
|
+
fresh.menuMessageId = menuMessageId;
|
|
8726
|
+
return;
|
|
8727
|
+
}
|
|
8728
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8729
|
+
}
|
|
8730
|
+
catch (err) {
|
|
8731
|
+
this.pendingModelSelects.delete(nonce);
|
|
8732
|
+
clearTimeout(timer);
|
|
8733
|
+
this.logger.warn({ err, instanceName: pending.instanceName }, "Expanded model menu failed");
|
|
8734
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
6638
8735
|
}
|
|
6639
8736
|
}
|
|
6640
8737
|
/** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
|
|
@@ -6656,8 +8753,14 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6656
8753
|
this.pendingModelSelects.delete(match[1]);
|
|
6657
8754
|
clearTimeout(pending.timer);
|
|
6658
8755
|
const model = match[2];
|
|
8756
|
+
// "More models…" is a navigation choice, not a model: swap the menu for the
|
|
8757
|
+
// full account catalog (live /v1/models with [1m] variants).
|
|
8758
|
+
if (model === "__more__") {
|
|
8759
|
+
await this.expandClaudeModelMenu(pending);
|
|
8760
|
+
return true;
|
|
8761
|
+
}
|
|
6659
8762
|
// Send immediate "⏳ Switching..." feedback, then apply in background.
|
|
6660
|
-
const progressText =
|
|
8763
|
+
const progressText = t("model.switching", pending.instanceName, model);
|
|
6661
8764
|
let progressMsgId;
|
|
6662
8765
|
if (pending.adapter && pending.adapterChatId) {
|
|
6663
8766
|
// TG path: turn the original menu into the progress message. This both
|
|
@@ -6694,7 +8797,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6694
8797
|
}
|
|
6695
8798
|
catch (err) {
|
|
6696
8799
|
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
6697
|
-
result =
|
|
8800
|
+
result = t("model.switch_failed", model, err instanceof Error ? err.message : String(err));
|
|
6698
8801
|
}
|
|
6699
8802
|
if (pending.adapter && pending.adapterChatId) {
|
|
6700
8803
|
if (progressMsgId) {
|
|
@@ -6791,17 +8894,17 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6791
8894
|
}
|
|
6792
8895
|
catch { /* treated as unsupported below */ }
|
|
6793
8896
|
if (strategy === "unsupported" || supported.length === 0) {
|
|
6794
|
-
return
|
|
8897
|
+
return t("effort.unsupported", backendName);
|
|
6795
8898
|
}
|
|
6796
8899
|
if (!FleetManager.EFFORT_LEVELS.includes(level)) {
|
|
6797
|
-
return
|
|
8900
|
+
return t("effort.unknown", level, FleetManager.EFFORT_LEVELS.join(", "));
|
|
6798
8901
|
}
|
|
6799
8902
|
const applied = FleetManager.clampEffort(level, supported);
|
|
6800
8903
|
if (!applied)
|
|
6801
|
-
return
|
|
8904
|
+
return t("effort.no_canonical", backendName);
|
|
6802
8905
|
const warn = applied === level
|
|
6803
8906
|
? ""
|
|
6804
|
-
:
|
|
8907
|
+
: t("effort.clamped", applied, level, backendName);
|
|
6805
8908
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6806
8909
|
// or the instance silently reverts on restart.
|
|
6807
8910
|
if (this.fleetConfig?.instances[instanceName]) {
|
|
@@ -6810,12 +8913,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6810
8913
|
}
|
|
6811
8914
|
if (strategy === "runtime") {
|
|
6812
8915
|
if (!this.instanceIpcClients.get(instanceName))
|
|
6813
|
-
return `${warn}
|
|
8916
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6814
8917
|
this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
|
|
6815
|
-
return `${warn}
|
|
8918
|
+
return `${warn}${t("effort.runtime_success", instanceName, applied)}`;
|
|
6816
8919
|
}
|
|
6817
8920
|
await this.restartSingleInstance(instanceName);
|
|
6818
|
-
return `${warn}
|
|
8921
|
+
return `${warn}${t("effort.restart_success", instanceName, applied)}`;
|
|
6819
8922
|
}
|
|
6820
8923
|
async applyModel(instanceName, model) {
|
|
6821
8924
|
const backendName = this.backendNameForInstance(instanceName);
|
|
@@ -6824,9 +8927,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6824
8927
|
strategy = createBackend(backendName, this.getInstanceDir(instanceName)).getModelSwitchStrategy?.(model) ?? "restart";
|
|
6825
8928
|
}
|
|
6826
8929
|
catch { /* default restart */ }
|
|
6827
|
-
const warn = isModelCompatible(backendName, model) ? "" :
|
|
8930
|
+
const warn = isModelCompatible(backendName, model) ? "" : t("model.pattern_warning", model, backendName);
|
|
6828
8931
|
if (strategy === "runtime" && !this.instanceIpcClients.get(instanceName)) {
|
|
6829
|
-
return `${warn}
|
|
8932
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6830
8933
|
}
|
|
6831
8934
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6832
8935
|
// or the instance silently reverts to the CLI default after a fleet restart.
|
|
@@ -6840,13 +8943,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6840
8943
|
persisted = true;
|
|
6841
8944
|
}
|
|
6842
8945
|
if (!persisted)
|
|
6843
|
-
return `${warn}
|
|
8946
|
+
return `${warn}${t("model.persist_failed", instanceName)}`;
|
|
6844
8947
|
if (strategy === "runtime") {
|
|
6845
8948
|
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
6846
|
-
return `${warn}
|
|
8949
|
+
return `${warn}${t("model.runtime_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6847
8950
|
}
|
|
6848
8951
|
await this.restartSingleInstance(instanceName);
|
|
6849
|
-
return `${warn}
|
|
8952
|
+
return `${warn}${t("model.restart_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6850
8953
|
}
|
|
6851
8954
|
/**
|
|
6852
8955
|
* The trailing "Current effort: …" line for a /model reply.
|
|
@@ -6860,10 +8963,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6860
8963
|
return "";
|
|
6861
8964
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6862
8965
|
if (!effort)
|
|
6863
|
-
return "
|
|
8966
|
+
return `\n${t("effort.current_default")}`;
|
|
6864
8967
|
return source === "fleet-default"
|
|
6865
|
-
? `\
|
|
6866
|
-
: `\
|
|
8968
|
+
? `\n${t("effort.current_fleet", effort)}`
|
|
8969
|
+
: `\n${t("effort.current", effort)}`;
|
|
6867
8970
|
}
|
|
6868
8971
|
/** Read recent chat log for agent context */
|
|
6869
8972
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
@@ -6898,11 +9001,21 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
6898
9001
|
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
6899
9002
|
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
6900
9003
|
if (!this.classicChannels)
|
|
6901
|
-
return "
|
|
9004
|
+
return t("classic.manager_unavailable");
|
|
6902
9005
|
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
6903
9006
|
const generalId = this.findGeneralInstance(adapterId);
|
|
6904
|
-
if (generalId)
|
|
6905
|
-
|
|
9007
|
+
if (generalId) {
|
|
9008
|
+
// Fire-and-forget, exactly as the notifyInstanceTopic it replaces: this
|
|
9009
|
+
// function's return value is the rejection shown to the user, and it
|
|
9010
|
+
// must not wait on posting buttons into the General topic.
|
|
9011
|
+
void this.promptClassicApproval({
|
|
9012
|
+
generalName: generalId,
|
|
9013
|
+
message: t("alert.unauth_guild", guildId, userId),
|
|
9014
|
+
groupId: String(guildId),
|
|
9015
|
+
scope: "guild",
|
|
9016
|
+
userId: userId ? String(userId) : undefined,
|
|
9017
|
+
}).catch(err => this.logger.warn({ err, guildId }, "Classic approval prompt failed"));
|
|
9018
|
+
}
|
|
6906
9019
|
return t("classic.not_authorized_guild");
|
|
6907
9020
|
}
|
|
6908
9021
|
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
@@ -7045,6 +9158,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7045
9158
|
return;
|
|
7046
9159
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
7047
9160
|
ensureWorkspaceGit(workDir);
|
|
9161
|
+
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
7048
9162
|
const config = {
|
|
7049
9163
|
...DEFAULT_INSTANCE_CONFIG,
|
|
7050
9164
|
...this.fleetConfig?.defaults,
|
|
@@ -7052,6 +9166,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7052
9166
|
lightweight: true,
|
|
7053
9167
|
...(backend ? { backend } : {}),
|
|
7054
9168
|
...(model ? { model } : {}),
|
|
9169
|
+
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
9170
|
+
...(classicIdentity?.description ? { description: classicIdentity.description } : {}),
|
|
7055
9171
|
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
7056
9172
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
7057
9173
|
};
|
|
@@ -7065,7 +9181,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7065
9181
|
return blocker;
|
|
7066
9182
|
const classicChannels = this.classicChannels;
|
|
7067
9183
|
if (!classicChannels)
|
|
7068
|
-
return "
|
|
9184
|
+
return t("classic.manager_unavailable");
|
|
7069
9185
|
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
7070
9186
|
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
7071
9187
|
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
@@ -7087,7 +9203,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7087
9203
|
/** Handle /stop slash command — unregister classic channel */
|
|
7088
9204
|
async handleClassicStop(channelId, adapterId) {
|
|
7089
9205
|
if (!this.classicChannels)
|
|
7090
|
-
return "
|
|
9206
|
+
return t("classic.manager_unavailable");
|
|
7091
9207
|
const ch = this.classicChannels.unregister(channelId, adapterId);
|
|
7092
9208
|
if (!ch)
|
|
7093
9209
|
return t("classic.no_agent");
|
|
@@ -7119,6 +9235,21 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7119
9235
|
// await in the same tick as the signal handler, so no event can slip in.
|
|
7120
9236
|
this.shuttingDown = true;
|
|
7121
9237
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
9238
|
+
// Release held delivery promises before awaiting daemon shutdown, then
|
|
9239
|
+
// reject spawn work which has not started. Otherwise a storm backoff could
|
|
9240
|
+
// make `agend stop` wait forever for its own queue.
|
|
9241
|
+
this.stormWindow.shutdown();
|
|
9242
|
+
this.spawnGate.shutdown();
|
|
9243
|
+
for (const pending of this.startupRetries.values())
|
|
9244
|
+
clearTimeout(pending.timer);
|
|
9245
|
+
this.startupRetries.clear();
|
|
9246
|
+
for (const pending of this.startupRetryNotices.values())
|
|
9247
|
+
clearTimeout(pending.timer);
|
|
9248
|
+
this.startupRetryNotices.clear();
|
|
9249
|
+
if (this.stormOpenNotifyTimer) {
|
|
9250
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
9251
|
+
this.stormOpenNotifyTimer = null;
|
|
9252
|
+
}
|
|
7122
9253
|
sdNotifyBlocking("STOPPING=1");
|
|
7123
9254
|
if (this.watchdogTimer) {
|
|
7124
9255
|
clearInterval(this.watchdogTimer);
|
|
@@ -7134,6 +9265,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7134
9265
|
this.clearStatuslineWatchers();
|
|
7135
9266
|
this.costGuard?.stop();
|
|
7136
9267
|
this.dailySummary?.stop();
|
|
9268
|
+
this.dailyTipScheduler?.stop();
|
|
9269
|
+
this.dailyTipScheduler = null;
|
|
7137
9270
|
if (this.updateCheckTimer) {
|
|
7138
9271
|
clearTimeout(this.updateCheckTimer);
|
|
7139
9272
|
clearInterval(this.updateCheckTimer);
|
|
@@ -7351,7 +9484,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7351
9484
|
/**
|
|
7352
9485
|
* Hot-reload: re-read fleet.yaml and reconcile running instances.
|
|
7353
9486
|
* Starts new, stops removed, restarts modified instances.
|
|
7354
|
-
*
|
|
9487
|
+
* Whitelisted runtime fields are pushed into live daemons; all other instance
|
|
9488
|
+
* fields, plus cold fleet-level settings, retain restart semantics.
|
|
7355
9489
|
*/
|
|
7356
9490
|
async reconcileInstances() {
|
|
7357
9491
|
if (!this.configPath)
|
|
@@ -7393,11 +9527,11 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7393
9527
|
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
7394
9528
|
// with no explanation anywhere they are looking.
|
|
7395
9529
|
const why = !validation.valid
|
|
7396
|
-
?
|
|
9530
|
+
? t("fleet.reload_validation", validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n"))
|
|
7397
9531
|
: unsafeEmpty
|
|
7398
|
-
?
|
|
7399
|
-
:
|
|
7400
|
-
this.notifyFleetError(
|
|
9532
|
+
? t("fleet.reload_removed_all", oldCount)
|
|
9533
|
+
: t("fleet.reload_removed_half", oldCount, newCount);
|
|
9534
|
+
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
7401
9535
|
return;
|
|
7402
9536
|
}
|
|
7403
9537
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -7405,9 +9539,20 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7405
9539
|
this.scheduler?.reload();
|
|
7406
9540
|
const newInstances = this.fleetConfig.instances;
|
|
7407
9541
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
7408
|
-
// Detect fleet-level
|
|
7409
|
-
|
|
7410
|
-
const
|
|
9542
|
+
// Detect fleet-level changes which still need a restart. Hot defaults are
|
|
9543
|
+
// reconciled below and must not produce a misleading restart warning.
|
|
9544
|
+
const oldDefaultColdWithFleetFields = oldConfig?.defaults
|
|
9545
|
+
? splitHotColdConfig(oldConfig.defaults).cold
|
|
9546
|
+
: {};
|
|
9547
|
+
const newDefaultColdWithFleetFields = this.fleetConfig?.defaults
|
|
9548
|
+
? splitHotColdConfig(this.fleetConfig.defaults).cold
|
|
9549
|
+
: {};
|
|
9550
|
+
// `tips` is a fleet-owned hot switch; it is read at post time and never
|
|
9551
|
+
// belongs in a daemon config or a "restart required" warning.
|
|
9552
|
+
const { tips: _oldTips, ...oldDefaultCold } = oldDefaultColdWithFleetFields;
|
|
9553
|
+
const { tips: _newTips, ...newDefaultCold } = newDefaultColdWithFleetFields;
|
|
9554
|
+
const oldFleetLevel = JSON.stringify({ channel: oldConfig?.channel, defaults: oldDefaultCold });
|
|
9555
|
+
const newFleetLevel = JSON.stringify({ channel: this.fleetConfig?.channel, defaults: newDefaultCold });
|
|
7411
9556
|
if (oldFleetLevel !== newFleetLevel) {
|
|
7412
9557
|
this.logger.warn("Fleet-level config changed (channel/defaults) — use /restart for full effect");
|
|
7413
9558
|
}
|
|
@@ -7419,22 +9564,43 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7419
9564
|
await this.stopInstance(name).catch(err => this.logger.error({ err, name }, "Failed to stop removed instance"));
|
|
7420
9565
|
}
|
|
7421
9566
|
}
|
|
7422
|
-
// Start new +
|
|
9567
|
+
// Start new + reconcile modified instances. Hot values are always sent as a
|
|
9568
|
+
// complete snapshot: Settings mutates FleetManager's config before SIGHUP,
|
|
9569
|
+
// so an old/new diff alone can miss the live daemon's stale value.
|
|
7423
9570
|
for (const [name, config] of Object.entries(newInstances)) {
|
|
7424
9571
|
if (!this.daemons.has(name)) {
|
|
7425
9572
|
// New instance — startInstance already calls connectIpcToInstance
|
|
7426
9573
|
this.logger.info({ name }, "New instance in config — starting");
|
|
7427
|
-
await this.
|
|
9574
|
+
await this.startInstanceUnattended(name, config, topicMode, "new instance");
|
|
7428
9575
|
}
|
|
7429
9576
|
else if (oldConfig?.instances[name]) {
|
|
7430
|
-
|
|
7431
|
-
|
|
9577
|
+
const daemon = this.daemons.get(name);
|
|
9578
|
+
const runtimeConfig = daemon.getConfigSnapshot?.() ?? oldConfig.instances[name];
|
|
9579
|
+
const oldParts = splitHotColdConfig(runtimeConfig);
|
|
9580
|
+
const newParts = splitHotColdConfig(config);
|
|
9581
|
+
// Every field not explicitly classified hot is cold by default.
|
|
9582
|
+
if (!isDeepStrictEqual(oldParts.cold, newParts.cold)) {
|
|
7432
9583
|
this.logger.info({ name }, "Instance config changed — restarting");
|
|
7433
9584
|
await this.stopInstance(name).catch(() => { });
|
|
7434
|
-
await this.
|
|
9585
|
+
await this.startInstanceUnattended(name, config, topicMode, "modified instance");
|
|
9586
|
+
}
|
|
9587
|
+
else if (!isDeepStrictEqual(oldParts.hot, newParts.hot)) {
|
|
9588
|
+
const update = hotConfigUpdate(config);
|
|
9589
|
+
const ipc = this.instanceIpcClients.get(name);
|
|
9590
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
9591
|
+
if (!sent) {
|
|
9592
|
+
// Daemon is in-process, so a reconnect gap must not leave runtime
|
|
9593
|
+
// state stale. Normal operation still uses the explicit IPC contract.
|
|
9594
|
+
daemon.applyConfigUpdate(update);
|
|
9595
|
+
this.logger.warn({ name }, "Config-update IPC unavailable — applied hot config in-process");
|
|
9596
|
+
}
|
|
9597
|
+
this.logger.info({ name, fields: [...HOT_INSTANCE_CONFIG_KEYS] }, "Instance hot config reloaded");
|
|
7435
9598
|
}
|
|
7436
9599
|
}
|
|
7437
9600
|
}
|
|
9601
|
+
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
9602
|
+
// currently idle instances instead of waiting for a future state edge.
|
|
9603
|
+
this.enforceWarmCap();
|
|
7438
9604
|
this.logger.info({ running: this.daemons.size, configured: Object.keys(newInstances).length }, "Reconcile complete");
|
|
7439
9605
|
}
|
|
7440
9606
|
async restartInstances() {
|
|
@@ -7515,14 +9681,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7515
9681
|
const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
7516
9682
|
const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
7517
9683
|
for (const [name, cfg] of restartGenerals) {
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
if (this.daemons.has(name))
|
|
7521
|
-
restartProgress.markReady();
|
|
7522
|
-
}
|
|
7523
|
-
catch (err) {
|
|
7524
|
-
this.logger.error({ err, name }, "Failed to start general instance");
|
|
7525
|
-
}
|
|
9684
|
+
if (await this.startInstanceUnattended(name, cfg, topicMode, "general instance"))
|
|
9685
|
+
restartProgress.markReady();
|
|
7526
9686
|
}
|
|
7527
9687
|
// General is ready again; now its topic can own the live progress message.
|
|
7528
9688
|
await restartProgress.start(progressTarget);
|
|
@@ -7543,14 +9703,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7543
9703
|
while (idx < channels.length) {
|
|
7544
9704
|
const batch = channels.slice(idx, idx + concurrency);
|
|
7545
9705
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
7546
|
-
|
|
7547
|
-
|
|
7548
|
-
if (this.daemons.has(ch.instanceName))
|
|
7549
|
-
restartProgress.markReady();
|
|
7550
|
-
}
|
|
7551
|
-
catch (err) {
|
|
7552
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
7553
|
-
}
|
|
9706
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
9707
|
+
restartProgress.markReady();
|
|
7554
9708
|
}));
|
|
7555
9709
|
idx += concurrency;
|
|
7556
9710
|
}
|
|
@@ -7674,7 +9828,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7674
9828
|
if (generalId) {
|
|
7675
9829
|
// No release URL — Discord's SuppressEmbeds proved unreliable and the
|
|
7676
9830
|
// link preview looked bad. Version + /update instruction is enough.
|
|
7677
|
-
this.notifyInstanceTopic(generalId, t("update.
|
|
9831
|
+
this.notifyInstanceTopic(generalId, t("update.available_current", `v${target}`, `v${currentVersion}`));
|
|
7678
9832
|
}
|
|
7679
9833
|
}
|
|
7680
9834
|
}
|
|
@@ -7808,63 +9962,81 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7808
9962
|
}
|
|
7809
9963
|
// Fleet API (enriched for agent board)
|
|
7810
9964
|
if (req.method === "GET" && req.url === "/api/fleet") {
|
|
7811
|
-
|
|
7812
|
-
|
|
7813
|
-
|
|
7814
|
-
|
|
7815
|
-
|
|
7816
|
-
|
|
7817
|
-
|
|
7818
|
-
|
|
7819
|
-
|
|
7820
|
-
|
|
7821
|
-
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
|
|
7825
|
-
|
|
7826
|
-
|
|
7827
|
-
|
|
7828
|
-
|
|
7829
|
-
const
|
|
7830
|
-
|
|
7831
|
-
|
|
7832
|
-
|
|
7833
|
-
|
|
7834
|
-
|
|
7835
|
-
|
|
7836
|
-
|
|
7837
|
-
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
|
|
9965
|
+
try {
|
|
9966
|
+
const sysInfo = this.getSysInfo();
|
|
9967
|
+
const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
|
|
9968
|
+
const fleetNames = new Set(fleetInstances.map(inst => inst.name));
|
|
9969
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
9970
|
+
.filter(channel => !fleetNames.has(channel.instanceName))
|
|
9971
|
+
.map(channel => ({
|
|
9972
|
+
name: channel.instanceName,
|
|
9973
|
+
status: this.getInstanceStatus(channel.instanceName),
|
|
9974
|
+
state: this.getInstanceExecutionState(channel.instanceName),
|
|
9975
|
+
ipc: this.instanceIpcClients.has(channel.instanceName),
|
|
9976
|
+
costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
|
|
9977
|
+
rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
|
|
9978
|
+
classic: true,
|
|
9979
|
+
classicName: channel.name,
|
|
9980
|
+
channelId: channel.channelId,
|
|
9981
|
+
adapterId: channel.adapterId ?? null,
|
|
9982
|
+
}));
|
|
9983
|
+
const enriched = [...fleetInstances, ...classicInstances].map(inst => {
|
|
9984
|
+
const config = this.fleetConfig?.instances[inst.name];
|
|
9985
|
+
const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
|
|
9986
|
+
const lastActivity = inst.classic
|
|
9987
|
+
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
9988
|
+
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
9989
|
+
const backend = this.backendNameForInstance(inst.name);
|
|
9990
|
+
const resolvedModel = this.resolveInstanceModel(inst.name);
|
|
9991
|
+
const effortStrategy = this.effortStrategyFor(inst.name);
|
|
9992
|
+
const resolvedEffort = this.resolveInstanceEffort(inst.name);
|
|
9993
|
+
// Find claimed tasks for this instance
|
|
9994
|
+
let currentTask = null;
|
|
9995
|
+
try {
|
|
9996
|
+
const tasks = this.scheduler?.db.listTasks({ assignee: inst.name, status: "claimed" });
|
|
9997
|
+
if (tasks?.length)
|
|
9998
|
+
currentTask = tasks[0].title;
|
|
9999
|
+
}
|
|
10000
|
+
catch (err) {
|
|
10001
|
+
this.logger.debug({ err, name: inst.name }, "Scheduler listTasks failed (/api/fleet)");
|
|
10002
|
+
}
|
|
10003
|
+
return {
|
|
10004
|
+
...inst,
|
|
10005
|
+
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
10006
|
+
backend,
|
|
10007
|
+
// Settings renders these runtime-effective values rather than the
|
|
10008
|
+
// sparse user-authored YAML. `auto` means the supported CLI is
|
|
10009
|
+
// using its own effort default; null is reserved for unsupported.
|
|
10010
|
+
model: resolvedModel.model,
|
|
10011
|
+
model_display: resolvedModel.display,
|
|
10012
|
+
model_source: resolvedModel.source,
|
|
10013
|
+
effort: effortStrategy === "unsupported" ? null : (resolvedEffort.effort ?? "auto"),
|
|
10014
|
+
effort_supported: effortStrategy !== "unsupported",
|
|
10015
|
+
tool_set: config?.tool_set ?? "full",
|
|
10016
|
+
general_topic: config?.general_topic ?? false,
|
|
10017
|
+
// User activity is persisted by the daemon, so both the board and
|
|
10018
|
+
// auto-pause retain an accurate age across fleet restarts.
|
|
10019
|
+
lastActivity,
|
|
10020
|
+
currentTask,
|
|
10021
|
+
idle: this.getInstanceIdle(inst.name),
|
|
10022
|
+
state: this.getInstanceExecutionState(inst.name),
|
|
10023
|
+
};
|
|
10024
|
+
});
|
|
10025
|
+
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
10026
|
+
res.writeHead(200);
|
|
10027
|
+
res.end(JSON.stringify({
|
|
10028
|
+
...sysInfo,
|
|
10029
|
+
version: this.currentVersion,
|
|
10030
|
+
instances: enriched,
|
|
10031
|
+
}));
|
|
10032
|
+
}
|
|
10033
|
+
catch (err) {
|
|
10034
|
+
this.logger.error({ err }, "/api/fleet failed");
|
|
10035
|
+
if (!res.headersSent) {
|
|
10036
|
+
res.writeHead(500);
|
|
10037
|
+
res.end("Internal Server Error");
|
|
7846
10038
|
}
|
|
7847
|
-
|
|
7848
|
-
...inst,
|
|
7849
|
-
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
7850
|
-
backend,
|
|
7851
|
-
tool_set: config?.tool_set ?? "full",
|
|
7852
|
-
general_topic: config?.general_topic ?? false,
|
|
7853
|
-
// User activity is persisted by the daemon, so both the board and
|
|
7854
|
-
// auto-pause retain an accurate age across fleet restarts.
|
|
7855
|
-
lastActivity,
|
|
7856
|
-
currentTask,
|
|
7857
|
-
idle: this.getInstanceIdle(inst.name),
|
|
7858
|
-
state: this.getInstanceExecutionState(inst.name),
|
|
7859
|
-
};
|
|
7860
|
-
});
|
|
7861
|
-
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
7862
|
-
res.writeHead(200);
|
|
7863
|
-
res.end(JSON.stringify({
|
|
7864
|
-
...sysInfo,
|
|
7865
|
-
version: this.currentVersion,
|
|
7866
|
-
instances: enriched,
|
|
7867
|
-
}));
|
|
10039
|
+
}
|
|
7868
10040
|
return;
|
|
7869
10041
|
}
|
|
7870
10042
|
// Activity API
|
|
@@ -7994,7 +10166,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
7994
10166
|
if (err.code === "EADDRINUSE") {
|
|
7995
10167
|
if (this.healthPortRetried) {
|
|
7996
10168
|
this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
|
|
7997
|
-
this.notifyFleetError(
|
|
10169
|
+
this.notifyFleetError(t("dashboard.port_in_use", port));
|
|
7998
10170
|
return;
|
|
7999
10171
|
}
|
|
8000
10172
|
this.healthPortRetried = true;
|
|
@@ -8020,7 +10192,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8020
10192
|
return;
|
|
8021
10193
|
}
|
|
8022
10194
|
this.logger.error({ err, port }, "Health server error");
|
|
8023
|
-
this.notifyFleetError(
|
|
10195
|
+
this.notifyFleetError(t("dashboard.server_failed", err.message));
|
|
8024
10196
|
});
|
|
8025
10197
|
this.healthServer.listen(port, "127.0.0.1", () => markListening());
|
|
8026
10198
|
}
|