@songsid/agend 2.1.4-beta.5 → 2.1.4-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +12 -8
- package/dist/backend/antigravity.js +189 -42
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +62 -23
- package/dist/backend/claude-code.js +296 -43
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +148 -8
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +52 -2
- package/dist/backend/kiro.js +267 -12
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.d.ts +27 -11
- package/dist/backend/opencode.js +65 -41
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +97 -3
- package/dist/backend/types.js +12 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/backend-outage.d.ts +61 -0
- package/dist/backend-outage.js +71 -0
- package/dist/backend-outage.js.map +1 -0
- package/dist/channel/adapters/discord.d.ts +44 -2
- package/dist/channel/adapters/discord.js +526 -78
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +30 -0
- package/dist/channel/adapters/telegram.js +221 -19
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/ipc-bridge.d.ts +2 -1
- package/dist/channel/ipc-bridge.js +33 -4
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/mcp-server.js +28 -25
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +4 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.js +62 -1
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/types.d.ts +31 -1
- package/dist/classic-channel-manager.d.ts +61 -3
- package/dist/classic-channel-manager.js +256 -31
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +109 -37
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +48 -7
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +12 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +6 -0
- package/dist/cross-instance-envelope.js +30 -0
- package/dist/cross-instance-envelope.js.map +1 -0
- package/dist/daemon.d.ts +429 -9
- package/dist/daemon.js +1843 -289
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +41 -0
- package/dist/doctor.js +267 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +54 -0
- package/dist/fleet-manager.d.ts +398 -11
- package/dist/fleet-manager.js +2756 -499
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/backend-providers/SKILL.md +114 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +3 -1
- package/dist/general-knowledge/skills/delegation-playbook/SKILL.md +52 -0
- package/dist/general-knowledge/skills/development-workflow/SKILL.md +28 -0
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -0
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +64 -16
- package/dist/general-knowledge/skills/multi-channel/SKILL.md +1 -0
- package/dist/general-knowledge/skills/scheduling/SKILL.md +1 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +172 -8
- package/dist/general-knowledge/skills/tui-effort/SKILL.md +1 -0
- package/dist/general-knowledge/skills/worker-collaboration/SKILL.md +30 -0
- package/dist/instance-lifecycle.d.ts +99 -0
- package/dist/instance-lifecycle.js +339 -11
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +12 -0
- package/dist/instructions.js +35 -1
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +1004 -288
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +91 -0
- package/dist/login-flows.js +170 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/network-family.d.ts +18 -0
- package/dist/network-family.js +20 -0
- package/dist/network-family.js.map +1 -0
- package/dist/outbound-handlers.d.ts +8 -0
- package/dist/outbound-handlers.js +101 -18
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -1
- package/dist/outbound-schemas.js +9 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +52 -0
- package/dist/pane-input-residue.js +107 -0
- package/dist/pane-input-residue.js.map +1 -0
- package/dist/reply-dedup.d.ts +7 -8
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/spawn-gate.d.ts +30 -0
- package/dist/spawn-gate.js +79 -0
- package/dist/spawn-gate.js.map +1 -0
- package/dist/storm-window.d.ts +83 -0
- package/dist/storm-window.js +251 -0
- package/dist/storm-window.js.map +1 -0
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +16 -0
- package/dist/tmux-manager.js +110 -25
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-progress.d.ts +40 -0
- package/dist/tool-progress.js +289 -0
- package/dist/tool-progress.js.map +1 -0
- package/dist/topic-commands.d.ts +42 -4
- package/dist/topic-commands.js +372 -67
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.d.ts +15 -2
- package/dist/transcript-monitor.js +63 -17
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/transcript-sources.d.ts +131 -0
- package/dist/transcript-sources.js +580 -0
- package/dist/transcript-sources.js.map +1 -0
- package/dist/types.d.ts +13 -0
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +200 -60
- package/dist/ui/view.html +147 -30
- package/dist/usage/format-rich.d.ts +1 -1
- package/dist/usage/format-rich.js +28 -24
- package/dist/usage/format-rich.js.map +1 -1
- package/dist/usage/i18n-keys.d.ts +7 -0
- package/dist/usage/i18n-keys.js +34 -0
- package/dist/usage/i18n-keys.js.map +1 -0
- package/dist/usage/i18n.d.ts +5 -0
- package/dist/usage/i18n.js +27 -0
- package/dist/usage/i18n.js.map +1 -0
- package/dist/usage/providers.d.ts +21 -0
- package/dist/usage/providers.js +153 -75
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +11 -3
- package/dist/usage/usage-api.js +61 -24
- package/dist/usage/usage-api.js.map +1 -1
- package/dist/workflow-templates/default.md +2 -1
- package/package.json +2 -2
package/dist/fleet-manager.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { spawnSync } from "node:child_process";
|
|
4
|
+
import { freemem } from "node:os";
|
|
3
5
|
import { createServer } from "node:http";
|
|
4
6
|
import { join, dirname, basename } from "node:path";
|
|
5
7
|
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
10
|
import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
|
|
8
11
|
import { formatUpdateProgress } from "./update-progress.js";
|
|
9
12
|
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
10
13
|
import { readFleetMemory } from "./process-memory.js";
|
|
11
14
|
import { ReplyDeduper } from "./reply-dedup.js";
|
|
12
|
-
import { isScalar, parseDocument } from "yaml";
|
|
15
|
+
import { isMap, isScalar, parseDocument } from "yaml";
|
|
13
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
14
17
|
const __dirname = dirname(__filename);
|
|
15
18
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
@@ -30,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
30
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
31
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
32
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
33
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE,
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
34
37
|
import { DailySummary } from "./daily-summary.js";
|
|
35
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
36
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -42,9 +45,11 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
|
42
45
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
43
46
|
import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
|
|
44
47
|
import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
45
|
-
import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
|
|
48
|
+
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
|
+
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
|
+
import { LoginSession } from "./login-manager.js";
|
|
46
51
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
47
|
-
import { setLocale, detectLocale, t } from "./locale.js";
|
|
52
|
+
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
48
53
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
49
54
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
50
55
|
import { validateFleetConfig } from "./config-validator.js";
|
|
@@ -54,6 +59,11 @@ import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
|
54
59
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
55
60
|
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
56
61
|
import { RestartProgress } from "./restart-progress.js";
|
|
62
|
+
import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
|
|
63
|
+
import { StormWindow } from "./storm-window.js";
|
|
64
|
+
import { SpawnGate } from "./spawn-gate.js";
|
|
65
|
+
import { BackendOutageTracker } from "./backend-outage.js";
|
|
66
|
+
import { canUnlockAdvancedTips, DailyTipScheduler, selectTip, visibleTipLevels, } from "./tips.js";
|
|
57
67
|
import { getTmuxSession } from "./config.js";
|
|
58
68
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
59
69
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -106,6 +116,8 @@ const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
|
|
|
106
116
|
* button alone (the idle edge retires it when the run really ends).
|
|
107
117
|
*/
|
|
108
118
|
const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
|
|
119
|
+
/** Bound for the daemon to capture the pane and answer a post-reply state query. */
|
|
120
|
+
const REPLY_STATE_REFRESH_TIMEOUT_MS = 2_000;
|
|
109
121
|
/**
|
|
110
122
|
* The daemon only broadcasts execution state on TRANSITIONS, so a long
|
|
111
123
|
* single-state run sends nothing for hours. The idle backstop therefore pokes a
|
|
@@ -135,6 +147,8 @@ const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
|
|
|
135
147
|
* exactly one progress message per turn, and it is the cancel button itself.
|
|
136
148
|
*/
|
|
137
149
|
const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
|
|
150
|
+
/** Floor between tool-progress-driven bubble edits (Telegram flood safety). */
|
|
151
|
+
const TOOL_PROGRESS_EDIT_MIN_MS = 4_000;
|
|
138
152
|
/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
|
|
139
153
|
/**
|
|
140
154
|
* Default delay before the button starts showing elapsed time. Configurable via
|
|
@@ -160,6 +174,30 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
160
174
|
* emoji never changes the documented delivery-state protocol.
|
|
161
175
|
*/
|
|
162
176
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
177
|
+
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
178
|
+
"tool_progress",
|
|
179
|
+
"mcp_proxy_reply",
|
|
180
|
+
"auto_pause_after",
|
|
181
|
+
"warm_cap",
|
|
182
|
+
"display_name",
|
|
183
|
+
"description",
|
|
184
|
+
"tags",
|
|
185
|
+
"log_level",
|
|
186
|
+
]);
|
|
187
|
+
function splitHotColdConfig(config) {
|
|
188
|
+
const hot = {};
|
|
189
|
+
const cold = {};
|
|
190
|
+
for (const [key, value] of Object.entries(config)) {
|
|
191
|
+
(HOT_INSTANCE_CONFIG_KEYS.has(key) ? hot : cold)[key] = value;
|
|
192
|
+
}
|
|
193
|
+
return { hot, cold };
|
|
194
|
+
}
|
|
195
|
+
function hotConfigUpdate(config) {
|
|
196
|
+
const update = {};
|
|
197
|
+
for (const key of HOT_INSTANCE_CONFIG_KEYS)
|
|
198
|
+
update[key] = config[key] ?? null;
|
|
199
|
+
return update;
|
|
200
|
+
}
|
|
163
201
|
/**
|
|
164
202
|
* How long a delivery waits out a disconnected instance IPC before giving up.
|
|
165
203
|
*
|
|
@@ -175,8 +213,19 @@ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
|
|
|
175
213
|
const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
|
|
176
214
|
const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
|
|
177
215
|
const HANG_CALLBACK_PREFIX = "hang:";
|
|
178
|
-
|
|
216
|
+
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
217
|
+
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
218
|
+
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
219
|
+
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
220
|
+
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
221
|
+
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
222
|
+
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
223
|
+
const LOGIN_CONFIRM_CALLBACK_PREFIX = "login-confirm:";
|
|
224
|
+
const INSTALL_LOGIN_CALLBACK_PREFIX = "install-login:";
|
|
225
|
+
const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
|
|
226
|
+
/** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
|
|
179
227
|
const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
|
|
228
|
+
const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
|
|
180
229
|
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
181
230
|
export class FleetManager {
|
|
182
231
|
dataDir;
|
|
@@ -184,6 +233,10 @@ export class FleetManager {
|
|
|
184
233
|
static sighupHandlerInstalled = false;
|
|
185
234
|
children = new Map();
|
|
186
235
|
lifecycle;
|
|
236
|
+
stormWindow;
|
|
237
|
+
spawnGate;
|
|
238
|
+
/** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
|
|
239
|
+
backendOutage = new BackendOutageTracker();
|
|
187
240
|
/** Live view of lifecycle.daemons — used throughout; not deprecated. */
|
|
188
241
|
get daemons() { return this.lifecycle.daemons; }
|
|
189
242
|
fleetConfig = null;
|
|
@@ -223,6 +276,7 @@ export class FleetManager {
|
|
|
223
276
|
costGuard = null;
|
|
224
277
|
statuslineWatcher;
|
|
225
278
|
dailySummary = null;
|
|
279
|
+
dailyTipScheduler = null;
|
|
226
280
|
webhookEmitter = null;
|
|
227
281
|
// Topic icon + auto-archive state
|
|
228
282
|
topicIcons = {};
|
|
@@ -235,6 +289,13 @@ export class FleetManager {
|
|
|
235
289
|
warmCapEvicting = new Set();
|
|
236
290
|
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
237
291
|
idleGatedDeliveryTails = new Map();
|
|
292
|
+
/**
|
|
293
|
+
* Per-instance cancellation epoch for deliveries which have not reached the
|
|
294
|
+
* daemon yet. A cancel advances the epoch; queued work captures the old value
|
|
295
|
+
* and becomes a no-op, while messages arriving after the click capture the new
|
|
296
|
+
* value and remain deliverable.
|
|
297
|
+
*/
|
|
298
|
+
deliveryEpochs = new Map();
|
|
238
299
|
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
239
300
|
lastDeliveryAt = new Map();
|
|
240
301
|
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
@@ -251,10 +312,34 @@ export class FleetManager {
|
|
|
251
312
|
replyDeduper = new ReplyDeduper();
|
|
252
313
|
/** instanceName → what it is doing right now, when the backend can tell us. */
|
|
253
314
|
instanceActivity = new Map();
|
|
315
|
+
/** instanceName → this turn's tool list (multi-line), for the bubble. */
|
|
316
|
+
instanceProgress = new Map();
|
|
254
317
|
/** instanceName → tail of deliveries waiting for its IPC to come back. */
|
|
255
318
|
ipcWaitTails = new Map();
|
|
256
319
|
/** instanceName → restart currently executing; concurrent callers join it. */
|
|
257
320
|
restartsInFlight = new Map();
|
|
321
|
+
/**
|
|
322
|
+
* Delayed automatic retries for instances whose startup failed. Before this a
|
|
323
|
+
* failed start was logged once and the instance stayed `stopped` until an
|
|
324
|
+
* operator noticed (2026-09-03: 4 kiro instances, all victims of the same
|
|
325
|
+
* backend outage during a post-update herd). instanceName → pending retry.
|
|
326
|
+
*/
|
|
327
|
+
startupRetries = new Map();
|
|
328
|
+
/** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
|
|
329
|
+
explicitStopGeneration = new Map();
|
|
330
|
+
/** instanceName → outage hand-off currently executing; a repeat joins it. */
|
|
331
|
+
handOffsInFlight = new Map();
|
|
332
|
+
/** instanceName → explicit stop currently executing; the outage hand-off waits for it. */
|
|
333
|
+
stopsInFlight = new Map();
|
|
334
|
+
/** Aggregation window for the "N instances failed to start" notice. */
|
|
335
|
+
startupRetryNotices = new Map();
|
|
336
|
+
/** Backoff between automatic startup retries; the last step repeats while the backend is down. */
|
|
337
|
+
static STARTUP_RETRY_BACKOFF_MS = [60_000, 5 * 60_000, 15 * 60_000];
|
|
338
|
+
/** Hard cap on automatic startup retries (3 backoff steps + up to 3 more during a backend outage). */
|
|
339
|
+
static STARTUP_RETRY_MAX_ATTEMPTS = 6;
|
|
340
|
+
/** Re-check interval when a retry is due but the tmux storm window still blocks spawns. */
|
|
341
|
+
static STARTUP_RETRY_STORM_DEFER_MS = 60_000;
|
|
342
|
+
static STARTUP_RETRY_NOTICE_AGGREGATE_MS = 1_000;
|
|
258
343
|
// Last user message delivered to each instance — used to react ✅ on completion.
|
|
259
344
|
lastInboundMsg = new Map();
|
|
260
345
|
topicArchiver;
|
|
@@ -291,12 +376,14 @@ export class FleetManager {
|
|
|
291
376
|
updateProgressTimer = null;
|
|
292
377
|
updateProgressEditRunning = false;
|
|
293
378
|
lastUpdateProgressText = null;
|
|
379
|
+
updateCompletionTipText = null;
|
|
294
380
|
eventLogPruneTimer = null;
|
|
295
381
|
logRotateTimer = null;
|
|
296
382
|
/** Days of event/activity history to keep. */
|
|
297
383
|
static EVENT_LOG_RETENTION_DAYS = 30;
|
|
298
384
|
watchdogTimer = null;
|
|
299
385
|
startedAt = 0;
|
|
386
|
+
stormOpenNotifyTimer = null;
|
|
300
387
|
// Mirror topic: buffer cross-instance messages, flush every 3s
|
|
301
388
|
mirrorBuffer = [];
|
|
302
389
|
mirrorTimer = null;
|
|
@@ -312,11 +399,63 @@ export class FleetManager {
|
|
|
312
399
|
process.on("SIGHUP", () => FleetManager.signalTarget?.handleSighup());
|
|
313
400
|
FleetManager.sighupHandlerInstalled = true;
|
|
314
401
|
}
|
|
402
|
+
this.stormWindow = new StormWindow();
|
|
403
|
+
this.spawnGate = new SpawnGate({
|
|
404
|
+
storm: this.stormWindow,
|
|
405
|
+
concurrency: () => this.spawnConcurrency(),
|
|
406
|
+
staggerMs: () => this.fleetConfig?.defaults?.startup?.stagger_delay_ms ?? 500,
|
|
407
|
+
});
|
|
408
|
+
this.bindStormWindowEvents();
|
|
315
409
|
this.lifecycle = new InstanceLifecycle(this);
|
|
316
410
|
this.topicCommands = new TopicCommands(this);
|
|
317
411
|
this.topicArchiver = new TopicArchiver(this);
|
|
318
412
|
this.statuslineWatcher = new StatuslineWatcher(this);
|
|
319
413
|
}
|
|
414
|
+
spawnConcurrency() {
|
|
415
|
+
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
416
|
+
if (explicit != null)
|
|
417
|
+
return Math.max(1, Math.min(20, explicit));
|
|
418
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
419
|
+
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
420
|
+
}
|
|
421
|
+
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
422
|
+
bindStormWindowEvents() {
|
|
423
|
+
this.stormWindow.on("opened", (snapshot) => {
|
|
424
|
+
for (const [name, daemon] of this.daemons) {
|
|
425
|
+
if (!daemon.isPaused)
|
|
426
|
+
this.stormWindow.addAffected(name);
|
|
427
|
+
}
|
|
428
|
+
this.logger.error({ ...snapshot }, "tmux server storm opened — holding respawns and delivery");
|
|
429
|
+
if (this.stormOpenNotifyTimer)
|
|
430
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
431
|
+
this.stormOpenNotifyTimer = setTimeout(() => {
|
|
432
|
+
this.stormOpenNotifyTimer = null;
|
|
433
|
+
const current = this.stormWindow.snapshot();
|
|
434
|
+
this.notifyFleetError(t("storm.opened", current.affected.length, this.formatStormDelay(current.backoffMs)));
|
|
435
|
+
}, 1_000);
|
|
436
|
+
this.stormOpenNotifyTimer.unref?.();
|
|
437
|
+
});
|
|
438
|
+
this.stormWindow.on("extended", (snapshot) => {
|
|
439
|
+
this.logger.error({ ...snapshot }, "tmux server storm repeated — backoff extended");
|
|
440
|
+
this.notifyFleetError(t("storm.extended", snapshot.crashCount, this.formatStormDelay(snapshot.backoffMs)));
|
|
441
|
+
});
|
|
442
|
+
this.stormWindow.on("recovery_due", (snapshot) => {
|
|
443
|
+
this.logger.warn({ ...snapshot }, "tmux storm backoff elapsed — rolling recovery started");
|
|
444
|
+
});
|
|
445
|
+
this.stormWindow.on("closed", (snapshot, reason) => {
|
|
446
|
+
const unresolved = snapshot.affected.filter(name => !snapshot.recovered.includes(name));
|
|
447
|
+
this.logger.info({ ...snapshot, reason, unresolved }, "tmux server storm closed");
|
|
448
|
+
this.notifyFleetError(t(reason === "timeout" ? "storm.timed_out" : "storm.recovered", snapshot.recovered.length, snapshot.affected.length, unresolved.length > 0 ? unresolved.join(", ") : t("storm.none")));
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
formatStormDelay(ms) {
|
|
452
|
+
if (ms >= 60_000)
|
|
453
|
+
return `${Math.round(ms / 60_000)}m`;
|
|
454
|
+
return `${Math.round(ms / 1_000)}s`;
|
|
455
|
+
}
|
|
456
|
+
stormSuppressed(kind) {
|
|
457
|
+
return this.stormWindow.shouldSuppress(kind);
|
|
458
|
+
}
|
|
320
459
|
handleSighup() {
|
|
321
460
|
this.logger.info("Received SIGHUP, hot-reloading config...");
|
|
322
461
|
if (!this.startupComplete) {
|
|
@@ -339,7 +478,7 @@ export class FleetManager {
|
|
|
339
478
|
// fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
|
|
340
479
|
this.logger.error({ err }, "SIGHUP config reload failed");
|
|
341
480
|
const message = err instanceof Error ? err.message : String(err);
|
|
342
|
-
this.notifyFleetError(
|
|
481
|
+
this.notifyFleetError(t("fleet.reload_failed", message));
|
|
343
482
|
})
|
|
344
483
|
.finally(() => {
|
|
345
484
|
this.reconcileInFlight = null;
|
|
@@ -589,6 +728,57 @@ export class FleetManager {
|
|
|
589
728
|
child.once("error", err => this.failUpdateProgress(err.message));
|
|
590
729
|
child.unref();
|
|
591
730
|
}
|
|
731
|
+
async handleTipsSlash(data, adapterId) {
|
|
732
|
+
if (!this.fleetConfig)
|
|
733
|
+
return;
|
|
734
|
+
const mode = typeof data.options?.mode === "string" ? data.options.mode : "";
|
|
735
|
+
if (mode === "advanced on") {
|
|
736
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
737
|
+
await data.respond(t("permission.denied"));
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
await data.respond(this.unlockAdvancedTips(data.userId)
|
|
741
|
+
? t("tips.advanced.unlocked")
|
|
742
|
+
: t("tips.unavailable"));
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
745
|
+
if (mode === "on" || mode === "off") {
|
|
746
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
747
|
+
await data.respond(t("permission.denied"));
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
this.fleetConfig.defaults.tips = mode === "on";
|
|
751
|
+
this.saveFleetConfig();
|
|
752
|
+
await data.respond(t(mode === "on" ? "tips.enabled" : "tips.disabled"));
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
756
|
+
if (!adapter) {
|
|
757
|
+
await data.respond(t("tips.unavailable"));
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
const targetName = this.resolveSlashTarget(data.channelId, adapterId)
|
|
761
|
+
?? this.findGeneralInstance(adapterId)
|
|
762
|
+
?? "general";
|
|
763
|
+
// Discord slash interactions are deferred ephemerally, but the Tip itself
|
|
764
|
+
// belongs in the channel where /tips was invoked. Post there directly,
|
|
765
|
+
// then delete the empty acknowledgement instead of leaving a slash corpse.
|
|
766
|
+
//
|
|
767
|
+
// The callback binding uses (chatId, threadId) that match what the adapter
|
|
768
|
+
// emits on callback_query. Discord emits (guildId, channelId); Telegram
|
|
769
|
+
// emits (supergroup chatId, topic threadId). For /tips invoked in an
|
|
770
|
+
// arbitrary channel, pass the canonical group id as chatId and the
|
|
771
|
+
// invocation channel as threadId, exactly as sendTipToGeneral() does.
|
|
772
|
+
const chatId = this.getGroupIdForInstance(targetName) || data.channelId;
|
|
773
|
+
const result = await this.promptTip(targetName, adapter, chatId, data.channelId);
|
|
774
|
+
if (result === "posted" && data.dismissResponse) {
|
|
775
|
+
await data.dismissResponse();
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
await data.respond(t(result === "posted"
|
|
779
|
+
? "tips.posted"
|
|
780
|
+
: result === "empty" ? "tips.empty" : "tips.unavailable"));
|
|
781
|
+
}
|
|
592
782
|
/** Admin-only full conversation reset for fleet-topic and Classic instances. */
|
|
593
783
|
async handleClearSlash(data, adapterId) {
|
|
594
784
|
if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
|
|
@@ -600,7 +790,14 @@ export class FleetManager {
|
|
|
600
790
|
await data.respond(t("classic.no_agent"));
|
|
601
791
|
return;
|
|
602
792
|
}
|
|
603
|
-
|
|
793
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
794
|
+
if (!adapter) {
|
|
795
|
+
await data.respond(t("clear.prompt_unavailable"));
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
const chatId = String(this.getChannelConfig(adapterId)?.group_id ?? data.channelId);
|
|
799
|
+
const fallback = await this.promptClearConfirmation(name, data.channelId, adapter, chatId, data.channelId);
|
|
800
|
+
await data.respond(fallback ?? t("clear.confirm_posted"));
|
|
604
801
|
}
|
|
605
802
|
/** Get the adapter bound to an instance, falling back to primary adapter */
|
|
606
803
|
getAdapterForInstance(name) {
|
|
@@ -734,6 +931,39 @@ export class FleetManager {
|
|
|
734
931
|
return null;
|
|
735
932
|
return this.instanceStateCache.get(name)?.state ?? null;
|
|
736
933
|
}
|
|
934
|
+
/**
|
|
935
|
+
* Subscription providers currently relevant to this fleet. Stopped/crashed
|
|
936
|
+
* rows do not keep a usage card alive, while persisted paused instances do.
|
|
937
|
+
* Classic channels use their effective backend merge chain, not the raw row.
|
|
938
|
+
*/
|
|
939
|
+
getActiveUsageProviderIds() {
|
|
940
|
+
const providers = new Set();
|
|
941
|
+
for (const backend of this.getActiveBackendIds()) {
|
|
942
|
+
const provider = usageProviderIdForBackend(backend);
|
|
943
|
+
if (provider)
|
|
944
|
+
providers.add(provider);
|
|
945
|
+
}
|
|
946
|
+
return providers;
|
|
947
|
+
}
|
|
948
|
+
/** Effective backends with a running or persisted-paused fleet/Classic instance. */
|
|
949
|
+
getActiveBackendIds() {
|
|
950
|
+
const backends = new Set();
|
|
951
|
+
const add = (name, backend) => {
|
|
952
|
+
const status = this.getInstanceStatus(name);
|
|
953
|
+
if (status !== "running" && status !== "paused")
|
|
954
|
+
return;
|
|
955
|
+
if (backend)
|
|
956
|
+
backends.add(backend);
|
|
957
|
+
};
|
|
958
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
959
|
+
// loadFleetConfig() has already merged the fleet default into each row.
|
|
960
|
+
add(name, config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
961
|
+
}
|
|
962
|
+
for (const channel of this.classicChannels?.getAll() ?? []) {
|
|
963
|
+
add(channel.instanceName, this.classicChannels?.getBackendByInstance(channel.instanceName, this.fleetConfig?.defaults?.backend));
|
|
964
|
+
}
|
|
965
|
+
return backends;
|
|
966
|
+
}
|
|
737
967
|
isClassicInstance(name) {
|
|
738
968
|
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
739
969
|
}
|
|
@@ -846,7 +1076,7 @@ export class FleetManager {
|
|
|
846
1076
|
.finally(() => this.warmCapEvicting.delete(victim));
|
|
847
1077
|
}
|
|
848
1078
|
}
|
|
849
|
-
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
1079
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0, cancelled) {
|
|
850
1080
|
const isReady = () => {
|
|
851
1081
|
const snapshot = this.instanceStateCache.get(instanceName);
|
|
852
1082
|
return snapshot?.state === "idle"
|
|
@@ -868,8 +1098,12 @@ export class FleetManager {
|
|
|
868
1098
|
this.instanceIdleWaiters.delete(instanceName);
|
|
869
1099
|
resolve(idle);
|
|
870
1100
|
};
|
|
871
|
-
const check = () => {
|
|
872
|
-
|
|
1101
|
+
const check = () => {
|
|
1102
|
+
if (cancelled?.())
|
|
1103
|
+
finish(false);
|
|
1104
|
+
else if (isReady())
|
|
1105
|
+
finish(true);
|
|
1106
|
+
};
|
|
873
1107
|
const query = () => {
|
|
874
1108
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
875
1109
|
if (ipc?.connected) {
|
|
@@ -885,24 +1119,90 @@ export class FleetManager {
|
|
|
885
1119
|
query();
|
|
886
1120
|
});
|
|
887
1121
|
}
|
|
888
|
-
|
|
1122
|
+
/**
|
|
1123
|
+
* Ask the daemon to capture the pane before answering, then wait for any state
|
|
1124
|
+
* report produced after this request. The ordinary query is intentionally
|
|
1125
|
+
* cache-only (idle gates call it frequently); this opt-in refresh is reserved
|
|
1126
|
+
* for lifecycle decisions where a stale state would strand UI.
|
|
1127
|
+
*/
|
|
1128
|
+
refreshInstanceExecutionState(instanceName, timeoutMs) {
|
|
1129
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1130
|
+
if (!ipc?.connected)
|
|
1131
|
+
return Promise.resolve(false);
|
|
1132
|
+
const previous = this.instanceStateCache.get(instanceName);
|
|
1133
|
+
return new Promise(resolve => {
|
|
1134
|
+
let settled = false;
|
|
1135
|
+
const finish = (refreshed) => {
|
|
1136
|
+
if (settled)
|
|
1137
|
+
return;
|
|
1138
|
+
settled = true;
|
|
1139
|
+
clearTimeout(timeout);
|
|
1140
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
1141
|
+
waiters?.delete(check);
|
|
1142
|
+
if (waiters?.size === 0)
|
|
1143
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
1144
|
+
resolve(refreshed);
|
|
1145
|
+
};
|
|
1146
|
+
const check = () => {
|
|
1147
|
+
if (this.instanceStateCache.get(instanceName) !== previous)
|
|
1148
|
+
finish(true);
|
|
1149
|
+
};
|
|
1150
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
1151
|
+
waiters.add(check);
|
|
1152
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
1153
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
1154
|
+
timeout.unref?.();
|
|
1155
|
+
const sent = ipc.send({
|
|
1156
|
+
type: "query_instance_state",
|
|
1157
|
+
requestId: `reply-grace-${Date.now()}`,
|
|
1158
|
+
refresh: true,
|
|
1159
|
+
});
|
|
1160
|
+
if (!sent)
|
|
1161
|
+
finish(false);
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch) {
|
|
1165
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1166
|
+
return;
|
|
1167
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1168
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1169
|
+
return;
|
|
889
1170
|
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
890
1171
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
891
1172
|
const wakeStartedAt = Date.now();
|
|
892
1173
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1174
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1175
|
+
return;
|
|
893
1176
|
// Waking added one to the warm count — make room by evicting a different
|
|
894
1177
|
// LRU idle instance (never this one; it's about to work).
|
|
895
1178
|
this.enforceWarmCap(instanceName);
|
|
896
1179
|
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
897
1180
|
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
898
1181
|
}
|
|
899
|
-
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
1182
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter, () => !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch));
|
|
1183
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch)) {
|
|
1184
|
+
this.logger.info({ instanceName }, "Pending delivery dropped by user cancel");
|
|
1185
|
+
return;
|
|
1186
|
+
}
|
|
1187
|
+
// A server crash can land while waitForInstanceIdle is pending. Re-check
|
|
1188
|
+
// immediately before the old timeout path would force text into a boot UI.
|
|
1189
|
+
await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
|
|
1190
|
+
if (this.shuttingDown || !this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1191
|
+
return;
|
|
900
1192
|
if (!idle) {
|
|
901
1193
|
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
902
1194
|
}
|
|
903
|
-
await this.sendWhenConnected(instanceName, payload);
|
|
1195
|
+
await this.sendWhenConnected(instanceName, payload, deliveryEpoch);
|
|
1196
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1197
|
+
return;
|
|
904
1198
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
905
1199
|
}
|
|
1200
|
+
async holdDeliveryForStorm(instanceName, deliveryEpoch) {
|
|
1201
|
+
if (!this.stormWindow.isDeliveryHeld(instanceName))
|
|
1202
|
+
return;
|
|
1203
|
+
this.logger.warn({ instanceName, deliveryEpoch }, "Delivery held during tmux server recovery");
|
|
1204
|
+
await this.stormWindow.waitForDeliveryAllowed(instanceName);
|
|
1205
|
+
}
|
|
906
1206
|
/**
|
|
907
1207
|
* Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
|
|
908
1208
|
*
|
|
@@ -919,7 +1219,9 @@ export class FleetManager {
|
|
|
919
1219
|
* instance, *including* when the socket happens to be up: otherwise a message
|
|
920
1220
|
* arriving after the reconnect could overtake one that has been waiting for it.
|
|
921
1221
|
*/
|
|
922
|
-
async sendWhenConnected(instanceName, payload) {
|
|
1222
|
+
async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
1223
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1224
|
+
return;
|
|
923
1225
|
const queued = this.ipcWaitTails.get(instanceName);
|
|
924
1226
|
if (!queued) {
|
|
925
1227
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -928,7 +1230,11 @@ export class FleetManager {
|
|
|
928
1230
|
}
|
|
929
1231
|
const attempt = (queued ?? Promise.resolve())
|
|
930
1232
|
.catch(() => { })
|
|
931
|
-
.then(() =>
|
|
1233
|
+
.then(() => {
|
|
1234
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1235
|
+
return;
|
|
1236
|
+
return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch);
|
|
1237
|
+
});
|
|
932
1238
|
// The chain stores a settled-either-way promise so one failed delivery cannot
|
|
933
1239
|
// wedge every later one, and so `queued` above is safe to await unguarded.
|
|
934
1240
|
const tail = attempt.catch(() => { });
|
|
@@ -945,10 +1251,12 @@ export class FleetManager {
|
|
|
945
1251
|
}
|
|
946
1252
|
}
|
|
947
1253
|
/** Poll for the instance's IPC to come back, then send. Throws if it does not. */
|
|
948
|
-
async sendAfterIpcReturns(instanceName, payload) {
|
|
1254
|
+
async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName)) {
|
|
949
1255
|
const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
|
|
950
1256
|
let warned = false;
|
|
951
1257
|
for (;;) {
|
|
1258
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1259
|
+
return;
|
|
952
1260
|
// Re-read every round: a reconnect replaces the IpcClient object entirely,
|
|
953
1261
|
// so a cached reference would stay dead forever.
|
|
954
1262
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
@@ -966,6 +1274,8 @@ export class FleetManager {
|
|
|
966
1274
|
}
|
|
967
1275
|
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
968
1276
|
async deliverToInstance(instanceName, payload, options = {}) {
|
|
1277
|
+
const deliveryEpoch = this.getDeliveryEpoch(instanceName);
|
|
1278
|
+
const deliveryPayload = { ...payload, delivery_epoch: deliveryEpoch };
|
|
969
1279
|
const meta = payload.meta && typeof payload.meta === "object"
|
|
970
1280
|
? payload.meta
|
|
971
1281
|
: undefined;
|
|
@@ -977,16 +1287,20 @@ export class FleetManager {
|
|
|
977
1287
|
if (!waitForIdle) {
|
|
978
1288
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
979
1289
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
1290
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1291
|
+
return;
|
|
980
1292
|
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
981
1293
|
}
|
|
982
|
-
await this.sendWhenConnected(instanceName,
|
|
1294
|
+
await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch);
|
|
1295
|
+
if (!this.isDeliveryEpochCurrent(instanceName, deliveryEpoch))
|
|
1296
|
+
return;
|
|
983
1297
|
// A cross-instance item arriving before the daemon observes this turn as
|
|
984
1298
|
// working must not trust the stale idle snapshot from before the send.
|
|
985
1299
|
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
986
1300
|
return;
|
|
987
1301
|
}
|
|
988
1302
|
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
989
|
-
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName,
|
|
1303
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch));
|
|
990
1304
|
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
991
1305
|
try {
|
|
992
1306
|
await delivery;
|
|
@@ -1037,6 +1351,9 @@ export class FleetManager {
|
|
|
1037
1351
|
* across a fleet restart.
|
|
1038
1352
|
*/
|
|
1039
1353
|
resumePaused = false) {
|
|
1354
|
+
// Any start supersedes a pending automatic retry (it would otherwise fire
|
|
1355
|
+
// into a running instance — harmless, but noisy — or race this start).
|
|
1356
|
+
this.cancelStartupRetry(name);
|
|
1040
1357
|
if (resumePaused && this.lifecycle.isPaused(name)) {
|
|
1041
1358
|
await this.lifecycle.wake(name, 30_000);
|
|
1042
1359
|
// A successful wake clears the persisted pause marker and produces a
|
|
@@ -1065,19 +1382,28 @@ export class FleetManager {
|
|
|
1065
1382
|
this.logger.info({ name }, "Instance already running, skipping");
|
|
1066
1383
|
return;
|
|
1067
1384
|
}
|
|
1385
|
+
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
1068
1386
|
if (config.general_topic) {
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1387
|
+
this.ensureGeneralInstructions(config.working_directory, backend, name);
|
|
1388
|
+
}
|
|
1389
|
+
else if (kind === "fleet-topic") {
|
|
1390
|
+
// Workers get only role-eligible on-demand skills. Classic instances are
|
|
1391
|
+
// deliberately excluded: their workspace and conversation lifecycle are
|
|
1392
|
+
// managed independently from fleet-topic workers.
|
|
1393
|
+
try {
|
|
1394
|
+
const skillsWorkDir = this.resolveKnowledgeWorkDir(config.working_directory, backend, name);
|
|
1395
|
+
this.syncRoleSkills(skillsWorkDir, backend, "worker");
|
|
1396
|
+
}
|
|
1397
|
+
catch (err) {
|
|
1398
|
+
// Skill publishing is additive. A read-only or temporarily unavailable
|
|
1399
|
+
// workspace must not turn an otherwise valid worker startup into a
|
|
1400
|
+
// fleet outage.
|
|
1401
|
+
this.logger.warn({ err, name, backend }, "Failed to sync worker skills — continuing startup");
|
|
1075
1402
|
}
|
|
1076
|
-
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
1077
1403
|
}
|
|
1078
1404
|
await this.lifecycle.start(name, config, topicMode, {
|
|
1079
1405
|
kind,
|
|
1080
|
-
backend
|
|
1406
|
+
backend,
|
|
1081
1407
|
model: this.resolveInstanceModel(name).display,
|
|
1082
1408
|
});
|
|
1083
1409
|
// Only clear a stale process status after a real start succeeded. Clearing
|
|
@@ -1123,86 +1449,243 @@ export class FleetManager {
|
|
|
1123
1449
|
}
|
|
1124
1450
|
if (runnableEntries.length === 0)
|
|
1125
1451
|
return;
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1452
|
+
if (this.fleetConfig?.defaults?.startup?.concurrency == null) {
|
|
1453
|
+
this.logger.info({
|
|
1454
|
+
concurrency: this.spawnConcurrency(),
|
|
1455
|
+
freeMemMB: Math.round(freemem() / (1024 * 1024)),
|
|
1456
|
+
totalInstances: runnableEntries.length,
|
|
1457
|
+
}, "Adaptive startup concurrency");
|
|
1458
|
+
}
|
|
1459
|
+
await Promise.all(runnableEntries.map(([name, config]) => this.spawnGate.run({
|
|
1460
|
+
instanceName: name,
|
|
1461
|
+
workingDirectory: config.working_directory,
|
|
1462
|
+
reason: "startup",
|
|
1463
|
+
}, async () => {
|
|
1464
|
+
if (await this.startInstanceUnattended(name, config, topicMode, "instance"))
|
|
1465
|
+
onReady?.(name);
|
|
1466
|
+
})));
|
|
1467
|
+
}
|
|
1468
|
+
/**
|
|
1469
|
+
* Start from an UNATTENDED path (fleet startup, full restart, config
|
|
1470
|
+
* reconcile): nobody is watching the result, so a failure is logged and
|
|
1471
|
+
* handed to the delayed automatic retry instead of leaving the instance
|
|
1472
|
+
* `stopped` forever. Explicit operator/API starts call startInstance directly
|
|
1473
|
+
* and keep their synchronous error. Returns whether the instance is up.
|
|
1474
|
+
*/
|
|
1475
|
+
async startInstanceUnattended(name, config, topicMode, what) {
|
|
1476
|
+
try {
|
|
1477
|
+
await this.startInstance(name, config, topicMode);
|
|
1478
|
+
return this.daemons.has(name);
|
|
1136
1479
|
}
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1480
|
+
catch (err) {
|
|
1481
|
+
this.logger.error({ err, name }, `Failed to start ${what}`);
|
|
1482
|
+
this.scheduleStartupRetry(name, 0);
|
|
1483
|
+
return false;
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
// ── Delayed automatic startup retries ──────────────────────────────────
|
|
1487
|
+
/**
|
|
1488
|
+
* Schedule attempt `attempt` (0-based) of the automatic startup retry for an
|
|
1489
|
+
* instance whose start just failed. Backoff 1 → 5 → 15 min; while the
|
|
1490
|
+
* instance's backend is known to be unreachable the 15-min step repeats up to
|
|
1491
|
+
* STARTUP_RETRY_MAX_ATTEMPTS, after which we give up with one notice. Each
|
|
1492
|
+
* retry re-checks the world (still configured, not running, not paused, no
|
|
1493
|
+
* tmux storm) and runs through the SpawnGate as "recovery" — so a herd of
|
|
1494
|
+
* failed instances comes back at the gate's concurrency, never all at once.
|
|
1495
|
+
*/
|
|
1496
|
+
scheduleStartupRetry(name, attempt) {
|
|
1497
|
+
if (this.shuttingDown)
|
|
1498
|
+
return;
|
|
1499
|
+
if (this.startupRetries.has(name))
|
|
1500
|
+
return;
|
|
1501
|
+
const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
|
|
1502
|
+
const outage = this.backendOutage.isActive(this.backendNameOf(name));
|
|
1503
|
+
const exhausted = attempt >= backoff.length && !(outage && attempt < FleetManager.STARTUP_RETRY_MAX_ATTEMPTS);
|
|
1504
|
+
if (attempt >= FleetManager.STARTUP_RETRY_MAX_ATTEMPTS || exhausted) {
|
|
1505
|
+
this.logger.error({ name, attempts: attempt }, "Giving up automatic startup retries");
|
|
1506
|
+
this.queueStartupRetryNotice("gave_up", name, 0);
|
|
1507
|
+
return;
|
|
1508
|
+
}
|
|
1509
|
+
const delayMs = backoff[Math.min(attempt, backoff.length - 1)];
|
|
1510
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, delayMs);
|
|
1511
|
+
timer.unref?.();
|
|
1512
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1513
|
+
this.logger.warn({ name, attempt: attempt + 1, delayMs, backendOutage: outage }, "Startup failed — automatic retry scheduled");
|
|
1514
|
+
if (attempt === 0)
|
|
1515
|
+
this.queueStartupRetryNotice("scheduled", name, delayMs);
|
|
1516
|
+
}
|
|
1517
|
+
/** Drop a pending automatic retry (an operator start/stop/restart supersedes it). */
|
|
1518
|
+
cancelStartupRetry(name) {
|
|
1519
|
+
const pending = this.startupRetries.get(name);
|
|
1520
|
+
if (!pending)
|
|
1521
|
+
return;
|
|
1522
|
+
clearTimeout(pending.timer);
|
|
1523
|
+
this.startupRetries.delete(name);
|
|
1524
|
+
this.logger.info({ name }, "Pending automatic startup retry cancelled");
|
|
1525
|
+
}
|
|
1526
|
+
/** Pending automatic retry, if any (status display / tests). */
|
|
1527
|
+
pendingStartupRetry(name) {
|
|
1528
|
+
const pending = this.startupRetries.get(name);
|
|
1529
|
+
return pending ? { attempt: pending.attempt } : null;
|
|
1530
|
+
}
|
|
1531
|
+
/**
|
|
1532
|
+
* A daemon's crash-respawn hit the backend outage (startup_backend_unreachable):
|
|
1533
|
+
* stop THAT daemon and schedule the delayed retry. Serialized against the
|
|
1534
|
+
* operator paths: an in-flight restart is awaited first (it replaces the
|
|
1535
|
+
* daemon itself), the stop is identity-checked so a fresh daemon registered
|
|
1536
|
+
* meanwhile is never deleted, and an explicit stop/restart that began during
|
|
1537
|
+
* the hand-off (generation bump) owns the outcome — no retry is scheduled
|
|
1538
|
+
* behind an operator's back.
|
|
1539
|
+
*/
|
|
1540
|
+
async handOffToStartupRetry(name, daemon) {
|
|
1541
|
+
const joined = this.handOffsInFlight.get(name);
|
|
1542
|
+
if (joined)
|
|
1543
|
+
return joined;
|
|
1544
|
+
const run = (async () => {
|
|
1545
|
+
// Let any operator restart/stop that is already executing finish first:
|
|
1546
|
+
// a restart replaces the daemon itself, a stop removes it — either way
|
|
1547
|
+
// the identity check below then sees "not ours any more" and we do
|
|
1548
|
+
// nothing. Loop: one operation can be chained behind another.
|
|
1549
|
+
for (;;) {
|
|
1550
|
+
const inFlight = this.restartsInFlight.get(name) ?? this.stopsInFlight.get(name);
|
|
1551
|
+
if (!inFlight)
|
|
1552
|
+
break;
|
|
1553
|
+
await inFlight.catch(() => { });
|
|
1554
|
+
}
|
|
1555
|
+
const stopGen = this.explicitStopGeneration.get(name) ?? 0;
|
|
1556
|
+
if (this.lifecycle.daemons.get(name) !== daemon)
|
|
1557
|
+
return; // already replaced or stopped
|
|
1558
|
+
await this.lifecycle.stopIfCurrent(name, daemon);
|
|
1559
|
+
if (this.shuttingDown)
|
|
1560
|
+
return;
|
|
1561
|
+
if ((this.explicitStopGeneration.get(name) ?? 0) !== stopGen) {
|
|
1562
|
+
this.logger.info({ name }, "Outage hand-off superseded by an explicit stop/restart — no automatic retry");
|
|
1157
1563
|
return;
|
|
1158
1564
|
}
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1565
|
+
if (this.lifecycle.daemons.has(name) || this.lifecycle.isPaused(name))
|
|
1566
|
+
return;
|
|
1567
|
+
this.scheduleStartupRetry(name, 0);
|
|
1568
|
+
})().finally(() => this.handOffsInFlight.delete(name));
|
|
1569
|
+
this.handOffsInFlight.set(name, run);
|
|
1570
|
+
return run;
|
|
1571
|
+
}
|
|
1572
|
+
async runStartupRetry(name, attempt) {
|
|
1573
|
+
this.startupRetries.delete(name);
|
|
1574
|
+
if (this.shuttingDown)
|
|
1575
|
+
return;
|
|
1576
|
+
if (!this.isConfiguredInstance(name))
|
|
1577
|
+
return; // removed from fleet.yaml / classic channel meanwhile
|
|
1578
|
+
if (this.lifecycle.isPaused(name) || this.daemons.has(name))
|
|
1579
|
+
return; // operator acted meanwhile
|
|
1580
|
+
if (this.stormWindow.isSpawnBlocked()) {
|
|
1581
|
+
// The tmux server is being restarted; joining that herd is what we are
|
|
1582
|
+
// trying to avoid. Same attempt again after the storm's own recovery.
|
|
1583
|
+
const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, FleetManager.STARTUP_RETRY_STORM_DEFER_MS);
|
|
1584
|
+
timer.unref?.();
|
|
1585
|
+
this.startupRetries.set(name, { attempt, timer });
|
|
1586
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Startup retry deferred — tmux storm window is blocking spawns");
|
|
1587
|
+
return;
|
|
1588
|
+
}
|
|
1589
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
1590
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
1591
|
+
try {
|
|
1592
|
+
await this.spawnGate.run({
|
|
1593
|
+
instanceName: name,
|
|
1594
|
+
workingDirectory: this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name),
|
|
1595
|
+
reason: "recovery",
|
|
1596
|
+
}, () => this.startConfiguredInstance(name, topicMode));
|
|
1597
|
+
if (this.daemons.has(name)) {
|
|
1598
|
+
this.logger.info({ name, attempt: attempt + 1 }, "Automatic startup retry succeeded");
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
catch (err) {
|
|
1602
|
+
this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
|
|
1603
|
+
this.scheduleStartupRetry(name, attempt + 1);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
/** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
|
|
1607
|
+
isConfiguredInstance(name) {
|
|
1608
|
+
if (this.fleetConfig?.instances[name])
|
|
1609
|
+
return true;
|
|
1610
|
+
return !!this.classicChannels?.getAll().some(channel => channel.instanceName === name);
|
|
1611
|
+
}
|
|
1612
|
+
/**
|
|
1613
|
+
* Kind-aware start for the automatic retry: fleet-topic instances come from
|
|
1614
|
+
* fleet.yaml, ClassicBot instances exist only in the classic channel manager
|
|
1615
|
+
* and must be rebuilt through startClassicInstance (a fleet.yaml lookup alone
|
|
1616
|
+
* silently dropped them — the retry timer fired and nothing happened).
|
|
1617
|
+
*/
|
|
1618
|
+
async startConfiguredInstance(name, topicMode) {
|
|
1619
|
+
const config = this.fleetConfig?.instances[name];
|
|
1620
|
+
if (config) {
|
|
1621
|
+
await this.startInstance(name, config, topicMode);
|
|
1622
|
+
return;
|
|
1623
|
+
}
|
|
1624
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
1625
|
+
if (!channel || !this.classicChannels)
|
|
1626
|
+
throw new Error(`Instance '${name}' is no longer configured`);
|
|
1627
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Unattended ClassicBot start (fleet startup batch, full-restart batch, the
|
|
1631
|
+
* classicBot.yaml reconcile): same contract as startInstanceUnattended —
|
|
1632
|
+
* failures are logged and handed to the delayed automatic retry, whose
|
|
1633
|
+
* kind-aware startConfiguredInstance rebuilds the Classic instance. Returns
|
|
1634
|
+
* whether the instance is up.
|
|
1635
|
+
*/
|
|
1636
|
+
async startClassicInstanceUnattended(ch, what) {
|
|
1637
|
+
try {
|
|
1638
|
+
await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1639
|
+
return this.daemons.has(ch.instanceName);
|
|
1640
|
+
}
|
|
1641
|
+
catch (err) {
|
|
1642
|
+
this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
|
|
1643
|
+
this.scheduleStartupRetry(ch.instanceName, 0);
|
|
1644
|
+
return false;
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
backendNameOf(name) {
|
|
1648
|
+
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1649
|
+
const configured = this.fleetConfig?.instances[name]?.backend;
|
|
1650
|
+
if (configured)
|
|
1651
|
+
return configured;
|
|
1652
|
+
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1653
|
+
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1654
|
+
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1655
|
+
}
|
|
1656
|
+
return fleetDefault ?? "claude-code";
|
|
1657
|
+
}
|
|
1658
|
+
/**
|
|
1659
|
+
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1660
|
+
* fails many instances within the same second. Two notices per incident at
|
|
1661
|
+
* most — "N failed, retrying in X" and, if it comes to that, "gave up on N".
|
|
1662
|
+
*/
|
|
1663
|
+
queueStartupRetryNotice(kind, name, delayMs) {
|
|
1664
|
+
const pending = this.startupRetryNotices.get(kind);
|
|
1665
|
+
if (pending) {
|
|
1666
|
+
if (!pending.names.includes(name))
|
|
1667
|
+
pending.names.push(name);
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
const timer = setTimeout(() => {
|
|
1671
|
+
const entry = this.startupRetryNotices.get(kind);
|
|
1672
|
+
this.startupRetryNotices.delete(kind);
|
|
1673
|
+
if (!entry)
|
|
1674
|
+
return;
|
|
1675
|
+
const list = entry.names.join(", ");
|
|
1676
|
+
if (kind === "scheduled") {
|
|
1677
|
+
let text = t("fleet.startup_retry_scheduled", entry.names.length, list, this.formatStormDelay(entry.delayMs));
|
|
1678
|
+
const downBackends = [...new Set(entry.names.map(n => this.backendNameOf(n)))].filter(b => this.backendOutage.isActive(b));
|
|
1679
|
+
if (downBackends.length)
|
|
1680
|
+
text += `\n${t("fleet.startup_retry_outage", downBackends.join(", "))}`;
|
|
1681
|
+
this.notifyFleetError(text);
|
|
1682
|
+
}
|
|
1683
|
+
else {
|
|
1684
|
+
this.notifyFleetError(t("fleet.startup_retry_gave_up", entry.names.length, list));
|
|
1685
|
+
}
|
|
1686
|
+
}, FleetManager.STARTUP_RETRY_NOTICE_AGGREGATE_MS);
|
|
1687
|
+
timer.unref?.();
|
|
1688
|
+
this.startupRetryNotices.set(kind, { names: [name], delayMs, timer });
|
|
1206
1689
|
}
|
|
1207
1690
|
runnableStartupCount(fleet, includeClassic) {
|
|
1208
1691
|
const names = this.configuredStartupInstanceNames(fleet, includeClassic);
|
|
@@ -1237,6 +1720,8 @@ export class FleetManager {
|
|
|
1237
1720
|
};
|
|
1238
1721
|
}
|
|
1239
1722
|
async stopInstance(name) {
|
|
1723
|
+
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
1724
|
+
this.cancelStartupRetry(name);
|
|
1240
1725
|
this.failoverActive.delete(name);
|
|
1241
1726
|
this.cancelIdleButtonRetirement(name);
|
|
1242
1727
|
this.instanceStateCache.delete(name);
|
|
@@ -1248,12 +1733,20 @@ export class FleetManager {
|
|
|
1248
1733
|
// interactive-prompt / clean-exit events whose handlers post fresh prompts
|
|
1249
1734
|
// while the stop is still awaiting (the TOCTOU sol's review called out).
|
|
1250
1735
|
this.clearNoncePromptsForInstance(name);
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1736
|
+
// Published so the outage hand-off can wait for an explicit stop that began
|
|
1737
|
+
// BEFORE it (the generation fence alone only catches stops that begin after
|
|
1738
|
+
// the hand-off snapshotted it).
|
|
1739
|
+
const run = (async () => {
|
|
1740
|
+
try {
|
|
1741
|
+
await this.lifecycle.stop(name);
|
|
1742
|
+
}
|
|
1743
|
+
finally {
|
|
1744
|
+
this.clearNoncePromptsForInstance(name);
|
|
1745
|
+
}
|
|
1746
|
+
})().finally(() => { if (this.stopsInFlight.get(name) === run)
|
|
1747
|
+
this.stopsInFlight.delete(name); });
|
|
1748
|
+
this.stopsInFlight.set(name, run);
|
|
1749
|
+
return run;
|
|
1257
1750
|
}
|
|
1258
1751
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
1259
1752
|
async restartSingleInstance(name, opts) {
|
|
@@ -1266,7 +1759,12 @@ export class FleetManager {
|
|
|
1266
1759
|
this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
|
|
1267
1760
|
return inFlight;
|
|
1268
1761
|
}
|
|
1269
|
-
const
|
|
1762
|
+
const workingDirectory = this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name);
|
|
1763
|
+
const run = this.spawnGate.run({
|
|
1764
|
+
instanceName: name,
|
|
1765
|
+
workingDirectory,
|
|
1766
|
+
reason: "restart",
|
|
1767
|
+
}, () => this.doRestartSingleInstance(name, opts))
|
|
1270
1768
|
.finally(() => this.restartsInFlight.delete(name));
|
|
1271
1769
|
this.restartsInFlight.set(name, run);
|
|
1272
1770
|
return run;
|
|
@@ -1284,6 +1782,7 @@ export class FleetManager {
|
|
|
1284
1782
|
this.writeFreshStartMarker(name);
|
|
1285
1783
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
1286
1784
|
await this.startInstance(name, config, topicMode ?? false);
|
|
1785
|
+
this.stormWindow.markRecovered(name);
|
|
1287
1786
|
return;
|
|
1288
1787
|
}
|
|
1289
1788
|
// Classic instance fallback
|
|
@@ -1296,6 +1795,7 @@ export class FleetManager {
|
|
|
1296
1795
|
if (opts?.freshStart)
|
|
1297
1796
|
this.writeFreshStartMarker(name);
|
|
1298
1797
|
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1798
|
+
this.stormWindow.markRecovered(name);
|
|
1299
1799
|
return;
|
|
1300
1800
|
}
|
|
1301
1801
|
throw new Error(`Instance not found: ${name}`);
|
|
@@ -1361,6 +1861,7 @@ export class FleetManager {
|
|
|
1361
1861
|
messageId,
|
|
1362
1862
|
});
|
|
1363
1863
|
this.lastUpdateProgressText = null;
|
|
1864
|
+
this.updateCompletionTipText = null;
|
|
1364
1865
|
this.startUpdateProgressMonitor(adapter);
|
|
1365
1866
|
}
|
|
1366
1867
|
failUpdateProgress(message) {
|
|
@@ -1384,7 +1885,17 @@ export class FleetManager {
|
|
|
1384
1885
|
const adapter = this.adapters.get(target.adapterId) ?? (initialAdapter?.id === target.adapterId ? initialAdapter : undefined);
|
|
1385
1886
|
if (!adapter)
|
|
1386
1887
|
return;
|
|
1387
|
-
|
|
1888
|
+
let text = formatUpdateProgress(marker);
|
|
1889
|
+
// No-restart updates complete in the old fleet process, so they never
|
|
1890
|
+
// reach RestartProgress.finish(). Append the same optional tip here.
|
|
1891
|
+
if (marker.progress.stage === "complete" && this.tipsEnabled()) {
|
|
1892
|
+
if (this.updateCompletionTipText === null) {
|
|
1893
|
+
const tip = this.pickAvailableTip();
|
|
1894
|
+
this.updateCompletionTipText = tip ? this.formatTip(tip) : "";
|
|
1895
|
+
}
|
|
1896
|
+
if (this.updateCompletionTipText)
|
|
1897
|
+
text += `\n\n${this.updateCompletionTipText}`;
|
|
1898
|
+
}
|
|
1388
1899
|
if (text === this.lastUpdateProgressText)
|
|
1389
1900
|
return;
|
|
1390
1901
|
this.updateProgressEditRunning = true;
|
|
@@ -1425,6 +1936,7 @@ export class FleetManager {
|
|
|
1425
1936
|
// Rotate fleet.log if oversized (before any logging)
|
|
1426
1937
|
rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
|
|
1427
1938
|
const fleet = this.loadConfig(configPath);
|
|
1939
|
+
this.slimFleetConfigAtStartup();
|
|
1428
1940
|
setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
|
|
1429
1941
|
const savedUpdateProgress = readUpdateProgress(this.dataDir);
|
|
1430
1942
|
const pendingUpdateProgress = savedUpdateProgress
|
|
@@ -1441,6 +1953,10 @@ export class FleetManager {
|
|
|
1441
1953
|
const { getTmuxSocketName: getSocket } = await import("./paths.js");
|
|
1442
1954
|
TmuxManager.setSocketName(getSocket());
|
|
1443
1955
|
await TmuxManager.ensureSession(getTmuxSession());
|
|
1956
|
+
// Pre-flight (advisory, fire-and-forget): warm the per-backend auth cache
|
|
1957
|
+
// before instances spawn, so a CLI that comes up on its sign-in screen gets
|
|
1958
|
+
// an instant 🔑 verdict instead of decaying into crash/MCP-died noise.
|
|
1959
|
+
this.lifecycle.primeAuthVerification(Object.values(fleet.instances).map(config => config.backend ?? fleet.defaults?.backend ?? "claude-code"));
|
|
1444
1960
|
// Start tmux control mode client for idle detection
|
|
1445
1961
|
if (!this.controlClient) {
|
|
1446
1962
|
this.controlClient = new TmuxControlClient(getTmuxSession(), 2000, this.logger);
|
|
@@ -1482,9 +1998,8 @@ export class FleetManager {
|
|
|
1482
1998
|
// routing does NOT go through the routing engine (single-key, can't hold two
|
|
1483
1999
|
// bots in one channel) — it resolves per-bot via getInstanceByChannel.
|
|
1484
2000
|
this.classicChannels = new ClassicChannelManager(this.dataDir, this.logger);
|
|
1485
|
-
const
|
|
1486
|
-
|
|
1487
|
-
this.classicChannels.setPrimaryAdapterId(primaryCh.id ?? primaryCh.type);
|
|
2001
|
+
const classicAdapters = fleet.channels?.length ? fleet.channels : (fleet.channel ? [fleet.channel] : []);
|
|
2002
|
+
this.classicChannels.configureAdapters(classicAdapters);
|
|
1488
2003
|
// Restore the persisted bot binding so replies/cancel go through the right
|
|
1489
2004
|
// bot after a restart (before this, inbound would re-bind lazily).
|
|
1490
2005
|
for (const ch of this.classicChannels.getAll()) {
|
|
@@ -1521,7 +2036,10 @@ export class FleetManager {
|
|
|
1521
2036
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1522
2037
|
// Small delay to let tmux window clean up
|
|
1523
2038
|
await new Promise(r => setTimeout(r, 2000));
|
|
1524
|
-
|
|
2039
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
2040
|
+
// unattended helper reads them from it and schedules the delayed
|
|
2041
|
+
// retry on failure like every other unattended start.
|
|
2042
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1525
2043
|
}
|
|
1526
2044
|
}
|
|
1527
2045
|
}
|
|
@@ -1653,7 +2171,7 @@ export class FleetManager {
|
|
|
1653
2171
|
const generalDir = join(getAgendHome(), name);
|
|
1654
2172
|
mkdirSync(generalDir, { recursive: true });
|
|
1655
2173
|
const backendName = fleet.defaults.backend ?? "claude-code";
|
|
1656
|
-
this.ensureGeneralInstructions(generalDir, backendName);
|
|
2174
|
+
this.ensureGeneralInstructions(generalDir, backendName, name);
|
|
1657
2175
|
fleet.instances[name] = {
|
|
1658
2176
|
...DEFAULT_INSTANCE_CONFIG,
|
|
1659
2177
|
working_directory: generalDir,
|
|
@@ -1672,6 +2190,13 @@ export class FleetManager {
|
|
|
1672
2190
|
this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule) => this.handleScheduleTrigger(schedule), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
|
|
1673
2191
|
this.scheduler.init();
|
|
1674
2192
|
this.logger.info("Scheduler initialized");
|
|
2193
|
+
// Tips share the daily-report clock but remain an independent internal
|
|
2194
|
+
// job: disabling daily_summary must not disable tips (and vice versa).
|
|
2195
|
+
// The callback checks defaults.tips at fire time, so /tips on|off is hot.
|
|
2196
|
+
this.dailyTipScheduler = new DailyTipScheduler(summaryConfig, costGuardConfig.timezone, () => this.sendTipToGeneral().catch(err => {
|
|
2197
|
+
this.logger.warn({ err }, "Failed to send daily tip");
|
|
2198
|
+
}));
|
|
2199
|
+
this.dailyTipScheduler.start();
|
|
1675
2200
|
// Inject active decisions as env var for MCP instructions.
|
|
1676
2201
|
// Snapshotted at startup — new decisions via post_decision are available
|
|
1677
2202
|
// through list_decisions tool but not auto-injected until restart.
|
|
@@ -1701,6 +2226,9 @@ export class FleetManager {
|
|
|
1701
2226
|
}
|
|
1702
2227
|
catch (err) {
|
|
1703
2228
|
this.logger.error({ err, name }, "Failed to start general instance");
|
|
2229
|
+
// General is the most important instance to bring back: it also gets
|
|
2230
|
+
// the delayed automatic retry (the topic notice below still goes out).
|
|
2231
|
+
this.scheduleStartupRetry(name, 0);
|
|
1704
2232
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1705
2233
|
const topicId = cfg.topic_id ? String(cfg.topic_id) : undefined;
|
|
1706
2234
|
if (this.adapter && topicId) {
|
|
@@ -1820,14 +2348,8 @@ export class FleetManager {
|
|
|
1820
2348
|
while (idx < channels.length) {
|
|
1821
2349
|
const batch = channels.slice(idx, idx + concurrency);
|
|
1822
2350
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
if (this.daemons.has(ch.instanceName))
|
|
1826
|
-
startupProgress.markReady();
|
|
1827
|
-
}
|
|
1828
|
-
catch (err) {
|
|
1829
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
1830
|
-
}
|
|
2351
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
2352
|
+
startupProgress.markReady();
|
|
1831
2353
|
}));
|
|
1832
2354
|
idx += concurrency;
|
|
1833
2355
|
}
|
|
@@ -1854,6 +2376,12 @@ export class FleetManager {
|
|
|
1854
2376
|
version: agendVersion,
|
|
1855
2377
|
pausedNames,
|
|
1856
2378
|
failedNames,
|
|
2379
|
+
tipText: pendingUpdateProgress && this.tipsEnabled()
|
|
2380
|
+
? (() => {
|
|
2381
|
+
const tip = this.pickAvailableTip();
|
|
2382
|
+
return tip ? this.formatTip(tip) : undefined;
|
|
2383
|
+
})()
|
|
2384
|
+
: undefined,
|
|
1857
2385
|
});
|
|
1858
2386
|
if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
|
|
1859
2387
|
let text;
|
|
@@ -2015,20 +2543,33 @@ export class FleetManager {
|
|
|
2015
2543
|
notifyAdapterFailure(adapterId, error) {
|
|
2016
2544
|
const generalId = this.findGeneralInstance();
|
|
2017
2545
|
if (generalId) {
|
|
2018
|
-
this.notifyInstanceTopic(generalId,
|
|
2546
|
+
this.notifyInstanceTopic(generalId, t("adapter.start_failed", adapterId, error));
|
|
2019
2547
|
}
|
|
2020
2548
|
}
|
|
2021
2549
|
/** Notify admin that a retried adapter reconnected. */
|
|
2022
2550
|
notifyAdapterRecovery(adapterId, attempts) {
|
|
2023
2551
|
const generalId = this.findGeneralInstance();
|
|
2024
2552
|
if (generalId) {
|
|
2025
|
-
this.notifyInstanceTopic(generalId,
|
|
2553
|
+
this.notifyInstanceTopic(generalId, t("adapter.reconnected", adapterId, attempts));
|
|
2026
2554
|
}
|
|
2027
2555
|
}
|
|
2028
2556
|
/** Get adapter states for /status visibility. */
|
|
2029
2557
|
getAdapterStates() {
|
|
2030
2558
|
return this.adapterState;
|
|
2031
2559
|
}
|
|
2560
|
+
bindAdapterHealth(adapter, adapterId) {
|
|
2561
|
+
adapter.on("gateway_health", (snapshot) => {
|
|
2562
|
+
const previous = this.adapterState.get(adapterId);
|
|
2563
|
+
const status = snapshot.status === "connected" ? "connected"
|
|
2564
|
+
: snapshot.status === "stopped" ? "failed"
|
|
2565
|
+
: "retrying";
|
|
2566
|
+
this.adapterState.set(adapterId, {
|
|
2567
|
+
status,
|
|
2568
|
+
retryCount: previous?.retryCount ?? 0,
|
|
2569
|
+
lastError: status === "connected" ? undefined : snapshot.lastReconnectReason ?? previous?.lastError,
|
|
2570
|
+
});
|
|
2571
|
+
});
|
|
2572
|
+
}
|
|
2032
2573
|
/**
|
|
2033
2574
|
* Real, checkable fleet health for `/health` and the operator.
|
|
2034
2575
|
*
|
|
@@ -2058,10 +2599,17 @@ export class FleetManager {
|
|
|
2058
2599
|
counts.stopped++;
|
|
2059
2600
|
}
|
|
2060
2601
|
const states = {};
|
|
2602
|
+
const details = {};
|
|
2061
2603
|
let connected = 0;
|
|
2062
2604
|
for (const [id, state] of this.adapterState) {
|
|
2063
|
-
|
|
2064
|
-
if (
|
|
2605
|
+
const snapshot = this.adapters.get(id)?.getHealthSnapshot?.();
|
|
2606
|
+
if (snapshot)
|
|
2607
|
+
details[id] = snapshot;
|
|
2608
|
+
const effectiveStatus = snapshot?.status === "connected" ? "connected"
|
|
2609
|
+
: snapshot && snapshot.status !== "stopped" ? "retrying"
|
|
2610
|
+
: state.status;
|
|
2611
|
+
states[id] = effectiveStatus;
|
|
2612
|
+
if (effectiveStatus === "connected")
|
|
2065
2613
|
connected++;
|
|
2066
2614
|
}
|
|
2067
2615
|
const problems = [];
|
|
@@ -2069,9 +2617,9 @@ export class FleetManager {
|
|
|
2069
2617
|
problems.push("no channel adapter is connected");
|
|
2070
2618
|
if (counts.crashed > 0)
|
|
2071
2619
|
problems.push(`${counts.crashed} instance(s) crashed`);
|
|
2072
|
-
for (const [id, state] of
|
|
2073
|
-
if (state
|
|
2074
|
-
problems.push(`adapter ${id} is ${state
|
|
2620
|
+
for (const [id, state] of Object.entries(states)) {
|
|
2621
|
+
if (state !== "connected")
|
|
2622
|
+
problems.push(`adapter ${id} is ${state}`);
|
|
2075
2623
|
}
|
|
2076
2624
|
if (!this.startupComplete)
|
|
2077
2625
|
problems.push("startup has not completed");
|
|
@@ -2084,7 +2632,7 @@ export class FleetManager {
|
|
|
2084
2632
|
status,
|
|
2085
2633
|
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
2086
2634
|
instances: counts,
|
|
2087
|
-
adapters: { total: this.adapterState.size, connected, states },
|
|
2635
|
+
adapters: { total: this.adapterState.size, connected, states, details },
|
|
2088
2636
|
startupComplete: this.startupComplete,
|
|
2089
2637
|
memory: readFleetMemory(),
|
|
2090
2638
|
problems,
|
|
@@ -2113,6 +2661,7 @@ export class FleetManager {
|
|
|
2113
2661
|
const world = new AdapterWorld(adapterId, this.adapter, accessManager, channelConfig);
|
|
2114
2662
|
this.worlds.set(adapterId, world);
|
|
2115
2663
|
this.adapters.set(adapterId, this.adapter);
|
|
2664
|
+
this.bindAdapterHealth(this.adapter, adapterId);
|
|
2116
2665
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2117
2666
|
await this.handleInboundMessage(msg);
|
|
2118
2667
|
}, this.logger, "adapter.message"));
|
|
@@ -2120,6 +2669,24 @@ export class FleetManager {
|
|
|
2120
2669
|
await this.handleInboundReaction(r);
|
|
2121
2670
|
}, this.logger, "adapter.reaction"));
|
|
2122
2671
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
2672
|
+
if (await this.handleTipDismiss(data, adapterId, this.adapter ?? undefined))
|
|
2673
|
+
return;
|
|
2674
|
+
if (await this.handleTipUnlock(data, adapterId, this.adapter ?? undefined))
|
|
2675
|
+
return;
|
|
2676
|
+
if (await this.handleLoginBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2677
|
+
return;
|
|
2678
|
+
if (await this.handleInstallBackendSelect(data, adapterId, this.adapter ?? undefined))
|
|
2679
|
+
return;
|
|
2680
|
+
if (await this.handleClassicApproval(data, adapterId, this.adapter ?? undefined))
|
|
2681
|
+
return;
|
|
2682
|
+
if (await this.handleLoginMenuSelect(data, adapterId, this.adapter ?? undefined))
|
|
2683
|
+
return;
|
|
2684
|
+
if (await this.handleLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2685
|
+
return;
|
|
2686
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2687
|
+
return;
|
|
2688
|
+
if (await this.handleClearConfirmation(data, adapterId, this.adapter ?? undefined))
|
|
2689
|
+
return;
|
|
2123
2690
|
if (await this.handleExitRestartPrompt(data, adapterId, this.adapter ?? undefined))
|
|
2124
2691
|
return;
|
|
2125
2692
|
if (await this.handleInteractivePromptAssist(data, adapterId, this.adapter ?? undefined))
|
|
@@ -2210,6 +2777,49 @@ export class FleetManager {
|
|
|
2210
2777
|
const result = await this.topicCommands.sendCompact(name);
|
|
2211
2778
|
await data.respond(result);
|
|
2212
2779
|
}
|
|
2780
|
+
else if (data.command === "steer") {
|
|
2781
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2782
|
+
if (!name) {
|
|
2783
|
+
await data.respond(t("classic.no_agent"));
|
|
2784
|
+
return;
|
|
2785
|
+
}
|
|
2786
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
2787
|
+
if (!steerText) {
|
|
2788
|
+
await data.respond(t("steer.usage"));
|
|
2789
|
+
return;
|
|
2790
|
+
}
|
|
2791
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
2792
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
2793
|
+
// rerouting the instance's replies to the slash context.
|
|
2794
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
2795
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2796
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2797
|
+
});
|
|
2798
|
+
await data.respond(result);
|
|
2799
|
+
}
|
|
2800
|
+
else if (data.command === "btw") {
|
|
2801
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2802
|
+
if (!name) {
|
|
2803
|
+
await data.respond(t("classic.no_agent"));
|
|
2804
|
+
return;
|
|
2805
|
+
}
|
|
2806
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
2807
|
+
if (!btwText) {
|
|
2808
|
+
await data.respond(t("btw.usage"));
|
|
2809
|
+
return;
|
|
2810
|
+
}
|
|
2811
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
2812
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2813
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2814
|
+
});
|
|
2815
|
+
await data.respond(result);
|
|
2816
|
+
}
|
|
2817
|
+
else if (data.command === "login") {
|
|
2818
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
2819
|
+
}
|
|
2820
|
+
else if (data.command === "install-cli") {
|
|
2821
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
2822
|
+
}
|
|
2213
2823
|
else if (data.command === "clear") {
|
|
2214
2824
|
await this.handleClearSlash(data, adapterId);
|
|
2215
2825
|
}
|
|
@@ -2283,12 +2893,15 @@ export class FleetManager {
|
|
|
2283
2893
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2284
2894
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2285
2895
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2286
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
2896
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2287
2897
|
}
|
|
2288
2898
|
catch (err) {
|
|
2289
|
-
await data.respond(
|
|
2899
|
+
await data.respond(t("usage.failed", err.message));
|
|
2290
2900
|
}
|
|
2291
2901
|
}
|
|
2902
|
+
else if (data.command === "tips") {
|
|
2903
|
+
await this.handleTipsSlash(data, adapterId);
|
|
2904
|
+
}
|
|
2292
2905
|
else if (data.command === "status") {
|
|
2293
2906
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2294
2907
|
// instance's cost and IPC health.
|
|
@@ -2334,6 +2947,49 @@ export class FleetManager {
|
|
|
2334
2947
|
const result = await this.topicCommands.sendCompact(name);
|
|
2335
2948
|
await data.respond(result);
|
|
2336
2949
|
}
|
|
2950
|
+
else if (data.command === "steer") {
|
|
2951
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2952
|
+
if (!name) {
|
|
2953
|
+
await data.respond(t("classic.no_agent"));
|
|
2954
|
+
return;
|
|
2955
|
+
}
|
|
2956
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
2957
|
+
if (!steerText) {
|
|
2958
|
+
await data.respond(t("steer.usage"));
|
|
2959
|
+
return;
|
|
2960
|
+
}
|
|
2961
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
2962
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
2963
|
+
// rerouting the instance's replies to the slash context.
|
|
2964
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
2965
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2966
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2967
|
+
});
|
|
2968
|
+
await data.respond(result);
|
|
2969
|
+
}
|
|
2970
|
+
else if (data.command === "btw") {
|
|
2971
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
2972
|
+
if (!name) {
|
|
2973
|
+
await data.respond(t("classic.no_agent"));
|
|
2974
|
+
return;
|
|
2975
|
+
}
|
|
2976
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
2977
|
+
if (!btwText) {
|
|
2978
|
+
await data.respond(t("btw.usage"));
|
|
2979
|
+
return;
|
|
2980
|
+
}
|
|
2981
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
2982
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
2983
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
2984
|
+
});
|
|
2985
|
+
await data.respond(result);
|
|
2986
|
+
}
|
|
2987
|
+
else if (data.command === "login") {
|
|
2988
|
+
await this.handleLoginSlash(data, adapterId, this.adapter);
|
|
2989
|
+
}
|
|
2990
|
+
else if (data.command === "install-cli") {
|
|
2991
|
+
await this.handleInstallCliSlash(data, adapterId, this.adapter);
|
|
2992
|
+
}
|
|
2337
2993
|
}, this.logger, "adapter.slash_command"));
|
|
2338
2994
|
await this.topicCommands.registerBotCommands().catch(e => this.logger.warn({ err: e }, "registerBotCommands failed (non-fatal)"));
|
|
2339
2995
|
// Background-probe each backend's CLI env (version/models) → cli-env cache.
|
|
@@ -2360,13 +3016,14 @@ export class FleetManager {
|
|
|
2360
3016
|
}, this.logger, "adapter.handler_error"));
|
|
2361
3017
|
this.adapter.on("error", (err) => {
|
|
2362
3018
|
this.logger.error({ err }, "Primary adapter fatal error");
|
|
2363
|
-
this.restartAdapter(this.adapter,
|
|
3019
|
+
this.restartAdapter(this.adapter, adapterId).catch(() => { });
|
|
2364
3020
|
});
|
|
2365
|
-
this.adapter.on("new_group_detected", safeHandler((data) => {
|
|
3021
|
+
this.adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2366
3022
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2367
3023
|
const generalId = this.findGeneralInstance();
|
|
3024
|
+
// No user to promote: the bot was just added, nobody has run /start yet.
|
|
2368
3025
|
if (generalId)
|
|
2369
|
-
this.
|
|
3026
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2370
3027
|
}, this.logger, "adapter.new_group_detected"));
|
|
2371
3028
|
// Start adapter AFTER all event listeners are registered (started event sets botUsername)
|
|
2372
3029
|
await this.adapter.start();
|
|
@@ -2402,6 +3059,7 @@ export class FleetManager {
|
|
|
2402
3059
|
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2403
3060
|
this.worlds.set(adapterId, world);
|
|
2404
3061
|
this.adapters.set(adapterId, adapter);
|
|
3062
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2405
3063
|
// Wire up event handlers (same as primary, routes through shared handleInboundMessage)
|
|
2406
3064
|
adapter.on("message", safeHandler(async (msg) => {
|
|
2407
3065
|
await this.handleInboundMessage(msg);
|
|
@@ -2410,20 +3068,38 @@ export class FleetManager {
|
|
|
2410
3068
|
await this.handleInboundReaction(r);
|
|
2411
3069
|
}, this.logger, `adapter[${adapterId}].reaction`));
|
|
2412
3070
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
2413
|
-
if (await this.
|
|
3071
|
+
if (await this.handleTipDismiss(data, adapterId, adapter))
|
|
2414
3072
|
return;
|
|
2415
|
-
if (await this.
|
|
3073
|
+
if (await this.handleTipUnlock(data, adapterId, adapter))
|
|
2416
3074
|
return;
|
|
2417
|
-
if (await this.
|
|
3075
|
+
if (await this.handleLoginBackendSelect(data, adapterId, adapter))
|
|
2418
3076
|
return;
|
|
2419
|
-
if (await this.
|
|
3077
|
+
if (await this.handleInstallBackendSelect(data, adapterId, adapter))
|
|
2420
3078
|
return;
|
|
2421
|
-
if (await this.
|
|
3079
|
+
if (await this.handleClassicApproval(data, adapterId, adapter))
|
|
2422
3080
|
return;
|
|
2423
|
-
if (await this.
|
|
3081
|
+
if (await this.handleLoginMenuSelect(data, adapterId, adapter))
|
|
2424
3082
|
return;
|
|
2425
|
-
if (
|
|
2426
|
-
|
|
3083
|
+
if (await this.handleLoginConfirm(data, adapterId, adapter))
|
|
3084
|
+
return;
|
|
3085
|
+
if (await this.handleInstallLoginConfirm(data, adapterId, adapter))
|
|
3086
|
+
return;
|
|
3087
|
+
if (await this.handleClearConfirmation(data, adapterId, adapter))
|
|
3088
|
+
return;
|
|
3089
|
+
if (await this.handleExitRestartPrompt(data, adapterId, adapter))
|
|
3090
|
+
return;
|
|
3091
|
+
if (await this.handleInteractivePromptAssist(data, adapterId, adapter))
|
|
3092
|
+
return;
|
|
3093
|
+
if (await this.handleClassicBackendSelection(data))
|
|
3094
|
+
return;
|
|
3095
|
+
if (await this.handleModelSelection(data))
|
|
3096
|
+
return;
|
|
3097
|
+
if (await this.handleEffortSelection(data))
|
|
3098
|
+
return;
|
|
3099
|
+
if (await this.handleHangPrompt(data, adapterId, adapter))
|
|
3100
|
+
return;
|
|
3101
|
+
if (data.callbackData.startsWith("cancel:")) {
|
|
3102
|
+
this.handleCancelClick(data.callbackData.slice("cancel:".length), adapter, data);
|
|
2427
3103
|
return;
|
|
2428
3104
|
}
|
|
2429
3105
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
@@ -2563,12 +3239,15 @@ export class FleetManager {
|
|
|
2563
3239
|
const { getUsageSnapshot } = await import("./usage/usage-api.js");
|
|
2564
3240
|
const { renderUsageMarkdown } = await import("./usage/format-rich.js");
|
|
2565
3241
|
// slash_command is Discord-only; editReply renders Markdown natively.
|
|
2566
|
-
await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
|
|
3242
|
+
await data.respond(renderUsageMarkdown(await getUsageSnapshot(false, this.getActiveUsageProviderIds())));
|
|
2567
3243
|
}
|
|
2568
3244
|
catch (err) {
|
|
2569
|
-
await data.respond(
|
|
3245
|
+
await data.respond(t("usage.failed", err.message));
|
|
2570
3246
|
}
|
|
2571
3247
|
}
|
|
3248
|
+
else if (data.command === "tips") {
|
|
3249
|
+
await this.handleTipsSlash(data, adapterId);
|
|
3250
|
+
}
|
|
2572
3251
|
else if (data.command === "status") {
|
|
2573
3252
|
// Admin-gated (like the topic path): the merged table shows every
|
|
2574
3253
|
// instance's cost and IPC health.
|
|
@@ -2614,11 +3293,50 @@ export class FleetManager {
|
|
|
2614
3293
|
const result = await this.topicCommands.sendCompact(name);
|
|
2615
3294
|
await data.respond(result);
|
|
2616
3295
|
}
|
|
3296
|
+
else if (data.command === "steer") {
|
|
3297
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3298
|
+
if (!name) {
|
|
3299
|
+
await data.respond(t("classic.no_agent"));
|
|
3300
|
+
return;
|
|
3301
|
+
}
|
|
3302
|
+
const steerText = String(data.options?.message ?? "").trim();
|
|
3303
|
+
if (!steerText) {
|
|
3304
|
+
await data.respond(t("steer.usage"));
|
|
3305
|
+
return;
|
|
3306
|
+
}
|
|
3307
|
+
// chat_id/message_id stay empty: a slash interaction has no channel
|
|
3308
|
+
// message to react to, and an empty chat_id keeps updateLastChat from
|
|
3309
|
+
// rerouting the instance's replies to the slash context.
|
|
3310
|
+
const result = this.topicCommands.sendSteer(name, steerText, {
|
|
3311
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3312
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3313
|
+
});
|
|
3314
|
+
await data.respond(result);
|
|
3315
|
+
}
|
|
3316
|
+
else if (data.command === "btw") {
|
|
3317
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
3318
|
+
if (!name) {
|
|
3319
|
+
await data.respond(t("classic.no_agent"));
|
|
3320
|
+
return;
|
|
3321
|
+
}
|
|
3322
|
+
const btwText = String(data.options?.message ?? "").trim();
|
|
3323
|
+
if (!btwText) {
|
|
3324
|
+
await data.respond(t("btw.usage"));
|
|
3325
|
+
return;
|
|
3326
|
+
}
|
|
3327
|
+
const result = this.topicCommands.sendBtw(name, btwText, {
|
|
3328
|
+
chatId: "", messageId: "", username: data.username ?? "user",
|
|
3329
|
+
userId: data.userId ?? "", threadId: undefined, adapterId, source: "discord",
|
|
3330
|
+
});
|
|
3331
|
+
await data.respond(result);
|
|
3332
|
+
}
|
|
3333
|
+
else if (data.command === "login") {
|
|
3334
|
+
await this.handleLoginSlash(data, adapterId, adapter);
|
|
3335
|
+
}
|
|
3336
|
+
else if (data.command === "install-cli") {
|
|
3337
|
+
await this.handleInstallCliSlash(data, adapterId, adapter);
|
|
3338
|
+
}
|
|
2617
3339
|
}, this.logger, `adapter[${adapterId}].slash_command`));
|
|
2618
|
-
await adapter.start();
|
|
2619
|
-
if (channelConfig.group_id) {
|
|
2620
|
-
adapter.setChatId(String(channelConfig.group_id));
|
|
2621
|
-
}
|
|
2622
3340
|
adapter.on("started", safeHandler((username, userId) => {
|
|
2623
3341
|
this.logger.info(`[${adapterId}] Bot @${username} polling started.`);
|
|
2624
3342
|
const world = this.worlds.get(adapterId);
|
|
@@ -2628,16 +3346,21 @@ export class FleetManager {
|
|
|
2628
3346
|
world.botUserId = userId;
|
|
2629
3347
|
}
|
|
2630
3348
|
}, this.logger, `adapter[${adapterId}].started`));
|
|
2631
|
-
adapter.on("new_group_detected", safeHandler((data) => {
|
|
3349
|
+
adapter.on("new_group_detected", safeHandler(async (data) => {
|
|
2632
3350
|
const adminMsg = t("alert.bot_added", data.groupTitle, data.groupId, data.source);
|
|
2633
3351
|
const generalId = this.findGeneralInstance(adapterId);
|
|
2634
3352
|
if (generalId)
|
|
2635
|
-
this.
|
|
3353
|
+
await this.promptClassicApproval({ generalName: generalId, message: adminMsg, groupId: data.groupId, scope: data.source === "telegram" ? "group" : "guild" });
|
|
2636
3354
|
}, this.logger, `adapter[${adapterId}].new_group_detected`));
|
|
2637
3355
|
adapter.on("error", (err) => {
|
|
2638
3356
|
this.logger.error({ err, adapterId }, "Additional adapter fatal error");
|
|
2639
3357
|
this.restartAdapter(adapter, adapterId).catch(() => { });
|
|
2640
3358
|
});
|
|
3359
|
+
// Register lifecycle listeners before login; a fast ready/error must not be lost.
|
|
3360
|
+
await adapter.start();
|
|
3361
|
+
if (channelConfig.group_id) {
|
|
3362
|
+
adapter.setChatId(String(channelConfig.group_id));
|
|
3363
|
+
}
|
|
2641
3364
|
this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
|
|
2642
3365
|
}
|
|
2643
3366
|
/** Connect IPC to a single instance with all handlers */
|
|
@@ -2731,6 +3454,42 @@ export class FleetManager {
|
|
|
2731
3454
|
else if (msg.type === "instance_activity") {
|
|
2732
3455
|
this.cacheInstanceActivity(name, msg.activity);
|
|
2733
3456
|
}
|
|
3457
|
+
else if (msg.type === "instance_progress") {
|
|
3458
|
+
this.cacheInstanceProgress(name, msg.progress || null);
|
|
3459
|
+
}
|
|
3460
|
+
else if (msg.type === "cross_instance_delivery_failed") {
|
|
3461
|
+
const senderSession = String(msg.senderSession ?? "");
|
|
3462
|
+
const targetInstance = String(msg.targetInstance ?? name);
|
|
3463
|
+
const correlationId = String(msg.correlationId ?? "unknown");
|
|
3464
|
+
const error = String(msg.error ?? "unknown delivery failure");
|
|
3465
|
+
const senderInstance = this.instanceIpcClients.has(senderSession)
|
|
3466
|
+
? senderSession
|
|
3467
|
+
: this.sessionRegistry.get(senderSession);
|
|
3468
|
+
this.logger.error({ senderSession, targetInstance, correlationId, error }, "Cross-instance pane delivery failed after queue acceptance");
|
|
3469
|
+
this.eventLog?.insert(targetInstance, "cross_instance_delivery_failed", {
|
|
3470
|
+
from: senderSession, correlation_id: correlationId, error,
|
|
3471
|
+
});
|
|
3472
|
+
if (senderInstance) {
|
|
3473
|
+
this.notifyInstanceTopic(senderInstance, t("cross_instance.delivery_failed_notice", targetInstance, error, correlationId));
|
|
3474
|
+
void this.deliverToInstance(senderInstance, {
|
|
3475
|
+
type: "fleet_inbound",
|
|
3476
|
+
targetSession: senderSession,
|
|
3477
|
+
content: t("cross_instance.delivery_failed_agent", targetInstance, error, correlationId),
|
|
3478
|
+
meta: {
|
|
3479
|
+
chat_id: "",
|
|
3480
|
+
message_id: `delivery-failed-${Date.now()}`,
|
|
3481
|
+
user: "AgEnD",
|
|
3482
|
+
user_id: "system:agend",
|
|
3483
|
+
ts: new Date().toISOString(),
|
|
3484
|
+
thread_id: "",
|
|
3485
|
+
request_kind: "update",
|
|
3486
|
+
correlation_id: correlationId,
|
|
3487
|
+
},
|
|
3488
|
+
}, { waitForIdle: true }).catch(deliveryError => {
|
|
3489
|
+
this.logger.error({ err: deliveryError, senderSession, correlationId }, "Could not deliver cross-instance failure status back to sender");
|
|
3490
|
+
});
|
|
3491
|
+
}
|
|
3492
|
+
}
|
|
2734
3493
|
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
2735
3494
|
this.cacheInstanceExecutionState(name, msg);
|
|
2736
3495
|
if (msg.type === "instance_state_response") {
|
|
@@ -2844,6 +3603,26 @@ export class FleetManager {
|
|
|
2844
3603
|
const previous = this.adapterState.get(id);
|
|
2845
3604
|
this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
|
|
2846
3605
|
try {
|
|
3606
|
+
if (adapter.reconnectGateway) {
|
|
3607
|
+
try {
|
|
3608
|
+
// Discord requires a fresh Client after destroy(). Its adapter owns the
|
|
3609
|
+
// single-flight, generation fence and bounded IDENTIFY backoff, so all
|
|
3610
|
+
// watchdog/manual/error triggers must converge here instead of stop/start.
|
|
3611
|
+
await adapter.reconnectGateway(previous?.lastError ?? "fleet adapter restart");
|
|
3612
|
+
this.adapterState.set(id, { status: "connected", retryCount: 0 });
|
|
3613
|
+
this.logger.info({ id }, "Adapter gateway rebuilt successfully");
|
|
3614
|
+
}
|
|
3615
|
+
catch (err) {
|
|
3616
|
+
if (!this.ipcStoppingInstances.has("__fleet_stopping__")) {
|
|
3617
|
+
this.adapterState.set(id, {
|
|
3618
|
+
status: "failed",
|
|
3619
|
+
retryCount: previous?.retryCount ?? 0,
|
|
3620
|
+
lastError: err?.message ?? String(err),
|
|
3621
|
+
});
|
|
3622
|
+
}
|
|
3623
|
+
}
|
|
3624
|
+
return;
|
|
3625
|
+
}
|
|
2847
3626
|
for (let attempt = 1;; attempt++) {
|
|
2848
3627
|
if (this.ipcStoppingInstances.has("__fleet_stopping__"))
|
|
2849
3628
|
return;
|
|
@@ -3081,7 +3860,13 @@ export class FleetManager {
|
|
|
3081
3860
|
const adminMsg = t("alert.new_group", groupTitle, chatId, msg.username, msg.userId, msg.source);
|
|
3082
3861
|
const generalId = this.findGeneralInstance(msg.adapterId);
|
|
3083
3862
|
if (generalId) {
|
|
3084
|
-
this.
|
|
3863
|
+
await this.promptClassicApproval({
|
|
3864
|
+
// Guarded by isGroupAllowed → this belongs in allowed_groups,
|
|
3865
|
+
// NOT allowed_guilds; writing the latter would change the file
|
|
3866
|
+
// without unblocking the group.
|
|
3867
|
+
generalName: generalId, message: adminMsg, groupId: String(chatId),
|
|
3868
|
+
scope: "group", userId: msg.userId,
|
|
3869
|
+
});
|
|
3085
3870
|
}
|
|
3086
3871
|
await msgAdapter?.sendText(chatId, t("classic.access_requested"));
|
|
3087
3872
|
return;
|
|
@@ -3178,6 +3963,42 @@ export class FleetManager {
|
|
|
3178
3963
|
await msgAdapter?.sendText(chatId, result);
|
|
3179
3964
|
return;
|
|
3180
3965
|
}
|
|
3966
|
+
// /steer — interject into the running turn. Not admin-gated: anyone who
|
|
3967
|
+
// can talk to this agent can send it a message; steer only changes when
|
|
3968
|
+
// it lands, and it keeps the full [user:] formatting (unlike /raw).
|
|
3969
|
+
if (text === "/steer" || text.startsWith("/steer ") || text.startsWith("/steer@")) {
|
|
3970
|
+
const steerName = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
3971
|
+
if (!steerName) {
|
|
3972
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
3973
|
+
return;
|
|
3974
|
+
}
|
|
3975
|
+
const steerContent = text.replace(/^\/steer(@\S+)?/, "").trim();
|
|
3976
|
+
if (!steerContent) {
|
|
3977
|
+
await msgAdapter?.sendText(chatId, t("steer.usage"));
|
|
3978
|
+
return;
|
|
3979
|
+
}
|
|
3980
|
+
const result = this.topicCommands.sendSteer(steerName, steerContent, msg);
|
|
3981
|
+
await msgAdapter?.sendText(chatId, result);
|
|
3982
|
+
return;
|
|
3983
|
+
}
|
|
3984
|
+
// /btw — Claude Code's native side-thread command. Like /steer it is
|
|
3985
|
+
// not admin-gated, but it remains a distinct raw CLI command so Claude
|
|
3986
|
+
// does not fold the question into the active task.
|
|
3987
|
+
if (text === "/btw" || text.startsWith("/btw ") || text.startsWith("/btw@")) {
|
|
3988
|
+
const btwName = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
3989
|
+
if (!btwName) {
|
|
3990
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
3991
|
+
return;
|
|
3992
|
+
}
|
|
3993
|
+
const btwContent = text.replace(/^\/btw(@\S+)?/, "").trim();
|
|
3994
|
+
if (!btwContent) {
|
|
3995
|
+
await msgAdapter?.sendText(chatId, t("btw.usage"));
|
|
3996
|
+
return;
|
|
3997
|
+
}
|
|
3998
|
+
const result = this.topicCommands.sendBtw(btwName, btwContent, msg);
|
|
3999
|
+
await msgAdapter?.sendText(chatId, result);
|
|
4000
|
+
return;
|
|
4001
|
+
}
|
|
3181
4002
|
// Handle /clear command (admin only) — unlike /compact this starts a
|
|
3182
4003
|
// fresh conversation and intentionally discards the current history.
|
|
3183
4004
|
if (text === "/clear" || text.startsWith("/clear@")) {
|
|
@@ -3190,7 +4011,11 @@ export class FleetManager {
|
|
|
3190
4011
|
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
3191
4012
|
return;
|
|
3192
4013
|
}
|
|
3193
|
-
|
|
4014
|
+
if (!msgAdapter)
|
|
4015
|
+
return;
|
|
4016
|
+
const fallback = await this.promptClearConfirmation(clearName, chatId, msgAdapter, chatId);
|
|
4017
|
+
if (fallback)
|
|
4018
|
+
await msgAdapter.sendText(chatId, fallback);
|
|
3194
4019
|
return;
|
|
3195
4020
|
}
|
|
3196
4021
|
// Handle /cancel command
|
|
@@ -3201,7 +4026,7 @@ export class FleetManager {
|
|
|
3201
4026
|
return;
|
|
3202
4027
|
}
|
|
3203
4028
|
const ok = this.cancelInstance(cancelName);
|
|
3204
|
-
await msgAdapter?.sendText(chatId, ok ?
|
|
4029
|
+
await msgAdapter?.sendText(chatId, ok ? t("cancel.sent", cancelName) : t("cancel.not_running", cancelName));
|
|
3205
4030
|
return;
|
|
3206
4031
|
}
|
|
3207
4032
|
// Handle /ctx command
|
|
@@ -3238,7 +4063,7 @@ export class FleetManager {
|
|
|
3238
4063
|
const backend = this.classicChannels.getBackendByInstance(saveName, this.fleetConfig?.defaults?.backend);
|
|
3239
4064
|
const cmd = saveCommandForBackend(backend, filename);
|
|
3240
4065
|
if (!cmd) {
|
|
3241
|
-
await msgAdapter?.sendText(chatId,
|
|
4066
|
+
await msgAdapter?.sendText(chatId, t("save.unsupported"));
|
|
3242
4067
|
return;
|
|
3243
4068
|
}
|
|
3244
4069
|
this.pasteRawToClassicInstance(saveName, cmd);
|
|
@@ -3456,10 +4281,10 @@ export class FleetManager {
|
|
|
3456
4281
|
return;
|
|
3457
4282
|
let warning = "";
|
|
3458
4283
|
if (rl.five_hour_pct >= 95) {
|
|
3459
|
-
warning =
|
|
4284
|
+
warning = t("rate_limit.five_hour", instanceName, Math.round(rl.five_hour_pct));
|
|
3460
4285
|
}
|
|
3461
4286
|
else if (rl.seven_day_pct >= 95) {
|
|
3462
|
-
warning =
|
|
4287
|
+
warning = t("rate_limit.weekly", instanceName, Math.round(rl.seven_day_pct));
|
|
3463
4288
|
}
|
|
3464
4289
|
if (!warning)
|
|
3465
4290
|
return;
|
|
@@ -3474,8 +4299,6 @@ export class FleetManager {
|
|
|
3474
4299
|
}
|
|
3475
4300
|
/** Handle outbound tool calls from a daemon instance */
|
|
3476
4301
|
async handleOutboundFromInstance(instanceName, msg) {
|
|
3477
|
-
if (this.worlds.size === 0)
|
|
3478
|
-
return;
|
|
3479
4302
|
this.touchActivity(instanceName);
|
|
3480
4303
|
this.setTopicIcon(instanceName, "green");
|
|
3481
4304
|
const tool = msg.tool;
|
|
@@ -3485,13 +4308,21 @@ export class FleetManager {
|
|
|
3485
4308
|
const senderSessionName = msg.senderSessionName;
|
|
3486
4309
|
const respond = (result, error) => {
|
|
3487
4310
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
4311
|
+
let sent = false;
|
|
3488
4312
|
if (fleetRequestId) {
|
|
3489
|
-
ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error });
|
|
4313
|
+
sent = ipc?.send({ type: "fleet_outbound_response", fleetRequestId, result, error }) ?? false;
|
|
3490
4314
|
}
|
|
3491
4315
|
else {
|
|
3492
|
-
ipc?.send({ type: "fleet_outbound_response", requestId, result, error });
|
|
4316
|
+
sent = ipc?.send({ type: "fleet_outbound_response", requestId, result, error }) ?? false;
|
|
4317
|
+
}
|
|
4318
|
+
if (!sent) {
|
|
4319
|
+
this.logger.warn({ instanceName, tool, requestId, fleetRequestId, error }, "Fleet outbound result could not be returned — instance IPC is disconnected");
|
|
3493
4320
|
}
|
|
3494
4321
|
};
|
|
4322
|
+
if (this.worlds.size === 0) {
|
|
4323
|
+
respond(null, "Channel adapters are not ready — retry shortly");
|
|
4324
|
+
return;
|
|
4325
|
+
}
|
|
3495
4326
|
// Resolve threadId: use sender's topic_id if sender is a known fleet instance,
|
|
3496
4327
|
// fall back to general topic if sender is unknown, or IPC owner if no sender.
|
|
3497
4328
|
const senderInstanceName = senderSessionName && this.fleetConfig?.instances[senderSessionName]
|
|
@@ -3536,17 +4367,30 @@ export class FleetManager {
|
|
|
3536
4367
|
if (tool === "reply") {
|
|
3537
4368
|
const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
|
|
3538
4369
|
if (ticket.duplicate) {
|
|
3539
|
-
this.logger.info({ instanceName }, "
|
|
4370
|
+
this.logger.info({ instanceName }, "Concurrent duplicate reply joined — awaiting the original send's outcome");
|
|
3540
4371
|
ticket.subscribe(respond);
|
|
3541
4372
|
return;
|
|
3542
4373
|
}
|
|
3543
4374
|
const original = respond;
|
|
3544
4375
|
const respondAndRecord = (result, error) => {
|
|
3545
4376
|
ticket.complete(result, error);
|
|
4377
|
+
// Return the platform outcome first. Bookkeeping below must never turn
|
|
4378
|
+
// a confirmed Discord/Telegram POST into a tool error if a secondary
|
|
4379
|
+
// log/button side effect happens to throw.
|
|
3546
4380
|
original(result, error);
|
|
4381
|
+
// The adapter resolves only after the platform POST returns. Keep
|
|
4382
|
+
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4383
|
+
// a routed-but-failed reply is not a delivered reply.
|
|
4384
|
+
if (!error && result != null) {
|
|
4385
|
+
try {
|
|
4386
|
+
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4387
|
+
}
|
|
4388
|
+
catch (err) {
|
|
4389
|
+
this.logger.warn({ err, instanceName }, "Reply delivered but post-delivery bookkeeping failed");
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
3547
4392
|
};
|
|
3548
4393
|
if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
|
|
3549
|
-
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
3550
4394
|
return;
|
|
3551
4395
|
}
|
|
3552
4396
|
// routeToolCall knows "reply"; not handling it means the world changed.
|
|
@@ -3583,7 +4427,7 @@ export class FleetManager {
|
|
|
3583
4427
|
this.clearCancelButton(instanceName);
|
|
3584
4428
|
}
|
|
3585
4429
|
else {
|
|
3586
|
-
void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
|
|
4430
|
+
void this.sendCancelButton(instanceName, undefined, true).then(() => this.armReplyGrace(instanceName));
|
|
3587
4431
|
}
|
|
3588
4432
|
this.reactDone(instanceName);
|
|
3589
4433
|
const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
|
|
@@ -3620,7 +4464,11 @@ export class FleetManager {
|
|
|
3620
4464
|
if (silent) {
|
|
3621
4465
|
const ipc = this.instanceIpcClients.get(target);
|
|
3622
4466
|
if (ipc) {
|
|
3623
|
-
ipc.send({
|
|
4467
|
+
ipc.send({
|
|
4468
|
+
type: "raw_paste",
|
|
4469
|
+
content: message,
|
|
4470
|
+
delivery_epoch: this.getDeliveryEpoch(target),
|
|
4471
|
+
});
|
|
3624
4472
|
this.scheduler.recordRun(id, "delivered");
|
|
3625
4473
|
this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
|
|
3626
4474
|
}
|
|
@@ -3635,10 +4483,19 @@ export class FleetManager {
|
|
|
3635
4483
|
const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
|
|
3636
4484
|
const deliver = async () => {
|
|
3637
4485
|
try {
|
|
4486
|
+
// A schedule has no live inbound adapter context. Seed the daemon with
|
|
4487
|
+
// the target instance's configured world so replies use its persona
|
|
4488
|
+
// after a fresh start instead of falling back to channels[0].
|
|
4489
|
+
const adapterId = this.getInstanceAdapterId(target);
|
|
3638
4490
|
await this.deliverToInstance(target, {
|
|
3639
4491
|
type: "fleet_schedule_trigger",
|
|
3640
4492
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
3641
|
-
meta: {
|
|
4493
|
+
meta: {
|
|
4494
|
+
chat_id: reply_chat_id,
|
|
4495
|
+
thread_id: reply_thread_id,
|
|
4496
|
+
user: "scheduler",
|
|
4497
|
+
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4498
|
+
},
|
|
3642
4499
|
}, { waitForIdle: true });
|
|
3643
4500
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
3644
4501
|
void this.sendCancelButton(target);
|
|
@@ -3786,7 +4643,29 @@ export class FleetManager {
|
|
|
3786
4643
|
}
|
|
3787
4644
|
/** Resolve display name for an instance, fallback to instance name. */
|
|
3788
4645
|
resolveDisplayName(instanceName) {
|
|
3789
|
-
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4646
|
+
return this.fleetConfig?.instances[instanceName]?.display_name
|
|
4647
|
+
?? this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName)?.displayName
|
|
4648
|
+
?? instanceName;
|
|
4649
|
+
}
|
|
4650
|
+
/** Persist identity to the instance's actual config store. Classic instances
|
|
4651
|
+
* are registry rows in classicBot.yaml, not fleet.yaml instance entries. */
|
|
4652
|
+
setInstanceDisplayName(instanceName, displayName) {
|
|
4653
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4654
|
+
if (fleetInstance) {
|
|
4655
|
+
fleetInstance.display_name = displayName;
|
|
4656
|
+
this.saveFleetConfig();
|
|
4657
|
+
return true;
|
|
4658
|
+
}
|
|
4659
|
+
return this.classicChannels?.setDisplayNameByInstance(instanceName, displayName) ?? false;
|
|
4660
|
+
}
|
|
4661
|
+
setInstanceDescription(instanceName, description) {
|
|
4662
|
+
const fleetInstance = this.fleetConfig?.instances[instanceName];
|
|
4663
|
+
if (fleetInstance) {
|
|
4664
|
+
fleetInstance.description = description;
|
|
4665
|
+
this.saveFleetConfig();
|
|
4666
|
+
return true;
|
|
4667
|
+
}
|
|
4668
|
+
return this.classicChannels?.setDescriptionByInstance(instanceName, description) ?? false;
|
|
3790
4669
|
}
|
|
3791
4670
|
handleSetDisplayName(instanceName, msg) {
|
|
3792
4671
|
const fleetRequestId = msg.fleetRequestId;
|
|
@@ -3799,8 +4678,10 @@ export class FleetManager {
|
|
|
3799
4678
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: "Name must be 1-30 characters" });
|
|
3800
4679
|
return;
|
|
3801
4680
|
}
|
|
3802
|
-
this.
|
|
3803
|
-
|
|
4681
|
+
if (!this.setInstanceDisplayName(instanceName, displayName)) {
|
|
4682
|
+
ipc.send({ type: "fleet_display_name_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4683
|
+
return;
|
|
4684
|
+
}
|
|
3804
4685
|
this.logger.info({ instanceName, displayName }, "Display name set");
|
|
3805
4686
|
ipc.send({ type: "fleet_display_name_response", fleetRequestId, result: { display_name: displayName } });
|
|
3806
4687
|
}
|
|
@@ -3815,8 +4696,10 @@ export class FleetManager {
|
|
|
3815
4696
|
ipc.send({ type: "fleet_description_response", fleetRequestId, error: "Description cannot be empty" });
|
|
3816
4697
|
return;
|
|
3817
4698
|
}
|
|
3818
|
-
this.
|
|
3819
|
-
|
|
4699
|
+
if (!this.setInstanceDescription(instanceName, description)) {
|
|
4700
|
+
ipc.send({ type: "fleet_description_response", fleetRequestId, error: `Instance '${instanceName}' not found` });
|
|
4701
|
+
return;
|
|
4702
|
+
}
|
|
3820
4703
|
this.logger.info({ instanceName, description: description.slice(0, 80) }, "Description set");
|
|
3821
4704
|
ipc.send({ type: "fleet_description_response", fleetRequestId, result: { description } });
|
|
3822
4705
|
}
|
|
@@ -3945,8 +4828,8 @@ export class FleetManager {
|
|
|
3945
4828
|
return { error: "Fleet config not available" };
|
|
3946
4829
|
if (!name || name.length > 30)
|
|
3947
4830
|
return { error: "Name must be 1-30 characters" };
|
|
3948
|
-
this.
|
|
3949
|
-
|
|
4831
|
+
if (!this.setInstanceDisplayName(instance, name))
|
|
4832
|
+
return { error: `Instance '${instance}' not found` };
|
|
3950
4833
|
return { display_name: name };
|
|
3951
4834
|
}
|
|
3952
4835
|
async handleSetDescriptionHttp(instance, description) {
|
|
@@ -3954,8 +4837,8 @@ export class FleetManager {
|
|
|
3954
4837
|
return { error: "Fleet config not available" };
|
|
3955
4838
|
if (!description)
|
|
3956
4839
|
return { error: "Description cannot be empty" };
|
|
3957
|
-
this.
|
|
3958
|
-
|
|
4840
|
+
if (!this.setInstanceDescription(instance, description))
|
|
4841
|
+
return { error: `Instance '${instance}' not found` };
|
|
3959
4842
|
return { description };
|
|
3960
4843
|
}
|
|
3961
4844
|
summarizeToolCall(tool, args) {
|
|
@@ -4087,7 +4970,8 @@ export class FleetManager {
|
|
|
4087
4970
|
}
|
|
4088
4971
|
/**
|
|
4089
4972
|
* Patch only values changed in the effective config into the original YAML
|
|
4090
|
-
* document. Unknown keys
|
|
4973
|
+
* document. Unknown keys and comments remain untouched; redundant
|
|
4974
|
+
* non-identity instance leaves are canonicalized to inheritance afterward.
|
|
4091
4975
|
*/
|
|
4092
4976
|
saveFleetConfig(explicitPatches = []) {
|
|
4093
4977
|
if (!this.fleetConfig || !this.configPath)
|
|
@@ -4103,9 +4987,9 @@ export class FleetManager {
|
|
|
4103
4987
|
}
|
|
4104
4988
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4105
4989
|
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
4106
|
-
// Settings edits are expressed against the raw config
|
|
4107
|
-
//
|
|
4108
|
-
//
|
|
4990
|
+
// Settings edits are expressed against the raw config, so apply them before
|
|
4991
|
+
// canonicalization. A non-identity value equal to its inherited default is
|
|
4992
|
+
// intentionally stored as inheritance rather than an explicit duplicate.
|
|
4109
4993
|
for (const patch of explicitPatches) {
|
|
4110
4994
|
if (patch.remove) {
|
|
4111
4995
|
// YAML's deleteIn throws when an inherited nested key has no raw parent
|
|
@@ -4120,7 +5004,24 @@ export class FleetManager {
|
|
|
4120
5004
|
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
4121
5005
|
}
|
|
4122
5006
|
}
|
|
5007
|
+
const rawAfterPatches = this.rawFleetDocument.toJS();
|
|
5008
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(rawAfterPatches);
|
|
5009
|
+
for (const path of redundantPaths) {
|
|
5010
|
+
if (this.rawFleetDocument.hasIn(path))
|
|
5011
|
+
this.rawFleetDocument.deleteIn(path);
|
|
5012
|
+
// Avoid leaving empty operational maps such as `terminal: {}` while
|
|
5013
|
+
// preserving the instance mapping itself and all surrounding comments.
|
|
5014
|
+
for (let depth = path.length - 1; depth > 2; depth--) {
|
|
5015
|
+
const parentPath = path.slice(0, depth);
|
|
5016
|
+
const parent = this.rawFleetDocument.getIn(parentPath, true);
|
|
5017
|
+
if (!isMap(parent) || parent.items.length > 0)
|
|
5018
|
+
break;
|
|
5019
|
+
this.rawFleetDocument.deleteIn(parentPath);
|
|
5020
|
+
}
|
|
5021
|
+
}
|
|
4123
5022
|
const output = String(this.rawFleetDocument);
|
|
5023
|
+
if (redundantPaths.length > 0)
|
|
5024
|
+
this.writeFleetConfigBackup(source);
|
|
4124
5025
|
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
4125
5026
|
writeFileSync(tempPath, output, "utf-8");
|
|
4126
5027
|
if (existsSync(this.configPath))
|
|
@@ -4128,7 +5029,36 @@ export class FleetManager {
|
|
|
4128
5029
|
renameSync(tempPath, this.configPath);
|
|
4129
5030
|
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
4130
5031
|
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
4131
|
-
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
5032
|
+
this.logger.info({ path: this.configPath, strippedDefaults: redundantPaths.length }, "Saved fleet config (lossless patch)");
|
|
5033
|
+
}
|
|
5034
|
+
/** One-time upgrade migration; invalid YAML is never rewritten. */
|
|
5035
|
+
slimFleetConfigAtStartup() {
|
|
5036
|
+
const redundantPaths = collectRedundantInstanceDefaultPaths(this.rawFleetConfig);
|
|
5037
|
+
if (redundantPaths.length === 0)
|
|
5038
|
+
return;
|
|
5039
|
+
const validation = validateFleetConfig(this.rawFleetConfig);
|
|
5040
|
+
if (!validation.valid) {
|
|
5041
|
+
this.logger.warn({ errors: validation.errors, redundantDefaults: redundantPaths.length }, "Skipping fleet.yaml default slimming because the raw config is invalid");
|
|
5042
|
+
return;
|
|
5043
|
+
}
|
|
5044
|
+
try {
|
|
5045
|
+
this.saveFleetConfig();
|
|
5046
|
+
this.logger.info({ strippedDefaults: redundantPaths.length, backup: `${this.configPath}.bak` }, "Slimmed redundant instance defaults in fleet.yaml");
|
|
5047
|
+
}
|
|
5048
|
+
catch (err) {
|
|
5049
|
+
// A migration must not turn a previously bootable fleet into an outage.
|
|
5050
|
+
this.logger.warn({ err }, "Could not slim fleet.yaml; continuing with the original config");
|
|
5051
|
+
}
|
|
5052
|
+
}
|
|
5053
|
+
writeFleetConfigBackup(source) {
|
|
5054
|
+
if (!this.configPath)
|
|
5055
|
+
return;
|
|
5056
|
+
const backupPath = `${this.configPath}.bak`;
|
|
5057
|
+
const tempPath = `${backupPath}.tmp-${process.pid}`;
|
|
5058
|
+
writeFileSync(tempPath, source, "utf-8");
|
|
5059
|
+
if (existsSync(this.configPath))
|
|
5060
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
5061
|
+
renameSync(tempPath, backupPath);
|
|
4132
5062
|
}
|
|
4133
5063
|
patchFleetDocument(document, path, before, after) {
|
|
4134
5064
|
if (Object.is(before, after))
|
|
@@ -4180,7 +5110,12 @@ export class FleetManager {
|
|
|
4180
5110
|
currentNode.value = after;
|
|
4181
5111
|
}
|
|
4182
5112
|
else {
|
|
4183
|
-
|
|
5113
|
+
// Use a YAML collection node for newly-added objects. Passing the raw
|
|
5114
|
+
// object creates a scalar wrapper: it serializes, but nested hasIn /
|
|
5115
|
+
// deleteIn cannot traverse it (notably when slimming create_instance).
|
|
5116
|
+
document.setIn(path, after !== null && typeof after === "object"
|
|
5117
|
+
? document.createNode(after)
|
|
5118
|
+
: after);
|
|
4184
5119
|
}
|
|
4185
5120
|
}
|
|
4186
5121
|
}
|
|
@@ -4271,7 +5206,7 @@ export class FleetManager {
|
|
|
4271
5206
|
from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct,
|
|
4272
5207
|
});
|
|
4273
5208
|
this.webhookEmitter?.emit("model_failover", name, { from: primaryModel, to: fallbackModel, five_hour_pct: fiveHourPct });
|
|
4274
|
-
this.notifyInstanceTopic(name,
|
|
5209
|
+
this.notifyInstanceTopic(name, t("failover.triggered", fiveHourPct, fallbackModel, primaryModel));
|
|
4275
5210
|
}
|
|
4276
5211
|
else if (fiveHourPct < FleetManager.FAILOVER_RECOVER_PCT && currentFailover) {
|
|
4277
5212
|
// Recover: switch back to primary
|
|
@@ -4282,7 +5217,7 @@ export class FleetManager {
|
|
|
4282
5217
|
restored: primaryModel, five_hour_pct: fiveHourPct,
|
|
4283
5218
|
});
|
|
4284
5219
|
this.webhookEmitter?.emit("model_recovered", name, { restored: primaryModel, five_hour_pct: fiveHourPct });
|
|
4285
|
-
this.notifyInstanceTopic(name,
|
|
5220
|
+
this.notifyInstanceTopic(name, t("failover.recovered", fiveHourPct, primaryModel));
|
|
4286
5221
|
}
|
|
4287
5222
|
}
|
|
4288
5223
|
toggleFleetCollab(instanceName) {
|
|
@@ -4336,9 +5271,9 @@ export class FleetManager {
|
|
|
4336
5271
|
if (output)
|
|
4337
5272
|
return output;
|
|
4338
5273
|
if (e.code === "ENOENT") {
|
|
4339
|
-
return "
|
|
5274
|
+
return t("doctor.cli_missing");
|
|
4340
5275
|
}
|
|
4341
|
-
return stripAnsi(e.message ?? "").trim() || "
|
|
5276
|
+
return stripAnsi(e.message ?? "").trim() || t("doctor.failed");
|
|
4342
5277
|
}
|
|
4343
5278
|
}
|
|
4344
5279
|
/** Drop event/activity rows older than the retention window. Best-effort. */
|
|
@@ -4440,7 +5375,7 @@ export class FleetManager {
|
|
|
4440
5375
|
this.fleetErrorNotices.delete(oldest);
|
|
4441
5376
|
}
|
|
4442
5377
|
const body = suppressed > 0
|
|
4443
|
-
? `${text}\n(
|
|
5378
|
+
? `${text}\n${t("fleet.error_suppressed", suppressed, Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000))}`
|
|
4444
5379
|
: text;
|
|
4445
5380
|
// Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
|
|
4446
5381
|
// and a fleet-level fault is exactly when the General may be down. The topic
|
|
@@ -4490,12 +5425,13 @@ export class FleetManager {
|
|
|
4490
5425
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
4491
5426
|
}
|
|
4492
5427
|
}
|
|
4493
|
-
// ── Nonce-armed button prompts (hang /
|
|
5428
|
+
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
4494
5429
|
//
|
|
4495
5430
|
// One shared lifecycle for every "notification with decision buttons":
|
|
4496
|
-
// post with a 128-bit nonce, arm a
|
|
4497
|
-
// exact adapter+chat+thread+message that created it, require fleet admin
|
|
4498
|
-
//
|
|
5431
|
+
// post with a 128-bit nonce, arm a bounded expiry, bind the click to the
|
|
5432
|
+
// exact adapter+chat+thread+message that created it, require fleet admin for
|
|
5433
|
+
// actions that mutate runtime state (Tips are the harmless exception),
|
|
5434
|
+
// consume exactly once. The features differ only in what they post
|
|
4499
5435
|
// and what a consumed click does.
|
|
4500
5436
|
/**
|
|
4501
5437
|
* Post decision buttons whose callback ids are `<prefix><nonce>:<action>`.
|
|
@@ -4505,8 +5441,14 @@ export class FleetManager {
|
|
|
4505
5441
|
*/
|
|
4506
5442
|
async postNonceButtonPrompt(opts) {
|
|
4507
5443
|
// 16 bytes = the 128-bit capability the design claims. Telegram's 64-byte
|
|
4508
|
-
// callback_data cap still holds
|
|
4509
|
-
// + 32 hex + ":confirm" (8) = 59
|
|
5444
|
+
// callback_data cap still holds, with two prefixes tied at the longest:
|
|
5445
|
+
// "interactive-assist:" (19) + 32 hex + ":confirm" (8) = 59, and
|
|
5446
|
+
// "install-select:" (15) + 32 hex + ":" + the longest backend name
|
|
5447
|
+
// ("claude-code"/"antigravity", 11) = 59. The longest overall is
|
|
5448
|
+
// "classic-approve:" (16) + 32 hex + ":allow-admin" (12) = 60, leaving 4
|
|
5449
|
+
// bytes. A longer prefix or action would be silently rejected by Telegram —
|
|
5450
|
+
// both sets are pinned by callback_data assertions in
|
|
5451
|
+
// install-backend-menu.test.ts and classic-approve-buttons.test.ts.
|
|
4510
5452
|
const nonce = randomBytes(16).toString("hex");
|
|
4511
5453
|
const entry = {
|
|
4512
5454
|
prefix: opts.prefix,
|
|
@@ -4536,6 +5478,17 @@ export class FleetManager {
|
|
|
4536
5478
|
message: opts.message,
|
|
4537
5479
|
choices: opts.choices.map(c => ({ id: `${opts.prefix}${nonce}:${c.action}`, label: c.label })),
|
|
4538
5480
|
}, opts.threadId ? { threadId: opts.threadId } : undefined);
|
|
5481
|
+
// Bind the nonce to the provider's canonical delivery address, not the
|
|
5482
|
+
// logical routing input. Telegram, for example, represents General as
|
|
5483
|
+
// AgEnD topic "1" on input but omits message_thread_id on the wire and in
|
|
5484
|
+
// callback queries. Exact callback matching below remains fail-closed.
|
|
5485
|
+
entry.chatId = sent.chatId;
|
|
5486
|
+
// Preserve compatibility with adapters that predate SentMessage.threadId
|
|
5487
|
+
// and omit the optional property altogether. An explicit undefined is a
|
|
5488
|
+
// canonical flat/root delivery context (notably Telegram General).
|
|
5489
|
+
if (Object.prototype.hasOwnProperty.call(sent, "threadId")) {
|
|
5490
|
+
entry.threadId = sent.threadId;
|
|
5491
|
+
}
|
|
4539
5492
|
entry.messageId = sent.messageId;
|
|
4540
5493
|
return nonce;
|
|
4541
5494
|
}
|
|
@@ -4548,19 +5501,18 @@ export class FleetManager {
|
|
|
4548
5501
|
}
|
|
4549
5502
|
}
|
|
4550
5503
|
/**
|
|
4551
|
-
*
|
|
5504
|
+
* Validate a nonce-armed callback and claim it exactly once.
|
|
4552
5505
|
*
|
|
4553
5506
|
* Returns:
|
|
4554
5507
|
* - null — the callback is not for this prefix; try the next handler
|
|
4555
5508
|
* - "consumed" — for this prefix but stale/denied/malformed; stop dispatch
|
|
4556
|
-
* - the entry+action — the click is authorized and
|
|
4557
|
-
* from the map before any await, so double clicks cannot act twice)
|
|
5509
|
+
* - the entry+action — the click is authorized and claimed before any await
|
|
4558
5510
|
*
|
|
4559
5511
|
* A stale click (expired nonce, or a pre-upgrade button whose payload no
|
|
4560
5512
|
* longer parses) collapses the clicked message so the dead button stops
|
|
4561
5513
|
* inviting clicks — the same courtesy the cancel button extends.
|
|
4562
5514
|
*/
|
|
4563
|
-
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter) {
|
|
5515
|
+
consumeNonceCallback(prefix, actionRe, data, callbackAdapterId, receivingAdapter, staleHandling) {
|
|
4564
5516
|
if (!data.callbackData.startsWith(prefix))
|
|
4565
5517
|
return null;
|
|
4566
5518
|
const match = data.callbackData.match(actionRe);
|
|
@@ -4572,23 +5524,52 @@ export class FleetManager {
|
|
|
4572
5524
|
pending = undefined;
|
|
4573
5525
|
if (!match || !pending) {
|
|
4574
5526
|
const adapter = receivingAdapter ?? this.adapter;
|
|
4575
|
-
|
|
5527
|
+
if (staleHandling?.keepText && adapter?.removeMessageButtons) {
|
|
5528
|
+
adapter.removeMessageButtons(data.chatId, data.messageId, data.threadId)
|
|
5529
|
+
.catch(() => { });
|
|
5530
|
+
}
|
|
5531
|
+
else {
|
|
5532
|
+
adapter?.editMessageRemoveButtons?.(data.chatId, data.messageId, t("buttons.stale"), data.threadId).catch(() => { });
|
|
5533
|
+
}
|
|
5534
|
+
if (staleHandling?.notice) {
|
|
5535
|
+
adapter?.sendText(data.chatId, staleHandling.notice, { threadId: data.threadId })
|
|
5536
|
+
.catch(() => { });
|
|
5537
|
+
}
|
|
4576
5538
|
return "consumed";
|
|
4577
5539
|
}
|
|
4578
5540
|
// Bind the capability to the exact message/world that created it. Telegram
|
|
4579
|
-
// keyboards are visible to everyone, so
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
5541
|
+
// keyboards are visible to everyone, so mutating actions require fleet admin;
|
|
5542
|
+
// a Tip acknowledgement only records that the shared content was read.
|
|
5543
|
+
const isAuthorized = data.userId
|
|
5544
|
+
? pending.allowAnyUser
|
|
5545
|
+
? true
|
|
5546
|
+
: pending.authChannelId
|
|
5547
|
+
? this.isModelAdmin(data.userId, pending.authChannelId, callbackAdapterId)
|
|
5548
|
+
: this.isFleetAdmin(data.userId, callbackAdapterId)
|
|
5549
|
+
: false;
|
|
5550
|
+
const mismatchedFields = [];
|
|
5551
|
+
if (pending.adapterId !== callbackAdapterId)
|
|
5552
|
+
mismatchedFields.push("adapterId");
|
|
5553
|
+
if (data.chatId !== pending.chatId)
|
|
5554
|
+
mismatchedFields.push("chatId");
|
|
5555
|
+
if (pending.threadId != null && data.threadId !== pending.threadId)
|
|
5556
|
+
mismatchedFields.push("threadId");
|
|
5557
|
+
if (pending.messageId != null && data.messageId !== pending.messageId)
|
|
5558
|
+
mismatchedFields.push("messageId");
|
|
5559
|
+
if (!isAuthorized)
|
|
5560
|
+
mismatchedFields.push("authorization");
|
|
5561
|
+
if (mismatchedFields.length > 0) {
|
|
4586
5562
|
// Deliberately does NOT consume the nonce: the real admin can still click.
|
|
4587
|
-
this.logger.warn({
|
|
5563
|
+
this.logger.warn({
|
|
5564
|
+
instanceName: pending.instanceName,
|
|
5565
|
+
prefix,
|
|
5566
|
+
userId: data.userId,
|
|
5567
|
+
mismatchedFields: mismatchedFields.join(","),
|
|
5568
|
+
}, "Rejected unauthorized or mismatched button callback");
|
|
4588
5569
|
return "consumed";
|
|
4589
5570
|
}
|
|
4590
|
-
// Claim before any await: double clicks and duplicate callback delivery
|
|
4591
|
-
// never act twice.
|
|
5571
|
+
// Claim before any await: double clicks and duplicate callback delivery
|
|
5572
|
+
// can never act twice for state-changing actions.
|
|
4592
5573
|
this.pendingNonceButtons.delete(match[1]);
|
|
4593
5574
|
if (pending.timer)
|
|
4594
5575
|
clearTimeout(pending.timer);
|
|
@@ -4609,6 +5590,270 @@ export class FleetManager {
|
|
|
4609
5590
|
this.logger.warn({ err, instanceName: pending.instanceName }, "Failed to retire prompt buttons");
|
|
4610
5591
|
}
|
|
4611
5592
|
}
|
|
5593
|
+
tipsEnabled() {
|
|
5594
|
+
return this.fleetConfig?.defaults.tips !== false;
|
|
5595
|
+
}
|
|
5596
|
+
readTipState() {
|
|
5597
|
+
if (!this.scheduler)
|
|
5598
|
+
return null;
|
|
5599
|
+
try {
|
|
5600
|
+
return {
|
|
5601
|
+
dismissed: this.scheduler.db.listDismissedTipIds(),
|
|
5602
|
+
advancedUnlocked: this.scheduler.db.isAdvancedTipsUnlocked(),
|
|
5603
|
+
};
|
|
5604
|
+
}
|
|
5605
|
+
catch (err) {
|
|
5606
|
+
// Tips are additive. A damaged/locked scheduler DB must never block fleet
|
|
5607
|
+
// startup, update completion, or a General command.
|
|
5608
|
+
this.logger.warn({ err }, "Failed to read tip state");
|
|
5609
|
+
return null;
|
|
5610
|
+
}
|
|
5611
|
+
}
|
|
5612
|
+
pickAvailableTip() {
|
|
5613
|
+
const state = this.readTipState();
|
|
5614
|
+
return state
|
|
5615
|
+
? selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds())
|
|
5616
|
+
: null;
|
|
5617
|
+
}
|
|
5618
|
+
tipText(tip) {
|
|
5619
|
+
return getLocale() === "zh-TW" ? tip.text_zh : tip.text_en;
|
|
5620
|
+
}
|
|
5621
|
+
formatTip(tip) {
|
|
5622
|
+
return `💡 ${t("tips.label")}: ${this.tipText(tip)}`;
|
|
5623
|
+
}
|
|
5624
|
+
/** Persistently enable advanced tips through the admin-only slash command. */
|
|
5625
|
+
unlockAdvancedTips(userId) {
|
|
5626
|
+
if (!this.scheduler)
|
|
5627
|
+
return false;
|
|
5628
|
+
try {
|
|
5629
|
+
this.scheduler.db.unlockAdvancedTips(userId);
|
|
5630
|
+
this.eventLog?.insert(this.findGeneralInstance() ?? "general", "tips_advanced_unlocked", {
|
|
5631
|
+
userId,
|
|
5632
|
+
source: "command",
|
|
5633
|
+
});
|
|
5634
|
+
return true;
|
|
5635
|
+
}
|
|
5636
|
+
catch (err) {
|
|
5637
|
+
this.logger.warn({ err, userId }, "Failed to unlock advanced tips from command");
|
|
5638
|
+
return false;
|
|
5639
|
+
}
|
|
5640
|
+
}
|
|
5641
|
+
/** Post one fresh nonce-armed tip in a known General channel. */
|
|
5642
|
+
async promptTip(generalName, adapter, chatId, threadId) {
|
|
5643
|
+
const state = this.readTipState();
|
|
5644
|
+
if (!state)
|
|
5645
|
+
return "unavailable";
|
|
5646
|
+
if (!state.advancedUnlocked
|
|
5647
|
+
&& visibleTipLevels(true).has("advanced")
|
|
5648
|
+
&& canUnlockAdvancedTips(state.dismissed)) {
|
|
5649
|
+
return await this.promptAdvancedTipUnlock(generalName, adapter, chatId, threadId)
|
|
5650
|
+
? "posted"
|
|
5651
|
+
: "unavailable";
|
|
5652
|
+
}
|
|
5653
|
+
const tip = selectTip(state.dismissed, Math.random, state.advancedUnlocked, this.getActiveBackendIds());
|
|
5654
|
+
if (!tip)
|
|
5655
|
+
return this.scheduler ? "empty" : "unavailable";
|
|
5656
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5657
|
+
prefix: TIP_DISMISS_CALLBACK_PREFIX,
|
|
5658
|
+
alertType: "tip",
|
|
5659
|
+
instanceName: generalName,
|
|
5660
|
+
adapter,
|
|
5661
|
+
adapterId: adapter.id,
|
|
5662
|
+
chatId,
|
|
5663
|
+
threadId,
|
|
5664
|
+
message: this.formatTip(tip),
|
|
5665
|
+
choices: [
|
|
5666
|
+
{ action: "dismiss", label: t("tips.dismiss") },
|
|
5667
|
+
{ action: "confused", label: t("tips.confused") },
|
|
5668
|
+
],
|
|
5669
|
+
// Expiry removes only the stale button; the useful tip remains readable.
|
|
5670
|
+
expiredText: this.formatTip(tip),
|
|
5671
|
+
extra: { allowAnyUser: true, tipId: tip.id },
|
|
5672
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5673
|
+
});
|
|
5674
|
+
return nonce ? "posted" : "unavailable";
|
|
5675
|
+
}
|
|
5676
|
+
async promptAdvancedTipUnlock(generalName, adapter, chatId, threadId) {
|
|
5677
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5678
|
+
prefix: TIP_UNLOCK_CALLBACK_PREFIX,
|
|
5679
|
+
alertType: "tip",
|
|
5680
|
+
instanceName: generalName,
|
|
5681
|
+
adapter,
|
|
5682
|
+
adapterId: adapter.id,
|
|
5683
|
+
chatId,
|
|
5684
|
+
threadId,
|
|
5685
|
+
message: t("tips.advanced.unlock_prompt"),
|
|
5686
|
+
choices: [{ action: "unlock", label: t("tips.advanced.unlock") }],
|
|
5687
|
+
expiredText: t("tips.advanced.expired"),
|
|
5688
|
+
extra: { allowAnyUser: true },
|
|
5689
|
+
timeoutMs: TIP_BUTTON_TIMEOUT_MS,
|
|
5690
|
+
});
|
|
5691
|
+
return nonce !== null;
|
|
5692
|
+
}
|
|
5693
|
+
async sendTipToGeneral() {
|
|
5694
|
+
if (!this.tipsEnabled())
|
|
5695
|
+
return;
|
|
5696
|
+
const generalName = this.findGeneralInstance();
|
|
5697
|
+
if (!generalName)
|
|
5698
|
+
return;
|
|
5699
|
+
const adapter = this.getAdapterForInstance(generalName);
|
|
5700
|
+
const chatId = this.getGroupIdForInstance(generalName);
|
|
5701
|
+
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
5702
|
+
if (!adapter || !chatId)
|
|
5703
|
+
return;
|
|
5704
|
+
const result = await this.promptTip(generalName, adapter, chatId, topicId != null ? String(topicId) : undefined);
|
|
5705
|
+
if (result === "unavailable") {
|
|
5706
|
+
this.logger.warn({ generalName }, "Daily tip could not be posted");
|
|
5707
|
+
}
|
|
5708
|
+
}
|
|
5709
|
+
async handleTipDismiss(data, callbackAdapterId, receivingAdapter) {
|
|
5710
|
+
const claimed = this.consumeNonceCallback(TIP_DISMISS_CALLBACK_PREFIX, /^tip-dismiss:([0-9a-f]+):(dismiss|confused)$/, data, callbackAdapterId, receivingAdapter,
|
|
5711
|
+
// A restart orphans the expiry timer, so tip buttons can outlive their
|
|
5712
|
+
// nonce by days. Keep the tip text readable and always tell the clicker
|
|
5713
|
+
// what to do next (the keyboard edit itself can be refused: Telegram
|
|
5714
|
+
// rejects edits on messages older than 48h).
|
|
5715
|
+
{ keepText: true, notice: t("tips.expired_use_tips") });
|
|
5716
|
+
if (claimed === null)
|
|
5717
|
+
return false;
|
|
5718
|
+
if (claimed === "consumed")
|
|
5719
|
+
return true;
|
|
5720
|
+
const { entry: pending } = claimed;
|
|
5721
|
+
if (claimed.action === "confused") {
|
|
5722
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5723
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip feedback has incomplete persistence context");
|
|
5724
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5725
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId });
|
|
5726
|
+
return true;
|
|
5727
|
+
}
|
|
5728
|
+
try {
|
|
5729
|
+
this.scheduler.db.recordTipFeedback(data.userId, pending.tipId, "confused");
|
|
5730
|
+
}
|
|
5731
|
+
catch (err) {
|
|
5732
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip feedback");
|
|
5733
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, pending.expiredText);
|
|
5734
|
+
await pending.adapter.sendText(pending.chatId, t("tips.unavailable"), { threadId: pending.threadId }).catch(() => { });
|
|
5735
|
+
return true;
|
|
5736
|
+
}
|
|
5737
|
+
this.logger.info({
|
|
5738
|
+
tipId: pending.tipId,
|
|
5739
|
+
userId: data.userId,
|
|
5740
|
+
feedbackType: "confused",
|
|
5741
|
+
}, "Tip feedback recorded");
|
|
5742
|
+
this.eventLog?.insert(pending.instanceName, "tip_feedback", {
|
|
5743
|
+
tipId: pending.tipId,
|
|
5744
|
+
userId: data.userId,
|
|
5745
|
+
feedbackType: "confused",
|
|
5746
|
+
});
|
|
5747
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5748
|
+
// Confusion is feedback, not a dismissal. Keep the Tip readable while
|
|
5749
|
+
// retiring this one-shot prompt; it remains eligible for future draws.
|
|
5750
|
+
pending.expiredText);
|
|
5751
|
+
await pending.adapter.sendText(pending.chatId, t("tips.feedback_recorded"), { threadId: pending.threadId }).catch(err => this.logger.warn({ err, tipId: pending.tipId }, "Failed to acknowledge tip feedback"));
|
|
5752
|
+
return true;
|
|
5753
|
+
}
|
|
5754
|
+
if (!pending.tipId || !data.userId || !this.scheduler) {
|
|
5755
|
+
this.logger.error({ tipId: pending.tipId, userId: data.userId }, "Tip dismissal has incomplete persistence context");
|
|
5756
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5757
|
+
return true;
|
|
5758
|
+
}
|
|
5759
|
+
try {
|
|
5760
|
+
this.scheduler.db.dismissTip(data.userId, pending.tipId);
|
|
5761
|
+
}
|
|
5762
|
+
catch (err) {
|
|
5763
|
+
this.logger.warn({ err, tipId: pending.tipId, userId: data.userId }, "Failed to persist tip dismissal");
|
|
5764
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5765
|
+
return true;
|
|
5766
|
+
}
|
|
5767
|
+
this.eventLog?.insert(pending.instanceName, "tip_dismissed", {
|
|
5768
|
+
tipId: pending.tipId,
|
|
5769
|
+
userId: data.userId,
|
|
5770
|
+
});
|
|
5771
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId,
|
|
5772
|
+
// Dismissal changes future selection only. Keep the useful tip text in
|
|
5773
|
+
// chat history and remove just the now-consumed button. Append a short
|
|
5774
|
+
// confirmation so the user sees feedback (unlike the pre-#605 version
|
|
5775
|
+
// that replaced the entire message with just the confirmation line).
|
|
5776
|
+
`${pending.expiredText}\n\n${t("tips.dismissed")}`);
|
|
5777
|
+
const state = this.readTipState();
|
|
5778
|
+
if (state && !state.advancedUnlocked && canUnlockAdvancedTips(state.dismissed)) {
|
|
5779
|
+
await this.promptAdvancedTipUnlock(pending.instanceName, pending.adapter, pending.chatId, pending.threadId);
|
|
5780
|
+
}
|
|
5781
|
+
return true;
|
|
5782
|
+
}
|
|
5783
|
+
async handleTipUnlock(data, callbackAdapterId, receivingAdapter) {
|
|
5784
|
+
const claimed = this.consumeNonceCallback(TIP_UNLOCK_CALLBACK_PREFIX, /^tip-unlock:([0-9a-f]+):(unlock)$/, data, callbackAdapterId, receivingAdapter, { keepText: true, notice: t("tips.expired_use_tips") });
|
|
5785
|
+
if (claimed === null)
|
|
5786
|
+
return false;
|
|
5787
|
+
if (claimed === "consumed")
|
|
5788
|
+
return true;
|
|
5789
|
+
const { entry: pending } = claimed;
|
|
5790
|
+
if (!data.userId || !this.scheduler) {
|
|
5791
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5792
|
+
return true;
|
|
5793
|
+
}
|
|
5794
|
+
try {
|
|
5795
|
+
this.scheduler.db.unlockAdvancedTips(data.userId);
|
|
5796
|
+
}
|
|
5797
|
+
catch (err) {
|
|
5798
|
+
this.logger.warn({ err, userId: data.userId }, "Failed to unlock advanced tips");
|
|
5799
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.unavailable"));
|
|
5800
|
+
return true;
|
|
5801
|
+
}
|
|
5802
|
+
this.eventLog?.insert(pending.instanceName, "tips_advanced_unlocked", { userId: data.userId });
|
|
5803
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("tips.advanced.unlocked"));
|
|
5804
|
+
return true;
|
|
5805
|
+
}
|
|
5806
|
+
/** Post the destructive `/clear` confirmation in the invoking channel. */
|
|
5807
|
+
async promptClearConfirmation(instanceName, channelId, adapter, chatId, threadId) {
|
|
5808
|
+
if (!this.topicCommands.supportsClear(instanceName))
|
|
5809
|
+
return t("clear.unsupported");
|
|
5810
|
+
const adapterId = adapter.id;
|
|
5811
|
+
if (!adapterId)
|
|
5812
|
+
return t("clear.prompt_unavailable");
|
|
5813
|
+
const nonce = await this.postNonceButtonPrompt({
|
|
5814
|
+
prefix: CLEAR_CONFIRM_CALLBACK_PREFIX,
|
|
5815
|
+
alertType: "clear_confirm",
|
|
5816
|
+
instanceName,
|
|
5817
|
+
adapter,
|
|
5818
|
+
adapterId,
|
|
5819
|
+
chatId,
|
|
5820
|
+
threadId,
|
|
5821
|
+
message: t("clear.confirm_message", instanceName),
|
|
5822
|
+
choices: [
|
|
5823
|
+
{ action: "confirm", label: t("clear.confirm") },
|
|
5824
|
+
{ action: "cancel", label: t("clear.cancel") },
|
|
5825
|
+
],
|
|
5826
|
+
expiredText: t("clear.expired", instanceName),
|
|
5827
|
+
extra: { authChannelId: channelId },
|
|
5828
|
+
timeoutMs: CLEAR_CONFIRM_TIMEOUT_MS,
|
|
5829
|
+
});
|
|
5830
|
+
return nonce ? null : t("clear.prompt_unavailable");
|
|
5831
|
+
}
|
|
5832
|
+
/** Consume `/clear` Confirm/Cancel exactly once; only Confirm reaches IPC. */
|
|
5833
|
+
async handleClearConfirmation(data, callbackAdapterId, receivingAdapter) {
|
|
5834
|
+
const claimed = this.consumeNonceCallback(CLEAR_CONFIRM_CALLBACK_PREFIX, /^clear-confirm:([0-9a-f]+):(confirm|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
5835
|
+
if (claimed === null)
|
|
5836
|
+
return false;
|
|
5837
|
+
if (claimed === "consumed")
|
|
5838
|
+
return true;
|
|
5839
|
+
const { entry: pending, action } = claimed;
|
|
5840
|
+
this.eventLog?.insert(pending.instanceName, "clear_action", { action, userId: data.userId });
|
|
5841
|
+
if (action === "cancel") {
|
|
5842
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("clear.cancelled", pending.instanceName));
|
|
5843
|
+
return true;
|
|
5844
|
+
}
|
|
5845
|
+
await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("clear.clearing", pending.instanceName));
|
|
5846
|
+
try {
|
|
5847
|
+
const result = await this.topicCommands.sendClear(pending.instanceName);
|
|
5848
|
+
await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, result, pending.threadId);
|
|
5849
|
+
}
|
|
5850
|
+
catch (err) {
|
|
5851
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
5852
|
+
this.logger.error({ err, instanceName: pending.instanceName }, "Clear command failed");
|
|
5853
|
+
await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("clear.failed", pending.instanceName, message), pending.threadId).catch(editErr => this.logger.warn({ err: editErr, instanceName: pending.instanceName }, "Failed to show clear command error"));
|
|
5854
|
+
}
|
|
5855
|
+
return true;
|
|
5856
|
+
}
|
|
4612
5857
|
/**
|
|
4613
5858
|
* Drop every pending prompt that refers to an instance being stopped or
|
|
4614
5859
|
* restarted — a restart offer for an instance the operator just restarted
|
|
@@ -4835,7 +6080,7 @@ export class FleetManager {
|
|
|
4835
6080
|
const force = target.kind === "classic" && !!data.options?.force;
|
|
4836
6081
|
const cmd = saveCommandForBackend(backend, filename, force);
|
|
4837
6082
|
if (!cmd) {
|
|
4838
|
-
await data.respond(
|
|
6083
|
+
await data.respond(t("save.unsupported"));
|
|
4839
6084
|
return;
|
|
4840
6085
|
}
|
|
4841
6086
|
if (target.kind === "classic") {
|
|
@@ -4888,7 +6133,7 @@ export class FleetManager {
|
|
|
4888
6133
|
}
|
|
4889
6134
|
return false;
|
|
4890
6135
|
}
|
|
4891
|
-
async sendCancelButton(instanceName, correlationId) {
|
|
6136
|
+
async sendCancelButton(instanceName, correlationId, preserveProgress = false) {
|
|
4892
6137
|
// Post first, retire after (see the tail of this method). Retiring up front
|
|
4893
6138
|
// meant that from the delete until the new message came back — a chat API
|
|
4894
6139
|
// round trip, and every reply goes through here — the instance had NO live
|
|
@@ -4949,6 +6194,10 @@ export class FleetManager {
|
|
|
4949
6194
|
// not re-edit identical text — which put a "(edited)" mark on Discord
|
|
4950
6195
|
// with nothing visibly changed.
|
|
4951
6196
|
lastProgressText: "👀 處理中…",
|
|
6197
|
+
// A reply can re-post the same in-flight bubble below the reply. Carry
|
|
6198
|
+
// its current list into that replacement; fresh inbound work must start
|
|
6199
|
+
// empty even if the daemon's reset broadcast is still in flight.
|
|
6200
|
+
toolProgress: preserveProgress ? this.instanceProgress.get(instanceName) : undefined,
|
|
4952
6201
|
};
|
|
4953
6202
|
this.startProgressTicker(entry);
|
|
4954
6203
|
// Idle-check backstop: every 5min, if the instance is idle, retire the
|
|
@@ -5002,16 +6251,28 @@ export class FleetManager {
|
|
|
5002
6251
|
static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
|
|
5003
6252
|
if (elapsedMs < minElapsedMs)
|
|
5004
6253
|
return "👀 處理中…";
|
|
6254
|
+
const elapsed = FleetManager.formatProgressElapsed(elapsedMs);
|
|
6255
|
+
const detail = FleetManager.sanitizeActivity(activity);
|
|
6256
|
+
return detail
|
|
6257
|
+
? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
|
|
6258
|
+
: `⏳ 處理中… (已進行 ${elapsed})`;
|
|
6259
|
+
}
|
|
6260
|
+
/** Render elapsed time consistently in live and retained progress bubbles. */
|
|
6261
|
+
static formatProgressElapsed(elapsedMs) {
|
|
5005
6262
|
const totalSeconds = Math.floor(elapsedMs / 1000);
|
|
5006
6263
|
const minutes = Math.floor(totalSeconds / 60);
|
|
5007
6264
|
const seconds = totalSeconds % 60;
|
|
5008
|
-
|
|
6265
|
+
return minutes >= 60
|
|
5009
6266
|
? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
|
5010
6267
|
: `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
6268
|
+
}
|
|
6269
|
+
/** Final read-only form of a bubble that contains semantic tool history.
|
|
6270
|
+
* Neutral wording is intentional: the same retirement primitive is used for
|
|
6271
|
+
* normal completion, a mid-turn bubble replacement, and user cancellation,
|
|
6272
|
+
* so claiming every retained checkpoint is "completed" would be false. */
|
|
6273
|
+
composeRetiredBubbleText(entry) {
|
|
6274
|
+
const elapsed = FleetManager.formatProgressElapsed(Date.now() - (entry.startedAt ?? Date.now()));
|
|
6275
|
+
return `🧾 工具歷程 (記錄至 ${elapsed})\n${entry.toolProgress}`;
|
|
5015
6276
|
}
|
|
5016
6277
|
/**
|
|
5017
6278
|
* Make a tool summary safe to paste into a channel message.
|
|
@@ -5045,9 +6306,91 @@ export class FleetManager {
|
|
|
5045
6306
|
this.instanceActivity.delete(name);
|
|
5046
6307
|
}
|
|
5047
6308
|
/**
|
|
5048
|
-
*
|
|
5049
|
-
*
|
|
5050
|
-
*
|
|
6309
|
+
* Cache the turn's tool-progress list and push it into the instance's live
|
|
6310
|
+
* bubble, coalesced so a burst of tool events cannot flood the Bot API
|
|
6311
|
+
* (Telegram's flood limit is per-chat across ALL forum topics, so edits are
|
|
6312
|
+
* rate-limited per bubble AND ride behind the daemon-side 3s coalescer).
|
|
6313
|
+
*/
|
|
6314
|
+
cacheInstanceProgress(name, progress) {
|
|
6315
|
+
if (progress)
|
|
6316
|
+
this.instanceProgress.set(name, progress);
|
|
6317
|
+
else
|
|
6318
|
+
this.instanceProgress.delete(name);
|
|
6319
|
+
for (const entry of this.cancelButtons.values()) {
|
|
6320
|
+
if (entry.instanceName !== name)
|
|
6321
|
+
continue;
|
|
6322
|
+
// Empty means the daemon completed/reset the turn. Clear the live cache
|
|
6323
|
+
// for the next turn, but retain this bubble's last non-empty list so its
|
|
6324
|
+
// retirement can preserve the history instead of deleting the message.
|
|
6325
|
+
if (progress)
|
|
6326
|
+
entry.toolProgress = progress;
|
|
6327
|
+
this.scheduleProgressEdit(entry);
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
/** At most one progress-driven edit per bubble per TOOL_PROGRESS_EDIT_MIN_MS. */
|
|
6331
|
+
scheduleProgressEdit(entry) {
|
|
6332
|
+
if (entry.retiring)
|
|
6333
|
+
return;
|
|
6334
|
+
const since = Date.now() - (entry.lastProgressEditAt ?? 0);
|
|
6335
|
+
if (since >= TOOL_PROGRESS_EDIT_MIN_MS) {
|
|
6336
|
+
this.refreshBubble(entry);
|
|
6337
|
+
return;
|
|
6338
|
+
}
|
|
6339
|
+
if (entry.progressEditTimer)
|
|
6340
|
+
return; // trailing edit already scheduled
|
|
6341
|
+
entry.progressEditTimer = setTimeout(() => {
|
|
6342
|
+
entry.progressEditTimer = undefined;
|
|
6343
|
+
this.refreshBubble(entry);
|
|
6344
|
+
}, TOOL_PROGRESS_EDIT_MIN_MS - since);
|
|
6345
|
+
entry.progressEditTimer.unref?.();
|
|
6346
|
+
}
|
|
6347
|
+
/**
|
|
6348
|
+
* The ONE composer for the bubble text. Both writers — the elapsed-time
|
|
6349
|
+
* ticker and the tool-progress push — go through here; two independent
|
|
6350
|
+
* renderers editing the same message is how the progress list used to get
|
|
6351
|
+
* wiped by the next elapsed tick (#528 trap 2).
|
|
6352
|
+
*/
|
|
6353
|
+
composeBubbleText(entry) {
|
|
6354
|
+
return FleetManager.bubbleText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs(), entry.toolProgress);
|
|
6355
|
+
}
|
|
6356
|
+
/** Pure composition of header + tool list, exposed for tests. */
|
|
6357
|
+
static bubbleText(elapsedMs, activity, minElapsedMs, progress) {
|
|
6358
|
+
const header = FleetManager.progressText(elapsedMs,
|
|
6359
|
+
// The single-line activity detail is redundant once a tool list exists.
|
|
6360
|
+
progress ? undefined : activity, minElapsedMs);
|
|
6361
|
+
return progress ? `${header}\n${progress}` : header;
|
|
6362
|
+
}
|
|
6363
|
+
/** Recompose and edit the bubble in place; skips when nothing changed. */
|
|
6364
|
+
refreshBubble(entry) {
|
|
6365
|
+
if (entry.retiring || !this.cancelButtons.has(entry.messageId)) {
|
|
6366
|
+
clearInterval(entry.progressTimer);
|
|
6367
|
+
return;
|
|
6368
|
+
}
|
|
6369
|
+
const text = this.composeBubbleText(entry);
|
|
6370
|
+
if (text === entry.lastProgressText)
|
|
6371
|
+
return; // nothing changed — skip the API call
|
|
6372
|
+
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
6373
|
+
if (!adapter?.editAlert)
|
|
6374
|
+
return;
|
|
6375
|
+
entry.lastProgressText = text;
|
|
6376
|
+
entry.lastProgressEditAt = Date.now();
|
|
6377
|
+
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
6378
|
+
type: "cancel",
|
|
6379
|
+
instanceName: entry.instanceName,
|
|
6380
|
+
message: text,
|
|
6381
|
+
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
6382
|
+
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
6383
|
+
.catch(err => {
|
|
6384
|
+
// A failed progress edit must never escalate: the button still works and
|
|
6385
|
+
// the next tick retries. Common causes are a deleted message or a
|
|
6386
|
+
// rate limit.
|
|
6387
|
+
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
6388
|
+
});
|
|
6389
|
+
}
|
|
6390
|
+
/**
|
|
6391
|
+
* Refresh the button's text in place while the instance keeps working.
|
|
6392
|
+
*
|
|
6393
|
+
* Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
|
|
5051
6394
|
* and the Bot API treats that as "clear the keyboard" — so editing with it would
|
|
5052
6395
|
* delete the very cancel button this is trying to keep alive.
|
|
5053
6396
|
*/
|
|
@@ -5061,31 +6404,7 @@ export class FleetManager {
|
|
|
5061
6404
|
return PROGRESS_MIN_ELAPSED_MS;
|
|
5062
6405
|
}
|
|
5063
6406
|
startProgressTicker(entry) {
|
|
5064
|
-
const tick = () =>
|
|
5065
|
-
if (!this.cancelButtons.has(entry.messageId)) {
|
|
5066
|
-
clearInterval(entry.progressTimer);
|
|
5067
|
-
return;
|
|
5068
|
-
}
|
|
5069
|
-
const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
|
|
5070
|
-
if (text === entry.lastProgressText)
|
|
5071
|
-
return; // nothing changed — skip the API call
|
|
5072
|
-
const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
|
|
5073
|
-
if (!adapter?.editAlert)
|
|
5074
|
-
return;
|
|
5075
|
-
entry.lastProgressText = text;
|
|
5076
|
-
adapter.editAlert(entry.chatId, entry.messageId, {
|
|
5077
|
-
type: "cancel",
|
|
5078
|
-
instanceName: entry.instanceName,
|
|
5079
|
-
message: text,
|
|
5080
|
-
choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
|
|
5081
|
-
}, entry.threadId ? { threadId: entry.threadId } : undefined)
|
|
5082
|
-
.catch(err => {
|
|
5083
|
-
// A failed progress edit must never escalate: the button still works and
|
|
5084
|
-
// the next tick retries. Common causes are a deleted message or a
|
|
5085
|
-
// rate limit.
|
|
5086
|
-
this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
|
|
5087
|
-
});
|
|
5088
|
-
};
|
|
6407
|
+
const tick = () => this.refreshBubble(entry);
|
|
5089
6408
|
entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
|
|
5090
6409
|
entry.progressTimer.unref?.();
|
|
5091
6410
|
// One extra tick right when the threshold passes, so a 30s threshold shows
|
|
@@ -5099,8 +6418,11 @@ export class FleetManager {
|
|
|
5099
6418
|
}
|
|
5100
6419
|
/**
|
|
5101
6420
|
* After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
|
|
5102
|
-
* it has not, retire its button.
|
|
5103
|
-
*
|
|
6421
|
+
* it has not, retire its button. The daemon is asked for a fresh pane capture
|
|
6422
|
+
* before deciding. Reading only the transition cache stranded the first
|
|
6423
|
+
* post-restart bubble when its startup "working" report never got a matching
|
|
6424
|
+
* idle edge. Re-arming replaces the previous timer, so a burst of replies ends
|
|
6425
|
+
* with exactly one pending check.
|
|
5104
6426
|
*/
|
|
5105
6427
|
armReplyGrace(instanceName) {
|
|
5106
6428
|
for (const entry of this.cancelButtons.values()) {
|
|
@@ -5112,14 +6434,26 @@ export class FleetManager {
|
|
|
5112
6434
|
entry.replyGraceTimer = undefined;
|
|
5113
6435
|
if (!this.cancelButtons.has(entry.messageId))
|
|
5114
6436
|
return;
|
|
5115
|
-
|
|
5116
|
-
return; // resumed — a long run keeps its button
|
|
5117
|
-
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
5118
|
-
this.retireButton(entry);
|
|
6437
|
+
void this.finishReplyGrace(instanceName, entry);
|
|
5119
6438
|
}, REPLY_RETIRE_GRACE_MS);
|
|
5120
6439
|
entry.replyGraceTimer.unref?.();
|
|
5121
6440
|
}
|
|
5122
6441
|
}
|
|
6442
|
+
async finishReplyGrace(instanceName, entry) {
|
|
6443
|
+
const refreshed = await this.refreshInstanceExecutionState(instanceName, REPLY_STATE_REFRESH_TIMEOUT_MS);
|
|
6444
|
+
if (!this.cancelButtons.has(entry.messageId) || entry.retiring)
|
|
6445
|
+
return;
|
|
6446
|
+
if (!refreshed) {
|
|
6447
|
+
// Fail safe: without an authoritative answer, retain a potentially live
|
|
6448
|
+
// Cancel button. The 5-minute/30-minute/24-hour safety nets still apply.
|
|
6449
|
+
this.logger.debug({ instanceName, messageId: entry.messageId }, "Cancel reply-grace state refresh timed out");
|
|
6450
|
+
return;
|
|
6451
|
+
}
|
|
6452
|
+
if (!this.getInstanceIdle(instanceName))
|
|
6453
|
+
return; // genuine long run keeps its button
|
|
6454
|
+
this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
|
|
6455
|
+
this.retireButton(entry);
|
|
6456
|
+
}
|
|
5123
6457
|
/** Retire (delete) every cancel button belonging to an instance. */
|
|
5124
6458
|
retireInstanceButtons(instanceName) {
|
|
5125
6459
|
// Snapshot first — retireButton may delete entries from the map on success.
|
|
@@ -5141,7 +6475,7 @@ export class FleetManager {
|
|
|
5141
6475
|
this.deleteButtonMessage(entry)
|
|
5142
6476
|
.then(() => {
|
|
5143
6477
|
this.discardButton(entry);
|
|
5144
|
-
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId }, "Cancel button
|
|
6478
|
+
this.logger.info({ instanceName: entry.instanceName, messageId: entry.messageId, historyPreserved: Boolean(entry.toolProgress) }, "Cancel button retired");
|
|
5145
6479
|
})
|
|
5146
6480
|
.catch((err) => this.scheduleButtonRetry(entry, err));
|
|
5147
6481
|
}
|
|
@@ -5153,6 +6487,8 @@ export class FleetManager {
|
|
|
5153
6487
|
clearInterval(entry.idleCheckTimer);
|
|
5154
6488
|
if (entry.progressTimer)
|
|
5155
6489
|
clearInterval(entry.progressTimer);
|
|
6490
|
+
if (entry.progressEditTimer)
|
|
6491
|
+
clearTimeout(entry.progressEditTimer);
|
|
5156
6492
|
if (entry.replyGraceTimer)
|
|
5157
6493
|
clearTimeout(entry.replyGraceTimer);
|
|
5158
6494
|
this.cancelButtons.delete(entry.messageId);
|
|
@@ -5230,12 +6566,21 @@ export class FleetManager {
|
|
|
5230
6566
|
// Continue the same retire cycle (bypass the retiring-guard in retireButton).
|
|
5231
6567
|
entry.retryTimer = setTimeout(() => this.attemptButtonDelete(entry), CANCEL_BTN_RETRY_INTERVAL_MS);
|
|
5232
6568
|
}
|
|
5233
|
-
/**
|
|
6569
|
+
/** Retire one cancel bubble via its own adapter. A bubble that accumulated
|
|
6570
|
+
* tool progress is kept as a read-only history message; a plain bubble keeps
|
|
6571
|
+
* the legacy delete/edit-to-checkmark behavior. Resolves on success and
|
|
5234
6572
|
* rejects on failure so the caller can retry. */
|
|
5235
6573
|
deleteButtonMessage(e) {
|
|
5236
6574
|
const adapter = (e.adapterId ? this.worlds.get(e.adapterId)?.adapter : undefined) ?? this.adapter;
|
|
5237
6575
|
if (!adapter)
|
|
5238
6576
|
return Promise.reject(new Error("no adapter for cancel button"));
|
|
6577
|
+
if (e.toolProgress && adapter.editMessageRemoveButtons) {
|
|
6578
|
+
return adapter.editMessageRemoveButtons(e.chatId, e.messageId, this.composeRetiredBubbleText(e), e.threadId);
|
|
6579
|
+
}
|
|
6580
|
+
// All production adapters currently implement editMessageRemoveButtons.
|
|
6581
|
+
// A third-party adapter without it degrades to the legacy delete behavior:
|
|
6582
|
+
// removing a live Cancel control is safer than retaining an actionable,
|
|
6583
|
+
// untracked button forever.
|
|
5239
6584
|
if (adapter.deleteMessage)
|
|
5240
6585
|
return adapter.deleteMessage(e.chatId, e.messageId, e.threadId);
|
|
5241
6586
|
if (adapter.editMessageRemoveButtons)
|
|
@@ -5287,11 +6632,604 @@ export class FleetManager {
|
|
|
5287
6632
|
const daemon = this.daemons.get(instanceName);
|
|
5288
6633
|
if (!daemon)
|
|
5289
6634
|
return false;
|
|
6635
|
+
const deliveryEpoch = this.cancelPendingDeliveries(instanceName);
|
|
6636
|
+
daemon.clearPendingDeliveries?.(deliveryEpoch);
|
|
5290
6637
|
daemon.sendEscape().catch(e => this.logger.warn({ err: e, instanceName }, "sendEscape failed"));
|
|
5291
6638
|
this.lastInboundMsg.delete(instanceName);
|
|
5292
6639
|
this.clearCancelButton(instanceName);
|
|
5293
6640
|
return true;
|
|
5294
6641
|
}
|
|
6642
|
+
getDeliveryEpoch(instanceName) {
|
|
6643
|
+
return this.deliveryEpochs.get(instanceName) ?? 0;
|
|
6644
|
+
}
|
|
6645
|
+
isDeliveryEpochCurrent(instanceName, deliveryEpoch) {
|
|
6646
|
+
return deliveryEpoch === this.getDeliveryEpoch(instanceName);
|
|
6647
|
+
}
|
|
6648
|
+
/** Invalidate work queued before a user cancel and wake idle-gate waiters. */
|
|
6649
|
+
cancelPendingDeliveries(instanceName) {
|
|
6650
|
+
const next = this.getDeliveryEpoch(instanceName) + 1;
|
|
6651
|
+
this.deliveryEpochs.set(instanceName, next);
|
|
6652
|
+
for (const check of this.instanceIdleWaiters.get(instanceName) ?? [])
|
|
6653
|
+
check();
|
|
6654
|
+
return next;
|
|
6655
|
+
}
|
|
6656
|
+
// ── Remote CLI login (`/login`) ──────────────────────────────────────────
|
|
6657
|
+
//
|
|
6658
|
+
// One session at a time, in a dedicated tmux window — never an instance
|
|
6659
|
+
// pane. Credentials are per-backend (codex homes symlink auth.json to the
|
|
6660
|
+
// shared ~/.codex; the other CLIs use one real home), so a single sign-in
|
|
6661
|
+
// repairs every instance of that backend, and instance delivery, pane-state
|
|
6662
|
+
// detection, tool progress, and mcp_proxy_reply never observe login output.
|
|
6663
|
+
activeLogin = null;
|
|
6664
|
+
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
6665
|
+
async promptLoginBackends(chat) {
|
|
6666
|
+
const configured = new Set();
|
|
6667
|
+
for (const [, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
6668
|
+
configured.add(config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
6669
|
+
}
|
|
6670
|
+
const choices = Object.keys(LOGIN_FLOWS)
|
|
6671
|
+
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
6672
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6673
|
+
if (choices.length === 0) {
|
|
6674
|
+
await chat.adapter.sendText(chat.chatId, t("login.unsupported", [...configured].join(", ")), { threadId: chat.threadId });
|
|
6675
|
+
return;
|
|
6676
|
+
}
|
|
6677
|
+
await this.postNonceButtonPrompt({
|
|
6678
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6679
|
+
alertType: "login",
|
|
6680
|
+
instanceName: "login",
|
|
6681
|
+
adapter: chat.adapter,
|
|
6682
|
+
adapterId: chat.adapterId,
|
|
6683
|
+
chatId: chat.chatId,
|
|
6684
|
+
threadId: chat.threadId,
|
|
6685
|
+
message: t("login.choose_backend"),
|
|
6686
|
+
choices,
|
|
6687
|
+
expiredText: t("buttons.stale"),
|
|
6688
|
+
});
|
|
6689
|
+
}
|
|
6690
|
+
/**
|
|
6691
|
+
* Ask the General topic to approve a ClassicBot access request.
|
|
6692
|
+
*
|
|
6693
|
+
* Two affirmative buttons rather than one, when the trigger identified a
|
|
6694
|
+
* user. Allowing a group to talk and granting that person ClassicBot admin
|
|
6695
|
+
* (start/stop/model on classic channels — NOT fleet admin, which reads
|
|
6696
|
+
* fleet.yaml `channel.access.allowed_users`) have different blast radii, and
|
|
6697
|
+
* bundling them would force anyone who wants only the cheap one to accept the
|
|
6698
|
+
* expensive one or go edit YAML by hand.
|
|
6699
|
+
*
|
|
6700
|
+
* Built on postNonceButtonPrompt, so it inherits the canonical-address
|
|
6701
|
+
* binding (#682) that makes these buttons answer in a Telegram General topic.
|
|
6702
|
+
*/
|
|
6703
|
+
async promptClassicApproval(opts) {
|
|
6704
|
+
const adapter = this.getAdapterForInstance(opts.generalName);
|
|
6705
|
+
// An instance with no world binding yet (fresh restart, or a fleet whose
|
|
6706
|
+
// primary `channel:` block carries the config) still has a usable adapter —
|
|
6707
|
+
// fall back to that adapter's own id rather than dropping the buttons. The
|
|
6708
|
+
// id only has to match what the callback arrives with, which is this same
|
|
6709
|
+
// adapter.
|
|
6710
|
+
const adapterId = this.getInstanceAdapterId(opts.generalName) ?? adapter?.id;
|
|
6711
|
+
const chatId = this.getGroupIdForInstance(opts.generalName);
|
|
6712
|
+
const topicId = this.fleetConfig?.instances[opts.generalName]?.topic_id;
|
|
6713
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6714
|
+
// Never lose the notification because the buttons could not be addressed.
|
|
6715
|
+
this.notifyInstanceTopic(opts.generalName, opts.message);
|
|
6716
|
+
return;
|
|
6717
|
+
}
|
|
6718
|
+
const choices = [{ action: "allow", label: t("classic.approve_group") }];
|
|
6719
|
+
if (opts.userId)
|
|
6720
|
+
choices.push({ action: "allow-admin", label: t("classic.approve_group_admin") });
|
|
6721
|
+
choices.push({ action: "ignore", label: t("classic.approve_ignore") });
|
|
6722
|
+
await this.postNonceButtonPrompt({
|
|
6723
|
+
prefix: CLASSIC_APPROVE_CALLBACK_PREFIX,
|
|
6724
|
+
alertType: "classic_approve",
|
|
6725
|
+
instanceName: opts.generalName,
|
|
6726
|
+
adapter,
|
|
6727
|
+
adapterId,
|
|
6728
|
+
chatId,
|
|
6729
|
+
threadId: topicId != null ? String(topicId) : undefined,
|
|
6730
|
+
message: opts.message,
|
|
6731
|
+
choices,
|
|
6732
|
+
expiredText: t("buttons.stale"),
|
|
6733
|
+
extra: { classicGroupId: opts.groupId, classicUserId: opts.userId, classicScope: opts.scope },
|
|
6734
|
+
});
|
|
6735
|
+
}
|
|
6736
|
+
/**
|
|
6737
|
+
* Apply an approved ClassicBot access request.
|
|
6738
|
+
*
|
|
6739
|
+
* Writes through ClassicChannelManager's typed mutators rather than handing
|
|
6740
|
+
* General a YAML edit: the id is stored via String() so a Discord snowflake
|
|
6741
|
+
* cannot land as a YAML integer and lose precision, the existing save() path
|
|
6742
|
+
* is reused, in-memory state is correct immediately (no waiting on the 30s
|
|
6743
|
+
* reload poll), and a click cannot silently do nothing because General
|
|
6744
|
+
* happened to be paused. General is still told the outcome.
|
|
6745
|
+
*/
|
|
6746
|
+
async applyClassicApproval(entry, groupId, adminUserId) {
|
|
6747
|
+
const classic = this.classicChannels;
|
|
6748
|
+
if (!classic) {
|
|
6749
|
+
this.notifyInstanceTopic(entry.instanceName, t("classic.approve_failed", groupId));
|
|
6750
|
+
return;
|
|
6751
|
+
}
|
|
6752
|
+
const scope = entry.classicScope ?? "guild";
|
|
6753
|
+
const access = scope === "group" ? classic.allowGroup(groupId) : classic.allowGuild(groupId);
|
|
6754
|
+
const admin = adminUserId ? classic.addAdminUser(adminUserId) : null;
|
|
6755
|
+
const lines = [t(access === "added" ? "classic.approve_done_group"
|
|
6756
|
+
: access === "already" ? "classic.approve_done_group_already"
|
|
6757
|
+
: "classic.approve_done_group_open", groupId)];
|
|
6758
|
+
if (admin) {
|
|
6759
|
+
lines.push(t(admin === "added" ? "classic.approve_done_admin" : "classic.approve_done_admin_already", adminUserId ?? ""));
|
|
6760
|
+
}
|
|
6761
|
+
this.notifyInstanceTopic(entry.instanceName, lines.join("\n"));
|
|
6762
|
+
}
|
|
6763
|
+
/** Consume a ClassicBot approval button. */
|
|
6764
|
+
async handleClassicApproval(data, callbackAdapterId, receivingAdapter) {
|
|
6765
|
+
const claimed = this.consumeNonceCallback(CLASSIC_APPROVE_CALLBACK_PREFIX, /^classic-approve:([0-9a-f]+):(allow|allow-admin|ignore)$/, data, callbackAdapterId, receivingAdapter);
|
|
6766
|
+
if (claimed === null)
|
|
6767
|
+
return false;
|
|
6768
|
+
if (claimed === "consumed")
|
|
6769
|
+
return true;
|
|
6770
|
+
const { entry, action } = claimed;
|
|
6771
|
+
const groupId = entry.classicGroupId ?? "";
|
|
6772
|
+
const userId = entry.classicUserId;
|
|
6773
|
+
if (action === "ignore") {
|
|
6774
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("classic.approve_ignored", groupId));
|
|
6775
|
+
return true;
|
|
6776
|
+
}
|
|
6777
|
+
const grantAdmin = action === "allow-admin" && !!userId;
|
|
6778
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, grantAdmin ? t("classic.approve_applying_admin", groupId, userId ?? "")
|
|
6779
|
+
: t("classic.approve_applying", groupId));
|
|
6780
|
+
await this.applyClassicApproval(entry, groupId, grantAdmin ? userId : undefined);
|
|
6781
|
+
return true;
|
|
6782
|
+
}
|
|
6783
|
+
/**
|
|
6784
|
+
* Backend chooser for a bare `/install-cli`, mirroring promptLoginBackends so
|
|
6785
|
+
* both commands feel the same. Built on postNonceButtonPrompt rather than the
|
|
6786
|
+
* `/model` selection coordinator: that is the mechanism `/login` already uses,
|
|
6787
|
+
* and the one whose canonical-address binding (#682) makes the buttons answer
|
|
6788
|
+
* in a Telegram General topic.
|
|
6789
|
+
*
|
|
6790
|
+
* Unlike the login chooser this does NOT filter to backends the fleet already
|
|
6791
|
+
* runs. Installing is how you get a backend you do not have yet, so filtering
|
|
6792
|
+
* by configured backends would hide the only entry the admin came for.
|
|
6793
|
+
*
|
|
6794
|
+
* gemini-cli is omitted: it is deprecated (see backend/factory.ts). Typing
|
|
6795
|
+
* `/install-cli gemini-cli` still works — this only stops recommending it.
|
|
6796
|
+
*/
|
|
6797
|
+
async promptInstallBackends(chat) {
|
|
6798
|
+
const choices = Object.keys(BACKEND_INSTALLATION_INFO)
|
|
6799
|
+
.filter(backend => backend !== "gemini-cli")
|
|
6800
|
+
.map(backend => ({ action: backend, label: backend }));
|
|
6801
|
+
await this.postNonceButtonPrompt({
|
|
6802
|
+
prefix: INSTALL_CALLBACK_PREFIX,
|
|
6803
|
+
alertType: "install",
|
|
6804
|
+
instanceName: "install",
|
|
6805
|
+
adapter: chat.adapter,
|
|
6806
|
+
adapterId: chat.adapterId,
|
|
6807
|
+
chatId: chat.chatId,
|
|
6808
|
+
threadId: chat.threadId,
|
|
6809
|
+
message: t("install.choose_backend"),
|
|
6810
|
+
choices,
|
|
6811
|
+
expiredText: t("buttons.stale"),
|
|
6812
|
+
});
|
|
6813
|
+
}
|
|
6814
|
+
/** Backend chooser button → start that backend's install session. */
|
|
6815
|
+
async handleInstallBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
6816
|
+
const claimed = this.consumeNonceCallback(INSTALL_CALLBACK_PREFIX, /^install-select:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
6817
|
+
if (claimed === null)
|
|
6818
|
+
return false;
|
|
6819
|
+
if (claimed === "consumed")
|
|
6820
|
+
return true;
|
|
6821
|
+
const { entry, action: backend } = claimed;
|
|
6822
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.starting_backend", backend));
|
|
6823
|
+
const text = await this.startInstallSession(backend, {
|
|
6824
|
+
adapter: entry.adapter,
|
|
6825
|
+
adapterId: entry.adapterId,
|
|
6826
|
+
chatId: entry.chatId,
|
|
6827
|
+
threadId: entry.threadId,
|
|
6828
|
+
});
|
|
6829
|
+
if (text)
|
|
6830
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
6831
|
+
return true;
|
|
6832
|
+
}
|
|
6833
|
+
/**
|
|
6834
|
+
* Start a login session for one backend. Caller enforces admin.
|
|
6835
|
+
* Returns a status line to post, or null when a confirmation prompt was
|
|
6836
|
+
* posted instead (auth still valid — see the pre-check below).
|
|
6837
|
+
*/
|
|
6838
|
+
async startLoginSession(backendArg, chat, opts = {}) {
|
|
6839
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
6840
|
+
const flow = LOGIN_FLOWS[backend];
|
|
6841
|
+
if (!flow)
|
|
6842
|
+
return t("login.unsupported", backendArg);
|
|
6843
|
+
if (this.activeLogin)
|
|
6844
|
+
return t("login.busy", this.activeLogin.backend);
|
|
6845
|
+
// Install sessions share the window namespace — never run both at once.
|
|
6846
|
+
if (this.activeInstall)
|
|
6847
|
+
return t("install.busy");
|
|
6848
|
+
// Token-free pre-check (5s cap): re-login while auth still works is
|
|
6849
|
+
// usually a mistake, so it needs a confirmed click. An invalid OR
|
|
6850
|
+
// uncertain result (timeout, missing binary) proceeds straight to login —
|
|
6851
|
+
// an unreliable probe must never block the re-login the admin asked for.
|
|
6852
|
+
if (!opts.skipAuthCheck && flow.authCheck) {
|
|
6853
|
+
const status = await checkAuthStatus(flow.authCheck);
|
|
6854
|
+
if (status === "valid") {
|
|
6855
|
+
await this.postNonceButtonPrompt({
|
|
6856
|
+
prefix: LOGIN_CONFIRM_CALLBACK_PREFIX,
|
|
6857
|
+
alertType: "login",
|
|
6858
|
+
instanceName: backend,
|
|
6859
|
+
adapter: chat.adapter,
|
|
6860
|
+
adapterId: chat.adapterId,
|
|
6861
|
+
chatId: chat.chatId,
|
|
6862
|
+
threadId: chat.threadId,
|
|
6863
|
+
message: t("login.still_valid", backend),
|
|
6864
|
+
choices: [
|
|
6865
|
+
{ action: "go", label: t("login.relogin_go") },
|
|
6866
|
+
{ action: "cancel", label: t("login.relogin_cancel") },
|
|
6867
|
+
],
|
|
6868
|
+
expiredText: t("buttons.stale"),
|
|
6869
|
+
});
|
|
6870
|
+
return null;
|
|
6871
|
+
}
|
|
6872
|
+
}
|
|
6873
|
+
return this.launchLoginSession(flow, backend, chat);
|
|
6874
|
+
}
|
|
6875
|
+
/** Create the login window and session (pre-check already settled). */
|
|
6876
|
+
async launchLoginSession(flow, backend, chat) {
|
|
6877
|
+
if (this.activeLogin)
|
|
6878
|
+
return t("login.busy", this.activeLogin.backend);
|
|
6879
|
+
// Install sessions share the window namespace — never run both at once.
|
|
6880
|
+
if (this.activeInstall)
|
|
6881
|
+
return t("install.busy");
|
|
6882
|
+
const sessionName = getTmuxSession();
|
|
6883
|
+
await TmuxManager.ensureSession(sessionName);
|
|
6884
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
6885
|
+
const session = new LoginSession(flow, tmux, {
|
|
6886
|
+
onMenu: async (options) => {
|
|
6887
|
+
await this.postNonceButtonPrompt({
|
|
6888
|
+
prefix: LOGIN_MENU_CALLBACK_PREFIX,
|
|
6889
|
+
alertType: "login",
|
|
6890
|
+
instanceName: backend,
|
|
6891
|
+
adapter: chat.adapter,
|
|
6892
|
+
adapterId: chat.adapterId,
|
|
6893
|
+
chatId: chat.chatId,
|
|
6894
|
+
threadId: chat.threadId,
|
|
6895
|
+
message: t("login.choose_provider"),
|
|
6896
|
+
choices: options.map((label, index) => ({ action: String(index), label })),
|
|
6897
|
+
expiredText: t("buttons.stale"),
|
|
6898
|
+
});
|
|
6899
|
+
},
|
|
6900
|
+
onAuthHint: async (url, code) => {
|
|
6901
|
+
await this.sendLoginSecret(chat, backend, url, code);
|
|
6902
|
+
},
|
|
6903
|
+
onNeedInput: async (promptExcerpt) => {
|
|
6904
|
+
await chat.adapter.sendText(chat.chatId, t("login.need_input", backend, promptExcerpt), { threadId: chat.threadId }).catch(() => { });
|
|
6905
|
+
},
|
|
6906
|
+
onDone: async ({ ok, detail }) => {
|
|
6907
|
+
this.activeLogin = null;
|
|
6908
|
+
let text;
|
|
6909
|
+
if (ok) {
|
|
6910
|
+
const { woken, restarted } = await this.recoverBackendInstances(backend);
|
|
6911
|
+
const none = t("login.none");
|
|
6912
|
+
text = t("login.success", backend, woken.length ? woken.join(", ") : none, restarted.length ? restarted.join(", ") : none);
|
|
6913
|
+
}
|
|
6914
|
+
else if (detail === "cancelled") {
|
|
6915
|
+
// The cancel command's own reply already announced this — a second
|
|
6916
|
+
// message here was a duplicate.
|
|
6917
|
+
return;
|
|
6918
|
+
}
|
|
6919
|
+
else {
|
|
6920
|
+
text = t("login.failed", backend, detail);
|
|
6921
|
+
}
|
|
6922
|
+
await chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
6923
|
+
},
|
|
6924
|
+
}, this.logger);
|
|
6925
|
+
// Claim the slot before the first await so two admins racing /login cannot
|
|
6926
|
+
// both create windows; release on startup failure.
|
|
6927
|
+
this.activeLogin = { session, backend, chat };
|
|
6928
|
+
try {
|
|
6929
|
+
await session.start();
|
|
6930
|
+
}
|
|
6931
|
+
catch (err) {
|
|
6932
|
+
this.activeLogin = null;
|
|
6933
|
+
return t("login.failed", backend, err.message);
|
|
6934
|
+
}
|
|
6935
|
+
return t("login.started", backend);
|
|
6936
|
+
}
|
|
6937
|
+
/** `/login code <text>` — paste admin-supplied text into the login window. */
|
|
6938
|
+
async loginSubmitInput(text) {
|
|
6939
|
+
if (!this.activeLogin)
|
|
6940
|
+
return t("login.no_session");
|
|
6941
|
+
const ok = await this.activeLogin.session.submitInput(text);
|
|
6942
|
+
return ok ? t("login.input_sent") : t("login.input_failed");
|
|
6943
|
+
}
|
|
6944
|
+
/** `/login cancel` — abort the active session and remove its window. */
|
|
6945
|
+
async cancelLoginSession() {
|
|
6946
|
+
if (!this.activeLogin)
|
|
6947
|
+
return t("login.no_session");
|
|
6948
|
+
const backend = this.activeLogin.backend;
|
|
6949
|
+
await this.activeLogin.session.cancel();
|
|
6950
|
+
return t("login.cancelled", backend);
|
|
6951
|
+
}
|
|
6952
|
+
/**
|
|
6953
|
+
* The URL (+ code) is a live credential: whoever completes it binds THEIR
|
|
6954
|
+
* account to this fleet's CLI. Telegram gets an HTML spoiler (same treatment
|
|
6955
|
+
* as the dashboard token); other adapters get plain text with the warning.
|
|
6956
|
+
*/
|
|
6957
|
+
async sendLoginSecret(chat, backend, url, code) {
|
|
6958
|
+
const codeLine = code ? `\n${t("login.auth_code", code)}` : "";
|
|
6959
|
+
try {
|
|
6960
|
+
if (chat.adapter.type === "telegram") {
|
|
6961
|
+
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
6962
|
+
await chat.adapter.sendText(chat.chatId, `${esc(t("login.auth_hint", backend))}\n<tg-spoiler>${esc(url)}${esc(codeLine)}</tg-spoiler>`, { threadId: chat.threadId, format: "html" });
|
|
6963
|
+
}
|
|
6964
|
+
else {
|
|
6965
|
+
await chat.adapter.sendText(chat.chatId, `${t("login.auth_hint", backend)}\n${url}${codeLine}`, { threadId: chat.threadId });
|
|
6966
|
+
}
|
|
6967
|
+
}
|
|
6968
|
+
catch (err) {
|
|
6969
|
+
this.logger.warn({ err: err.message, backend }, "Failed to deliver login URL");
|
|
6970
|
+
}
|
|
6971
|
+
}
|
|
6972
|
+
/**
|
|
6973
|
+
* After a successful login: wake the paused instances of that backend AND
|
|
6974
|
+
* restart the running ones — a live CLI holds the old token in memory and
|
|
6975
|
+
* only re-reads credentials on process start (a paused instance's CLI is
|
|
6976
|
+
* already dead, so waking it respawns with the new token for free).
|
|
6977
|
+
*/
|
|
6978
|
+
async recoverBackendInstances(backend) {
|
|
6979
|
+
const woken = [];
|
|
6980
|
+
const restarted = [];
|
|
6981
|
+
for (const [name, config] of Object.entries(this.fleetConfig?.instances ?? {})) {
|
|
6982
|
+
const effective = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
6983
|
+
if (effective !== backend)
|
|
6984
|
+
continue;
|
|
6985
|
+
const status = this.getInstanceStatus(name);
|
|
6986
|
+
try {
|
|
6987
|
+
if (status === "paused") {
|
|
6988
|
+
if (this.daemons.has(name))
|
|
6989
|
+
await this.lifecycle.wake(name, 30_000);
|
|
6990
|
+
else
|
|
6991
|
+
await this.startPersistedPausedInstance(name);
|
|
6992
|
+
woken.push(name);
|
|
6993
|
+
}
|
|
6994
|
+
else if (status === "running") {
|
|
6995
|
+
await this.restartSingleInstance(name);
|
|
6996
|
+
restarted.push(name);
|
|
6997
|
+
}
|
|
6998
|
+
}
|
|
6999
|
+
catch (err) {
|
|
7000
|
+
this.logger.warn({ err: err.message, name, status }, "Post-login recovery failed");
|
|
7001
|
+
}
|
|
7002
|
+
}
|
|
7003
|
+
return { woken, restarted };
|
|
7004
|
+
}
|
|
7005
|
+
/** Backend chooser button → start that backend's login session. */
|
|
7006
|
+
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7007
|
+
const claimed = this.consumeNonceCallback(LOGIN_CALLBACK_PREFIX, /^login:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
|
|
7008
|
+
if (claimed === null)
|
|
7009
|
+
return false;
|
|
7010
|
+
if (claimed === "consumed")
|
|
7011
|
+
return true;
|
|
7012
|
+
const { entry, action: backend } = claimed;
|
|
7013
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7014
|
+
const text = await this.startLoginSession(backend, {
|
|
7015
|
+
adapter: entry.adapter,
|
|
7016
|
+
adapterId: entry.adapterId,
|
|
7017
|
+
chatId: entry.chatId,
|
|
7018
|
+
threadId: entry.threadId,
|
|
7019
|
+
});
|
|
7020
|
+
if (text)
|
|
7021
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7022
|
+
return true;
|
|
7023
|
+
}
|
|
7024
|
+
/** Re-login confirmation (auth pre-check said credentials still work). */
|
|
7025
|
+
async handleLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7026
|
+
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
7027
|
+
if (claimed === null)
|
|
7028
|
+
return false;
|
|
7029
|
+
if (claimed === "consumed")
|
|
7030
|
+
return true;
|
|
7031
|
+
const { entry, action } = claimed;
|
|
7032
|
+
const backend = entry.instanceName;
|
|
7033
|
+
if (action === "cancel") {
|
|
7034
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.cancelled", backend));
|
|
7035
|
+
return true;
|
|
7036
|
+
}
|
|
7037
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7038
|
+
const text = await this.startLoginSession(backend, {
|
|
7039
|
+
adapter: entry.adapter,
|
|
7040
|
+
adapterId: entry.adapterId,
|
|
7041
|
+
chatId: entry.chatId,
|
|
7042
|
+
threadId: entry.threadId,
|
|
7043
|
+
}, { skipAuthCheck: true });
|
|
7044
|
+
if (text)
|
|
7045
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7046
|
+
return true;
|
|
7047
|
+
}
|
|
7048
|
+
/** Kiro provider button → drive the CLI's arrow-key selector. */
|
|
7049
|
+
async handleLoginMenuSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7050
|
+
const claimed = this.consumeNonceCallback(LOGIN_MENU_CALLBACK_PREFIX, /^login-menu:([0-9a-f]+):(\d)$/, data, callbackAdapterId, receivingAdapter);
|
|
7051
|
+
if (claimed === null)
|
|
7052
|
+
return false;
|
|
7053
|
+
if (claimed === "consumed")
|
|
7054
|
+
return true;
|
|
7055
|
+
const { entry, action } = claimed;
|
|
7056
|
+
const index = Number(action);
|
|
7057
|
+
const label = this.activeLogin?.session.flow.menu?.options[index] ?? action;
|
|
7058
|
+
const ok = await this.activeLogin?.session.selectMenuOption(index) ?? false;
|
|
7059
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, ok ? t("login.provider_selected", label) : t("login.no_session"));
|
|
7060
|
+
return true;
|
|
7061
|
+
}
|
|
7062
|
+
// ── Remote CLI install (`/install-cli`) ──────────────────────────────────
|
|
7063
|
+
//
|
|
7064
|
+
// Same dedicated-window model as /login (and the same LoginSession state
|
|
7065
|
+
// machine — an install is a login flow with no auth hints): run the
|
|
7066
|
+
// installer, judge by exit code, verify the binary on a fresh login shell,
|
|
7067
|
+
// then offer to chain straight into /login.
|
|
7068
|
+
activeInstall = null;
|
|
7069
|
+
/** Start a CLI install session. Caller enforces admin. */
|
|
7070
|
+
async startInstallSession(backendArg, chat) {
|
|
7071
|
+
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7072
|
+
const info = BACKEND_INSTALLATION_INFO[backend];
|
|
7073
|
+
if (!info)
|
|
7074
|
+
return t("install.unsupported", backendArg);
|
|
7075
|
+
if (checkBinaryInstalled(info.binary))
|
|
7076
|
+
return t("install.already", backend, info.binary);
|
|
7077
|
+
if (this.activeInstall)
|
|
7078
|
+
return t("install.busy");
|
|
7079
|
+
if (this.activeLogin)
|
|
7080
|
+
return t("login.busy", this.activeLogin.backend);
|
|
7081
|
+
const sessionName = getTmuxSession();
|
|
7082
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7083
|
+
const tmux = new TmuxManager(sessionName, "");
|
|
7084
|
+
// A synthetic login flow: success is decided by the installer's exit code
|
|
7085
|
+
// (LoginSession treats a clean exit as success), never by pane text — and
|
|
7086
|
+
// installer output that happens to contain a URL must not be forwarded as
|
|
7087
|
+
// an auth hint, hence the no-op events below.
|
|
7088
|
+
const flow = {
|
|
7089
|
+
backend,
|
|
7090
|
+
command: info.install,
|
|
7091
|
+
successPattern: /$^/,
|
|
7092
|
+
timeoutMs: 10 * 60 * 1000,
|
|
7093
|
+
};
|
|
7094
|
+
const session = new LoginSession(flow, tmux, {
|
|
7095
|
+
onMenu: () => { },
|
|
7096
|
+
onAuthHint: () => { },
|
|
7097
|
+
onNeedInput: () => { },
|
|
7098
|
+
onDone: async ({ ok, detail }) => {
|
|
7099
|
+
this.activeInstall = null;
|
|
7100
|
+
if (!ok) {
|
|
7101
|
+
// A cancel is user-initiated — the cancel command's own reply already
|
|
7102
|
+
// said so; a second message here would be a duplicate.
|
|
7103
|
+
if (detail !== "cancelled") {
|
|
7104
|
+
await chat.adapter.sendText(chat.chatId, t("install.failed", backend, detail), { threadId: chat.threadId }).catch(() => { });
|
|
7105
|
+
}
|
|
7106
|
+
return;
|
|
7107
|
+
}
|
|
7108
|
+
// The installer may only have added the binary to a profile PATH; a
|
|
7109
|
+
// fresh login shell sees that, the fleet process's PATH may not.
|
|
7110
|
+
if (!this.verifyBinaryOnLoginShell(info.binary)) {
|
|
7111
|
+
await chat.adapter.sendText(chat.chatId, t("install.verify_failed", backend, info.binary), { threadId: chat.threadId }).catch(() => { });
|
|
7112
|
+
return;
|
|
7113
|
+
}
|
|
7114
|
+
if (!LOGIN_FLOWS[backend]) {
|
|
7115
|
+
await chat.adapter.sendText(chat.chatId, t("install.success_no_login", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7116
|
+
return;
|
|
7117
|
+
}
|
|
7118
|
+
await this.postNonceButtonPrompt({
|
|
7119
|
+
prefix: INSTALL_LOGIN_CALLBACK_PREFIX,
|
|
7120
|
+
alertType: "login",
|
|
7121
|
+
instanceName: backend,
|
|
7122
|
+
adapter: chat.adapter,
|
|
7123
|
+
adapterId: chat.adapterId,
|
|
7124
|
+
chatId: chat.chatId,
|
|
7125
|
+
threadId: chat.threadId,
|
|
7126
|
+
message: t("install.login_prompt", backend),
|
|
7127
|
+
choices: [
|
|
7128
|
+
{ action: "go", label: t("install.login_now") },
|
|
7129
|
+
{ action: "later", label: t("install.later") },
|
|
7130
|
+
],
|
|
7131
|
+
expiredText: t("install.later_ack", backend),
|
|
7132
|
+
});
|
|
7133
|
+
},
|
|
7134
|
+
}, this.logger);
|
|
7135
|
+
this.activeInstall = { session, backend, chat };
|
|
7136
|
+
try {
|
|
7137
|
+
await session.start();
|
|
7138
|
+
}
|
|
7139
|
+
catch (err) {
|
|
7140
|
+
this.activeInstall = null;
|
|
7141
|
+
return t("install.failed", backend, err.message);
|
|
7142
|
+
}
|
|
7143
|
+
return t("install.started", backend);
|
|
7144
|
+
}
|
|
7145
|
+
/** `/install-cli cancel` — abort the active install and remove its window. */
|
|
7146
|
+
async cancelInstallSession() {
|
|
7147
|
+
if (!this.activeInstall)
|
|
7148
|
+
return t("install.no_session");
|
|
7149
|
+
const backend = this.activeInstall.backend;
|
|
7150
|
+
await this.activeInstall.session.cancel();
|
|
7151
|
+
return t("install.cancelled", backend);
|
|
7152
|
+
}
|
|
7153
|
+
/** `command -v` on a login shell, so PATH additions from rc files count. */
|
|
7154
|
+
verifyBinaryOnLoginShell(binary) {
|
|
7155
|
+
try {
|
|
7156
|
+
const result = spawnSync("bash", ["-lc", `command -v ${binary}`], { timeout: 10_000, stdio: "pipe" });
|
|
7157
|
+
return result.status === 0;
|
|
7158
|
+
}
|
|
7159
|
+
catch {
|
|
7160
|
+
return false;
|
|
7161
|
+
}
|
|
7162
|
+
}
|
|
7163
|
+
/** "Sign in now?" button after a successful install. */
|
|
7164
|
+
async handleInstallLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7165
|
+
const claimed = this.consumeNonceCallback(INSTALL_LOGIN_CALLBACK_PREFIX, /^install-login:([0-9a-f]+):(go|later)$/, data, callbackAdapterId, receivingAdapter);
|
|
7166
|
+
if (claimed === null)
|
|
7167
|
+
return false;
|
|
7168
|
+
if (claimed === "consumed")
|
|
7169
|
+
return true;
|
|
7170
|
+
const { entry, action } = claimed;
|
|
7171
|
+
const backend = entry.instanceName;
|
|
7172
|
+
if (action === "later") {
|
|
7173
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.later_ack", backend));
|
|
7174
|
+
return true;
|
|
7175
|
+
}
|
|
7176
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("login.starting_backend", backend));
|
|
7177
|
+
const text = await this.startLoginSession(backend, {
|
|
7178
|
+
adapter: entry.adapter,
|
|
7179
|
+
adapterId: entry.adapterId,
|
|
7180
|
+
chatId: entry.chatId,
|
|
7181
|
+
threadId: entry.threadId,
|
|
7182
|
+
});
|
|
7183
|
+
if (text)
|
|
7184
|
+
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7185
|
+
return true;
|
|
7186
|
+
}
|
|
7187
|
+
/** Discord native `/login` slash — shared by every adapter dispatch block. */
|
|
7188
|
+
async handleLoginSlash(data, adapterId, adapter) {
|
|
7189
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7190
|
+
await data.respond(t("permission.denied"));
|
|
7191
|
+
return;
|
|
7192
|
+
}
|
|
7193
|
+
const chat = { adapter, adapterId, chatId: data.channelId };
|
|
7194
|
+
if (data.options?.cancel === true) {
|
|
7195
|
+
await data.respond(await this.cancelLoginSession());
|
|
7196
|
+
return;
|
|
7197
|
+
}
|
|
7198
|
+
const code = String(data.options?.code ?? "").trim();
|
|
7199
|
+
if (code) {
|
|
7200
|
+
await data.respond(await this.loginSubmitInput(code));
|
|
7201
|
+
return;
|
|
7202
|
+
}
|
|
7203
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7204
|
+
if (backend) {
|
|
7205
|
+
const text = await this.startLoginSession(backend, chat);
|
|
7206
|
+
await data.respond(text ?? t("login.confirm_posted"));
|
|
7207
|
+
return;
|
|
7208
|
+
}
|
|
7209
|
+
await this.promptLoginBackends(chat);
|
|
7210
|
+
await data.respond(t("login.chooser_posted"));
|
|
7211
|
+
}
|
|
7212
|
+
/** Discord native `/install-cli` slash — shared by every dispatch block. */
|
|
7213
|
+
async handleInstallCliSlash(data, adapterId, adapter) {
|
|
7214
|
+
if (!data.userId || !this.isFleetAdmin(data.userId, adapterId)) {
|
|
7215
|
+
await data.respond(t("permission.denied"));
|
|
7216
|
+
return;
|
|
7217
|
+
}
|
|
7218
|
+
if (data.options?.cancel === true) {
|
|
7219
|
+
await data.respond(await this.cancelInstallSession());
|
|
7220
|
+
return;
|
|
7221
|
+
}
|
|
7222
|
+
const backend = String(data.options?.backend ?? "").trim();
|
|
7223
|
+
if (!backend) {
|
|
7224
|
+
// Bare call: offer the backends instead of printing a usage line the user
|
|
7225
|
+
// then has to retype. A Discord slash that DID pick the native `backend`
|
|
7226
|
+
// choice never lands here, so the two paths cannot both fire.
|
|
7227
|
+
await this.promptInstallBackends({ adapter, adapterId, chatId: data.channelId });
|
|
7228
|
+
await data.respond(t("install.chooser_posted"));
|
|
7229
|
+
return;
|
|
7230
|
+
}
|
|
7231
|
+
await data.respond(await this.startInstallSession(backend, { adapter, adapterId, chatId: data.channelId }));
|
|
7232
|
+
}
|
|
5295
7233
|
queueMirrorMessage(text) {
|
|
5296
7234
|
const mirrorTopicId = this.fleetConfig?.channel?.mirror_topic_id;
|
|
5297
7235
|
if (mirrorTopicId == null || !this.adapter)
|
|
@@ -5411,6 +7349,10 @@ export class FleetManager {
|
|
|
5411
7349
|
"gemini-cli": "GEMINI.md",
|
|
5412
7350
|
"opencode": "AGENTS.md",
|
|
5413
7351
|
"kiro-cli": ".kiro/steering/project.md",
|
|
7352
|
+
// Grok reads AGENTS.md project docs; agy reads .agents/agents.md — the
|
|
7353
|
+
// same files their writeConfig() appends fleet instructions to.
|
|
7354
|
+
"grok": "AGENTS.md",
|
|
7355
|
+
"antigravity": ".agents/agents.md",
|
|
5414
7356
|
"mock": "CLAUDE.md",
|
|
5415
7357
|
};
|
|
5416
7358
|
static GENERAL_INSTRUCTIONS = `# Fleet Coordinator
|
|
@@ -5420,146 +7362,33 @@ You route tasks, manage instances, enforce policies, and synthesize results.
|
|
|
5420
7362
|
Do NOT modify project files directly — delegate file changes to the project's instance.
|
|
5421
7363
|
You CAN write code snippets, explain code, and answer technical questions directly.
|
|
5422
7364
|
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
## Task Classification
|
|
5426
|
-
|
|
5427
|
-
Classify every incoming request before acting.
|
|
5428
|
-
|
|
5429
|
-
### Handle Directly (ALL conditions must be true)
|
|
5430
|
-
|
|
5431
|
-
- No file system access needed
|
|
5432
|
-
- No external execution needed
|
|
5433
|
-
- Answerable from static knowledge
|
|
5434
|
-
- ≤ 2 reasoning steps
|
|
5435
|
-
|
|
5436
|
-
Examples: Q&A, translation, fleet status queries, explaining a concept, writing code snippets.
|
|
5437
|
-
|
|
5438
|
-
### Delegate to 1 Instance
|
|
5439
|
-
|
|
5440
|
-
- Task scoped to a single project or repo
|
|
5441
|
-
- Requires file access, code changes, or execution
|
|
7365
|
+
## Task Routing
|
|
5442
7366
|
|
|
5443
|
-
|
|
7367
|
+
- **Handle directly**: no file/exec access needed, answerable from knowledge, ≤2 reasoning steps (Q&A, translation, status queries, code snippets).
|
|
7368
|
+
- **Delegate to 1 instance**: scoped to one project/repo, needs file access or execution.
|
|
7369
|
+
- **Coordinate multiple**: spans repos, outputs feed each other, or parallel helps (max 3 per task).
|
|
5444
7370
|
|
|
5445
|
-
|
|
5446
|
-
- Requires outputs from one instance to feed into another
|
|
5447
|
-
- Benefits from parallel execution (max 3 instances per task)
|
|
7371
|
+
Instance discovery order: list_teams() → list_instances() → describe_instance() → create_instance() only if nothing fits. Prefer reuse; never duplicate a running instance.
|
|
5448
7372
|
|
|
5449
|
-
|
|
7373
|
+
## Reply Contract
|
|
5450
7374
|
|
|
5451
|
-
|
|
5452
|
-
1. list_teams() → reuse existing teams first
|
|
5453
|
-
2. list_instances() → find by working_directory, description, or tags
|
|
5454
|
-
3. describe_instance() → confirm capabilities before delegating
|
|
5455
|
-
4. create_instance() → only if no suitable instance exists
|
|
5456
|
-
|
|
5457
|
-
Rules: prefer reuse over creation. Do NOT create duplicates of running instances.
|
|
5458
|
-
|
|
5459
|
-
-----
|
|
5460
|
-
|
|
5461
|
-
## Delegation Protocol
|
|
5462
|
-
|
|
5463
|
-
Every delegation via send_to_instance() MUST include:
|
|
5464
|
-
|
|
5465
|
-
1. Task scope — what exactly to do, bounded clearly
|
|
5466
|
-
2. Expected output — what to return and in what form
|
|
5467
|
-
3. Policy reminder — "Follow Development Workflow policy" (for code tasks)
|
|
5468
|
-
|
|
5469
|
-
### Loop Prevention
|
|
5470
|
-
|
|
5471
|
-
- Never re-delegate a task back to the instance that sent it to you
|
|
5472
|
-
- If a task has bounced 3 times, stop and solve locally or reduce scope
|
|
5473
|
-
|
|
5474
|
-
### Execution Strategy
|
|
5475
|
-
|
|
5476
|
-
Parallel — use only when tasks are independent with no shared state
|
|
5477
|
-
Sequential — use when one task's output feeds into the next
|
|
5478
|
-
|
|
5479
|
-
-----
|
|
5480
|
-
|
|
5481
|
-
## Result Handling
|
|
5482
|
-
|
|
5483
|
-
When an instance reports back, classify the outcome:
|
|
5484
|
-
|
|
5485
|
-
- Success → Summarize key results for user. Omit internal coordination noise.
|
|
5486
|
-
- Partial → State what succeeded, what remains, proposed next steps.
|
|
5487
|
-
- Failure → Retry up to 2 times. If still failing: try alternative instance, reduce scope, or return partial result clearly marked.
|
|
5488
|
-
- No response → Ping again after reasonable wait. If still silent: report to user with options.
|
|
5489
|
-
|
|
5490
|
-
### Output to User
|
|
5491
|
-
|
|
5492
|
-
Every final response to the user should contain:
|
|
5493
|
-
|
|
5494
|
-
- Result — the actual answer or deliverable
|
|
5495
|
-
- Gaps — anything incomplete or unresolved (omit if none)
|
|
5496
|
-
|
|
5497
|
-
-----
|
|
5498
|
-
|
|
5499
|
-
## Shared Decisions
|
|
5500
|
-
|
|
5501
|
-
Use post_decision() / list_decisions() for any choice that affects more than 1 instance, changes an API contract, introduces a new dependency, or alters deployment process.
|
|
5502
|
-
|
|
5503
|
-
When instances disagree, collect both viewpoints, make a decision, and record it via post_decision.
|
|
5504
|
-
|
|
5505
|
-
-----
|
|
7375
|
+
Every final response to the user contains: the result (the actual answer or deliverable) and gaps (anything incomplete — omit if none). Summarize instance reports; omit internal coordination noise.
|
|
5506
7376
|
|
|
5507
7377
|
## After Restart
|
|
5508
7378
|
|
|
5509
|
-
|
|
5510
|
-
1. list_instances() → rebuild fleet awareness
|
|
5511
|
-
2. list_teams() → restore team structure
|
|
5512
|
-
3. list_decisions() → reload policies and conventions
|
|
5513
|
-
|
|
5514
|
-
Only then handle incoming requests.
|
|
7379
|
+
BEFORE processing any new messages: 1. list_instances() 2. list_teams() 3. list_decisions(). Only then handle requests.
|
|
5515
7380
|
|
|
5516
|
-
|
|
7381
|
+
## Playbooks (on-demand skills)
|
|
5517
7382
|
|
|
5518
|
-
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
Remind instances of this policy when delegating code tasks.
|
|
5523
|
-
|
|
5524
|
-
### Workflow Stages
|
|
5525
|
-
Design Proposed → Design Approved → Implementation → Submit for Review → Under Review → Approved → Merge
|
|
5526
|
-
|
|
5527
|
-
### Policy Rules
|
|
5528
|
-
|
|
5529
|
-
1. Design before code — developer sends design proposal to reviewer before implementation. Consensus required before proceeding.
|
|
5530
|
-
2. Challenger pairing — every code task should have a developer + reviewer. Reviewer actively questions decisions and finds risks.
|
|
5531
|
-
3. Verify by execution — backend/CLI changes must be tested by running them. Do not trust documentation alone.
|
|
5532
|
-
4. Independent review — every merge requires code review from someone other than the author.
|
|
5533
|
-
5. Root cause first — bug fixes require confirmed root cause before proposing a fix.
|
|
5534
|
-
6. Merge conditions: tests pass, reviewer approved, branch and worktree cleaned up.
|
|
5535
|
-
|
|
5536
|
-
### Specialist Instance Rules
|
|
5537
|
-
|
|
5538
|
-
- Execute within defined scope only
|
|
5539
|
-
- Return structured output: result, assumptions, uncertainties, verification status
|
|
5540
|
-
- Do NOT create new instances without coordinator approval
|
|
5541
|
-
|
|
5542
|
-
-----
|
|
5543
|
-
|
|
5544
|
-
## Team Management
|
|
5545
|
-
|
|
5546
|
-
- Always check existing teams before creating new ones
|
|
5547
|
-
- Default to ephemeral teams (created for a specific task, dissolved after completion)
|
|
5548
|
-
- Clean up ephemeral teams and instances after task completion
|
|
5549
|
-
|
|
5550
|
-
-----
|
|
5551
|
-
|
|
5552
|
-
## Instance Configuration Tips
|
|
5553
|
-
|
|
5554
|
-
When users create specialized instances, suggest these configurations:
|
|
5555
|
-
|
|
5556
|
-
- **Reviewer instances**: Add \`pre_task_command: "/chat load reviewer-base"\` to reset context before each review, preventing influence from previous conversations.
|
|
5557
|
-
- **Collab mode**: For multi-bot channels, use \`/collab\` to enable @mention-based triggering.
|
|
5558
|
-
- **Cost control**: Set per-instance \`cost_guard\` for expensive backends.
|
|
7383
|
+
Detailed procedures live in your skills — consult them when the situation comes up rather than from memory:
|
|
7384
|
+
- **delegation-playbook** — delegation protocol, loop prevention, parallel vs sequential, result/failure handling, team management, instance configuration tips.
|
|
7385
|
+
- **development-workflow** — the fleet-wide code-change policy you enforce when delegating code tasks.
|
|
7386
|
+
Plus the operational skills (fleet-health, instance-lifecycle, scheduling, session-management, …).
|
|
5559
7387
|
`;
|
|
5560
7388
|
/** Ensure the general instance has its project instructions file + knowledge */
|
|
5561
|
-
ensureGeneralInstructions(workDir, backendName) {
|
|
7389
|
+
ensureGeneralInstructions(workDir, backendName, instanceName) {
|
|
5562
7390
|
const backend = backendName ?? "claude-code";
|
|
7391
|
+
workDir = this.resolveKnowledgeWorkDir(workDir, backend, instanceName);
|
|
5563
7392
|
const filename = FleetManager.INSTRUCTIONS_FILENAME[backend] ?? "CLAUDE.md";
|
|
5564
7393
|
const filePath = join(workDir, filename);
|
|
5565
7394
|
mkdirSync(dirname(filePath), { recursive: true });
|
|
@@ -5567,60 +7396,237 @@ When users create specialized instances, suggest these configurations:
|
|
|
5567
7396
|
writeFileSync(filePath, FleetManager.GENERAL_INSTRUCTIONS, "utf-8");
|
|
5568
7397
|
this.logger.info({ filePath }, "Created general instance instructions file");
|
|
5569
7398
|
}
|
|
5570
|
-
// Sync bundled knowledge files to general's steering
|
|
7399
|
+
// Sync bundled knowledge files to general's steering and skills directories.
|
|
5571
7400
|
this.syncGeneralKnowledge(workDir, backend);
|
|
5572
7401
|
}
|
|
5573
|
-
/**
|
|
7402
|
+
/** Resolve the workspace path a backend actually uses before publishing knowledge. */
|
|
7403
|
+
resolveKnowledgeWorkDir(workDir, backend, instanceName) {
|
|
7404
|
+
// Backend resolution may create/normalize the real cwd. Instructions and
|
|
7405
|
+
// skills must land where the CLI actually runs, not an assumed path.
|
|
7406
|
+
try {
|
|
7407
|
+
const resolved = createBackend(backend, join(getAgendHome(), "cli-env"))
|
|
7408
|
+
.resolveWorkingDirectory?.(workDir, instanceName);
|
|
7409
|
+
if (resolved)
|
|
7410
|
+
workDir = resolved;
|
|
7411
|
+
}
|
|
7412
|
+
catch { /* unknown backend name — keep the raw path */ }
|
|
7413
|
+
return workDir;
|
|
7414
|
+
}
|
|
7415
|
+
/**
|
|
7416
|
+
* Where each backend natively loads on-demand skills from, relative to the
|
|
7417
|
+
* workspace. Backends without a native skill mechanism (opencode, grok,
|
|
7418
|
+
* antigravity, gemini-cli) are deliberately absent: dropping files a CLI
|
|
7419
|
+
* never reads is clutter, not capability. Unknown directories are ignored
|
|
7420
|
+
* by older CLI versions, so publishing is fail-open across upgrades.
|
|
7421
|
+
*/
|
|
7422
|
+
static SKILLS_DIR_SEGMENTS = {
|
|
7423
|
+
"kiro-cli": [".kiro", "skills"],
|
|
7424
|
+
"claude-code": [".claude", "skills"],
|
|
7425
|
+
"codex": [".agents", "skills"],
|
|
7426
|
+
// Live-verified: OpenCode and Antigravity scan .agents/skills; Grok's
|
|
7427
|
+
// vendor-canonical location is .grok/skills.
|
|
7428
|
+
"opencode": [".agents", "skills"],
|
|
7429
|
+
"grok": [".grok", "skills"],
|
|
7430
|
+
"antigravity": [".agents", "skills"],
|
|
7431
|
+
};
|
|
7432
|
+
/** Copy general-knowledge steering + all role-eligible skills to General. */
|
|
5574
7433
|
syncGeneralKnowledge(workDir, backend) {
|
|
5575
7434
|
const knowledgeDir = join(dirname(fileURLToPath(import.meta.url)), "general-knowledge");
|
|
5576
7435
|
if (!existsSync(knowledgeDir))
|
|
5577
7436
|
return;
|
|
5578
|
-
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
7437
|
+
this.syncGeneralSteering(workDir, backend, join(knowledgeDir, "steering"));
|
|
7438
|
+
this.syncRoleSkills(workDir, backend, "general", knowledgeDir);
|
|
7439
|
+
this.logger.debug({ knowledgeDir, workDir, backend }, "Synced general knowledge files");
|
|
7440
|
+
}
|
|
7441
|
+
/** Publish only the bundled skills eligible for an instance role. */
|
|
7442
|
+
syncRoleSkills(workDir, backend, role, knowledgeDir) {
|
|
7443
|
+
const skillSegments = FleetManager.SKILLS_DIR_SEGMENTS[backend];
|
|
7444
|
+
if (!skillSegments)
|
|
7445
|
+
return;
|
|
7446
|
+
const root = knowledgeDir ?? join(dirname(fileURLToPath(import.meta.url)), "general-knowledge");
|
|
7447
|
+
if (!existsSync(root))
|
|
7448
|
+
return;
|
|
7449
|
+
// Before managed-skill manifests existed, only Kiro General received
|
|
7450
|
+
// bundled skills. Allow that one legacy layout to be adopted so upgrades
|
|
7451
|
+
// can keep those copies current; other backends never had unmanaged
|
|
7452
|
+
// AgEnD-published skills and must retain the normal user-ownership guard.
|
|
7453
|
+
const adoptLegacyUnmanaged = backend === "kiro-cli" && role === "general";
|
|
7454
|
+
this.syncManagedSkills(join(workDir, ...skillSegments), join(root, "skills"), role, adoptLegacyUnmanaged);
|
|
7455
|
+
this.logger.debug({ workDir, backend, role }, "Synced role-based bundled skills");
|
|
7456
|
+
}
|
|
7457
|
+
/**
|
|
7458
|
+
* Steering (always-on rules like core-rules.md). Kiro loads a native
|
|
7459
|
+
* steering directory; every other backend gets the content embedded into
|
|
7460
|
+
* its instructions file (CLAUDE.md / AGENTS.md / …) inside a managed marker
|
|
7461
|
+
* block — the previous behavior dropped bare .md files in the workspace
|
|
7462
|
+
* root, which no CLI ever read. The block is replaced in place on every
|
|
7463
|
+
* sync, so rule updates reach EXISTING workspaces; everything the user
|
|
7464
|
+
* wrote outside the markers is preserved byte-for-byte.
|
|
7465
|
+
*/
|
|
7466
|
+
syncGeneralSteering(workDir, backend, srcSteering) {
|
|
7467
|
+
if (!existsSync(srcSteering))
|
|
7468
|
+
return;
|
|
7469
|
+
const files = readdirSync(srcSteering).filter(f => f.endsWith(".md")).sort();
|
|
7470
|
+
if (files.length === 0)
|
|
7471
|
+
return;
|
|
7472
|
+
if (backend === "kiro-cli") {
|
|
7473
|
+
const steeringDir = join(workDir, ".kiro", "steering");
|
|
7474
|
+
mkdirSync(steeringDir, { recursive: true });
|
|
7475
|
+
for (const file of files) {
|
|
5589
7476
|
const dest = join(steeringDir, file);
|
|
5590
|
-
const newContent = readFileSync(
|
|
7477
|
+
const newContent = readFileSync(join(srcSteering, file), "utf-8");
|
|
5591
7478
|
try {
|
|
5592
7479
|
if (existsSync(dest) && readFileSync(dest, "utf-8") === newContent)
|
|
5593
7480
|
continue;
|
|
5594
7481
|
}
|
|
5595
|
-
catch { }
|
|
7482
|
+
catch { /* rewrite */ }
|
|
5596
7483
|
writeFileSync(dest, newContent);
|
|
5597
7484
|
}
|
|
7485
|
+
return;
|
|
5598
7486
|
}
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
|
|
5608
|
-
|
|
5609
|
-
|
|
5610
|
-
|
|
5611
|
-
|
|
5612
|
-
|
|
5613
|
-
|
|
5614
|
-
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
7487
|
+
const filename = FleetManager.INSTRUCTIONS_FILENAME[backend] ?? "CLAUDE.md";
|
|
7488
|
+
const instructionsPath = join(workDir, filename);
|
|
7489
|
+
const body = files.map(f => readFileSync(join(srcSteering, f), "utf-8").trim()).join("\n\n");
|
|
7490
|
+
const block = `${FleetManager.STEERING_BLOCK_BEGIN}\n${body}\n${FleetManager.STEERING_BLOCK_END}`;
|
|
7491
|
+
let existing = "";
|
|
7492
|
+
try {
|
|
7493
|
+
existing = existsSync(instructionsPath) ? readFileSync(instructionsPath, "utf-8") : "";
|
|
7494
|
+
}
|
|
7495
|
+
catch { /* treat as empty */ }
|
|
7496
|
+
const beginAt = existing.indexOf(FleetManager.STEERING_BLOCK_BEGIN);
|
|
7497
|
+
const endAt = existing.indexOf(FleetManager.STEERING_BLOCK_END);
|
|
7498
|
+
let next;
|
|
7499
|
+
if (beginAt !== -1 && endAt !== -1 && endAt > beginAt) {
|
|
7500
|
+
next = existing.slice(0, beginAt) + block + existing.slice(endAt + FleetManager.STEERING_BLOCK_END.length);
|
|
7501
|
+
}
|
|
7502
|
+
else {
|
|
7503
|
+
next = existing.trimEnd() + (existing.trim() ? "\n\n" : "") + block + "\n";
|
|
7504
|
+
}
|
|
7505
|
+
if (next !== existing) {
|
|
7506
|
+
mkdirSync(dirname(instructionsPath), { recursive: true });
|
|
7507
|
+
writeFileSync(instructionsPath, next);
|
|
7508
|
+
}
|
|
7509
|
+
}
|
|
7510
|
+
static STEERING_BLOCK_BEGIN = "<!-- >>> agend:core-rules — managed by AgEnD; edits inside this block are overwritten -->";
|
|
7511
|
+
static STEERING_BLOCK_END = "<!-- <<< agend:core-rules -->";
|
|
7512
|
+
/**
|
|
7513
|
+
* Publish AgEnD's bundled skills into a CLI's native skills directory,
|
|
7514
|
+
* owning ONLY what we published. A manifest records which skill names AgEnD
|
|
7515
|
+
* wrote; a bundled rename/removal deletes the stale managed copy, while a
|
|
7516
|
+
* skill the user created by hand is never listed and therefore never
|
|
7517
|
+
* touched — even if a future bundle happens to reuse its name (the sync
|
|
7518
|
+
* then skips it and logs, rather than overwrite the user's work). The sole
|
|
7519
|
+
* migration exception is the pre-manifest Kiro General layout explicitly
|
|
7520
|
+
* selected by adoptLegacyUnmanaged.
|
|
7521
|
+
*/
|
|
7522
|
+
syncManagedSkills(destSkills, srcSkills, role, adoptLegacyUnmanaged = false) {
|
|
7523
|
+
if (!existsSync(srcSkills))
|
|
7524
|
+
return;
|
|
7525
|
+
mkdirSync(destSkills, { recursive: true });
|
|
7526
|
+
const manifestPath = join(destSkills, ".agend-managed-skills.json");
|
|
7527
|
+
const hadManifest = existsSync(manifestPath);
|
|
7528
|
+
let previouslyManaged = [];
|
|
7529
|
+
try {
|
|
7530
|
+
const parsed = JSON.parse(readFileSync(manifestPath, "utf-8"));
|
|
7531
|
+
if (Array.isArray(parsed))
|
|
7532
|
+
previouslyManaged = parsed.filter(n => typeof n === "string");
|
|
7533
|
+
}
|
|
7534
|
+
catch { /* first sync, or corrupt manifest — treat as owning nothing */ }
|
|
7535
|
+
const bundled = readdirSync(srcSkills)
|
|
7536
|
+
.filter(name => existsSync(join(srcSkills, name, "SKILL.md")))
|
|
7537
|
+
.sort();
|
|
7538
|
+
// Pre-manifest Kiro General copied bundled skills directly into
|
|
7539
|
+
// .kiro/skills. If every existing skill is still a bundled name, this is
|
|
7540
|
+
// the unambiguous legacy layout: adopt it once, update it below, and write
|
|
7541
|
+
// the ownership manifest. Any extra skill name keeps the directory fully
|
|
7542
|
+
// on the user-owned/collision path.
|
|
7543
|
+
if (!hadManifest && adoptLegacyUnmanaged) {
|
|
7544
|
+
const existing = readdirSync(destSkills, { withFileTypes: true })
|
|
7545
|
+
.filter(entry => entry.isDirectory() && existsSync(join(destSkills, entry.name, "SKILL.md")))
|
|
7546
|
+
.map(entry => entry.name)
|
|
7547
|
+
.sort();
|
|
7548
|
+
if (existing.length > 0 && existing.every(name => bundled.includes(name))) {
|
|
7549
|
+
previouslyManaged = existing;
|
|
7550
|
+
this.logger.info({ skills: existing, destSkills }, "Adopted legacy AgEnD skills into managed ownership");
|
|
7551
|
+
}
|
|
7552
|
+
}
|
|
7553
|
+
const eligible = bundled.filter(name => {
|
|
7554
|
+
const roles = this.readManagedSkillRoles(join(srcSkills, name, "SKILL.md"));
|
|
7555
|
+
// General is the coordinator and receives both coordinator and worker
|
|
7556
|
+
// playbooks. Workers receive only skills explicitly marked for workers.
|
|
7557
|
+
return role === "general" || roles.includes("worker");
|
|
7558
|
+
});
|
|
7559
|
+
const managed = [];
|
|
7560
|
+
for (const name of eligible) {
|
|
7561
|
+
const destDir = join(destSkills, name);
|
|
7562
|
+
const dest = join(destDir, "SKILL.md");
|
|
7563
|
+
const isOurs = previouslyManaged.includes(name) || !existsSync(destDir);
|
|
7564
|
+
if (!isOurs) {
|
|
7565
|
+
// Name collision with a user-authored skill: theirs wins, loudly.
|
|
7566
|
+
this.logger.warn({ skill: name, destSkills }, "Skipping bundled skill — a skill of this name exists but was not published by AgEnD");
|
|
7567
|
+
continue;
|
|
7568
|
+
}
|
|
7569
|
+
managed.push(name);
|
|
7570
|
+
const newContent = readFileSync(join(srcSkills, name, "SKILL.md"), "utf-8");
|
|
7571
|
+
try {
|
|
7572
|
+
if (existsSync(dest) && readFileSync(dest, "utf-8") === newContent)
|
|
7573
|
+
continue;
|
|
7574
|
+
}
|
|
7575
|
+
catch { /* rewrite */ }
|
|
7576
|
+
mkdirSync(destDir, { recursive: true });
|
|
7577
|
+
writeFileSync(dest, newContent);
|
|
7578
|
+
}
|
|
7579
|
+
// Remove managed skills that are no longer bundled OR no longer eligible
|
|
7580
|
+
// for this role. This makes a shared → general-only metadata change take
|
|
7581
|
+
// effect on the next worker startup instead of leaving stale capability.
|
|
7582
|
+
for (const stale of previouslyManaged) {
|
|
7583
|
+
if (eligible.includes(stale))
|
|
7584
|
+
continue;
|
|
7585
|
+
try {
|
|
7586
|
+
rmSync(join(destSkills, stale), { recursive: true, force: true });
|
|
7587
|
+
this.logger.info({ skill: stale, destSkills }, "Removed retired AgEnD-managed skill");
|
|
7588
|
+
}
|
|
7589
|
+
catch (err) {
|
|
7590
|
+
this.logger.debug({ err, skill: stale }, "Failed to remove retired managed skill");
|
|
5621
7591
|
}
|
|
5622
7592
|
}
|
|
5623
|
-
|
|
7593
|
+
try {
|
|
7594
|
+
writeFileSync(manifestPath, JSON.stringify(managed, null, 2) + "\n");
|
|
7595
|
+
}
|
|
7596
|
+
catch (err) {
|
|
7597
|
+
this.logger.debug({ err, manifestPath }, "Failed to write managed-skills manifest");
|
|
7598
|
+
}
|
|
7599
|
+
}
|
|
7600
|
+
/** Read AgEnD's roles extension from SKILL.md YAML frontmatter. */
|
|
7601
|
+
readManagedSkillRoles(skillPath) {
|
|
7602
|
+
const fallback = ["general"];
|
|
7603
|
+
try {
|
|
7604
|
+
const content = readFileSync(skillPath, "utf-8");
|
|
7605
|
+
const match = content.match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
|
7606
|
+
if (!match)
|
|
7607
|
+
return fallback;
|
|
7608
|
+
const document = parseDocument(match[1]);
|
|
7609
|
+
if (document.errors.length > 0)
|
|
7610
|
+
throw document.errors[0];
|
|
7611
|
+
const frontmatter = document.toJS();
|
|
7612
|
+
if (!frontmatter || frontmatter.roles === undefined)
|
|
7613
|
+
return fallback;
|
|
7614
|
+
if (!Array.isArray(frontmatter.roles))
|
|
7615
|
+
throw new Error("roles must be an array");
|
|
7616
|
+
const roles = [...new Set(frontmatter.roles)]
|
|
7617
|
+
.filter((value) => value === "general" || value === "worker");
|
|
7618
|
+
if (roles.length !== frontmatter.roles.length || roles.length === 0) {
|
|
7619
|
+
throw new Error("roles must contain only general and/or worker");
|
|
7620
|
+
}
|
|
7621
|
+
return roles;
|
|
7622
|
+
}
|
|
7623
|
+
catch (err) {
|
|
7624
|
+
// Fail closed for workers: malformed or unknown metadata keeps the
|
|
7625
|
+
// backwards-compatible General-only behavior instead of leaking an
|
|
7626
|
+
// administrative skill into worker workspaces.
|
|
7627
|
+
this.logger.warn({ err, skillPath }, "Invalid bundled skill roles — defaulting to General only");
|
|
7628
|
+
return fallback;
|
|
7629
|
+
}
|
|
5624
7630
|
}
|
|
5625
7631
|
/** Fetch forum topic icon stickers and pick emoji IDs for each state */
|
|
5626
7632
|
async resolveTopicIcons() {
|
|
@@ -6052,6 +8058,24 @@ When users create specialized instances, suggest these configurations:
|
|
|
6052
8058
|
return null;
|
|
6053
8059
|
const probed = await be.probeCLIEnv({ workingDirectory: "", instanceDir: join(getAgendHome(), "cli-env"), instanceName: `probe-${backend}`, mcpServers: {} });
|
|
6054
8060
|
const env = { backend, probedAt: Date.now(), ...probed };
|
|
8061
|
+
// An empty result must never overwrite a catalog we already have. Some
|
|
8062
|
+
// probes hit the network (`agy models` fetches, 5s cap), so a slow moment
|
|
8063
|
+
// returns [] — and writing that would blank the list for the whole 24h
|
|
8064
|
+
// TTL, long after the CLI recovered. Observed live: a good 11-model
|
|
8065
|
+
// antigravity cache replaced by an empty one. Keep the known models and
|
|
8066
|
+
// let the fresher currentModel/version through.
|
|
8067
|
+
if (!env.models?.length) {
|
|
8068
|
+
const previous = this.readCliEnv(backend);
|
|
8069
|
+
if (previous?.models?.length)
|
|
8070
|
+
env.models = previous.models;
|
|
8071
|
+
}
|
|
8072
|
+
// Same protection for the extended catalog: one offline moment must not
|
|
8073
|
+
// blank a good account list for the whole cache TTL.
|
|
8074
|
+
if (!env.apiModels?.length) {
|
|
8075
|
+
const previous = this.readCliEnv(backend);
|
|
8076
|
+
if (previous?.apiModels?.length)
|
|
8077
|
+
env.apiModels = previous.apiModels;
|
|
8078
|
+
}
|
|
6055
8079
|
const path = this.cliEnvPath(backend);
|
|
6056
8080
|
mkdirSync(dirname(path), { recursive: true });
|
|
6057
8081
|
writeFileSync(path, JSON.stringify(env, null, 2));
|
|
@@ -6090,6 +8114,108 @@ When users create specialized instances, suggest these configurations:
|
|
|
6090
8114
|
const env = await this.probeBackend(backendName);
|
|
6091
8115
|
return env?.models ?? [];
|
|
6092
8116
|
}
|
|
8117
|
+
/**
|
|
8118
|
+
* Model catalog behind the `list_models` tool.
|
|
8119
|
+
*
|
|
8120
|
+
* The two scopes are not cosmetic. "global" is the account/CLI catalog served
|
|
8121
|
+
* from the startup probe cache; "instance" is resolved through that instance's
|
|
8122
|
+
* OWN backend config, and for a Codex instance on a custom provider that is a
|
|
8123
|
+
* different catalog entirely — `listModels()` reads models_cache.json out of
|
|
8124
|
+
* the instance's private CODEX_HOME. Answering such an instance with the
|
|
8125
|
+
* account list would name models its CLI rejects, which is exactly the
|
|
8126
|
+
* mistake this tool exists to prevent.
|
|
8127
|
+
*
|
|
8128
|
+
* `scope` always describes where the returned LIST came from, not what was
|
|
8129
|
+
* asked for: an instance query that falls back to the account catalog reports
|
|
8130
|
+
* scope "global" and says so in `note`, rather than implying instance-level
|
|
8131
|
+
* accuracy it does not have.
|
|
8132
|
+
*
|
|
8133
|
+
* Never throws — a model listing is an aid, and failing it must not fail a turn.
|
|
8134
|
+
*/
|
|
8135
|
+
async listModelCatalog(opts = {}) {
|
|
8136
|
+
const { instanceName } = opts;
|
|
8137
|
+
if (instanceName) {
|
|
8138
|
+
const backend = this.backendNameForInstance(instanceName);
|
|
8139
|
+
const resolved = this.resolveInstanceModel(instanceName);
|
|
8140
|
+
const currentModel = resolved.source === "unresolved" ? null : resolved.model;
|
|
8141
|
+
const provider = this.customProviderFor(instanceName, backend);
|
|
8142
|
+
const scoped = await this.instanceScopedModels(instanceName, backend);
|
|
8143
|
+
if (scoped.length) {
|
|
8144
|
+
return {
|
|
8145
|
+
backend, scope: "instance", instance: instanceName,
|
|
8146
|
+
current_model: currentModel, models: scoped, source: "live",
|
|
8147
|
+
...(provider ? { note: `Catalog read through this instance's ${backend} provider "${provider}" — it may differ from the account catalog.` } : {}),
|
|
8148
|
+
};
|
|
8149
|
+
}
|
|
8150
|
+
// No instance-local catalog (never launched, or the backend has no
|
|
8151
|
+
// per-instance list). The account catalog is the best available answer,
|
|
8152
|
+
// but it is labelled honestly rather than dressed up as instance scope.
|
|
8153
|
+
const global = await this.globalModelCatalog(backend);
|
|
8154
|
+
return {
|
|
8155
|
+
...global, instance: instanceName, current_model: currentModel,
|
|
8156
|
+
note: provider
|
|
8157
|
+
? `No instance-local catalog yet; showing the account catalog, which may NOT match this instance's ${backend} provider "${provider}".`
|
|
8158
|
+
: "No instance-local catalog yet; showing the account catalog.",
|
|
8159
|
+
};
|
|
8160
|
+
}
|
|
8161
|
+
return this.globalModelCatalog(opts.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code");
|
|
8162
|
+
}
|
|
8163
|
+
/** The custom provider an instance overrides its backend with, if any. */
|
|
8164
|
+
customProviderFor(instanceName, backend) {
|
|
8165
|
+
const opts = this.fleetConfig?.instances?.[instanceName]?.backend_options?.[backend]
|
|
8166
|
+
?? this.fleetConfig?.defaults?.backend_options?.[backend];
|
|
8167
|
+
const provider = opts?.provider;
|
|
8168
|
+
return typeof provider === "string" && provider.trim() ? provider.trim() : null;
|
|
8169
|
+
}
|
|
8170
|
+
/** Ask a backend for its catalog using ONE instance's real config. Never throws. */
|
|
8171
|
+
async instanceScopedModels(instanceName, backend) {
|
|
8172
|
+
try {
|
|
8173
|
+
const inst = this.fleetConfig?.instances?.[instanceName];
|
|
8174
|
+
const instanceDir = this.getInstanceDir(instanceName);
|
|
8175
|
+
const be = createBackend(backend, instanceDir);
|
|
8176
|
+
if (!be.listModels)
|
|
8177
|
+
return [];
|
|
8178
|
+
return await be.listModels({
|
|
8179
|
+
workingDirectory: inst?.working_directory ?? "",
|
|
8180
|
+
instanceDir,
|
|
8181
|
+
instanceName,
|
|
8182
|
+
mcpServers: {},
|
|
8183
|
+
model: inst?.model,
|
|
8184
|
+
backendOptions: inst?.backend_options?.[backend] ?? this.fleetConfig?.defaults?.backend_options?.[backend],
|
|
8185
|
+
}) ?? [];
|
|
8186
|
+
}
|
|
8187
|
+
catch {
|
|
8188
|
+
// listModels is documented never to throw, but a backend constructor can
|
|
8189
|
+
// (missing binary). A catalog is an aid; degrade to the account list.
|
|
8190
|
+
return [];
|
|
8191
|
+
}
|
|
8192
|
+
}
|
|
8193
|
+
/** Account-wide catalog: probe cache first, live probe on miss. */
|
|
8194
|
+
async globalModelCatalog(backend) {
|
|
8195
|
+
const cached = this.readCliEnv(backend);
|
|
8196
|
+
if (cached?.models?.length) {
|
|
8197
|
+
return {
|
|
8198
|
+
backend, scope: "global", current_model: cached.currentModel ?? null,
|
|
8199
|
+
models: cached.models, source: "cache",
|
|
8200
|
+
probed_at: new Date(cached.probedAt).toISOString(),
|
|
8201
|
+
};
|
|
8202
|
+
}
|
|
8203
|
+
const env = await this.probeBackend(backend);
|
|
8204
|
+
if (env?.models?.length) {
|
|
8205
|
+
return {
|
|
8206
|
+
backend, scope: "global", current_model: env.currentModel ?? null,
|
|
8207
|
+
models: env.models, source: "live",
|
|
8208
|
+
probed_at: new Date(env.probedAt).toISOString(),
|
|
8209
|
+
};
|
|
8210
|
+
}
|
|
8211
|
+
// Reported rather than thrown: "we could not enumerate" is a useful answer,
|
|
8212
|
+
// and the caller can still set a model by name (AgEnD passes it through).
|
|
8213
|
+
return {
|
|
8214
|
+
backend, scope: "global", current_model: env?.currentModel ?? null,
|
|
8215
|
+
models: [], source: "fallback",
|
|
8216
|
+
note: `Could not enumerate models for ${backend} (CLI missing, not logged in, or it offers no list). Model names are passed through to the CLI, so a known-good name still works.`,
|
|
8217
|
+
};
|
|
8218
|
+
}
|
|
6093
8219
|
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
6094
8220
|
/** Label an effort choice, marking the one currently configured. */
|
|
6095
8221
|
effortChoiceLabel(level, current) {
|
|
@@ -6098,10 +8224,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
6098
8224
|
effortMenuHeader(instanceName) {
|
|
6099
8225
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6100
8226
|
if (!effort)
|
|
6101
|
-
return "
|
|
8227
|
+
return t("effort.current_default");
|
|
6102
8228
|
return source === "fleet-default"
|
|
6103
|
-
?
|
|
6104
|
-
:
|
|
8229
|
+
? t("effort.current_fleet", effort)
|
|
8230
|
+
: t("effort.current", effort);
|
|
6105
8231
|
}
|
|
6106
8232
|
/** `/effort` — DC Select Menu, or apply directly when a level is given. */
|
|
6107
8233
|
async handleEffortSlash(data, adapterId) {
|
|
@@ -6122,11 +8248,11 @@ When users create specialized instances, suggest these configurations:
|
|
|
6122
8248
|
}
|
|
6123
8249
|
const levels = this.effortLevelsFor(name);
|
|
6124
8250
|
if (levels.length === 0) {
|
|
6125
|
-
await data.respond(
|
|
8251
|
+
await data.respond(t("effort.unsupported", this.backendNameForInstance(name)));
|
|
6126
8252
|
return;
|
|
6127
8253
|
}
|
|
6128
8254
|
if (!data.respondChoices) {
|
|
6129
|
-
await data.respond(
|
|
8255
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6130
8256
|
return;
|
|
6131
8257
|
}
|
|
6132
8258
|
const current = this.resolveInstanceEffort(name).effort;
|
|
@@ -6139,20 +8265,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
6139
8265
|
timer.unref?.();
|
|
6140
8266
|
this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
6141
8267
|
try {
|
|
6142
|
-
await data.respondChoices(
|
|
8268
|
+
await data.respondChoices(t("effort.menu", this.effortMenuHeader(name)), choices);
|
|
6143
8269
|
}
|
|
6144
8270
|
catch (err) {
|
|
6145
8271
|
this.pendingEffortSelects.delete(nonce);
|
|
6146
8272
|
clearTimeout(timer);
|
|
6147
8273
|
this.logger.warn({ err, instanceName: name }, "effort menu failed");
|
|
6148
|
-
await data.respond(
|
|
8274
|
+
await data.respond(t("effort.usage", levels.join("|")));
|
|
6149
8275
|
}
|
|
6150
8276
|
}
|
|
6151
8277
|
/** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
|
|
6152
8278
|
async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6153
8279
|
const levels = this.effortLevelsFor(instanceName);
|
|
6154
8280
|
if (levels.length === 0) {
|
|
6155
|
-
return
|
|
8281
|
+
return t("effort.unsupported", this.backendNameForInstance(instanceName));
|
|
6156
8282
|
}
|
|
6157
8283
|
const current = this.resolveInstanceEffort(instanceName).effort;
|
|
6158
8284
|
const nonce = randomBytes(6).toString("hex");
|
|
@@ -6168,13 +8294,13 @@ When users create specialized instances, suggest these configurations:
|
|
|
6168
8294
|
const p = this.pendingEffortSelects.get(nonce);
|
|
6169
8295
|
if (p) {
|
|
6170
8296
|
this.pendingEffortSelects.delete(nonce);
|
|
6171
|
-
p.respond("
|
|
8297
|
+
p.respond(t("effort.selection_expired")).catch(() => { });
|
|
6172
8298
|
}
|
|
6173
8299
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6174
8300
|
timer.unref?.();
|
|
6175
8301
|
this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6176
8302
|
try {
|
|
6177
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8303
|
+
const menuMessageId = await adapter.promptUser(chatId, t("effort.menu", this.effortMenuHeader(instanceName)), choices, { threadId });
|
|
6178
8304
|
const pending = this.pendingEffortSelects.get(nonce);
|
|
6179
8305
|
if (pending)
|
|
6180
8306
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6184,7 +8310,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6184
8310
|
this.pendingEffortSelects.delete(nonce);
|
|
6185
8311
|
clearTimeout(timer);
|
|
6186
8312
|
this.logger.warn({ err, instanceName }, "TG effort menu failed");
|
|
6187
|
-
return
|
|
8313
|
+
return t("effort.usage", levels.join("|"));
|
|
6188
8314
|
}
|
|
6189
8315
|
}
|
|
6190
8316
|
/** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
|
|
@@ -6205,7 +8331,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6205
8331
|
this.pendingEffortSelects.delete(match[1]);
|
|
6206
8332
|
clearTimeout(pending.timer);
|
|
6207
8333
|
const level = match[2];
|
|
6208
|
-
const progressText =
|
|
8334
|
+
const progressText = t("effort.setting", pending.instanceName, level);
|
|
6209
8335
|
let progressMsgId;
|
|
6210
8336
|
if (pending.adapter && pending.adapterChatId) {
|
|
6211
8337
|
const menuMessageId = pending.menuMessageId ?? data.messageId;
|
|
@@ -6237,7 +8363,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6237
8363
|
}
|
|
6238
8364
|
catch (err) {
|
|
6239
8365
|
this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
|
|
6240
|
-
result =
|
|
8366
|
+
result = t("effort.switch_failed", level, err instanceof Error ? err.message : String(err));
|
|
6241
8367
|
}
|
|
6242
8368
|
if (pending.adapter && pending.adapterChatId) {
|
|
6243
8369
|
if (progressMsgId) {
|
|
@@ -6274,32 +8400,36 @@ When users create specialized instances, suggest these configurations:
|
|
|
6274
8400
|
}
|
|
6275
8401
|
// No arg (or --refresh) → menu. Menu is DC-only this round (respondChoices); TG uses `/model <name>`.
|
|
6276
8402
|
if (!data.respondChoices) {
|
|
6277
|
-
await data.respond("
|
|
8403
|
+
await data.respond(t("model.usage"));
|
|
6278
8404
|
return;
|
|
6279
8405
|
}
|
|
6280
8406
|
const options = await this.getModelOptions(name, isRefresh);
|
|
6281
8407
|
if (options.length === 0) {
|
|
6282
|
-
await data.respond(
|
|
8408
|
+
await data.respond(t("model.list_unavailable", name));
|
|
6283
8409
|
return;
|
|
6284
8410
|
}
|
|
6285
8411
|
// Raw id for ✓-matching options; display resolves an inherited CLI default.
|
|
6286
8412
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(name);
|
|
6287
8413
|
const nonce = randomBytes(6).toString("hex");
|
|
6288
|
-
const
|
|
8414
|
+
const isClaude = this.backendNameForInstance(name) === "claude-code";
|
|
8415
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6289
8416
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6290
8417
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6291
8418
|
}));
|
|
8419
|
+
if (isClaude) {
|
|
8420
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8421
|
+
}
|
|
6292
8422
|
const timer = setTimeout(() => this.pendingModelSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6293
8423
|
timer.unref?.();
|
|
6294
|
-
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
8424
|
+
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond, respondChoices: data.respondChoices });
|
|
6295
8425
|
try {
|
|
6296
|
-
await data.respondChoices(
|
|
8426
|
+
await data.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
6297
8427
|
}
|
|
6298
8428
|
catch (err) {
|
|
6299
8429
|
this.pendingModelSelects.delete(nonce);
|
|
6300
8430
|
clearTimeout(timer);
|
|
6301
8431
|
this.logger.warn({ err, instanceName: name }, "model menu failed");
|
|
6302
|
-
await data.respond("
|
|
8432
|
+
await data.respond(t("model.usage"));
|
|
6303
8433
|
}
|
|
6304
8434
|
}
|
|
6305
8435
|
/**
|
|
@@ -6310,14 +8440,19 @@ When users create specialized instances, suggest these configurations:
|
|
|
6310
8440
|
async promptModelMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
|
|
6311
8441
|
const options = await this.getModelOptions(instanceName);
|
|
6312
8442
|
if (options.length === 0) {
|
|
6313
|
-
return
|
|
8443
|
+
return t("model.list_unavailable", instanceName);
|
|
6314
8444
|
}
|
|
6315
8445
|
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(instanceName);
|
|
6316
8446
|
const nonce = randomBytes(6).toString("hex");
|
|
6317
|
-
const
|
|
8447
|
+
const isClaude = this.backendNameForInstance(instanceName) === "claude-code";
|
|
8448
|
+
// Keep the more-models entry inside Discord's 25-option select cap.
|
|
8449
|
+
const choices = options.slice(0, isClaude ? 24 : 25).map(o => ({
|
|
6318
8450
|
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
6319
8451
|
label: this.modelChoiceLabel(o, currentModel),
|
|
6320
8452
|
}));
|
|
8453
|
+
if (isClaude) {
|
|
8454
|
+
choices.push({ id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:__more__`, label: t("model.more") });
|
|
8455
|
+
}
|
|
6321
8456
|
const respond = async (text) => {
|
|
6322
8457
|
await adapter.sendText(chatId, text, { threadId });
|
|
6323
8458
|
return undefined;
|
|
@@ -6326,13 +8461,13 @@ When users create specialized instances, suggest these configurations:
|
|
|
6326
8461
|
const p = this.pendingModelSelects.get(nonce);
|
|
6327
8462
|
if (p) {
|
|
6328
8463
|
this.pendingModelSelects.delete(nonce);
|
|
6329
|
-
p.respond("
|
|
8464
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
6330
8465
|
}
|
|
6331
8466
|
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
6332
8467
|
timer.unref?.();
|
|
6333
8468
|
this.pendingModelSelects.set(nonce, { instanceName, model: "", userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
|
|
6334
8469
|
try {
|
|
6335
|
-
const menuMessageId = await adapter.promptUser(chatId,
|
|
8470
|
+
const menuMessageId = await adapter.promptUser(chatId, t("model.menu", currentDisplay), choices, { threadId });
|
|
6336
8471
|
const pending = this.pendingModelSelects.get(nonce);
|
|
6337
8472
|
if (pending)
|
|
6338
8473
|
pending.menuMessageId = menuMessageId;
|
|
@@ -6342,7 +8477,63 @@ When users create specialized instances, suggest these configurations:
|
|
|
6342
8477
|
this.pendingModelSelects.delete(nonce);
|
|
6343
8478
|
clearTimeout(timer);
|
|
6344
8479
|
this.logger.warn({ err, instanceName }, "TG model menu failed");
|
|
6345
|
-
return
|
|
8480
|
+
return t("model.usage");
|
|
8481
|
+
}
|
|
8482
|
+
}
|
|
8483
|
+
/** Cached-or-live account catalog behind "/model → more models" (claude). */
|
|
8484
|
+
async claudeApiModelOptions() {
|
|
8485
|
+
const cached = this.readCliEnv("claude-code");
|
|
8486
|
+
if (cached?.apiModels?.length)
|
|
8487
|
+
return cached.apiModels;
|
|
8488
|
+
const env = await this.probeBackend("claude-code");
|
|
8489
|
+
return env?.apiModels ?? [];
|
|
8490
|
+
}
|
|
8491
|
+
/** Replace a consumed "/model" menu with the full account catalog tier. */
|
|
8492
|
+
async expandClaudeModelMenu(pending) {
|
|
8493
|
+
const expanded = await this.claudeApiModelOptions();
|
|
8494
|
+
if (!expanded.length) {
|
|
8495
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8496
|
+
return;
|
|
8497
|
+
}
|
|
8498
|
+
const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(pending.instanceName);
|
|
8499
|
+
const nonce = randomBytes(6).toString("hex");
|
|
8500
|
+
const choices = expanded.slice(0, 25).map(o => ({
|
|
8501
|
+
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
8502
|
+
label: this.modelChoiceLabel(o, currentModel),
|
|
8503
|
+
}));
|
|
8504
|
+
const timer = setTimeout(() => {
|
|
8505
|
+
const p = this.pendingModelSelects.get(nonce);
|
|
8506
|
+
if (p) {
|
|
8507
|
+
this.pendingModelSelects.delete(nonce);
|
|
8508
|
+
p.respond(t("model.selection_expired")).catch(() => { });
|
|
8509
|
+
}
|
|
8510
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
8511
|
+
timer.unref?.();
|
|
8512
|
+
this.pendingModelSelects.set(nonce, { ...pending, model: "", timer });
|
|
8513
|
+
try {
|
|
8514
|
+
if (pending.respondChoices) {
|
|
8515
|
+
// Discord select menu: edit the same interaction reply in place.
|
|
8516
|
+
await pending.respondChoices(t("model.menu", `**${currentDisplay}**`), choices);
|
|
8517
|
+
return;
|
|
8518
|
+
}
|
|
8519
|
+
if (pending.adapter && pending.adapterChatId) {
|
|
8520
|
+
// Telegram: retire the tier-1 keyboard, then post the expanded menu.
|
|
8521
|
+
if (pending.menuMessageId && pending.adapter.editMessageRemoveButtons) {
|
|
8522
|
+
await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, pending.menuMessageId, t("model.more"), pending.adapterThreadId).catch(() => { });
|
|
8523
|
+
}
|
|
8524
|
+
const menuMessageId = await pending.adapter.promptUser(pending.adapterChatId, t("model.menu", currentDisplay), choices, { threadId: pending.adapterThreadId });
|
|
8525
|
+
const fresh = this.pendingModelSelects.get(nonce);
|
|
8526
|
+
if (fresh)
|
|
8527
|
+
fresh.menuMessageId = menuMessageId;
|
|
8528
|
+
return;
|
|
8529
|
+
}
|
|
8530
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
8531
|
+
}
|
|
8532
|
+
catch (err) {
|
|
8533
|
+
this.pendingModelSelects.delete(nonce);
|
|
8534
|
+
clearTimeout(timer);
|
|
8535
|
+
this.logger.warn({ err, instanceName: pending.instanceName }, "Expanded model menu failed");
|
|
8536
|
+
await pending.respond(t("model.more_unavailable")).catch(() => { });
|
|
6346
8537
|
}
|
|
6347
8538
|
}
|
|
6348
8539
|
/** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
|
|
@@ -6364,8 +8555,14 @@ When users create specialized instances, suggest these configurations:
|
|
|
6364
8555
|
this.pendingModelSelects.delete(match[1]);
|
|
6365
8556
|
clearTimeout(pending.timer);
|
|
6366
8557
|
const model = match[2];
|
|
8558
|
+
// "More models…" is a navigation choice, not a model: swap the menu for the
|
|
8559
|
+
// full account catalog (live /v1/models with [1m] variants).
|
|
8560
|
+
if (model === "__more__") {
|
|
8561
|
+
await this.expandClaudeModelMenu(pending);
|
|
8562
|
+
return true;
|
|
8563
|
+
}
|
|
6367
8564
|
// Send immediate "⏳ Switching..." feedback, then apply in background.
|
|
6368
|
-
const progressText =
|
|
8565
|
+
const progressText = t("model.switching", pending.instanceName, model);
|
|
6369
8566
|
let progressMsgId;
|
|
6370
8567
|
if (pending.adapter && pending.adapterChatId) {
|
|
6371
8568
|
// TG path: turn the original menu into the progress message. This both
|
|
@@ -6402,7 +8599,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6402
8599
|
}
|
|
6403
8600
|
catch (err) {
|
|
6404
8601
|
this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
|
|
6405
|
-
result =
|
|
8602
|
+
result = t("model.switch_failed", model, err instanceof Error ? err.message : String(err));
|
|
6406
8603
|
}
|
|
6407
8604
|
if (pending.adapter && pending.adapterChatId) {
|
|
6408
8605
|
if (progressMsgId) {
|
|
@@ -6499,17 +8696,17 @@ When users create specialized instances, suggest these configurations:
|
|
|
6499
8696
|
}
|
|
6500
8697
|
catch { /* treated as unsupported below */ }
|
|
6501
8698
|
if (strategy === "unsupported" || supported.length === 0) {
|
|
6502
|
-
return
|
|
8699
|
+
return t("effort.unsupported", backendName);
|
|
6503
8700
|
}
|
|
6504
8701
|
if (!FleetManager.EFFORT_LEVELS.includes(level)) {
|
|
6505
|
-
return
|
|
8702
|
+
return t("effort.unknown", level, FleetManager.EFFORT_LEVELS.join(", "));
|
|
6506
8703
|
}
|
|
6507
8704
|
const applied = FleetManager.clampEffort(level, supported);
|
|
6508
8705
|
if (!applied)
|
|
6509
|
-
return
|
|
8706
|
+
return t("effort.no_canonical", backendName);
|
|
6510
8707
|
const warn = applied === level
|
|
6511
8708
|
? ""
|
|
6512
|
-
:
|
|
8709
|
+
: t("effort.clamped", applied, level, backendName);
|
|
6513
8710
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6514
8711
|
// or the instance silently reverts on restart.
|
|
6515
8712
|
if (this.fleetConfig?.instances[instanceName]) {
|
|
@@ -6518,12 +8715,12 @@ When users create specialized instances, suggest these configurations:
|
|
|
6518
8715
|
}
|
|
6519
8716
|
if (strategy === "runtime") {
|
|
6520
8717
|
if (!this.instanceIpcClients.get(instanceName))
|
|
6521
|
-
return `${warn}
|
|
8718
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6522
8719
|
this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
|
|
6523
|
-
return `${warn}
|
|
8720
|
+
return `${warn}${t("effort.runtime_success", instanceName, applied)}`;
|
|
6524
8721
|
}
|
|
6525
8722
|
await this.restartSingleInstance(instanceName);
|
|
6526
|
-
return `${warn}
|
|
8723
|
+
return `${warn}${t("effort.restart_success", instanceName, applied)}`;
|
|
6527
8724
|
}
|
|
6528
8725
|
async applyModel(instanceName, model) {
|
|
6529
8726
|
const backendName = this.backendNameForInstance(instanceName);
|
|
@@ -6532,9 +8729,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
6532
8729
|
strategy = createBackend(backendName, this.getInstanceDir(instanceName)).getModelSwitchStrategy?.(model) ?? "restart";
|
|
6533
8730
|
}
|
|
6534
8731
|
catch { /* default restart */ }
|
|
6535
|
-
const warn = isModelCompatible(backendName, model) ? "" :
|
|
8732
|
+
const warn = isModelCompatible(backendName, model) ? "" : t("model.pattern_warning", model, backendName);
|
|
6536
8733
|
if (strategy === "runtime" && !this.instanceIpcClients.get(instanceName)) {
|
|
6537
|
-
return `${warn}
|
|
8734
|
+
return `${warn}${t("effort.not_running", instanceName)}`;
|
|
6538
8735
|
}
|
|
6539
8736
|
// Persist either way: a runtime switch must survive the next respawn too,
|
|
6540
8737
|
// or the instance silently reverts to the CLI default after a fleet restart.
|
|
@@ -6548,13 +8745,13 @@ When users create specialized instances, suggest these configurations:
|
|
|
6548
8745
|
persisted = true;
|
|
6549
8746
|
}
|
|
6550
8747
|
if (!persisted)
|
|
6551
|
-
return `${warn}
|
|
8748
|
+
return `${warn}${t("model.persist_failed", instanceName)}`;
|
|
6552
8749
|
if (strategy === "runtime") {
|
|
6553
8750
|
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
6554
|
-
return `${warn}
|
|
8751
|
+
return `${warn}${t("model.runtime_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6555
8752
|
}
|
|
6556
8753
|
await this.restartSingleInstance(instanceName);
|
|
6557
|
-
return `${warn}
|
|
8754
|
+
return `${warn}${t("model.restart_success", instanceName, model)}${this.effortSuffix(instanceName)}`;
|
|
6558
8755
|
}
|
|
6559
8756
|
/**
|
|
6560
8757
|
* The trailing "Current effort: …" line for a /model reply.
|
|
@@ -6568,10 +8765,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
6568
8765
|
return "";
|
|
6569
8766
|
const { effort, source } = this.resolveInstanceEffort(instanceName);
|
|
6570
8767
|
if (!effort)
|
|
6571
|
-
return "
|
|
8768
|
+
return `\n${t("effort.current_default")}`;
|
|
6572
8769
|
return source === "fleet-default"
|
|
6573
|
-
? `\
|
|
6574
|
-
: `\
|
|
8770
|
+
? `\n${t("effort.current_fleet", effort)}`
|
|
8771
|
+
: `\n${t("effort.current", effort)}`;
|
|
6575
8772
|
}
|
|
6576
8773
|
/** Read recent chat log for agent context */
|
|
6577
8774
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
@@ -6606,11 +8803,21 @@ When users create specialized instances, suggest these configurations:
|
|
|
6606
8803
|
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
6607
8804
|
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
6608
8805
|
if (!this.classicChannels)
|
|
6609
|
-
return "
|
|
8806
|
+
return t("classic.manager_unavailable");
|
|
6610
8807
|
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
6611
8808
|
const generalId = this.findGeneralInstance(adapterId);
|
|
6612
|
-
if (generalId)
|
|
6613
|
-
|
|
8809
|
+
if (generalId) {
|
|
8810
|
+
// Fire-and-forget, exactly as the notifyInstanceTopic it replaces: this
|
|
8811
|
+
// function's return value is the rejection shown to the user, and it
|
|
8812
|
+
// must not wait on posting buttons into the General topic.
|
|
8813
|
+
void this.promptClassicApproval({
|
|
8814
|
+
generalName: generalId,
|
|
8815
|
+
message: t("alert.unauth_guild", guildId, userId),
|
|
8816
|
+
groupId: String(guildId),
|
|
8817
|
+
scope: "guild",
|
|
8818
|
+
userId: userId ? String(userId) : undefined,
|
|
8819
|
+
}).catch(err => this.logger.warn({ err, guildId }, "Classic approval prompt failed"));
|
|
8820
|
+
}
|
|
6614
8821
|
return t("classic.not_authorized_guild");
|
|
6615
8822
|
}
|
|
6616
8823
|
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
@@ -6753,6 +8960,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6753
8960
|
return;
|
|
6754
8961
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
6755
8962
|
ensureWorkspaceGit(workDir);
|
|
8963
|
+
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
6756
8964
|
const config = {
|
|
6757
8965
|
...DEFAULT_INSTANCE_CONFIG,
|
|
6758
8966
|
...this.fleetConfig?.defaults,
|
|
@@ -6760,6 +8968,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
6760
8968
|
lightweight: true,
|
|
6761
8969
|
...(backend ? { backend } : {}),
|
|
6762
8970
|
...(model ? { model } : {}),
|
|
8971
|
+
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
8972
|
+
...(classicIdentity?.description ? { description: classicIdentity.description } : {}),
|
|
6763
8973
|
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
6764
8974
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
6765
8975
|
};
|
|
@@ -6773,7 +8983,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6773
8983
|
return blocker;
|
|
6774
8984
|
const classicChannels = this.classicChannels;
|
|
6775
8985
|
if (!classicChannels)
|
|
6776
|
-
return "
|
|
8986
|
+
return t("classic.manager_unavailable");
|
|
6777
8987
|
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
6778
8988
|
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
6779
8989
|
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
@@ -6795,7 +9005,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
6795
9005
|
/** Handle /stop slash command — unregister classic channel */
|
|
6796
9006
|
async handleClassicStop(channelId, adapterId) {
|
|
6797
9007
|
if (!this.classicChannels)
|
|
6798
|
-
return "
|
|
9008
|
+
return t("classic.manager_unavailable");
|
|
6799
9009
|
const ch = this.classicChannels.unregister(channelId, adapterId);
|
|
6800
9010
|
if (!ch)
|
|
6801
9011
|
return t("classic.no_agent");
|
|
@@ -6827,6 +9037,21 @@ When users create specialized instances, suggest these configurations:
|
|
|
6827
9037
|
// await in the same tick as the signal handler, so no event can slip in.
|
|
6828
9038
|
this.shuttingDown = true;
|
|
6829
9039
|
this.ipcStoppingInstances.add("__fleet_stopping__");
|
|
9040
|
+
// Release held delivery promises before awaiting daemon shutdown, then
|
|
9041
|
+
// reject spawn work which has not started. Otherwise a storm backoff could
|
|
9042
|
+
// make `agend stop` wait forever for its own queue.
|
|
9043
|
+
this.stormWindow.shutdown();
|
|
9044
|
+
this.spawnGate.shutdown();
|
|
9045
|
+
for (const pending of this.startupRetries.values())
|
|
9046
|
+
clearTimeout(pending.timer);
|
|
9047
|
+
this.startupRetries.clear();
|
|
9048
|
+
for (const pending of this.startupRetryNotices.values())
|
|
9049
|
+
clearTimeout(pending.timer);
|
|
9050
|
+
this.startupRetryNotices.clear();
|
|
9051
|
+
if (this.stormOpenNotifyTimer) {
|
|
9052
|
+
clearTimeout(this.stormOpenNotifyTimer);
|
|
9053
|
+
this.stormOpenNotifyTimer = null;
|
|
9054
|
+
}
|
|
6830
9055
|
sdNotifyBlocking("STOPPING=1");
|
|
6831
9056
|
if (this.watchdogTimer) {
|
|
6832
9057
|
clearInterval(this.watchdogTimer);
|
|
@@ -6842,6 +9067,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
6842
9067
|
this.clearStatuslineWatchers();
|
|
6843
9068
|
this.costGuard?.stop();
|
|
6844
9069
|
this.dailySummary?.stop();
|
|
9070
|
+
this.dailyTipScheduler?.stop();
|
|
9071
|
+
this.dailyTipScheduler = null;
|
|
6845
9072
|
if (this.updateCheckTimer) {
|
|
6846
9073
|
clearTimeout(this.updateCheckTimer);
|
|
6847
9074
|
clearInterval(this.updateCheckTimer);
|
|
@@ -7059,7 +9286,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
7059
9286
|
/**
|
|
7060
9287
|
* Hot-reload: re-read fleet.yaml and reconcile running instances.
|
|
7061
9288
|
* Starts new, stops removed, restarts modified instances.
|
|
7062
|
-
*
|
|
9289
|
+
* Whitelisted runtime fields are pushed into live daemons; all other instance
|
|
9290
|
+
* fields, plus cold fleet-level settings, retain restart semantics.
|
|
7063
9291
|
*/
|
|
7064
9292
|
async reconcileInstances() {
|
|
7065
9293
|
if (!this.configPath)
|
|
@@ -7101,11 +9329,11 @@ When users create specialized instances, suggest these configurations:
|
|
|
7101
9329
|
// possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
|
|
7102
9330
|
// with no explanation anywhere they are looking.
|
|
7103
9331
|
const why = !validation.valid
|
|
7104
|
-
?
|
|
9332
|
+
? t("fleet.reload_validation", validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n"))
|
|
7105
9333
|
: unsafeEmpty
|
|
7106
|
-
?
|
|
7107
|
-
:
|
|
7108
|
-
this.notifyFleetError(
|
|
9334
|
+
? t("fleet.reload_removed_all", oldCount)
|
|
9335
|
+
: t("fleet.reload_removed_half", oldCount, newCount);
|
|
9336
|
+
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
7109
9337
|
return;
|
|
7110
9338
|
}
|
|
7111
9339
|
this.routing.rebuild(this.fleetConfig);
|
|
@@ -7113,9 +9341,20 @@ When users create specialized instances, suggest these configurations:
|
|
|
7113
9341
|
this.scheduler?.reload();
|
|
7114
9342
|
const newInstances = this.fleetConfig.instances;
|
|
7115
9343
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
7116
|
-
// Detect fleet-level
|
|
7117
|
-
|
|
7118
|
-
const
|
|
9344
|
+
// Detect fleet-level changes which still need a restart. Hot defaults are
|
|
9345
|
+
// reconciled below and must not produce a misleading restart warning.
|
|
9346
|
+
const oldDefaultColdWithFleetFields = oldConfig?.defaults
|
|
9347
|
+
? splitHotColdConfig(oldConfig.defaults).cold
|
|
9348
|
+
: {};
|
|
9349
|
+
const newDefaultColdWithFleetFields = this.fleetConfig?.defaults
|
|
9350
|
+
? splitHotColdConfig(this.fleetConfig.defaults).cold
|
|
9351
|
+
: {};
|
|
9352
|
+
// `tips` is a fleet-owned hot switch; it is read at post time and never
|
|
9353
|
+
// belongs in a daemon config or a "restart required" warning.
|
|
9354
|
+
const { tips: _oldTips, ...oldDefaultCold } = oldDefaultColdWithFleetFields;
|
|
9355
|
+
const { tips: _newTips, ...newDefaultCold } = newDefaultColdWithFleetFields;
|
|
9356
|
+
const oldFleetLevel = JSON.stringify({ channel: oldConfig?.channel, defaults: oldDefaultCold });
|
|
9357
|
+
const newFleetLevel = JSON.stringify({ channel: this.fleetConfig?.channel, defaults: newDefaultCold });
|
|
7119
9358
|
if (oldFleetLevel !== newFleetLevel) {
|
|
7120
9359
|
this.logger.warn("Fleet-level config changed (channel/defaults) — use /restart for full effect");
|
|
7121
9360
|
}
|
|
@@ -7127,22 +9366,43 @@ When users create specialized instances, suggest these configurations:
|
|
|
7127
9366
|
await this.stopInstance(name).catch(err => this.logger.error({ err, name }, "Failed to stop removed instance"));
|
|
7128
9367
|
}
|
|
7129
9368
|
}
|
|
7130
|
-
// Start new +
|
|
9369
|
+
// Start new + reconcile modified instances. Hot values are always sent as a
|
|
9370
|
+
// complete snapshot: Settings mutates FleetManager's config before SIGHUP,
|
|
9371
|
+
// so an old/new diff alone can miss the live daemon's stale value.
|
|
7131
9372
|
for (const [name, config] of Object.entries(newInstances)) {
|
|
7132
9373
|
if (!this.daemons.has(name)) {
|
|
7133
9374
|
// New instance — startInstance already calls connectIpcToInstance
|
|
7134
9375
|
this.logger.info({ name }, "New instance in config — starting");
|
|
7135
|
-
await this.
|
|
9376
|
+
await this.startInstanceUnattended(name, config, topicMode, "new instance");
|
|
7136
9377
|
}
|
|
7137
9378
|
else if (oldConfig?.instances[name]) {
|
|
7138
|
-
|
|
7139
|
-
|
|
9379
|
+
const daemon = this.daemons.get(name);
|
|
9380
|
+
const runtimeConfig = daemon.getConfigSnapshot?.() ?? oldConfig.instances[name];
|
|
9381
|
+
const oldParts = splitHotColdConfig(runtimeConfig);
|
|
9382
|
+
const newParts = splitHotColdConfig(config);
|
|
9383
|
+
// Every field not explicitly classified hot is cold by default.
|
|
9384
|
+
if (!isDeepStrictEqual(oldParts.cold, newParts.cold)) {
|
|
7140
9385
|
this.logger.info({ name }, "Instance config changed — restarting");
|
|
7141
9386
|
await this.stopInstance(name).catch(() => { });
|
|
7142
|
-
await this.
|
|
9387
|
+
await this.startInstanceUnattended(name, config, topicMode, "modified instance");
|
|
9388
|
+
}
|
|
9389
|
+
else if (!isDeepStrictEqual(oldParts.hot, newParts.hot)) {
|
|
9390
|
+
const update = hotConfigUpdate(config);
|
|
9391
|
+
const ipc = this.instanceIpcClients.get(name);
|
|
9392
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
9393
|
+
if (!sent) {
|
|
9394
|
+
// Daemon is in-process, so a reconnect gap must not leave runtime
|
|
9395
|
+
// state stale. Normal operation still uses the explicit IPC contract.
|
|
9396
|
+
daemon.applyConfigUpdate(update);
|
|
9397
|
+
this.logger.warn({ name }, "Config-update IPC unavailable — applied hot config in-process");
|
|
9398
|
+
}
|
|
9399
|
+
this.logger.info({ name, fields: [...HOT_INSTANCE_CONFIG_KEYS] }, "Instance hot config reloaded");
|
|
7143
9400
|
}
|
|
7144
9401
|
}
|
|
7145
9402
|
}
|
|
9403
|
+
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
9404
|
+
// currently idle instances instead of waiting for a future state edge.
|
|
9405
|
+
this.enforceWarmCap();
|
|
7146
9406
|
this.logger.info({ running: this.daemons.size, configured: Object.keys(newInstances).length }, "Reconcile complete");
|
|
7147
9407
|
}
|
|
7148
9408
|
async restartInstances() {
|
|
@@ -7223,14 +9483,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
7223
9483
|
const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
7224
9484
|
const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
7225
9485
|
for (const [name, cfg] of restartGenerals) {
|
|
7226
|
-
|
|
7227
|
-
|
|
7228
|
-
if (this.daemons.has(name))
|
|
7229
|
-
restartProgress.markReady();
|
|
7230
|
-
}
|
|
7231
|
-
catch (err) {
|
|
7232
|
-
this.logger.error({ err, name }, "Failed to start general instance");
|
|
7233
|
-
}
|
|
9486
|
+
if (await this.startInstanceUnattended(name, cfg, topicMode, "general instance"))
|
|
9487
|
+
restartProgress.markReady();
|
|
7234
9488
|
}
|
|
7235
9489
|
// General is ready again; now its topic can own the live progress message.
|
|
7236
9490
|
await restartProgress.start(progressTarget);
|
|
@@ -7251,14 +9505,8 @@ When users create specialized instances, suggest these configurations:
|
|
|
7251
9505
|
while (idx < channels.length) {
|
|
7252
9506
|
const batch = channels.slice(idx, idx + concurrency);
|
|
7253
9507
|
await Promise.allSettled(batch.map(async (ch) => {
|
|
7254
|
-
|
|
7255
|
-
|
|
7256
|
-
if (this.daemons.has(ch.instanceName))
|
|
7257
|
-
restartProgress.markReady();
|
|
7258
|
-
}
|
|
7259
|
-
catch (err) {
|
|
7260
|
-
this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
|
|
7261
|
-
}
|
|
9508
|
+
if (await this.startClassicInstanceUnattended(ch, "classic instance"))
|
|
9509
|
+
restartProgress.markReady();
|
|
7262
9510
|
}));
|
|
7263
9511
|
idx += concurrency;
|
|
7264
9512
|
}
|
|
@@ -7382,7 +9630,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
7382
9630
|
if (generalId) {
|
|
7383
9631
|
// No release URL — Discord's SuppressEmbeds proved unreliable and the
|
|
7384
9632
|
// link preview looked bad. Version + /update instruction is enough.
|
|
7385
|
-
this.notifyInstanceTopic(generalId, t("update.
|
|
9633
|
+
this.notifyInstanceTopic(generalId, t("update.available_current", `v${target}`, `v${currentVersion}`));
|
|
7386
9634
|
}
|
|
7387
9635
|
}
|
|
7388
9636
|
}
|
|
@@ -7539,9 +9787,10 @@ When users create specialized instances, suggest these configurations:
|
|
|
7539
9787
|
const lastActivity = inst.classic
|
|
7540
9788
|
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
7541
9789
|
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
7542
|
-
const backend = inst.
|
|
7543
|
-
|
|
7544
|
-
|
|
9790
|
+
const backend = this.backendNameForInstance(inst.name);
|
|
9791
|
+
const resolvedModel = this.resolveInstanceModel(inst.name);
|
|
9792
|
+
const effortStrategy = this.effortStrategyFor(inst.name);
|
|
9793
|
+
const resolvedEffort = this.resolveInstanceEffort(inst.name);
|
|
7545
9794
|
// Find claimed tasks for this instance
|
|
7546
9795
|
let currentTask = null;
|
|
7547
9796
|
try {
|
|
@@ -7556,6 +9805,14 @@ When users create specialized instances, suggest these configurations:
|
|
|
7556
9805
|
...inst,
|
|
7557
9806
|
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
7558
9807
|
backend,
|
|
9808
|
+
// Settings renders these runtime-effective values rather than the
|
|
9809
|
+
// sparse user-authored YAML. `auto` means the supported CLI is
|
|
9810
|
+
// using its own effort default; null is reserved for unsupported.
|
|
9811
|
+
model: resolvedModel.model,
|
|
9812
|
+
model_display: resolvedModel.display,
|
|
9813
|
+
model_source: resolvedModel.source,
|
|
9814
|
+
effort: effortStrategy === "unsupported" ? null : (resolvedEffort.effort ?? "auto"),
|
|
9815
|
+
effort_supported: effortStrategy !== "unsupported",
|
|
7559
9816
|
tool_set: config?.tool_set ?? "full",
|
|
7560
9817
|
general_topic: config?.general_topic ?? false,
|
|
7561
9818
|
// User activity is persisted by the daemon, so both the board and
|
|
@@ -7702,7 +9959,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
7702
9959
|
if (err.code === "EADDRINUSE") {
|
|
7703
9960
|
if (this.healthPortRetried) {
|
|
7704
9961
|
this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
|
|
7705
|
-
this.notifyFleetError(
|
|
9962
|
+
this.notifyFleetError(t("dashboard.port_in_use", port));
|
|
7706
9963
|
return;
|
|
7707
9964
|
}
|
|
7708
9965
|
this.healthPortRetried = true;
|
|
@@ -7728,7 +9985,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
7728
9985
|
return;
|
|
7729
9986
|
}
|
|
7730
9987
|
this.logger.error({ err, port }, "Health server error");
|
|
7731
|
-
this.notifyFleetError(
|
|
9988
|
+
this.notifyFleetError(t("dashboard.server_failed", err.message));
|
|
7732
9989
|
});
|
|
7733
9990
|
this.healthServer.listen(port, "127.0.0.1", () => markListening());
|
|
7734
9991
|
}
|