@songsid/agend 2.1.5-beta.8 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/claude-code.d.ts +12 -0
- package/dist/backend/claude-code.js +12 -0
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +25 -1
- package/dist/backend/codex.js +76 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.js +30 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +48 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +17 -2
- package/dist/channel/adapters/discord.js +64 -5
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +41 -2
- package/dist/channel/adapters/telegram.js +116 -20
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/factory.js +6 -1
- package/dist/channel/factory.js.map +1 -1
- package/dist/channel/types.d.ts +33 -1
- package/dist/classic-channel-manager.d.ts +23 -3
- package/dist/classic-channel-manager.js +35 -0
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/config-validator.js +23 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +5 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +143 -21
- package/dist/daemon.js +969 -185
- package/dist/daemon.js.map +1 -1
- package/dist/deadline.d.ts +23 -0
- package/dist/deadline.js +29 -0
- package/dist/deadline.js.map +1 -0
- package/dist/fleet-context.d.ts +15 -1
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +185 -3
- package/dist/fleet-manager.js +782 -127
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +26 -2
- package/dist/instance-lifecycle.d.ts +8 -2
- package/dist/instance-lifecycle.js +47 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instance-removal.d.ts +13 -0
- package/dist/instance-removal.js +14 -0
- package/dist/instance-removal.js.map +1 -0
- package/dist/instructions.js +2 -2
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/login-controller.d.ts +31 -4
- package/dist/login-controller.js +52 -4
- package/dist/login-controller.js.map +1 -1
- package/dist/login-flows.d.ts +29 -0
- package/dist/login-flows.js +48 -1
- package/dist/login-flows.js.map +1 -1
- package/dist/outbound-handlers.d.ts +1 -0
- package/dist/outbound-handlers.js +118 -29
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -0
- package/dist/outbound-schemas.js +3 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +25 -2
- package/dist/pane-input-residue.js +31 -2
- package/dist/pane-input-residue.js.map +1 -1
- package/dist/restart-progress.js +3 -21
- package/dist/restart-progress.js.map +1 -1
- package/dist/settings-api.d.ts +1 -1
- package/dist/settings-api.js +38 -6
- package/dist/settings-api.js.map +1 -1
- package/dist/topic-commands.d.ts +27 -2
- package/dist/topic-commands.js +37 -6
- package/dist/topic-commands.js.map +1 -1
- package/dist/turn-reply-guard.d.ts +41 -0
- package/dist/turn-reply-guard.js +77 -0
- package/dist/turn-reply-guard.js.map +1 -0
- package/dist/types.d.ts +7 -0
- package/dist/ui/dashboard.html +38 -10
- package/dist/ui/settings.html +93 -15
- package/dist/ui/view.html +35 -20
- package/dist/view-api.d.ts +1 -0
- package/dist/view-api.js +12 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.d.ts +22 -1
- package/dist/web-api.js +25 -1
- package/dist/web-api.js.map +1 -1
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
|
-
import { freemem, totalmem } from "node:os";
|
|
4
|
+
import { freemem, totalmem, cpus } from "node:os";
|
|
5
5
|
import { createServer } from "node:http";
|
|
6
6
|
import { join, dirname, basename } from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
@@ -33,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
33
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
34
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
35
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
36
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext, readStatuslineModel } from "./topic-commands.js";
|
|
37
37
|
import { DailySummary } from "./daily-summary.js";
|
|
38
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
39
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -48,12 +48,14 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
48
48
|
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
49
|
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
50
|
import { LoginSession } from "./login-manager.js";
|
|
51
|
-
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX } from "./login-controller.js";
|
|
51
|
+
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX, POST_LOGIN_RECOVERY_DEADLINE_MS, announcePostLoginRecovery } from "./login-controller.js";
|
|
52
|
+
import { runBeforeDeadline } from "./deadline.js";
|
|
52
53
|
import { LoginWindowLock } from "./login-window-lock.js";
|
|
53
54
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
54
55
|
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
55
56
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
56
57
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
58
|
+
import { assertExplicitInstanceRemoval } from "./instance-removal.js";
|
|
57
59
|
import { validateFleetConfig } from "./config-validator.js";
|
|
58
60
|
import { readLastInboundAt } from "./daemon.js";
|
|
59
61
|
import { clearPausedMarker } from "./pause-marker.js";
|
|
@@ -179,6 +181,7 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
179
181
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
180
182
|
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
181
183
|
"tool_progress",
|
|
184
|
+
"reply_completion_guard",
|
|
182
185
|
"mcp_proxy_reply",
|
|
183
186
|
"auto_pause_after",
|
|
184
187
|
"warm_cap",
|
|
@@ -219,7 +222,7 @@ const HANG_CALLBACK_PREFIX = "hang:";
|
|
|
219
222
|
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
220
223
|
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
221
224
|
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
222
|
-
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
225
|
+
export const LOGIN_CALLBACK_PREFIX = "login:";
|
|
223
226
|
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
224
227
|
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
225
228
|
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
@@ -262,6 +265,18 @@ const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
|
|
|
262
265
|
* The wait is announced before it starts, so it reads as progress, not a stall.
|
|
263
266
|
*/
|
|
264
267
|
export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
|
|
268
|
+
/**
|
|
269
|
+
* How many CLIs may cold-start at once, from BOTH memory and cores.
|
|
270
|
+
*
|
|
271
|
+
* Memory alone said 10 on any host with roughly 3GB free, so a three-core box
|
|
272
|
+
* started ten CLIs together, saturated the CPU, and healthy starts then missed
|
|
273
|
+
* their startup budget — which used to cost the user their conversation. Cores
|
|
274
|
+
* bound how many can actually make progress; memory bounds how many fit.
|
|
275
|
+
*/
|
|
276
|
+
export function deriveSpawnConcurrency(freeMemMB, cores) {
|
|
277
|
+
const byMemory = Math.floor(freeMemMB / 300);
|
|
278
|
+
return Math.max(2, Math.min(10, byMemory, Math.max(1, cores)));
|
|
279
|
+
}
|
|
265
280
|
export class FleetManager {
|
|
266
281
|
dataDir;
|
|
267
282
|
static signalTarget = null;
|
|
@@ -303,6 +318,33 @@ export class FleetManager {
|
|
|
303
318
|
reloadPending = false;
|
|
304
319
|
/** A running reconciliation; only one may mutate lifecycle/config state at a time. */
|
|
305
320
|
reconcileInFlight = null;
|
|
321
|
+
/** Topology checks are serialized separately from config reconciliation. */
|
|
322
|
+
topicCleanupInFlight = null;
|
|
323
|
+
topicCleanupGeneration = 0;
|
|
324
|
+
topicProbeWarnings = new Map();
|
|
325
|
+
/**
|
|
326
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
327
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
328
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
329
|
+
*/
|
|
330
|
+
topicProbeUnknownStreak = new Map();
|
|
331
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
332
|
+
static TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
333
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
334
|
+
static TOPIC_PROBE_ADAPTER_SCOPED_REASONS = new Set([
|
|
335
|
+
"owner-adapter-unavailable",
|
|
336
|
+
"owner-adapter-not-ready",
|
|
337
|
+
"owner-adapter-generation-changed",
|
|
338
|
+
"owner-adapter-changed-before-action",
|
|
339
|
+
"adapter-not-ready",
|
|
340
|
+
"adapter-not-initialized",
|
|
341
|
+
"adapter-generation-changed",
|
|
342
|
+
"topic-close-from-unready-adapter",
|
|
343
|
+
"topic-close-generation-changed",
|
|
344
|
+
// Telegram probe: the transport or Telegram itself is down, not one topic.
|
|
345
|
+
"transport-failed",
|
|
346
|
+
"provider-unavailable",
|
|
347
|
+
]);
|
|
306
348
|
logger = createLogger("info");
|
|
307
349
|
topicCommands;
|
|
308
350
|
// sessionName → instanceName mapping for external sessions
|
|
@@ -452,8 +494,7 @@ export class FleetManager {
|
|
|
452
494
|
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
453
495
|
if (explicit != null)
|
|
454
496
|
return Math.max(1, Math.min(20, explicit));
|
|
455
|
-
|
|
456
|
-
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
497
|
+
return deriveSpawnConcurrency(Math.round(freemem() / (1024 * 1024)), cpus().length);
|
|
457
498
|
}
|
|
458
499
|
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
459
500
|
bindStormWindowEvents() {
|
|
@@ -916,6 +957,45 @@ export class FleetManager {
|
|
|
916
957
|
}
|
|
917
958
|
return channels[0];
|
|
918
959
|
}
|
|
960
|
+
/**
|
|
961
|
+
* The configured world that owns `chatId`, when that is provably NOT the
|
|
962
|
+
* world `target` lives in. Returns undefined when they agree, when there is
|
|
963
|
+
* nothing to check, or when no configured channel claims the id.
|
|
964
|
+
*
|
|
965
|
+
* Deliberately one-sided: only a POSITIVE match against another channel's
|
|
966
|
+
* group id counts as foreign. A chat id that matches nothing may still be
|
|
967
|
+
* legitimate for this world (a classic channel, a DM), and treating
|
|
968
|
+
* "unrecognised" as "wrong" would stop seeding for cases that work today.
|
|
969
|
+
*
|
|
970
|
+
* Read from config rather than the live worlds map on purpose: a channel
|
|
971
|
+
* whose adapter failed to start still owns its group id, and the coordinates
|
|
972
|
+
* are just as unusable by the target's adapter either way.
|
|
973
|
+
*
|
|
974
|
+
* This is the other half of what scheduleSourceAdapter fixed. That one stops
|
|
975
|
+
* the trigger NOTICE being sent through the wrong bot; this one stops the
|
|
976
|
+
* same coordinates being planted as the target instance's reply context,
|
|
977
|
+
* which is what made its own replies fail until someone spoke to it (#752).
|
|
978
|
+
*/
|
|
979
|
+
scheduleChatWorldMismatch(target, chatId) {
|
|
980
|
+
if (!chatId)
|
|
981
|
+
return undefined;
|
|
982
|
+
const targetWorld = this.getInstanceAdapterId(target);
|
|
983
|
+
if (!targetWorld)
|
|
984
|
+
return undefined;
|
|
985
|
+
const channels = this.fleetConfig?.channels
|
|
986
|
+
?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
|
|
987
|
+
// ALL owners, not the first: a persona bot shares the primary's guild
|
|
988
|
+
// (quickstart writes `group_id: primary.group_id`), so one group id is
|
|
989
|
+
// legitimately claimed by two channels. Taking the first match called a
|
|
990
|
+
// persona instance's own guild "another world" and stopped seeding a
|
|
991
|
+
// context it can address perfectly well.
|
|
992
|
+
const owners = channels.filter(ch => ch.group_id != null && String(ch.group_id) === String(chatId));
|
|
993
|
+
if (owners.length === 0)
|
|
994
|
+
return undefined;
|
|
995
|
+
if (owners.some(ch => (ch.id ?? ch.type) === targetWorld))
|
|
996
|
+
return undefined;
|
|
997
|
+
return owners[0].id ?? owners[0].type;
|
|
998
|
+
}
|
|
919
999
|
/** Get the group_id for an instance's bound adapter */
|
|
920
1000
|
getGroupIdForInstance(name) {
|
|
921
1001
|
const adapterId = this.getInstanceAdapterId(name);
|
|
@@ -1426,8 +1506,27 @@ export class FleetManager {
|
|
|
1426
1506
|
await this.lifecycle.pause(name);
|
|
1427
1507
|
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
1428
1508
|
}
|
|
1509
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
1510
|
+
applyHotConfigUpdate(instanceName, update) {
|
|
1511
|
+
const daemon = this.daemons.get(instanceName);
|
|
1512
|
+
if (!daemon)
|
|
1513
|
+
return false;
|
|
1514
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1515
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
1516
|
+
if (!sent) {
|
|
1517
|
+
daemon.applyConfigUpdate(update);
|
|
1518
|
+
this.logger.warn({ name: instanceName }, "Config-update IPC unavailable — applied hot config in-process");
|
|
1519
|
+
}
|
|
1520
|
+
return true;
|
|
1521
|
+
}
|
|
1522
|
+
classicBehaviorUpdate(instanceName) {
|
|
1523
|
+
return {
|
|
1524
|
+
tool_progress: this.classicChannels?.getToolProgressByInstance(instanceName, this.fleetConfig?.defaults?.tool_progress) ?? "off",
|
|
1525
|
+
reply_completion_guard: this.classicChannels?.getReplyCompletionGuardByInstance(instanceName, this.fleetConfig?.defaults?.reply_completion_guard) ?? true,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1429
1528
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
1430
|
-
async restartClassicInstanceFromSettings(instanceName) {
|
|
1529
|
+
async restartClassicInstanceFromSettings(instanceName, changedFields = []) {
|
|
1431
1530
|
if (!this.classicChannels)
|
|
1432
1531
|
throw new Error("Classic channel manager not initialized");
|
|
1433
1532
|
const wasRunning = this.daemons.has(instanceName);
|
|
@@ -1439,10 +1538,77 @@ export class FleetManager {
|
|
|
1439
1538
|
throw new Error("Classic channel not found after reload");
|
|
1440
1539
|
if (!wasRunning)
|
|
1441
1540
|
return;
|
|
1541
|
+
const hotOnly = changedFields.length > 0
|
|
1542
|
+
&& changedFields.every(field => field === "tool_progress" || field === "reply_completion_guard");
|
|
1543
|
+
if (hotOnly) {
|
|
1544
|
+
this.applyHotConfigUpdate(instanceName, this.classicBehaviorUpdate(instanceName));
|
|
1545
|
+
this.logger.info({ instanceName, fields: changedFields }, "Classic instance hot config reloaded");
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1442
1548
|
await this.stopInstance(instanceName);
|
|
1443
1549
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
1444
1550
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1445
1551
|
}
|
|
1552
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
1553
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
1554
|
+
async reloadClassicConfigFromDisk() {
|
|
1555
|
+
try {
|
|
1556
|
+
if (!this.classicChannels)
|
|
1557
|
+
return;
|
|
1558
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
1559
|
+
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
1560
|
+
const oldBackends = new Map();
|
|
1561
|
+
const oldModels = new Map();
|
|
1562
|
+
const oldAutoPause = new Map();
|
|
1563
|
+
const oldToolProgress = new Map();
|
|
1564
|
+
const oldReplyGuard = new Map();
|
|
1565
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1566
|
+
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
1567
|
+
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
1568
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1569
|
+
oldToolProgress.set(ch.instanceName, this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress));
|
|
1570
|
+
oldReplyGuard.set(ch.instanceName, this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard));
|
|
1571
|
+
}
|
|
1572
|
+
if (!this.classicChannels.checkReload())
|
|
1573
|
+
return;
|
|
1574
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
1575
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
1576
|
+
this.reportClassicUnrecoverableIds();
|
|
1577
|
+
this.reregisterClassicChannels();
|
|
1578
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1579
|
+
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
1580
|
+
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
1581
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
1582
|
+
const newToolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
1583
|
+
const newReplyGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
1584
|
+
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
1585
|
+
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
1586
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
1587
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
1588
|
+
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
1589
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1590
|
+
// Small delay to let tmux window clean up
|
|
1591
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
1592
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
1593
|
+
// unattended helper reads them from it and schedules the delayed
|
|
1594
|
+
// retry on failure like every other unattended start.
|
|
1595
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1596
|
+
}
|
|
1597
|
+
else if (this.daemons.has(ch.instanceName)
|
|
1598
|
+
&& (oldToolProgress.get(ch.instanceName) !== newToolProgress
|
|
1599
|
+
|| oldReplyGuard.get(ch.instanceName) !== newReplyGuard)) {
|
|
1600
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
1601
|
+
tool_progress: newToolProgress,
|
|
1602
|
+
reply_completion_guard: newReplyGuard,
|
|
1603
|
+
});
|
|
1604
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic instance hot config reloaded");
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
catch (err) {
|
|
1609
|
+
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1446
1612
|
async startInstance(name, config, topicMode, kind = "fleet-topic",
|
|
1447
1613
|
/**
|
|
1448
1614
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -1745,15 +1911,31 @@ export class FleetManager {
|
|
|
1745
1911
|
}
|
|
1746
1912
|
backendNameOf(name) {
|
|
1747
1913
|
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1748
|
-
const
|
|
1749
|
-
|
|
1750
|
-
|
|
1914
|
+
const fleetInstance = this.fleetConfig?.instances[name];
|
|
1915
|
+
// A malformed/manual config can give a fleet and Classic entry the same
|
|
1916
|
+
// instance name. Fleet ownership wins, including its inherited default;
|
|
1917
|
+
// otherwise a Classic override could make backend-scoped recovery restart
|
|
1918
|
+
// the shared process under the wrong login result.
|
|
1919
|
+
if (fleetInstance)
|
|
1920
|
+
return fleetInstance.backend ?? fleetDefault ?? "claude-code";
|
|
1751
1921
|
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1752
1922
|
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1753
1923
|
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1754
1924
|
}
|
|
1755
1925
|
return fleetDefault ?? "claude-code";
|
|
1756
1926
|
}
|
|
1927
|
+
/**
|
|
1928
|
+
* Every configured instance whose backend may share credentials. ClassicBot
|
|
1929
|
+
* rows live only in classicBot.yaml, so backend-wide operations must not use
|
|
1930
|
+
* fleetConfig.instances as their roster. Set keeps a malformed name collision
|
|
1931
|
+
* from restarting the same process twice; backendNameOf defines ownership.
|
|
1932
|
+
*/
|
|
1933
|
+
configuredBackendInstanceNames() {
|
|
1934
|
+
const names = new Set(Object.keys(this.fleetConfig?.instances ?? {}));
|
|
1935
|
+
for (const channel of this.classicChannels?.getAll() ?? [])
|
|
1936
|
+
names.add(channel.instanceName);
|
|
1937
|
+
return [...names];
|
|
1938
|
+
}
|
|
1757
1939
|
/**
|
|
1758
1940
|
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1759
1941
|
* fails many instances within the same second. Two notices per incident at
|
|
@@ -2284,48 +2466,8 @@ export class FleetManager {
|
|
|
2284
2466
|
this.instanceWorldBinding.set(ch.instanceName, ch.adapterId);
|
|
2285
2467
|
}
|
|
2286
2468
|
// Poll classicBot.yaml for external changes every 30s
|
|
2287
|
-
this.classicReloadTimer = setInterval(
|
|
2288
|
-
|
|
2289
|
-
if (!this.classicChannels)
|
|
2290
|
-
return;
|
|
2291
|
-
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
2292
|
-
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
2293
|
-
const oldBackends = new Map();
|
|
2294
|
-
const oldModels = new Map();
|
|
2295
|
-
const oldAutoPause = new Map();
|
|
2296
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2297
|
-
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
2298
|
-
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
2299
|
-
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2300
|
-
}
|
|
2301
|
-
if (!this.classicChannels.checkReload())
|
|
2302
|
-
return;
|
|
2303
|
-
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2304
|
-
// throttle keeps a repeated report from flooding the topic.
|
|
2305
|
-
this.reportClassicUnrecoverableIds();
|
|
2306
|
-
this.reregisterClassicChannels();
|
|
2307
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2308
|
-
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
2309
|
-
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
2310
|
-
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
2311
|
-
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
2312
|
-
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
2313
|
-
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
2314
|
-
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
2315
|
-
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
2316
|
-
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
2317
|
-
// Small delay to let tmux window clean up
|
|
2318
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
2319
|
-
// The manager already holds the new backend/model/auto-pause; the
|
|
2320
|
-
// unattended helper reads them from it and schedules the delayed
|
|
2321
|
-
// retry on failure like every other unattended start.
|
|
2322
|
-
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
2323
|
-
}
|
|
2324
|
-
}
|
|
2325
|
-
}
|
|
2326
|
-
catch (err) {
|
|
2327
|
-
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
2328
|
-
}
|
|
2469
|
+
this.classicReloadTimer = setInterval(() => {
|
|
2470
|
+
void this.reloadClassicConfigFromDisk();
|
|
2329
2471
|
}, 30_000);
|
|
2330
2472
|
const costGuardConfig = {
|
|
2331
2473
|
...DEFAULT_COST_GUARD,
|
|
@@ -2353,10 +2495,7 @@ export class FleetManager {
|
|
|
2353
2495
|
...fleet.defaults.daily_summary,
|
|
2354
2496
|
};
|
|
2355
2497
|
this.dailySummary = new DailySummary(summaryConfig, costGuardConfig.timezone, (text) => {
|
|
2356
|
-
|
|
2357
|
-
return;
|
|
2358
|
-
this.adapter.sendText(String(this.fleetConfig.channel.group_id), text)
|
|
2359
|
-
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
2498
|
+
this.postDailySummary(text);
|
|
2360
2499
|
// Rotate classic channel chat logs daily
|
|
2361
2500
|
this.classicChannels?.rotateLogs();
|
|
2362
2501
|
this.rotateInboxes();
|
|
@@ -2954,10 +3093,11 @@ export class FleetManager {
|
|
|
2954
3093
|
accessManager,
|
|
2955
3094
|
inboxDir,
|
|
2956
3095
|
});
|
|
2957
|
-
const
|
|
3096
|
+
const adapter = this.adapter;
|
|
3097
|
+
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2958
3098
|
this.worlds.set(adapterId, world);
|
|
2959
|
-
this.adapters.set(adapterId,
|
|
2960
|
-
this.bindAdapterHealth(
|
|
3099
|
+
this.adapters.set(adapterId, adapter);
|
|
3100
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2961
3101
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2962
3102
|
await this.handleInboundMessage(msg);
|
|
2963
3103
|
}, this.logger, "adapter.message"));
|
|
@@ -3002,12 +3142,7 @@ export class FleetManager {
|
|
|
3002
3142
|
return;
|
|
3003
3143
|
}
|
|
3004
3144
|
}, this.logger, "adapter.callback_query"));
|
|
3005
|
-
this.adapter.
|
|
3006
|
-
// Skip unbind if we archived this topic ourselves
|
|
3007
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
3008
|
-
return;
|
|
3009
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
3010
|
-
}, this.logger, "adapter.topic_closed"));
|
|
3145
|
+
this.bindTopicClosedHandler(adapter, adapterId, "adapter.topic_closed");
|
|
3011
3146
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
3012
3147
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
3013
3148
|
if (data.command === "start") {
|
|
@@ -3398,11 +3533,7 @@ export class FleetManager {
|
|
|
3398
3533
|
return;
|
|
3399
3534
|
}
|
|
3400
3535
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
3401
|
-
|
|
3402
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
3403
|
-
return;
|
|
3404
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
3405
|
-
}, this.logger, `adapter[${adapterId}].topic_closed`));
|
|
3536
|
+
this.bindTopicClosedHandler(adapter, adapterId, `adapter[${adapterId}].topic_closed`);
|
|
3406
3537
|
// Slash commands: classic bot + admin commands
|
|
3407
3538
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
3408
3539
|
if (data.command === "start") {
|
|
@@ -4761,7 +4892,9 @@ export class FleetManager {
|
|
|
4761
4892
|
// The adapter resolves only after the platform POST returns. Keep
|
|
4762
4893
|
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4763
4894
|
// a routed-but-failed reply is not a delivered reply.
|
|
4764
|
-
|
|
4895
|
+
// `statusOnly` is daemon-owned envelope metadata, not an MCP argument:
|
|
4896
|
+
// an agent cannot invent it to suppress the normal completion marker.
|
|
4897
|
+
if (!error && result != null && msg.statusOnly !== true) {
|
|
4765
4898
|
try {
|
|
4766
4899
|
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4767
4900
|
}
|
|
@@ -4867,12 +5000,25 @@ export class FleetManager {
|
|
|
4867
5000
|
// the target instance's configured world so replies use its persona
|
|
4868
5001
|
// after a fresh start instead of falling back to channels[0].
|
|
4869
5002
|
const adapterId = this.getInstanceAdapterId(target);
|
|
5003
|
+
// ...but the stored reply coordinates belong to the chat the schedule
|
|
5004
|
+
// was CREATED in, which is not always the target's world. Pairing them
|
|
5005
|
+
// with the target's adapter hands one platform's ids to another's API:
|
|
5006
|
+
// a Telegram group + forum topic delivered to a Discord instance made
|
|
5007
|
+
// every reply on that turn fetch `/channels/<telegram topic>` and fail
|
|
5008
|
+
// with Unknown Channel, and because the seeding repeats on each trigger
|
|
5009
|
+
// it stayed broken until a real inbound overwrote the context (#752).
|
|
5010
|
+
const foreignWorld = this.scheduleChatWorldMismatch(target, reply_chat_id);
|
|
5011
|
+
if (foreignWorld) {
|
|
5012
|
+
this.logger.warn({ scheduleId: id, target, foreignWorld, targetWorld: adapterId, chatId: reply_chat_id }, "Schedule reply target belongs to another channel world — not seeding chat context; the instance keeps its own last known chat");
|
|
5013
|
+
}
|
|
4870
5014
|
await this.deliverToInstance(target, {
|
|
4871
5015
|
type: "fleet_schedule_trigger",
|
|
4872
5016
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
4873
5017
|
meta: {
|
|
4874
|
-
|
|
4875
|
-
|
|
5018
|
+
// Omitted on a mismatch: the daemon then keeps its persisted
|
|
5019
|
+
// last-chat, which is in the right world, instead of being
|
|
5020
|
+
// overwritten with coordinates the target's adapter cannot address.
|
|
5021
|
+
...(foreignWorld ? {} : { chat_id: reply_chat_id, thread_id: reply_thread_id }),
|
|
4876
5022
|
user: "scheduler",
|
|
4877
5023
|
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4878
5024
|
},
|
|
@@ -4904,19 +5050,69 @@ export class FleetManager {
|
|
|
4904
5050
|
this.scheduler.recordRun(id, "instance_offline", `retry ${retryCount}x failed`);
|
|
4905
5051
|
this.notifyScheduleFailure(schedule);
|
|
4906
5052
|
}
|
|
5053
|
+
/**
|
|
5054
|
+
* The adapter that can actually post into a chat, found by its group.
|
|
5055
|
+
*
|
|
5056
|
+
* A schedule records where it was created (reply_chat_id) separately from
|
|
5057
|
+
* what it triggers (target). Those need not share a platform: a Telegram
|
|
5058
|
+
* group can schedule a Discord-topic instance. Picking the adapter from the
|
|
5059
|
+
* target then sends a Telegram chat id through the Discord bot, which fails
|
|
5060
|
+
* with Unknown Channel — the source topic never hears that its schedule ran.
|
|
5061
|
+
*
|
|
5062
|
+
* When several bots share one guild, the primary wins: a persona should not
|
|
5063
|
+
* be the voice announcing fleet scheduling.
|
|
5064
|
+
*/
|
|
5065
|
+
adapterForChat(chatId) {
|
|
5066
|
+
const id = String(chatId);
|
|
5067
|
+
const matches = [...this.worlds.values()].filter(world => String(world.groupId) === id);
|
|
5068
|
+
if (matches.length === 0)
|
|
5069
|
+
return undefined;
|
|
5070
|
+
const primaryId = this.getPrimaryAdapterId();
|
|
5071
|
+
return (matches.find(world => world.id === primaryId) ?? matches[0]).adapter;
|
|
5072
|
+
}
|
|
5073
|
+
/**
|
|
5074
|
+
* The adapter that can answer a schedule in the chat it was created from.
|
|
5075
|
+
*
|
|
5076
|
+
* A schedule records its creator (source) and its trigger (target) separately,
|
|
5077
|
+
* and they need not share a platform — the live fleet has a Telegram group
|
|
5078
|
+
* scheduling a Discord-topic instance. Routing by target sends a Telegram chat
|
|
5079
|
+
* id through the Discord bot, which is one half of the Unknown Channel errors.
|
|
5080
|
+
*
|
|
5081
|
+
* The creator's own adapter comes first, and only when its world actually owns
|
|
5082
|
+
* that chat. Classic keeps its own identity, so a schedule made from a
|
|
5083
|
+
* persona-bound Classic channel is answered by that persona: the primary bot
|
|
5084
|
+
* may not even have access there, and would be the wrong voice if it did.
|
|
5085
|
+
* Falling back to the target's adapter is deliberately NOT an option — that is
|
|
5086
|
+
* the misroute itself; callers say why they stayed silent instead.
|
|
5087
|
+
*/
|
|
5088
|
+
scheduleSourceAdapter(schedule) {
|
|
5089
|
+
const chatId = String(schedule.reply_chat_id);
|
|
5090
|
+
const sourceAdapterId = schedule.source ? this.getInstanceAdapterId(schedule.source) : undefined;
|
|
5091
|
+
const sourceWorld = sourceAdapterId ? this.worlds.get(sourceAdapterId) : undefined;
|
|
5092
|
+
if (sourceWorld && String(sourceWorld.groupId) === chatId)
|
|
5093
|
+
return sourceWorld.adapter;
|
|
5094
|
+
return this.adapterForChat(chatId);
|
|
5095
|
+
}
|
|
4907
5096
|
notifySourceTopic(schedule) {
|
|
4908
|
-
const adapter = this.
|
|
4909
|
-
if (!adapter)
|
|
5097
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5098
|
+
if (!adapter) {
|
|
5099
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — trigger notice not sent");
|
|
4910
5100
|
return;
|
|
5101
|
+
}
|
|
4911
5102
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" triggered, target: ${schedule.target}`;
|
|
4912
5103
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4913
5104
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
4914
5105
|
}).catch((err) => this.logger.error({ err }, "Failed to send cross-instance notification"));
|
|
4915
5106
|
}
|
|
4916
5107
|
notifyScheduleFailure(schedule) {
|
|
4917
|
-
|
|
4918
|
-
|
|
5108
|
+
// Same resolver as the success path: a failure notice was still being sent
|
|
5109
|
+
// through the target's adapter, so a Telegram-created schedule for a
|
|
5110
|
+
// Discord target announced its failure into the wrong platform.
|
|
5111
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5112
|
+
if (!adapter) {
|
|
5113
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — failure notice not sent");
|
|
4919
5114
|
return;
|
|
5115
|
+
}
|
|
4920
5116
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" trigger failed: instance ${schedule.target} is offline.`;
|
|
4921
5117
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4922
5118
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
@@ -5327,26 +5523,261 @@ export class FleetManager {
|
|
|
5327
5523
|
sessionPruneTimer = null;
|
|
5328
5524
|
classicReloadTimer = null;
|
|
5329
5525
|
botUserId;
|
|
5330
|
-
/** Periodically check if bound topics still exist */
|
|
5526
|
+
/** Periodically check if bound topics still exist. Never deletes user data. */
|
|
5331
5527
|
startTopicCleanupPoller() {
|
|
5332
|
-
this.topicCleanupTimer
|
|
5333
|
-
|
|
5528
|
+
if (this.topicCleanupTimer)
|
|
5529
|
+
clearInterval(this.topicCleanupTimer);
|
|
5530
|
+
const generation = ++this.topicCleanupGeneration;
|
|
5531
|
+
this.topicCleanupTimer = setInterval(() => { void this.scheduleTopicCleanup(generation); }, 5 * 60_000);
|
|
5532
|
+
}
|
|
5533
|
+
/** Coalesce timer ticks; an outage must not create overlapping destructive-looking scans. */
|
|
5534
|
+
scheduleTopicCleanup(generation = this.topicCleanupGeneration) {
|
|
5535
|
+
if (this.topicCleanupInFlight)
|
|
5536
|
+
return this.topicCleanupInFlight;
|
|
5537
|
+
const run = this.runTopicCleanup(generation).finally(() => {
|
|
5538
|
+
if (this.topicCleanupInFlight === run)
|
|
5539
|
+
this.topicCleanupInFlight = null;
|
|
5540
|
+
});
|
|
5541
|
+
this.topicCleanupInFlight = run;
|
|
5542
|
+
return run;
|
|
5543
|
+
}
|
|
5544
|
+
confirmedProbeFence(adapterId, adapter) {
|
|
5545
|
+
const health = adapter.getHealthSnapshot?.();
|
|
5546
|
+
if (health) {
|
|
5547
|
+
return health.status === "connected" && health.isReady
|
|
5548
|
+
? { generation: health.generation }
|
|
5549
|
+
: null;
|
|
5550
|
+
}
|
|
5551
|
+
return this.adapterState.get(adapterId)?.status === "connected" ? {} : null;
|
|
5552
|
+
}
|
|
5553
|
+
sameProbeFence(adapterId, adapter, before, result) {
|
|
5554
|
+
const after = this.confirmedProbeFence(adapterId, adapter);
|
|
5555
|
+
if (!after)
|
|
5556
|
+
return false;
|
|
5557
|
+
if (before.generation !== after.generation)
|
|
5558
|
+
return false;
|
|
5559
|
+
return result?.generation === undefined || result.generation === after.generation;
|
|
5560
|
+
}
|
|
5561
|
+
topicProbeStreakKey(threadId, adapterId, reason) {
|
|
5562
|
+
return FleetManager.TOPIC_PROBE_ADAPTER_SCOPED_REASONS.has(reason)
|
|
5563
|
+
? `adapter:${adapterId ?? "unbound"}`
|
|
5564
|
+
: `thread:${threadId}`;
|
|
5565
|
+
}
|
|
5566
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
5567
|
+
clearTopicProbeUnknownStreak(threadId, adapterId) {
|
|
5568
|
+
this.topicProbeUnknownStreak.delete(`thread:${threadId}`);
|
|
5569
|
+
this.topicProbeUnknownStreak.delete(`adapter:${adapterId ?? "unbound"}`);
|
|
5570
|
+
}
|
|
5571
|
+
/**
|
|
5572
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
5573
|
+
* removal: unknown is always retained data. The only question is whether
|
|
5574
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
5575
|
+
* out of dozens per pass) must not — only a streak does.
|
|
5576
|
+
*
|
|
5577
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
5578
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
5579
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
5580
|
+
*/
|
|
5581
|
+
warnTopicProbeUnknown(instanceName, threadId, adapterId, reason, detail) {
|
|
5582
|
+
this.noteTopicProbeUnknown(this.topicProbeStreakKey(threadId, adapterId, reason), { instanceName, threadId, adapterId, reason, detail });
|
|
5583
|
+
}
|
|
5584
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
5585
|
+
newTopicProbePass() {
|
|
5586
|
+
return { unknown: new Map(), definite: new Set() };
|
|
5587
|
+
}
|
|
5588
|
+
passTopicProbeUnknown(pass, instanceName, threadId, adapterId, reason, detail) {
|
|
5589
|
+
const key = this.topicProbeStreakKey(threadId, adapterId, reason);
|
|
5590
|
+
// First unknown per key per pass wins; the rest of the routes on a dead
|
|
5591
|
+
// adapter are the same observation, not additional checks.
|
|
5592
|
+
if (!pass.unknown.has(key))
|
|
5593
|
+
pass.unknown.set(key, { instanceName, threadId, adapterId, reason, detail });
|
|
5594
|
+
}
|
|
5595
|
+
passTopicProbeDefinite(pass, threadId, adapterId) {
|
|
5596
|
+
pass.definite.add(`thread:${threadId}`);
|
|
5597
|
+
pass.definite.add(`adapter:${adapterId ?? "unbound"}`);
|
|
5598
|
+
}
|
|
5599
|
+
/**
|
|
5600
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
5601
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
5602
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
5603
|
+
* the routes happened to be probed in.
|
|
5604
|
+
*/
|
|
5605
|
+
applyTopicProbePass(pass) {
|
|
5606
|
+
for (const key of pass.definite)
|
|
5607
|
+
this.topicProbeUnknownStreak.delete(key);
|
|
5608
|
+
for (const [key, ctx] of pass.unknown) {
|
|
5609
|
+
if (pass.definite.has(key))
|
|
5610
|
+
continue;
|
|
5611
|
+
this.noteTopicProbeUnknown(key, ctx);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
noteTopicProbeUnknown(streakKey, { instanceName, threadId, adapterId, reason, detail }) {
|
|
5615
|
+
const streak = (this.topicProbeUnknownStreak.get(streakKey) ?? 0) + 1;
|
|
5616
|
+
this.topicProbeUnknownStreak.set(streakKey, streak);
|
|
5617
|
+
if (streak < FleetManager.TOPIC_PROBE_UNKNOWN_ESCALATION) {
|
|
5618
|
+
this.logger.debug({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence not confirmed this pass — transient, retaining instance and all data");
|
|
5619
|
+
return;
|
|
5620
|
+
}
|
|
5621
|
+
const key = `${adapterId ?? "unbound"}:${reason}`;
|
|
5622
|
+
const now = Date.now();
|
|
5623
|
+
const last = this.topicProbeWarnings.get(key) ?? 0;
|
|
5624
|
+
if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
|
|
5625
|
+
return;
|
|
5626
|
+
this.topicProbeWarnings.set(key, now);
|
|
5627
|
+
this.logger.error({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence could not be confirmed repeatedly — retaining instance and all data");
|
|
5628
|
+
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound", streak));
|
|
5629
|
+
}
|
|
5630
|
+
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
5631
|
+
async runTopicCleanup(generation) {
|
|
5632
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5633
|
+
return;
|
|
5634
|
+
const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
|
|
5635
|
+
const missing = [];
|
|
5636
|
+
const pass = this.newTopicProbePass();
|
|
5637
|
+
const skippedOnDemand = new Set();
|
|
5638
|
+
for (const [threadId, target] of snapshot) {
|
|
5639
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5334
5640
|
return;
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
|
|
5342
|
-
|
|
5343
|
-
|
|
5641
|
+
// The route may have been replaced while an earlier probe was in flight.
|
|
5642
|
+
if (this.routing.resolve(threadId) !== target)
|
|
5643
|
+
continue;
|
|
5644
|
+
const adapterId = this.getInstanceAdapterId(target.name);
|
|
5645
|
+
const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
|
|
5646
|
+
if (!adapterId || !adapter?.probeTopicPresence) {
|
|
5647
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-unavailable");
|
|
5648
|
+
continue;
|
|
5649
|
+
}
|
|
5650
|
+
// An on-demand adapter's probe is reserved for confirming a delivery
|
|
5651
|
+
// failure hint (handleProviderTopicClosed); the periodic scan leaves its
|
|
5652
|
+
// routes alone — no probe, no unknown, no streak, no notice.
|
|
5653
|
+
if (adapter.topicProbePolicy?.() === "on-demand") {
|
|
5654
|
+
if (!skippedOnDemand.has(adapterId)) {
|
|
5655
|
+
skippedOnDemand.add(adapterId);
|
|
5656
|
+
this.logger.debug({ adapterId }, "Topic scan skipping on-demand adapter — presence is confirmed only on delivery failure");
|
|
5344
5657
|
}
|
|
5345
|
-
|
|
5346
|
-
|
|
5658
|
+
continue;
|
|
5659
|
+
}
|
|
5660
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5661
|
+
if (!before) {
|
|
5662
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-not-ready");
|
|
5663
|
+
continue;
|
|
5664
|
+
}
|
|
5665
|
+
let result;
|
|
5666
|
+
try {
|
|
5667
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5668
|
+
}
|
|
5669
|
+
catch {
|
|
5670
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5671
|
+
}
|
|
5672
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5673
|
+
return;
|
|
5674
|
+
if (!this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5675
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-generation-changed");
|
|
5676
|
+
continue;
|
|
5677
|
+
}
|
|
5678
|
+
if (result.status === "unknown") {
|
|
5679
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, result.reason, result.detail);
|
|
5680
|
+
}
|
|
5681
|
+
else {
|
|
5682
|
+
this.passTopicProbeDefinite(pass, threadId, adapterId);
|
|
5683
|
+
if (result.status === "missing") {
|
|
5684
|
+
missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
|
|
5347
5685
|
}
|
|
5348
5686
|
}
|
|
5349
|
-
}
|
|
5687
|
+
}
|
|
5688
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5689
|
+
return;
|
|
5690
|
+
// One pass, one check: streaks move by at most one per key here.
|
|
5691
|
+
this.applyTopicProbePass(pass);
|
|
5692
|
+
if (missing.length === 0)
|
|
5693
|
+
return;
|
|
5694
|
+
if (missing.length > 1) {
|
|
5695
|
+
this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
|
|
5696
|
+
this.notifyFleetError(t("fleet.topic_probe_bulk", missing.length));
|
|
5697
|
+
return;
|
|
5698
|
+
}
|
|
5699
|
+
const item = missing[0];
|
|
5700
|
+
if (this.routing.resolve(item.threadId) !== item.target
|
|
5701
|
+
|| this.getInstanceAdapterId(item.target.name) !== item.adapterId
|
|
5702
|
+
|| this.adapters.get(item.adapterId) !== item.adapter)
|
|
5703
|
+
return;
|
|
5704
|
+
const current = this.confirmedProbeFence(item.adapterId, item.adapter);
|
|
5705
|
+
if (!current || (item.generation !== undefined && current.generation !== item.generation)) {
|
|
5706
|
+
this.warnTopicProbeUnknown(item.target.name, item.threadId, item.adapterId, "owner-adapter-changed-before-action");
|
|
5707
|
+
return;
|
|
5708
|
+
}
|
|
5709
|
+
this.topicCommands.handleTopicDeleted(item.threadId, {
|
|
5710
|
+
source: "provider-probe",
|
|
5711
|
+
adapterId: item.adapterId,
|
|
5712
|
+
generation: item.generation,
|
|
5713
|
+
});
|
|
5714
|
+
}
|
|
5715
|
+
/**
|
|
5716
|
+
* A gateway event is only a hint. Confirm it through the passive REST probe;
|
|
5717
|
+
* reconnecting gateways have emitted false channelDelete events in practice.
|
|
5718
|
+
*/
|
|
5719
|
+
async handleProviderTopicClosed(threadId, adapterId, adapter) {
|
|
5720
|
+
const target = this.routing.resolve(threadId);
|
|
5721
|
+
if (!target || !isProbeableRouteTarget(target))
|
|
5722
|
+
return;
|
|
5723
|
+
if (this.getInstanceAdapterId(target.name) !== adapterId)
|
|
5724
|
+
return;
|
|
5725
|
+
if (this.adapters.get(adapterId) !== adapter)
|
|
5726
|
+
return;
|
|
5727
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5728
|
+
if (!before || !adapter.probeTopicPresence) {
|
|
5729
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-from-unready-adapter");
|
|
5730
|
+
return;
|
|
5731
|
+
}
|
|
5732
|
+
let result;
|
|
5733
|
+
try {
|
|
5734
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5735
|
+
}
|
|
5736
|
+
catch {
|
|
5737
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5738
|
+
}
|
|
5739
|
+
if (this.routing.resolve(threadId) !== target
|
|
5740
|
+
|| this.getInstanceAdapterId(target.name) !== adapterId
|
|
5741
|
+
|| this.adapters.get(adapterId) !== adapter
|
|
5742
|
+
|| !this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5743
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
|
|
5744
|
+
return;
|
|
5745
|
+
}
|
|
5746
|
+
if (result.status === "unknown") {
|
|
5747
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason, result.detail);
|
|
5748
|
+
return;
|
|
5749
|
+
}
|
|
5750
|
+
this.clearTopicProbeUnknownStreak(threadId, adapterId);
|
|
5751
|
+
if (result.status !== "missing") {
|
|
5752
|
+
// The gateway said deleted, REST says present: a definite answer, so it
|
|
5753
|
+
// is not an unknown streak — but it is worth one debug line.
|
|
5754
|
+
this.logger.debug({ instanceName: target.name, threadId, adapterId }, "channelDelete hint contradicted by REST — topic present, nothing to do");
|
|
5755
|
+
return;
|
|
5756
|
+
}
|
|
5757
|
+
this.topicCommands.handleTopicDeleted(threadId, {
|
|
5758
|
+
source: "provider-event",
|
|
5759
|
+
adapterId,
|
|
5760
|
+
generation: result.generation,
|
|
5761
|
+
});
|
|
5762
|
+
}
|
|
5763
|
+
bindTopicClosedHandler(adapter, adapterId, label) {
|
|
5764
|
+
adapter.on("topic_closed", safeHandler(async (data) => {
|
|
5765
|
+
if (this.topicArchiver.isArchived(data.threadId))
|
|
5766
|
+
return;
|
|
5767
|
+
await this.handleProviderTopicClosed(data.threadId, adapterId, adapter);
|
|
5768
|
+
}, this.logger, label));
|
|
5769
|
+
}
|
|
5770
|
+
/**
|
|
5771
|
+
* Remove only the volatile route. The instance config, daemon, schedules,
|
|
5772
|
+
* teams, metadata directory, and working tree remain untouched until an
|
|
5773
|
+
* authenticated explicit deletion is requested.
|
|
5774
|
+
*/
|
|
5775
|
+
quarantineMissingTopic(threadId, target, evidence) {
|
|
5776
|
+
if (!isProbeableRouteTarget(target) || this.routing.resolve(threadId) !== target)
|
|
5777
|
+
return;
|
|
5778
|
+
this.routing.unregister(threadId);
|
|
5779
|
+
this.logger.error({ instanceName: target.name, threadId, ...evidence }, "Topic is confirmed missing — route quarantined; instance configuration and user data were retained");
|
|
5780
|
+
this.notifyFleetError(t("fleet.topic_quarantined", target.name, threadId));
|
|
5350
5781
|
}
|
|
5351
5782
|
/**
|
|
5352
5783
|
* Patch only values changed in the effective config into the original YAML
|
|
@@ -5499,7 +5930,8 @@ export class FleetManager {
|
|
|
5499
5930
|
}
|
|
5500
5931
|
}
|
|
5501
5932
|
}
|
|
5502
|
-
async removeInstance(name) {
|
|
5933
|
+
async removeInstance(name, authorization) {
|
|
5934
|
+
assertExplicitInstanceRemoval(authorization);
|
|
5503
5935
|
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
5504
5936
|
// else evicted deleted entries, so it grew for the life of the process.
|
|
5505
5937
|
forgetInstanceContext(name);
|
|
@@ -5525,7 +5957,7 @@ export class FleetManager {
|
|
|
5525
5957
|
}
|
|
5526
5958
|
}
|
|
5527
5959
|
}
|
|
5528
|
-
await this.lifecycle.remove(name);
|
|
5960
|
+
await this.lifecycle.remove(name, authorization);
|
|
5529
5961
|
// Clean up per-instance tracking maps so they don't grow unbounded
|
|
5530
5962
|
// as instances are created and deleted over the lifetime of the fleet.
|
|
5531
5963
|
this.lastActivity.delete(name);
|
|
@@ -5762,6 +6194,65 @@ export class FleetManager {
|
|
|
5762
6194
|
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5763
6195
|
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5764
6196
|
}
|
|
6197
|
+
/**
|
|
6198
|
+
* Where a fleet-wide notice can actually be posted, or null if nowhere.
|
|
6199
|
+
*
|
|
6200
|
+
* On Telegram a group id is itself a chat, so posting straight to it is
|
|
6201
|
+
* right. On Discord it is a *guild* id, and sending there makes the adapter
|
|
6202
|
+
* fetch a channel that does not exist — DiscordAPIError 10003 Unknown
|
|
6203
|
+
* Channel. That is why the daily summary never arrived on a Discord fleet:
|
|
6204
|
+
* it had been posting to the guild every night and only the catch handler
|
|
6205
|
+
* ever saw it.
|
|
6206
|
+
*
|
|
6207
|
+
* Discord therefore needs a real channel: the General topic (resolved from
|
|
6208
|
+
* config rather than findGeneralInstance, so a fleet-level fault can still be
|
|
6209
|
+
* reported while the General daemon is down), else the adapter's configured
|
|
6210
|
+
* general_channel_id. With neither, there is no safe target and the caller
|
|
6211
|
+
* should say so rather than send into a guaranteed failure.
|
|
6212
|
+
*/
|
|
6213
|
+
fleetNoticeTarget(adapterId) {
|
|
6214
|
+
const cfg = this.getChannelConfig(adapterId);
|
|
6215
|
+
const groupId = cfg?.group_id;
|
|
6216
|
+
if (groupId == null)
|
|
6217
|
+
return null;
|
|
6218
|
+
const chatId = String(groupId);
|
|
6219
|
+
// The General must belong to the SAME adapter as the group above. Taking
|
|
6220
|
+
// whichever General comes first in the instance map produced a mixed target
|
|
6221
|
+
// on a dual-platform fleet — a Telegram group id carrying a Discord channel
|
|
6222
|
+
// as its thread — and made the result depend on map insertion order.
|
|
6223
|
+
// Resolved from config, not from a live daemon, so a fleet-level fault is
|
|
6224
|
+
// still reportable while the General itself is down.
|
|
6225
|
+
const ownerId = cfg?.id ?? cfg?.type;
|
|
6226
|
+
const generalTopic = Object.entries(this.fleetConfig?.instances ?? {})
|
|
6227
|
+
.find(([name, instance]) => instance.general_topic === true
|
|
6228
|
+
&& this.getInstanceAdapterId(name) === ownerId)?.[1]?.topic_id;
|
|
6229
|
+
if (generalTopic != null)
|
|
6230
|
+
return { chatId, opts: { threadId: String(generalTopic) } };
|
|
6231
|
+
if (cfg?.type === "discord") {
|
|
6232
|
+
const configured = cfg.options?.general_channel_id;
|
|
6233
|
+
if (configured != null && String(configured)) {
|
|
6234
|
+
return { chatId, opts: { threadId: String(configured) } };
|
|
6235
|
+
}
|
|
6236
|
+
return null; // a guild id is not a channel; sending would always fail
|
|
6237
|
+
}
|
|
6238
|
+
return { chatId, opts: {} };
|
|
6239
|
+
}
|
|
6240
|
+
/**
|
|
6241
|
+
* Post the daily summary where fleet-wide notices go.
|
|
6242
|
+
*
|
|
6243
|
+
* A named method rather than an inline closure so a test can drive the real
|
|
6244
|
+
* thing: asserting on fleetNoticeTarget alone leaves the call site free to go
|
|
6245
|
+
* back to posting at the bare group id, which is the defect this replaced.
|
|
6246
|
+
*/
|
|
6247
|
+
postDailySummary(text) {
|
|
6248
|
+
const target = this.fleetNoticeTarget();
|
|
6249
|
+
if (!this.adapter || !target) {
|
|
6250
|
+
this.logger.warn("Daily summary has no postable target — set a General topic or channel.options.general_channel_id");
|
|
6251
|
+
return;
|
|
6252
|
+
}
|
|
6253
|
+
this.adapter.sendText(target.chatId, text, target.opts)
|
|
6254
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
6255
|
+
}
|
|
5765
6256
|
notifyFleetError(text) {
|
|
5766
6257
|
const now = Date.now();
|
|
5767
6258
|
const key = text.slice(0, 200);
|
|
@@ -5788,10 +6279,12 @@ export class FleetManager {
|
|
|
5788
6279
|
dispatched = this.notifyInstanceTopic(general, body);
|
|
5789
6280
|
}
|
|
5790
6281
|
else {
|
|
5791
|
-
// No General instance — fall back to the primary channel's
|
|
5792
|
-
|
|
5793
|
-
|
|
5794
|
-
|
|
6282
|
+
// No General instance — fall back to the primary channel's own notice
|
|
6283
|
+
// target. Posting to the bare group id looked right but is a guild id on
|
|
6284
|
+
// Discord, so every such fallback failed inside the catch handler.
|
|
6285
|
+
const target = this.fleetNoticeTarget();
|
|
6286
|
+
if (this.adapter && target) {
|
|
6287
|
+
this.adapter.sendText(target.chatId, body, target.opts)
|
|
5795
6288
|
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5796
6289
|
dispatched = true;
|
|
5797
6290
|
}
|
|
@@ -5850,13 +6343,16 @@ export class FleetManager {
|
|
|
5850
6343
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
5851
6344
|
return true;
|
|
5852
6345
|
}
|
|
5853
|
-
// Fallback:
|
|
5854
|
-
|
|
5855
|
-
|
|
6346
|
+
// Fallback: the instance has neither a topic nor a classic channel, so post
|
|
6347
|
+
// where fleet-wide notices go. Not the bare group id: on Discord that is a
|
|
6348
|
+
// guild, and the send fails inside the catch handler.
|
|
6349
|
+
const target = this.fleetNoticeTarget(this.getInstanceAdapterId(instanceName));
|
|
6350
|
+
if (target) {
|
|
6351
|
+
adapter.sendText(target.chatId, text, { ...target.opts, ...extraOpts })
|
|
5856
6352
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5857
6353
|
return true;
|
|
5858
6354
|
}
|
|
5859
|
-
this.logger.warn({ instanceName }, "No
|
|
6355
|
+
this.logger.warn({ instanceName }, "No postable target — instance topic notification not sent");
|
|
5860
6356
|
return false;
|
|
5861
6357
|
}
|
|
5862
6358
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
@@ -6417,6 +6913,41 @@ export class FleetManager {
|
|
|
6417
6913
|
extra: { generalName, promptKind: kind },
|
|
6418
6914
|
});
|
|
6419
6915
|
}
|
|
6916
|
+
/**
|
|
6917
|
+
* Offer a one-tap re-login next to an auth alert.
|
|
6918
|
+
*
|
|
6919
|
+
* The alert already names the remedy in words (`/login <backend>`), which
|
|
6920
|
+
* still leaves the user to retype it somewhere. The button routes into the
|
|
6921
|
+
* same chooser `/login` uses, so pressing it starts the flow in place.
|
|
6922
|
+
*
|
|
6923
|
+
* Backends with no remote login flow (opencode logs in from a terminal) get
|
|
6924
|
+
* no button — the alert's own wording already tells them what to run.
|
|
6925
|
+
*/
|
|
6926
|
+
async offerBackendLogin(targetInstance, backend) {
|
|
6927
|
+
if (!LOGIN_FLOWS[backend])
|
|
6928
|
+
return;
|
|
6929
|
+
const adapterId = this.getInstanceAdapterId(targetInstance);
|
|
6930
|
+
const adapter = this.getAdapterForInstance(targetInstance);
|
|
6931
|
+
const chatId = this.getGroupIdForInstance(targetInstance);
|
|
6932
|
+
const topicId = this.fleetConfig?.instances[targetInstance]?.topic_id;
|
|
6933
|
+
const threadId = topicId != null ? String(topicId) : undefined;
|
|
6934
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6935
|
+
this.logger.warn({ targetInstance, backend, adapterId, chatId }, "Cannot address the re-login button — the alert text still names the command");
|
|
6936
|
+
return;
|
|
6937
|
+
}
|
|
6938
|
+
await this.postNonceButtonPrompt({
|
|
6939
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6940
|
+
alertType: "login",
|
|
6941
|
+
instanceName: "login",
|
|
6942
|
+
adapter,
|
|
6943
|
+
adapterId,
|
|
6944
|
+
chatId,
|
|
6945
|
+
threadId,
|
|
6946
|
+
message: t("login.offer", backend),
|
|
6947
|
+
choices: [{ action: backend, label: t("login.offer_action", backend) }],
|
|
6948
|
+
expiredText: t("buttons.stale"),
|
|
6949
|
+
});
|
|
6950
|
+
}
|
|
6420
6951
|
/** Consume a General assist button exactly once. */
|
|
6421
6952
|
async handleInteractivePromptAssist(data, callbackAdapterId, receivingAdapter) {
|
|
6422
6953
|
const claimed = this.consumeNonceCallback(INTERACTIVE_ASSIST_CALLBACK_PREFIX, /^interactive-assist:([0-9a-f]+):(confirm|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
@@ -7132,8 +7663,8 @@ export class FleetManager {
|
|
|
7132
7663
|
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
7133
7664
|
async promptLoginBackends(chat) {
|
|
7134
7665
|
const configured = new Set();
|
|
7135
|
-
for (const
|
|
7136
|
-
configured.add(
|
|
7666
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
7667
|
+
configured.add(this.backendNameOf(name));
|
|
7137
7668
|
}
|
|
7138
7669
|
const choices = Object.keys(LOGIN_FLOWS)
|
|
7139
7670
|
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
@@ -7405,11 +7936,15 @@ export class FleetManager {
|
|
|
7405
7936
|
if (cleanupFailed) {
|
|
7406
7937
|
await chat.adapter.sendText(chat.chatId, t("login.web_cleanup_failed", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7407
7938
|
}
|
|
7939
|
+
const send = (text) => chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
7408
7940
|
let text;
|
|
7409
7941
|
if (ok) {
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7942
|
+
// Same order as the web-login path: the login result goes out first,
|
|
7943
|
+
// then the recovery reports its own outcome. Waiting for recovery to
|
|
7944
|
+
// build this message is what made a successful login look hung.
|
|
7945
|
+
await send(t("login.completed", backend));
|
|
7946
|
+
await announcePostLoginRecovery(backend, () => this.recoverBackendInstances(backend), send);
|
|
7947
|
+
return;
|
|
7413
7948
|
}
|
|
7414
7949
|
else if (detail === "cancelled") {
|
|
7415
7950
|
// The cancel command's own reply already announced this — a second
|
|
@@ -7533,32 +8068,53 @@ export class FleetManager {
|
|
|
7533
8068
|
* only re-reads credentials on process start (a paused instance's CLI is
|
|
7534
8069
|
* already dead, so waking it respawns with the new token for free).
|
|
7535
8070
|
*/
|
|
7536
|
-
|
|
8071
|
+
/**
|
|
8072
|
+
* Wake/restart every instance of a backend after a successful re-login.
|
|
8073
|
+
*
|
|
8074
|
+
* Bounded by a wall-clock deadline. This used to be an unbounded sequential
|
|
8075
|
+
* loop, and the caller only built its "login completed" message AFTER it
|
|
8076
|
+
* returned — so with several instances (or one slow restart) the user was
|
|
8077
|
+
* told nothing at all for minutes, concluded the login had hung, and
|
|
8078
|
+
* restarted things by hand. Instances that do not finish in time are NOT
|
|
8079
|
+
* cancelled: they are still coming back, and are reported as pending so the
|
|
8080
|
+
* message can say so instead of implying failure.
|
|
8081
|
+
*/
|
|
8082
|
+
async recoverBackendInstances(backend, deadlineMs = POST_LOGIN_RECOVERY_DEADLINE_MS) {
|
|
7537
8083
|
const woken = [];
|
|
7538
8084
|
const restarted = [];
|
|
7539
|
-
|
|
7540
|
-
|
|
7541
|
-
|
|
8085
|
+
const pending = [];
|
|
8086
|
+
const deadline = Date.now() + deadlineMs;
|
|
8087
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
8088
|
+
if (this.backendNameOf(name) !== backend)
|
|
7542
8089
|
continue;
|
|
7543
8090
|
const status = this.getInstanceStatus(name);
|
|
7544
|
-
|
|
8091
|
+
if (status !== "paused" && status !== "running")
|
|
8092
|
+
continue;
|
|
8093
|
+
const result = await runBeforeDeadline(async () => {
|
|
7545
8094
|
if (status === "paused") {
|
|
7546
8095
|
if (this.daemons.has(name))
|
|
7547
8096
|
await this.lifecycle.wake(name, 30_000);
|
|
7548
8097
|
else
|
|
7549
8098
|
await this.startPersistedPausedInstance(name);
|
|
7550
|
-
woken.push(name);
|
|
7551
8099
|
}
|
|
7552
|
-
else
|
|
8100
|
+
else {
|
|
7553
8101
|
await this.restartSingleInstance(name);
|
|
7554
|
-
restarted.push(name);
|
|
7555
8102
|
}
|
|
8103
|
+
}, deadline);
|
|
8104
|
+
if (result.status === "fulfilled") {
|
|
8105
|
+
(status === "paused" ? woken : restarted).push(name);
|
|
7556
8106
|
}
|
|
7557
|
-
|
|
7558
|
-
|
|
8107
|
+
else if (result.status === "timeout") {
|
|
8108
|
+
// Out of time: record it and stop waiting, but keep walking the list —
|
|
8109
|
+
// the remaining instances are checked against the same deadline and
|
|
8110
|
+
// fall straight through, so the caller still learns about all of them.
|
|
8111
|
+
pending.push(name);
|
|
8112
|
+
}
|
|
8113
|
+
else {
|
|
8114
|
+
this.logger.warn({ err: result.reason?.message, name, status }, "Post-login recovery failed");
|
|
7559
8115
|
}
|
|
7560
8116
|
}
|
|
7561
|
-
return { woken, restarted };
|
|
8117
|
+
return { woken, restarted, pending };
|
|
7562
8118
|
}
|
|
7563
8119
|
/** Backend chooser button → start that backend's login session. */
|
|
7564
8120
|
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
@@ -7970,7 +8526,7 @@ You CAN write code snippets, explain code, and answer technical questions direct
|
|
|
7970
8526
|
- **Delegate to 1 instance**: scoped to one project/repo, needs file access or execution.
|
|
7971
8527
|
- **Coordinate multiple**: spans repos, outputs feed each other, or parallel helps (max 3 per task).
|
|
7972
8528
|
|
|
7973
|
-
Instance discovery
|
|
8529
|
+
Instance discovery: start with list_instances(); follow the guidance in its response. Prefer reuse; never duplicate a running instance.
|
|
7974
8530
|
|
|
7975
8531
|
## Reply Contract
|
|
7976
8532
|
|
|
@@ -9598,11 +10154,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9598
10154
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
9599
10155
|
ensureWorkspaceGit(workDir);
|
|
9600
10156
|
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
10157
|
+
const toolProgress = classicIdentity
|
|
10158
|
+
? this.classicChannels?.getToolProgress(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.tool_progress)
|
|
10159
|
+
: this.fleetConfig?.defaults?.tool_progress;
|
|
10160
|
+
const replyCompletionGuard = classicIdentity
|
|
10161
|
+
? this.classicChannels?.getReplyCompletionGuard(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.reply_completion_guard)
|
|
10162
|
+
: this.fleetConfig?.defaults?.reply_completion_guard;
|
|
9601
10163
|
const config = {
|
|
9602
10164
|
...DEFAULT_INSTANCE_CONFIG,
|
|
9603
10165
|
...this.fleetConfig?.defaults,
|
|
9604
10166
|
working_directory: workDir,
|
|
9605
10167
|
lightweight: true,
|
|
10168
|
+
tool_progress: toolProgress ?? "off",
|
|
10169
|
+
reply_completion_guard: replyCompletionGuard ?? true,
|
|
9606
10170
|
...(backend ? { backend } : {}),
|
|
9607
10171
|
...(model ? { model } : {}),
|
|
9608
10172
|
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
@@ -9742,6 +10306,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9742
10306
|
clearInterval(this.topicCleanupTimer);
|
|
9743
10307
|
this.topicCleanupTimer = null;
|
|
9744
10308
|
}
|
|
10309
|
+
this.topicCleanupGeneration++;
|
|
9745
10310
|
if (this.sessionPruneTimer) {
|
|
9746
10311
|
clearInterval(this.sessionPruneTimer);
|
|
9747
10312
|
this.sessionPruneTimer = null;
|
|
@@ -9884,10 +10449,17 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9884
10449
|
// Keep that single message; the new process will adopt and finish it.
|
|
9885
10450
|
setUpdateProgressStage(this.dataDir, "stopping");
|
|
9886
10451
|
}
|
|
9887
|
-
const
|
|
9888
|
-
if (!trackedFullRestart &&
|
|
9889
|
-
|
|
9890
|
-
|
|
10452
|
+
const restartTarget = this.fleetNoticeTarget();
|
|
10453
|
+
if (!trackedFullRestart && this.adapter) {
|
|
10454
|
+
if (restartTarget) {
|
|
10455
|
+
await this.adapter.sendText(restartTarget.chatId, t("restart.full_initiated"), restartTarget.opts)
|
|
10456
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
|
|
10457
|
+
}
|
|
10458
|
+
else {
|
|
10459
|
+
// Say why nothing was posted. A restart that announces itself nowhere,
|
|
10460
|
+
// for a reason nobody logged, is the harder version of this bug.
|
|
10461
|
+
this.logger.warn("Full restart notice has no postable target — set a General topic or channel.options.general_channel_id");
|
|
10462
|
+
}
|
|
9891
10463
|
}
|
|
9892
10464
|
// Wait for idle with 5-minute timeout
|
|
9893
10465
|
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
|
|
@@ -9987,6 +10559,30 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9987
10559
|
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
9988
10560
|
return;
|
|
9989
10561
|
}
|
|
10562
|
+
// Classic behavior settings share the fleet defaults but are not entries
|
|
10563
|
+
// in fleet.yaml. Snapshot the old effective chain before reloading the
|
|
10564
|
+
// Classic file so SIGHUP can hot-apply either source without waiting for
|
|
10565
|
+
// the 30-second Classic poller.
|
|
10566
|
+
const oldClassicBehavior = new Map();
|
|
10567
|
+
if (this.classicChannels) {
|
|
10568
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10569
|
+
const runtimeConfig = this.daemons.get(ch.instanceName)?.getConfigSnapshot?.();
|
|
10570
|
+
oldClassicBehavior.set(ch.instanceName, {
|
|
10571
|
+
backend: this.classicChannels.getBackend(ch.channelId, ch.adapterId, oldConfig?.defaults?.backend),
|
|
10572
|
+
model: this.classicChannels.getModel(ch.channelId, ch.adapterId, oldConfig?.defaults?.model),
|
|
10573
|
+
autoPauseAfter: this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, oldConfig?.defaults?.auto_pause_after),
|
|
10574
|
+
// Settings mutates FleetManager's in-memory defaults before SIGHUP.
|
|
10575
|
+
// The live daemon is therefore the authority for the previous hot
|
|
10576
|
+
// values, exactly as in the fleet-topic reconciliation below.
|
|
10577
|
+
toolProgress: runtimeConfig?.tool_progress
|
|
10578
|
+
?? this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, oldConfig?.defaults?.tool_progress),
|
|
10579
|
+
replyCompletionGuard: runtimeConfig?.reply_completion_guard
|
|
10580
|
+
?? this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, oldConfig?.defaults?.reply_completion_guard),
|
|
10581
|
+
});
|
|
10582
|
+
}
|
|
10583
|
+
if (this.classicChannels.checkReload())
|
|
10584
|
+
this.reportClassicUnrecoverableIds();
|
|
10585
|
+
}
|
|
9990
10586
|
this.routing.rebuild(this.fleetConfig);
|
|
9991
10587
|
this.reregisterClassicChannels();
|
|
9992
10588
|
this.scheduler?.reload();
|
|
@@ -10051,6 +10647,35 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10051
10647
|
}
|
|
10052
10648
|
}
|
|
10053
10649
|
}
|
|
10650
|
+
// A Classic channel inherits fleet defaults beneath its own two levels.
|
|
10651
|
+
// Recompute that complete chain on SIGHUP. Only the two behavior switches
|
|
10652
|
+
// are hot; changes to backend/model/auto-pause retain the existing restart
|
|
10653
|
+
// semantics.
|
|
10654
|
+
if (this.classicChannels) {
|
|
10655
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10656
|
+
const old = oldClassicBehavior.get(ch.instanceName);
|
|
10657
|
+
if (!old || !this.daemons.has(ch.instanceName))
|
|
10658
|
+
continue;
|
|
10659
|
+
const backend = this.classicChannels.getBackend(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.backend);
|
|
10660
|
+
const model = this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model);
|
|
10661
|
+
const autoPauseAfter = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
10662
|
+
if (old.backend !== backend || old.model !== model || old.autoPauseAfter !== autoPauseAfter) {
|
|
10663
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic cold config changed — restarting");
|
|
10664
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
10665
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after fleet reload");
|
|
10666
|
+
continue;
|
|
10667
|
+
}
|
|
10668
|
+
const toolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
10669
|
+
const replyCompletionGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
10670
|
+
if (old.toolProgress !== toolProgress || old.replyCompletionGuard !== replyCompletionGuard) {
|
|
10671
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
10672
|
+
tool_progress: toolProgress,
|
|
10673
|
+
reply_completion_guard: replyCompletionGuard,
|
|
10674
|
+
});
|
|
10675
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic inherited hot config reloaded");
|
|
10676
|
+
}
|
|
10677
|
+
}
|
|
10678
|
+
}
|
|
10054
10679
|
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
10055
10680
|
// currently idle instances instead of waiting for a future state edge.
|
|
10056
10681
|
this.enforceWarmCap();
|
|
@@ -10668,11 +11293,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10668
11293
|
const instances = names.map(name => {
|
|
10669
11294
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
10670
11295
|
let cost = 0;
|
|
10671
|
-
let model = "";
|
|
10672
11296
|
try {
|
|
10673
11297
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
10674
11298
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
10675
|
-
model = data.model?.display_name ?? "";
|
|
10676
11299
|
}
|
|
10677
11300
|
catch (err) {
|
|
10678
11301
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
@@ -10685,8 +11308,40 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10685
11308
|
?? this.fleetConfig?.defaults?.backend
|
|
10686
11309
|
?? "claude-code");
|
|
10687
11310
|
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
10688
|
-
|
|
10689
|
-
|
|
11311
|
+
// context_pct: null when unavailable, not 0
|
|
11312
|
+
const context_pct = context ?? null;
|
|
11313
|
+
// Model: Only Claude Code has live statusline; others use the effective resolver.
|
|
11314
|
+
// readStatuslineModel provides the /ctx-aligned display_name+id combo for Claude.
|
|
11315
|
+
// Non-Claude backends must NOT read statusline.json model (may be stale from previous Claude run).
|
|
11316
|
+
const resolved = this.resolveInstanceModel(name);
|
|
11317
|
+
const liveModel = backend === "claude-code" ? readStatuslineModel(this.dataDir, name) : null;
|
|
11318
|
+
// Display value: live model for Claude, resolved.display for others (includes "auto (default)" for Kiro)
|
|
11319
|
+
const model = liveModel ?? resolved.display;
|
|
11320
|
+
// model_source: "live" when Claude statusline succeeded, else the resolver's source
|
|
11321
|
+
const model_source = liveModel ? "live" : resolved.source;
|
|
11322
|
+
// Effort: aligned with /ctx's effortLineFor — unsupported and antigravity don't show effort.
|
|
11323
|
+
const effortStrategy = this.effortStrategyFor(name);
|
|
11324
|
+
const isAgy = backend === "antigravity" || backend === "agy";
|
|
11325
|
+
const effortResolved = this.resolveInstanceEffort(name);
|
|
11326
|
+
// Only show effort if backend supports it and it's not antigravity
|
|
11327
|
+
const effort = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.effort;
|
|
11328
|
+
const effort_source = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.source;
|
|
11329
|
+
// Display name: fleet config → classic channel → undefined
|
|
11330
|
+
const display_name = classic
|
|
11331
|
+
? this.classicChannels?.getAll().find(ch => ch.instanceName === name)?.displayName
|
|
11332
|
+
: this.fleetConfig?.instances[name]?.display_name;
|
|
11333
|
+
return {
|
|
11334
|
+
name,
|
|
11335
|
+
display_name: display_name || undefined,
|
|
11336
|
+
status: this.getInstanceStatus(name),
|
|
11337
|
+
context_pct,
|
|
11338
|
+
cost,
|
|
11339
|
+
model,
|
|
11340
|
+
model_source,
|
|
11341
|
+
backend,
|
|
11342
|
+
effort,
|
|
11343
|
+
effort_source,
|
|
11344
|
+
};
|
|
10690
11345
|
});
|
|
10691
11346
|
return {
|
|
10692
11347
|
instances,
|