@songsid/agend 2.1.5-beta.9 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/claude-code.d.ts +12 -0
- package/dist/backend/claude-code.js +12 -0
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +25 -1
- package/dist/backend/codex.js +76 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.js +30 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +48 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +17 -2
- package/dist/channel/adapters/discord.js +64 -5
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +41 -2
- package/dist/channel/adapters/telegram.js +116 -20
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/factory.js +6 -1
- package/dist/channel/factory.js.map +1 -1
- package/dist/channel/types.d.ts +33 -1
- package/dist/classic-channel-manager.d.ts +23 -3
- package/dist/classic-channel-manager.js +35 -0
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/config-validator.js +23 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +5 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +143 -21
- package/dist/daemon.js +969 -185
- package/dist/daemon.js.map +1 -1
- package/dist/deadline.d.ts +23 -0
- package/dist/deadline.js +29 -0
- package/dist/deadline.js.map +1 -0
- package/dist/fleet-context.d.ts +15 -1
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +178 -3
- package/dist/fleet-manager.js +759 -119
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +26 -2
- package/dist/instance-lifecycle.d.ts +8 -2
- package/dist/instance-lifecycle.js +47 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instance-removal.d.ts +13 -0
- package/dist/instance-removal.js +14 -0
- package/dist/instance-removal.js.map +1 -0
- package/dist/instructions.js +2 -2
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/login-controller.d.ts +31 -4
- package/dist/login-controller.js +52 -4
- package/dist/login-controller.js.map +1 -1
- package/dist/login-flows.d.ts +29 -0
- package/dist/login-flows.js +48 -1
- package/dist/login-flows.js.map +1 -1
- package/dist/outbound-handlers.d.ts +1 -0
- package/dist/outbound-handlers.js +118 -29
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -0
- package/dist/outbound-schemas.js +3 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +25 -2
- package/dist/pane-input-residue.js +31 -2
- package/dist/pane-input-residue.js.map +1 -1
- package/dist/restart-progress.js +3 -21
- package/dist/restart-progress.js.map +1 -1
- package/dist/settings-api.d.ts +1 -1
- package/dist/settings-api.js +38 -6
- package/dist/settings-api.js.map +1 -1
- package/dist/topic-commands.d.ts +27 -2
- package/dist/topic-commands.js +37 -6
- package/dist/topic-commands.js.map +1 -1
- package/dist/turn-reply-guard.d.ts +41 -0
- package/dist/turn-reply-guard.js +77 -0
- package/dist/turn-reply-guard.js.map +1 -0
- package/dist/types.d.ts +7 -0
- package/dist/ui/dashboard.html +38 -10
- package/dist/ui/settings.html +93 -15
- package/dist/ui/view.html +35 -20
- package/dist/view-api.d.ts +1 -0
- package/dist/view-api.js +12 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.d.ts +22 -1
- package/dist/web-api.js +25 -1
- package/dist/web-api.js.map +1 -1
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
|
-
import { freemem, totalmem } from "node:os";
|
|
4
|
+
import { freemem, totalmem, cpus } from "node:os";
|
|
5
5
|
import { createServer } from "node:http";
|
|
6
6
|
import { join, dirname, basename } from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
@@ -33,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
33
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
34
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
35
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
36
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext, readStatuslineModel } from "./topic-commands.js";
|
|
37
37
|
import { DailySummary } from "./daily-summary.js";
|
|
38
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
39
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -48,12 +48,14 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
48
48
|
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
49
|
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
50
|
import { LoginSession } from "./login-manager.js";
|
|
51
|
-
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX } from "./login-controller.js";
|
|
51
|
+
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX, POST_LOGIN_RECOVERY_DEADLINE_MS, announcePostLoginRecovery } from "./login-controller.js";
|
|
52
|
+
import { runBeforeDeadline } from "./deadline.js";
|
|
52
53
|
import { LoginWindowLock } from "./login-window-lock.js";
|
|
53
54
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
54
55
|
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
55
56
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
56
57
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
58
|
+
import { assertExplicitInstanceRemoval } from "./instance-removal.js";
|
|
57
59
|
import { validateFleetConfig } from "./config-validator.js";
|
|
58
60
|
import { readLastInboundAt } from "./daemon.js";
|
|
59
61
|
import { clearPausedMarker } from "./pause-marker.js";
|
|
@@ -179,6 +181,7 @@ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
|
|
|
179
181
|
const IGNORED_REACTION_EMOJIS = new Set(["📷"]);
|
|
180
182
|
const HOT_INSTANCE_CONFIG_KEYS = new Set([
|
|
181
183
|
"tool_progress",
|
|
184
|
+
"reply_completion_guard",
|
|
182
185
|
"mcp_proxy_reply",
|
|
183
186
|
"auto_pause_after",
|
|
184
187
|
"warm_cap",
|
|
@@ -219,7 +222,7 @@ const HANG_CALLBACK_PREFIX = "hang:";
|
|
|
219
222
|
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
220
223
|
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
221
224
|
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
222
|
-
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
225
|
+
export const LOGIN_CALLBACK_PREFIX = "login:";
|
|
223
226
|
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
224
227
|
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
225
228
|
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
@@ -262,6 +265,18 @@ const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
|
|
|
262
265
|
* The wait is announced before it starts, so it reads as progress, not a stall.
|
|
263
266
|
*/
|
|
264
267
|
export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
|
|
268
|
+
/**
|
|
269
|
+
* How many CLIs may cold-start at once, from BOTH memory and cores.
|
|
270
|
+
*
|
|
271
|
+
* Memory alone said 10 on any host with roughly 3GB free, so a three-core box
|
|
272
|
+
* started ten CLIs together, saturated the CPU, and healthy starts then missed
|
|
273
|
+
* their startup budget — which used to cost the user their conversation. Cores
|
|
274
|
+
* bound how many can actually make progress; memory bounds how many fit.
|
|
275
|
+
*/
|
|
276
|
+
export function deriveSpawnConcurrency(freeMemMB, cores) {
|
|
277
|
+
const byMemory = Math.floor(freeMemMB / 300);
|
|
278
|
+
return Math.max(2, Math.min(10, byMemory, Math.max(1, cores)));
|
|
279
|
+
}
|
|
265
280
|
export class FleetManager {
|
|
266
281
|
dataDir;
|
|
267
282
|
static signalTarget = null;
|
|
@@ -303,6 +318,33 @@ export class FleetManager {
|
|
|
303
318
|
reloadPending = false;
|
|
304
319
|
/** A running reconciliation; only one may mutate lifecycle/config state at a time. */
|
|
305
320
|
reconcileInFlight = null;
|
|
321
|
+
/** Topology checks are serialized separately from config reconciliation. */
|
|
322
|
+
topicCleanupInFlight = null;
|
|
323
|
+
topicCleanupGeneration = 0;
|
|
324
|
+
topicProbeWarnings = new Map();
|
|
325
|
+
/**
|
|
326
|
+
* Consecutive unknown probe results per route (or per adapter for outage
|
|
327
|
+
* class reasons). A single transient never reaches the operator; only a
|
|
328
|
+
* streak of TOPIC_PROBE_UNKNOWN_ESCALATION does.
|
|
329
|
+
*/
|
|
330
|
+
topicProbeUnknownStreak = new Map();
|
|
331
|
+
/** Unknown results in a row before the operator is told. 3 × 5 min poller = 15 min. */
|
|
332
|
+
static TOPIC_PROBE_UNKNOWN_ESCALATION = 3;
|
|
333
|
+
/** Reasons that describe the adapter, not one topic — counted once per adapter. */
|
|
334
|
+
static TOPIC_PROBE_ADAPTER_SCOPED_REASONS = new Set([
|
|
335
|
+
"owner-adapter-unavailable",
|
|
336
|
+
"owner-adapter-not-ready",
|
|
337
|
+
"owner-adapter-generation-changed",
|
|
338
|
+
"owner-adapter-changed-before-action",
|
|
339
|
+
"adapter-not-ready",
|
|
340
|
+
"adapter-not-initialized",
|
|
341
|
+
"adapter-generation-changed",
|
|
342
|
+
"topic-close-from-unready-adapter",
|
|
343
|
+
"topic-close-generation-changed",
|
|
344
|
+
// Telegram probe: the transport or Telegram itself is down, not one topic.
|
|
345
|
+
"transport-failed",
|
|
346
|
+
"provider-unavailable",
|
|
347
|
+
]);
|
|
306
348
|
logger = createLogger("info");
|
|
307
349
|
topicCommands;
|
|
308
350
|
// sessionName → instanceName mapping for external sessions
|
|
@@ -452,8 +494,7 @@ export class FleetManager {
|
|
|
452
494
|
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
453
495
|
if (explicit != null)
|
|
454
496
|
return Math.max(1, Math.min(20, explicit));
|
|
455
|
-
|
|
456
|
-
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
497
|
+
return deriveSpawnConcurrency(Math.round(freemem() / (1024 * 1024)), cpus().length);
|
|
457
498
|
}
|
|
458
499
|
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
459
500
|
bindStormWindowEvents() {
|
|
@@ -916,6 +957,45 @@ export class FleetManager {
|
|
|
916
957
|
}
|
|
917
958
|
return channels[0];
|
|
918
959
|
}
|
|
960
|
+
/**
|
|
961
|
+
* The configured world that owns `chatId`, when that is provably NOT the
|
|
962
|
+
* world `target` lives in. Returns undefined when they agree, when there is
|
|
963
|
+
* nothing to check, or when no configured channel claims the id.
|
|
964
|
+
*
|
|
965
|
+
* Deliberately one-sided: only a POSITIVE match against another channel's
|
|
966
|
+
* group id counts as foreign. A chat id that matches nothing may still be
|
|
967
|
+
* legitimate for this world (a classic channel, a DM), and treating
|
|
968
|
+
* "unrecognised" as "wrong" would stop seeding for cases that work today.
|
|
969
|
+
*
|
|
970
|
+
* Read from config rather than the live worlds map on purpose: a channel
|
|
971
|
+
* whose adapter failed to start still owns its group id, and the coordinates
|
|
972
|
+
* are just as unusable by the target's adapter either way.
|
|
973
|
+
*
|
|
974
|
+
* This is the other half of what scheduleSourceAdapter fixed. That one stops
|
|
975
|
+
* the trigger NOTICE being sent through the wrong bot; this one stops the
|
|
976
|
+
* same coordinates being planted as the target instance's reply context,
|
|
977
|
+
* which is what made its own replies fail until someone spoke to it (#752).
|
|
978
|
+
*/
|
|
979
|
+
scheduleChatWorldMismatch(target, chatId) {
|
|
980
|
+
if (!chatId)
|
|
981
|
+
return undefined;
|
|
982
|
+
const targetWorld = this.getInstanceAdapterId(target);
|
|
983
|
+
if (!targetWorld)
|
|
984
|
+
return undefined;
|
|
985
|
+
const channels = this.fleetConfig?.channels
|
|
986
|
+
?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
|
|
987
|
+
// ALL owners, not the first: a persona bot shares the primary's guild
|
|
988
|
+
// (quickstart writes `group_id: primary.group_id`), so one group id is
|
|
989
|
+
// legitimately claimed by two channels. Taking the first match called a
|
|
990
|
+
// persona instance's own guild "another world" and stopped seeding a
|
|
991
|
+
// context it can address perfectly well.
|
|
992
|
+
const owners = channels.filter(ch => ch.group_id != null && String(ch.group_id) === String(chatId));
|
|
993
|
+
if (owners.length === 0)
|
|
994
|
+
return undefined;
|
|
995
|
+
if (owners.some(ch => (ch.id ?? ch.type) === targetWorld))
|
|
996
|
+
return undefined;
|
|
997
|
+
return owners[0].id ?? owners[0].type;
|
|
998
|
+
}
|
|
919
999
|
/** Get the group_id for an instance's bound adapter */
|
|
920
1000
|
getGroupIdForInstance(name) {
|
|
921
1001
|
const adapterId = this.getInstanceAdapterId(name);
|
|
@@ -1426,8 +1506,27 @@ export class FleetManager {
|
|
|
1426
1506
|
await this.lifecycle.pause(name);
|
|
1427
1507
|
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
1428
1508
|
}
|
|
1509
|
+
/** Deliver an already-resolved hot snapshot without depending on IPC timing. */
|
|
1510
|
+
applyHotConfigUpdate(instanceName, update) {
|
|
1511
|
+
const daemon = this.daemons.get(instanceName);
|
|
1512
|
+
if (!daemon)
|
|
1513
|
+
return false;
|
|
1514
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1515
|
+
const sent = ipc?.connected === true && ipc.send({ type: "config_update", config: update });
|
|
1516
|
+
if (!sent) {
|
|
1517
|
+
daemon.applyConfigUpdate(update);
|
|
1518
|
+
this.logger.warn({ name: instanceName }, "Config-update IPC unavailable — applied hot config in-process");
|
|
1519
|
+
}
|
|
1520
|
+
return true;
|
|
1521
|
+
}
|
|
1522
|
+
classicBehaviorUpdate(instanceName) {
|
|
1523
|
+
return {
|
|
1524
|
+
tool_progress: this.classicChannels?.getToolProgressByInstance(instanceName, this.fleetConfig?.defaults?.tool_progress) ?? "off",
|
|
1525
|
+
reply_completion_guard: this.classicChannels?.getReplyCompletionGuardByInstance(instanceName, this.fleetConfig?.defaults?.reply_completion_guard) ?? true,
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1429
1528
|
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
1430
|
-
async restartClassicInstanceFromSettings(instanceName) {
|
|
1529
|
+
async restartClassicInstanceFromSettings(instanceName, changedFields = []) {
|
|
1431
1530
|
if (!this.classicChannels)
|
|
1432
1531
|
throw new Error("Classic channel manager not initialized");
|
|
1433
1532
|
const wasRunning = this.daemons.has(instanceName);
|
|
@@ -1439,10 +1538,77 @@ export class FleetManager {
|
|
|
1439
1538
|
throw new Error("Classic channel not found after reload");
|
|
1440
1539
|
if (!wasRunning)
|
|
1441
1540
|
return;
|
|
1541
|
+
const hotOnly = changedFields.length > 0
|
|
1542
|
+
&& changedFields.every(field => field === "tool_progress" || field === "reply_completion_guard");
|
|
1543
|
+
if (hotOnly) {
|
|
1544
|
+
this.applyHotConfigUpdate(instanceName, this.classicBehaviorUpdate(instanceName));
|
|
1545
|
+
this.logger.info({ instanceName, fields: changedFields }, "Classic instance hot config reloaded");
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1442
1548
|
await this.stopInstance(instanceName);
|
|
1443
1549
|
await new Promise(resolve => setTimeout(resolve, 250));
|
|
1444
1550
|
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1445
1551
|
}
|
|
1552
|
+
/** Reload classicBot.yaml once. Kept callable so the periodic production
|
|
1553
|
+
* path is covered without relying on fake timers around startAll(). */
|
|
1554
|
+
async reloadClassicConfigFromDisk() {
|
|
1555
|
+
try {
|
|
1556
|
+
if (!this.classicChannels)
|
|
1557
|
+
return;
|
|
1558
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
1559
|
+
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
1560
|
+
const oldBackends = new Map();
|
|
1561
|
+
const oldModels = new Map();
|
|
1562
|
+
const oldAutoPause = new Map();
|
|
1563
|
+
const oldToolProgress = new Map();
|
|
1564
|
+
const oldReplyGuard = new Map();
|
|
1565
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1566
|
+
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
1567
|
+
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
1568
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
1569
|
+
oldToolProgress.set(ch.instanceName, this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress));
|
|
1570
|
+
oldReplyGuard.set(ch.instanceName, this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard));
|
|
1571
|
+
}
|
|
1572
|
+
if (!this.classicChannels.checkReload())
|
|
1573
|
+
return;
|
|
1574
|
+
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
1575
|
+
// throttle keeps a repeated report from flooding the topic.
|
|
1576
|
+
this.reportClassicUnrecoverableIds();
|
|
1577
|
+
this.reregisterClassicChannels();
|
|
1578
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
1579
|
+
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
1580
|
+
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
1581
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
1582
|
+
const newToolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
1583
|
+
const newReplyGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
1584
|
+
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
1585
|
+
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
1586
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
1587
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
1588
|
+
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
1589
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
1590
|
+
// Small delay to let tmux window clean up
|
|
1591
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
1592
|
+
// The manager already holds the new backend/model/auto-pause; the
|
|
1593
|
+
// unattended helper reads them from it and schedules the delayed
|
|
1594
|
+
// retry on failure like every other unattended start.
|
|
1595
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
1596
|
+
}
|
|
1597
|
+
else if (this.daemons.has(ch.instanceName)
|
|
1598
|
+
&& (oldToolProgress.get(ch.instanceName) !== newToolProgress
|
|
1599
|
+
|| oldReplyGuard.get(ch.instanceName) !== newReplyGuard)) {
|
|
1600
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
1601
|
+
tool_progress: newToolProgress,
|
|
1602
|
+
reply_completion_guard: newReplyGuard,
|
|
1603
|
+
});
|
|
1604
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic instance hot config reloaded");
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
catch (err) {
|
|
1609
|
+
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1446
1612
|
async startInstance(name, config, topicMode, kind = "fleet-topic",
|
|
1447
1613
|
/**
|
|
1448
1614
|
* Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
|
|
@@ -2300,48 +2466,8 @@ export class FleetManager {
|
|
|
2300
2466
|
this.instanceWorldBinding.set(ch.instanceName, ch.adapterId);
|
|
2301
2467
|
}
|
|
2302
2468
|
// Poll classicBot.yaml for external changes every 30s
|
|
2303
|
-
this.classicReloadTimer = setInterval(
|
|
2304
|
-
|
|
2305
|
-
if (!this.classicChannels)
|
|
2306
|
-
return;
|
|
2307
|
-
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
2308
|
-
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
2309
|
-
const oldBackends = new Map();
|
|
2310
|
-
const oldModels = new Map();
|
|
2311
|
-
const oldAutoPause = new Map();
|
|
2312
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2313
|
-
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
2314
|
-
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
2315
|
-
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
2316
|
-
}
|
|
2317
|
-
if (!this.classicChannels.checkReload())
|
|
2318
|
-
return;
|
|
2319
|
-
// A reload can introduce a bad id (hand edit) or clear one; the
|
|
2320
|
-
// throttle keeps a repeated report from flooding the topic.
|
|
2321
|
-
this.reportClassicUnrecoverableIds();
|
|
2322
|
-
this.reregisterClassicChannels();
|
|
2323
|
-
for (const ch of this.classicChannels.getAll()) {
|
|
2324
|
-
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
2325
|
-
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
2326
|
-
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
2327
|
-
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
2328
|
-
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
2329
|
-
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
2330
|
-
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
2331
|
-
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
2332
|
-
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
2333
|
-
// Small delay to let tmux window clean up
|
|
2334
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
2335
|
-
// The manager already holds the new backend/model/auto-pause; the
|
|
2336
|
-
// unattended helper reads them from it and schedules the delayed
|
|
2337
|
-
// retry on failure like every other unattended start.
|
|
2338
|
-
await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
|
|
2339
|
-
}
|
|
2340
|
-
}
|
|
2341
|
-
}
|
|
2342
|
-
catch (err) {
|
|
2343
|
-
this.logger.warn({ err }, "classicBot.yaml reload error");
|
|
2344
|
-
}
|
|
2469
|
+
this.classicReloadTimer = setInterval(() => {
|
|
2470
|
+
void this.reloadClassicConfigFromDisk();
|
|
2345
2471
|
}, 30_000);
|
|
2346
2472
|
const costGuardConfig = {
|
|
2347
2473
|
...DEFAULT_COST_GUARD,
|
|
@@ -2369,10 +2495,7 @@ export class FleetManager {
|
|
|
2369
2495
|
...fleet.defaults.daily_summary,
|
|
2370
2496
|
};
|
|
2371
2497
|
this.dailySummary = new DailySummary(summaryConfig, costGuardConfig.timezone, (text) => {
|
|
2372
|
-
|
|
2373
|
-
return;
|
|
2374
|
-
this.adapter.sendText(String(this.fleetConfig.channel.group_id), text)
|
|
2375
|
-
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
2498
|
+
this.postDailySummary(text);
|
|
2376
2499
|
// Rotate classic channel chat logs daily
|
|
2377
2500
|
this.classicChannels?.rotateLogs();
|
|
2378
2501
|
this.rotateInboxes();
|
|
@@ -2970,10 +3093,11 @@ export class FleetManager {
|
|
|
2970
3093
|
accessManager,
|
|
2971
3094
|
inboxDir,
|
|
2972
3095
|
});
|
|
2973
|
-
const
|
|
3096
|
+
const adapter = this.adapter;
|
|
3097
|
+
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2974
3098
|
this.worlds.set(adapterId, world);
|
|
2975
|
-
this.adapters.set(adapterId,
|
|
2976
|
-
this.bindAdapterHealth(
|
|
3099
|
+
this.adapters.set(adapterId, adapter);
|
|
3100
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2977
3101
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2978
3102
|
await this.handleInboundMessage(msg);
|
|
2979
3103
|
}, this.logger, "adapter.message"));
|
|
@@ -3018,12 +3142,7 @@ export class FleetManager {
|
|
|
3018
3142
|
return;
|
|
3019
3143
|
}
|
|
3020
3144
|
}, this.logger, "adapter.callback_query"));
|
|
3021
|
-
this.adapter.
|
|
3022
|
-
// Skip unbind if we archived this topic ourselves
|
|
3023
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
3024
|
-
return;
|
|
3025
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
3026
|
-
}, this.logger, "adapter.topic_closed"));
|
|
3145
|
+
this.bindTopicClosedHandler(adapter, adapterId, "adapter.topic_closed");
|
|
3027
3146
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
3028
3147
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
3029
3148
|
if (data.command === "start") {
|
|
@@ -3414,11 +3533,7 @@ export class FleetManager {
|
|
|
3414
3533
|
return;
|
|
3415
3534
|
}
|
|
3416
3535
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
3417
|
-
|
|
3418
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
3419
|
-
return;
|
|
3420
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
3421
|
-
}, this.logger, `adapter[${adapterId}].topic_closed`));
|
|
3536
|
+
this.bindTopicClosedHandler(adapter, adapterId, `adapter[${adapterId}].topic_closed`);
|
|
3422
3537
|
// Slash commands: classic bot + admin commands
|
|
3423
3538
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
3424
3539
|
if (data.command === "start") {
|
|
@@ -4777,7 +4892,9 @@ export class FleetManager {
|
|
|
4777
4892
|
// The adapter resolves only after the platform POST returns. Keep
|
|
4778
4893
|
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4779
4894
|
// a routed-but-failed reply is not a delivered reply.
|
|
4780
|
-
|
|
4895
|
+
// `statusOnly` is daemon-owned envelope metadata, not an MCP argument:
|
|
4896
|
+
// an agent cannot invent it to suppress the normal completion marker.
|
|
4897
|
+
if (!error && result != null && msg.statusOnly !== true) {
|
|
4781
4898
|
try {
|
|
4782
4899
|
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4783
4900
|
}
|
|
@@ -4883,12 +5000,25 @@ export class FleetManager {
|
|
|
4883
5000
|
// the target instance's configured world so replies use its persona
|
|
4884
5001
|
// after a fresh start instead of falling back to channels[0].
|
|
4885
5002
|
const adapterId = this.getInstanceAdapterId(target);
|
|
5003
|
+
// ...but the stored reply coordinates belong to the chat the schedule
|
|
5004
|
+
// was CREATED in, which is not always the target's world. Pairing them
|
|
5005
|
+
// with the target's adapter hands one platform's ids to another's API:
|
|
5006
|
+
// a Telegram group + forum topic delivered to a Discord instance made
|
|
5007
|
+
// every reply on that turn fetch `/channels/<telegram topic>` and fail
|
|
5008
|
+
// with Unknown Channel, and because the seeding repeats on each trigger
|
|
5009
|
+
// it stayed broken until a real inbound overwrote the context (#752).
|
|
5010
|
+
const foreignWorld = this.scheduleChatWorldMismatch(target, reply_chat_id);
|
|
5011
|
+
if (foreignWorld) {
|
|
5012
|
+
this.logger.warn({ scheduleId: id, target, foreignWorld, targetWorld: adapterId, chatId: reply_chat_id }, "Schedule reply target belongs to another channel world — not seeding chat context; the instance keeps its own last known chat");
|
|
5013
|
+
}
|
|
4886
5014
|
await this.deliverToInstance(target, {
|
|
4887
5015
|
type: "fleet_schedule_trigger",
|
|
4888
5016
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
4889
5017
|
meta: {
|
|
4890
|
-
|
|
4891
|
-
|
|
5018
|
+
// Omitted on a mismatch: the daemon then keeps its persisted
|
|
5019
|
+
// last-chat, which is in the right world, instead of being
|
|
5020
|
+
// overwritten with coordinates the target's adapter cannot address.
|
|
5021
|
+
...(foreignWorld ? {} : { chat_id: reply_chat_id, thread_id: reply_thread_id }),
|
|
4892
5022
|
user: "scheduler",
|
|
4893
5023
|
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4894
5024
|
},
|
|
@@ -4920,19 +5050,69 @@ export class FleetManager {
|
|
|
4920
5050
|
this.scheduler.recordRun(id, "instance_offline", `retry ${retryCount}x failed`);
|
|
4921
5051
|
this.notifyScheduleFailure(schedule);
|
|
4922
5052
|
}
|
|
5053
|
+
/**
|
|
5054
|
+
* The adapter that can actually post into a chat, found by its group.
|
|
5055
|
+
*
|
|
5056
|
+
* A schedule records where it was created (reply_chat_id) separately from
|
|
5057
|
+
* what it triggers (target). Those need not share a platform: a Telegram
|
|
5058
|
+
* group can schedule a Discord-topic instance. Picking the adapter from the
|
|
5059
|
+
* target then sends a Telegram chat id through the Discord bot, which fails
|
|
5060
|
+
* with Unknown Channel — the source topic never hears that its schedule ran.
|
|
5061
|
+
*
|
|
5062
|
+
* When several bots share one guild, the primary wins: a persona should not
|
|
5063
|
+
* be the voice announcing fleet scheduling.
|
|
5064
|
+
*/
|
|
5065
|
+
adapterForChat(chatId) {
|
|
5066
|
+
const id = String(chatId);
|
|
5067
|
+
const matches = [...this.worlds.values()].filter(world => String(world.groupId) === id);
|
|
5068
|
+
if (matches.length === 0)
|
|
5069
|
+
return undefined;
|
|
5070
|
+
const primaryId = this.getPrimaryAdapterId();
|
|
5071
|
+
return (matches.find(world => world.id === primaryId) ?? matches[0]).adapter;
|
|
5072
|
+
}
|
|
5073
|
+
/**
|
|
5074
|
+
* The adapter that can answer a schedule in the chat it was created from.
|
|
5075
|
+
*
|
|
5076
|
+
* A schedule records its creator (source) and its trigger (target) separately,
|
|
5077
|
+
* and they need not share a platform — the live fleet has a Telegram group
|
|
5078
|
+
* scheduling a Discord-topic instance. Routing by target sends a Telegram chat
|
|
5079
|
+
* id through the Discord bot, which is one half of the Unknown Channel errors.
|
|
5080
|
+
*
|
|
5081
|
+
* The creator's own adapter comes first, and only when its world actually owns
|
|
5082
|
+
* that chat. Classic keeps its own identity, so a schedule made from a
|
|
5083
|
+
* persona-bound Classic channel is answered by that persona: the primary bot
|
|
5084
|
+
* may not even have access there, and would be the wrong voice if it did.
|
|
5085
|
+
* Falling back to the target's adapter is deliberately NOT an option — that is
|
|
5086
|
+
* the misroute itself; callers say why they stayed silent instead.
|
|
5087
|
+
*/
|
|
5088
|
+
scheduleSourceAdapter(schedule) {
|
|
5089
|
+
const chatId = String(schedule.reply_chat_id);
|
|
5090
|
+
const sourceAdapterId = schedule.source ? this.getInstanceAdapterId(schedule.source) : undefined;
|
|
5091
|
+
const sourceWorld = sourceAdapterId ? this.worlds.get(sourceAdapterId) : undefined;
|
|
5092
|
+
if (sourceWorld && String(sourceWorld.groupId) === chatId)
|
|
5093
|
+
return sourceWorld.adapter;
|
|
5094
|
+
return this.adapterForChat(chatId);
|
|
5095
|
+
}
|
|
4923
5096
|
notifySourceTopic(schedule) {
|
|
4924
|
-
const adapter = this.
|
|
4925
|
-
if (!adapter)
|
|
5097
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5098
|
+
if (!adapter) {
|
|
5099
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — trigger notice not sent");
|
|
4926
5100
|
return;
|
|
5101
|
+
}
|
|
4927
5102
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" triggered, target: ${schedule.target}`;
|
|
4928
5103
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4929
5104
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
4930
5105
|
}).catch((err) => this.logger.error({ err }, "Failed to send cross-instance notification"));
|
|
4931
5106
|
}
|
|
4932
5107
|
notifyScheduleFailure(schedule) {
|
|
4933
|
-
|
|
4934
|
-
|
|
5108
|
+
// Same resolver as the success path: a failure notice was still being sent
|
|
5109
|
+
// through the target's adapter, so a Telegram-created schedule for a
|
|
5110
|
+
// Discord target announced its failure into the wrong platform.
|
|
5111
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5112
|
+
if (!adapter) {
|
|
5113
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — failure notice not sent");
|
|
4935
5114
|
return;
|
|
5115
|
+
}
|
|
4936
5116
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" trigger failed: instance ${schedule.target} is offline.`;
|
|
4937
5117
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4938
5118
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
@@ -5343,26 +5523,261 @@ export class FleetManager {
|
|
|
5343
5523
|
sessionPruneTimer = null;
|
|
5344
5524
|
classicReloadTimer = null;
|
|
5345
5525
|
botUserId;
|
|
5346
|
-
/** Periodically check if bound topics still exist */
|
|
5526
|
+
/** Periodically check if bound topics still exist. Never deletes user data. */
|
|
5347
5527
|
startTopicCleanupPoller() {
|
|
5348
|
-
this.topicCleanupTimer
|
|
5349
|
-
|
|
5528
|
+
if (this.topicCleanupTimer)
|
|
5529
|
+
clearInterval(this.topicCleanupTimer);
|
|
5530
|
+
const generation = ++this.topicCleanupGeneration;
|
|
5531
|
+
this.topicCleanupTimer = setInterval(() => { void this.scheduleTopicCleanup(generation); }, 5 * 60_000);
|
|
5532
|
+
}
|
|
5533
|
+
/** Coalesce timer ticks; an outage must not create overlapping destructive-looking scans. */
|
|
5534
|
+
scheduleTopicCleanup(generation = this.topicCleanupGeneration) {
|
|
5535
|
+
if (this.topicCleanupInFlight)
|
|
5536
|
+
return this.topicCleanupInFlight;
|
|
5537
|
+
const run = this.runTopicCleanup(generation).finally(() => {
|
|
5538
|
+
if (this.topicCleanupInFlight === run)
|
|
5539
|
+
this.topicCleanupInFlight = null;
|
|
5540
|
+
});
|
|
5541
|
+
this.topicCleanupInFlight = run;
|
|
5542
|
+
return run;
|
|
5543
|
+
}
|
|
5544
|
+
confirmedProbeFence(adapterId, adapter) {
|
|
5545
|
+
const health = adapter.getHealthSnapshot?.();
|
|
5546
|
+
if (health) {
|
|
5547
|
+
return health.status === "connected" && health.isReady
|
|
5548
|
+
? { generation: health.generation }
|
|
5549
|
+
: null;
|
|
5550
|
+
}
|
|
5551
|
+
return this.adapterState.get(adapterId)?.status === "connected" ? {} : null;
|
|
5552
|
+
}
|
|
5553
|
+
sameProbeFence(adapterId, adapter, before, result) {
|
|
5554
|
+
const after = this.confirmedProbeFence(adapterId, adapter);
|
|
5555
|
+
if (!after)
|
|
5556
|
+
return false;
|
|
5557
|
+
if (before.generation !== after.generation)
|
|
5558
|
+
return false;
|
|
5559
|
+
return result?.generation === undefined || result.generation === after.generation;
|
|
5560
|
+
}
|
|
5561
|
+
topicProbeStreakKey(threadId, adapterId, reason) {
|
|
5562
|
+
return FleetManager.TOPIC_PROBE_ADAPTER_SCOPED_REASONS.has(reason)
|
|
5563
|
+
? `adapter:${adapterId ?? "unbound"}`
|
|
5564
|
+
: `thread:${threadId}`;
|
|
5565
|
+
}
|
|
5566
|
+
/** A definite answer (present or missing) ends the unknown streak for that route and its adapter. */
|
|
5567
|
+
clearTopicProbeUnknownStreak(threadId, adapterId) {
|
|
5568
|
+
this.topicProbeUnknownStreak.delete(`thread:${threadId}`);
|
|
5569
|
+
this.topicProbeUnknownStreak.delete(`adapter:${adapterId ?? "unbound"}`);
|
|
5570
|
+
}
|
|
5571
|
+
/**
|
|
5572
|
+
* Record one unknown probe result. Nothing here can touch quarantine or
|
|
5573
|
+
* removal: unknown is always retained data. The only question is whether
|
|
5574
|
+
* the operator hears about it, and a single transient (one flaky HTTP call
|
|
5575
|
+
* out of dozens per pass) must not — only a streak does.
|
|
5576
|
+
*
|
|
5577
|
+
* Used directly for single-route events (channelDelete, the pre-action
|
|
5578
|
+
* fence). The periodic scan goes through a TopicProbePass instead, so one
|
|
5579
|
+
* pass over N routes of a dead adapter counts as ONE check, not N.
|
|
5580
|
+
*/
|
|
5581
|
+
warnTopicProbeUnknown(instanceName, threadId, adapterId, reason, detail) {
|
|
5582
|
+
this.noteTopicProbeUnknown(this.topicProbeStreakKey(threadId, adapterId, reason), { instanceName, threadId, adapterId, reason, detail });
|
|
5583
|
+
}
|
|
5584
|
+
/** One scan's worth of probe outcomes, applied to the streaks after the loop. */
|
|
5585
|
+
newTopicProbePass() {
|
|
5586
|
+
return { unknown: new Map(), definite: new Set() };
|
|
5587
|
+
}
|
|
5588
|
+
passTopicProbeUnknown(pass, instanceName, threadId, adapterId, reason, detail) {
|
|
5589
|
+
const key = this.topicProbeStreakKey(threadId, adapterId, reason);
|
|
5590
|
+
// First unknown per key per pass wins; the rest of the routes on a dead
|
|
5591
|
+
// adapter are the same observation, not additional checks.
|
|
5592
|
+
if (!pass.unknown.has(key))
|
|
5593
|
+
pass.unknown.set(key, { instanceName, threadId, adapterId, reason, detail });
|
|
5594
|
+
}
|
|
5595
|
+
passTopicProbeDefinite(pass, threadId, adapterId) {
|
|
5596
|
+
pass.definite.add(`thread:${threadId}`);
|
|
5597
|
+
pass.definite.add(`adapter:${adapterId ?? "unbound"}`);
|
|
5598
|
+
}
|
|
5599
|
+
/**
|
|
5600
|
+
* Apply a pass: a definite answer resets its keys, and an adapter that
|
|
5601
|
+
* answered for any route this pass is evidently alive, so an unknown for the
|
|
5602
|
+
* same adapter key in the same pass does not count — regardless of the order
|
|
5603
|
+
* the routes happened to be probed in.
|
|
5604
|
+
*/
|
|
5605
|
+
applyTopicProbePass(pass) {
|
|
5606
|
+
for (const key of pass.definite)
|
|
5607
|
+
this.topicProbeUnknownStreak.delete(key);
|
|
5608
|
+
for (const [key, ctx] of pass.unknown) {
|
|
5609
|
+
if (pass.definite.has(key))
|
|
5610
|
+
continue;
|
|
5611
|
+
this.noteTopicProbeUnknown(key, ctx);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
noteTopicProbeUnknown(streakKey, { instanceName, threadId, adapterId, reason, detail }) {
|
|
5615
|
+
const streak = (this.topicProbeUnknownStreak.get(streakKey) ?? 0) + 1;
|
|
5616
|
+
this.topicProbeUnknownStreak.set(streakKey, streak);
|
|
5617
|
+
if (streak < FleetManager.TOPIC_PROBE_UNKNOWN_ESCALATION) {
|
|
5618
|
+
this.logger.debug({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence not confirmed this pass — transient, retaining instance and all data");
|
|
5619
|
+
return;
|
|
5620
|
+
}
|
|
5621
|
+
const key = `${adapterId ?? "unbound"}:${reason}`;
|
|
5622
|
+
const now = Date.now();
|
|
5623
|
+
const last = this.topicProbeWarnings.get(key) ?? 0;
|
|
5624
|
+
if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
|
|
5625
|
+
return;
|
|
5626
|
+
this.topicProbeWarnings.set(key, now);
|
|
5627
|
+
this.logger.error({ instanceName, threadId, adapterId, reason, detail, streak }, "Topic presence could not be confirmed repeatedly — retaining instance and all data");
|
|
5628
|
+
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound", streak));
|
|
5629
|
+
}
|
|
5630
|
+
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
5631
|
+
async runTopicCleanup(generation) {
|
|
5632
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5633
|
+
return;
|
|
5634
|
+
const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
|
|
5635
|
+
const missing = [];
|
|
5636
|
+
const pass = this.newTopicProbePass();
|
|
5637
|
+
const skippedOnDemand = new Set();
|
|
5638
|
+
for (const [threadId, target] of snapshot) {
|
|
5639
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5350
5640
|
return;
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5354
|
-
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5358
|
-
|
|
5359
|
-
|
|
5641
|
+
// The route may have been replaced while an earlier probe was in flight.
|
|
5642
|
+
if (this.routing.resolve(threadId) !== target)
|
|
5643
|
+
continue;
|
|
5644
|
+
const adapterId = this.getInstanceAdapterId(target.name);
|
|
5645
|
+
const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
|
|
5646
|
+
if (!adapterId || !adapter?.probeTopicPresence) {
|
|
5647
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-unavailable");
|
|
5648
|
+
continue;
|
|
5649
|
+
}
|
|
5650
|
+
// An on-demand adapter's probe is reserved for confirming a delivery
|
|
5651
|
+
// failure hint (handleProviderTopicClosed); the periodic scan leaves its
|
|
5652
|
+
// routes alone — no probe, no unknown, no streak, no notice.
|
|
5653
|
+
if (adapter.topicProbePolicy?.() === "on-demand") {
|
|
5654
|
+
if (!skippedOnDemand.has(adapterId)) {
|
|
5655
|
+
skippedOnDemand.add(adapterId);
|
|
5656
|
+
this.logger.debug({ adapterId }, "Topic scan skipping on-demand adapter — presence is confirmed only on delivery failure");
|
|
5360
5657
|
}
|
|
5361
|
-
|
|
5362
|
-
|
|
5658
|
+
continue;
|
|
5659
|
+
}
|
|
5660
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5661
|
+
if (!before) {
|
|
5662
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-not-ready");
|
|
5663
|
+
continue;
|
|
5664
|
+
}
|
|
5665
|
+
let result;
|
|
5666
|
+
try {
|
|
5667
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5668
|
+
}
|
|
5669
|
+
catch {
|
|
5670
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5671
|
+
}
|
|
5672
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5673
|
+
return;
|
|
5674
|
+
if (!this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5675
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, "owner-adapter-generation-changed");
|
|
5676
|
+
continue;
|
|
5677
|
+
}
|
|
5678
|
+
if (result.status === "unknown") {
|
|
5679
|
+
this.passTopicProbeUnknown(pass, target.name, threadId, adapterId, result.reason, result.detail);
|
|
5680
|
+
}
|
|
5681
|
+
else {
|
|
5682
|
+
this.passTopicProbeDefinite(pass, threadId, adapterId);
|
|
5683
|
+
if (result.status === "missing") {
|
|
5684
|
+
missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
|
|
5363
5685
|
}
|
|
5364
5686
|
}
|
|
5365
|
-
}
|
|
5687
|
+
}
|
|
5688
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5689
|
+
return;
|
|
5690
|
+
// One pass, one check: streaks move by at most one per key here.
|
|
5691
|
+
this.applyTopicProbePass(pass);
|
|
5692
|
+
if (missing.length === 0)
|
|
5693
|
+
return;
|
|
5694
|
+
if (missing.length > 1) {
|
|
5695
|
+
this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
|
|
5696
|
+
this.notifyFleetError(t("fleet.topic_probe_bulk", missing.length));
|
|
5697
|
+
return;
|
|
5698
|
+
}
|
|
5699
|
+
const item = missing[0];
|
|
5700
|
+
if (this.routing.resolve(item.threadId) !== item.target
|
|
5701
|
+
|| this.getInstanceAdapterId(item.target.name) !== item.adapterId
|
|
5702
|
+
|| this.adapters.get(item.adapterId) !== item.adapter)
|
|
5703
|
+
return;
|
|
5704
|
+
const current = this.confirmedProbeFence(item.adapterId, item.adapter);
|
|
5705
|
+
if (!current || (item.generation !== undefined && current.generation !== item.generation)) {
|
|
5706
|
+
this.warnTopicProbeUnknown(item.target.name, item.threadId, item.adapterId, "owner-adapter-changed-before-action");
|
|
5707
|
+
return;
|
|
5708
|
+
}
|
|
5709
|
+
this.topicCommands.handleTopicDeleted(item.threadId, {
|
|
5710
|
+
source: "provider-probe",
|
|
5711
|
+
adapterId: item.adapterId,
|
|
5712
|
+
generation: item.generation,
|
|
5713
|
+
});
|
|
5714
|
+
}
|
|
5715
|
+
/**
|
|
5716
|
+
* A gateway event is only a hint. Confirm it through the passive REST probe;
|
|
5717
|
+
* reconnecting gateways have emitted false channelDelete events in practice.
|
|
5718
|
+
*/
|
|
5719
|
+
async handleProviderTopicClosed(threadId, adapterId, adapter) {
|
|
5720
|
+
const target = this.routing.resolve(threadId);
|
|
5721
|
+
if (!target || !isProbeableRouteTarget(target))
|
|
5722
|
+
return;
|
|
5723
|
+
if (this.getInstanceAdapterId(target.name) !== adapterId)
|
|
5724
|
+
return;
|
|
5725
|
+
if (this.adapters.get(adapterId) !== adapter)
|
|
5726
|
+
return;
|
|
5727
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5728
|
+
if (!before || !adapter.probeTopicPresence) {
|
|
5729
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-from-unready-adapter");
|
|
5730
|
+
return;
|
|
5731
|
+
}
|
|
5732
|
+
let result;
|
|
5733
|
+
try {
|
|
5734
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5735
|
+
}
|
|
5736
|
+
catch {
|
|
5737
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5738
|
+
}
|
|
5739
|
+
if (this.routing.resolve(threadId) !== target
|
|
5740
|
+
|| this.getInstanceAdapterId(target.name) !== adapterId
|
|
5741
|
+
|| this.adapters.get(adapterId) !== adapter
|
|
5742
|
+
|| !this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5743
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
|
|
5744
|
+
return;
|
|
5745
|
+
}
|
|
5746
|
+
if (result.status === "unknown") {
|
|
5747
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason, result.detail);
|
|
5748
|
+
return;
|
|
5749
|
+
}
|
|
5750
|
+
this.clearTopicProbeUnknownStreak(threadId, adapterId);
|
|
5751
|
+
if (result.status !== "missing") {
|
|
5752
|
+
// The gateway said deleted, REST says present: a definite answer, so it
|
|
5753
|
+
// is not an unknown streak — but it is worth one debug line.
|
|
5754
|
+
this.logger.debug({ instanceName: target.name, threadId, adapterId }, "channelDelete hint contradicted by REST — topic present, nothing to do");
|
|
5755
|
+
return;
|
|
5756
|
+
}
|
|
5757
|
+
this.topicCommands.handleTopicDeleted(threadId, {
|
|
5758
|
+
source: "provider-event",
|
|
5759
|
+
adapterId,
|
|
5760
|
+
generation: result.generation,
|
|
5761
|
+
});
|
|
5762
|
+
}
|
|
5763
|
+
bindTopicClosedHandler(adapter, adapterId, label) {
|
|
5764
|
+
adapter.on("topic_closed", safeHandler(async (data) => {
|
|
5765
|
+
if (this.topicArchiver.isArchived(data.threadId))
|
|
5766
|
+
return;
|
|
5767
|
+
await this.handleProviderTopicClosed(data.threadId, adapterId, adapter);
|
|
5768
|
+
}, this.logger, label));
|
|
5769
|
+
}
|
|
5770
|
+
/**
|
|
5771
|
+
* Remove only the volatile route. The instance config, daemon, schedules,
|
|
5772
|
+
* teams, metadata directory, and working tree remain untouched until an
|
|
5773
|
+
* authenticated explicit deletion is requested.
|
|
5774
|
+
*/
|
|
5775
|
+
quarantineMissingTopic(threadId, target, evidence) {
|
|
5776
|
+
if (!isProbeableRouteTarget(target) || this.routing.resolve(threadId) !== target)
|
|
5777
|
+
return;
|
|
5778
|
+
this.routing.unregister(threadId);
|
|
5779
|
+
this.logger.error({ instanceName: target.name, threadId, ...evidence }, "Topic is confirmed missing — route quarantined; instance configuration and user data were retained");
|
|
5780
|
+
this.notifyFleetError(t("fleet.topic_quarantined", target.name, threadId));
|
|
5366
5781
|
}
|
|
5367
5782
|
/**
|
|
5368
5783
|
* Patch only values changed in the effective config into the original YAML
|
|
@@ -5515,7 +5930,8 @@ export class FleetManager {
|
|
|
5515
5930
|
}
|
|
5516
5931
|
}
|
|
5517
5932
|
}
|
|
5518
|
-
async removeInstance(name) {
|
|
5933
|
+
async removeInstance(name, authorization) {
|
|
5934
|
+
assertExplicitInstanceRemoval(authorization);
|
|
5519
5935
|
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
5520
5936
|
// else evicted deleted entries, so it grew for the life of the process.
|
|
5521
5937
|
forgetInstanceContext(name);
|
|
@@ -5541,7 +5957,7 @@ export class FleetManager {
|
|
|
5541
5957
|
}
|
|
5542
5958
|
}
|
|
5543
5959
|
}
|
|
5544
|
-
await this.lifecycle.remove(name);
|
|
5960
|
+
await this.lifecycle.remove(name, authorization);
|
|
5545
5961
|
// Clean up per-instance tracking maps so they don't grow unbounded
|
|
5546
5962
|
// as instances are created and deleted over the lifetime of the fleet.
|
|
5547
5963
|
this.lastActivity.delete(name);
|
|
@@ -5778,6 +6194,65 @@ export class FleetManager {
|
|
|
5778
6194
|
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5779
6195
|
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5780
6196
|
}
|
|
6197
|
+
/**
|
|
6198
|
+
* Where a fleet-wide notice can actually be posted, or null if nowhere.
|
|
6199
|
+
*
|
|
6200
|
+
* On Telegram a group id is itself a chat, so posting straight to it is
|
|
6201
|
+
* right. On Discord it is a *guild* id, and sending there makes the adapter
|
|
6202
|
+
* fetch a channel that does not exist — DiscordAPIError 10003 Unknown
|
|
6203
|
+
* Channel. That is why the daily summary never arrived on a Discord fleet:
|
|
6204
|
+
* it had been posting to the guild every night and only the catch handler
|
|
6205
|
+
* ever saw it.
|
|
6206
|
+
*
|
|
6207
|
+
* Discord therefore needs a real channel: the General topic (resolved from
|
|
6208
|
+
* config rather than findGeneralInstance, so a fleet-level fault can still be
|
|
6209
|
+
* reported while the General daemon is down), else the adapter's configured
|
|
6210
|
+
* general_channel_id. With neither, there is no safe target and the caller
|
|
6211
|
+
* should say so rather than send into a guaranteed failure.
|
|
6212
|
+
*/
|
|
6213
|
+
fleetNoticeTarget(adapterId) {
|
|
6214
|
+
const cfg = this.getChannelConfig(adapterId);
|
|
6215
|
+
const groupId = cfg?.group_id;
|
|
6216
|
+
if (groupId == null)
|
|
6217
|
+
return null;
|
|
6218
|
+
const chatId = String(groupId);
|
|
6219
|
+
// The General must belong to the SAME adapter as the group above. Taking
|
|
6220
|
+
// whichever General comes first in the instance map produced a mixed target
|
|
6221
|
+
// on a dual-platform fleet — a Telegram group id carrying a Discord channel
|
|
6222
|
+
// as its thread — and made the result depend on map insertion order.
|
|
6223
|
+
// Resolved from config, not from a live daemon, so a fleet-level fault is
|
|
6224
|
+
// still reportable while the General itself is down.
|
|
6225
|
+
const ownerId = cfg?.id ?? cfg?.type;
|
|
6226
|
+
const generalTopic = Object.entries(this.fleetConfig?.instances ?? {})
|
|
6227
|
+
.find(([name, instance]) => instance.general_topic === true
|
|
6228
|
+
&& this.getInstanceAdapterId(name) === ownerId)?.[1]?.topic_id;
|
|
6229
|
+
if (generalTopic != null)
|
|
6230
|
+
return { chatId, opts: { threadId: String(generalTopic) } };
|
|
6231
|
+
if (cfg?.type === "discord") {
|
|
6232
|
+
const configured = cfg.options?.general_channel_id;
|
|
6233
|
+
if (configured != null && String(configured)) {
|
|
6234
|
+
return { chatId, opts: { threadId: String(configured) } };
|
|
6235
|
+
}
|
|
6236
|
+
return null; // a guild id is not a channel; sending would always fail
|
|
6237
|
+
}
|
|
6238
|
+
return { chatId, opts: {} };
|
|
6239
|
+
}
|
|
6240
|
+
/**
|
|
6241
|
+
* Post the daily summary where fleet-wide notices go.
|
|
6242
|
+
*
|
|
6243
|
+
* A named method rather than an inline closure so a test can drive the real
|
|
6244
|
+
* thing: asserting on fleetNoticeTarget alone leaves the call site free to go
|
|
6245
|
+
* back to posting at the bare group id, which is the defect this replaced.
|
|
6246
|
+
*/
|
|
6247
|
+
postDailySummary(text) {
|
|
6248
|
+
const target = this.fleetNoticeTarget();
|
|
6249
|
+
if (!this.adapter || !target) {
|
|
6250
|
+
this.logger.warn("Daily summary has no postable target — set a General topic or channel.options.general_channel_id");
|
|
6251
|
+
return;
|
|
6252
|
+
}
|
|
6253
|
+
this.adapter.sendText(target.chatId, text, target.opts)
|
|
6254
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
6255
|
+
}
|
|
5781
6256
|
notifyFleetError(text) {
|
|
5782
6257
|
const now = Date.now();
|
|
5783
6258
|
const key = text.slice(0, 200);
|
|
@@ -5804,10 +6279,12 @@ export class FleetManager {
|
|
|
5804
6279
|
dispatched = this.notifyInstanceTopic(general, body);
|
|
5805
6280
|
}
|
|
5806
6281
|
else {
|
|
5807
|
-
// No General instance — fall back to the primary channel's
|
|
5808
|
-
|
|
5809
|
-
|
|
5810
|
-
|
|
6282
|
+
// No General instance — fall back to the primary channel's own notice
|
|
6283
|
+
// target. Posting to the bare group id looked right but is a guild id on
|
|
6284
|
+
// Discord, so every such fallback failed inside the catch handler.
|
|
6285
|
+
const target = this.fleetNoticeTarget();
|
|
6286
|
+
if (this.adapter && target) {
|
|
6287
|
+
this.adapter.sendText(target.chatId, body, target.opts)
|
|
5811
6288
|
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5812
6289
|
dispatched = true;
|
|
5813
6290
|
}
|
|
@@ -5866,13 +6343,16 @@ export class FleetManager {
|
|
|
5866
6343
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
5867
6344
|
return true;
|
|
5868
6345
|
}
|
|
5869
|
-
// Fallback:
|
|
5870
|
-
|
|
5871
|
-
|
|
6346
|
+
// Fallback: the instance has neither a topic nor a classic channel, so post
|
|
6347
|
+
// where fleet-wide notices go. Not the bare group id: on Discord that is a
|
|
6348
|
+
// guild, and the send fails inside the catch handler.
|
|
6349
|
+
const target = this.fleetNoticeTarget(this.getInstanceAdapterId(instanceName));
|
|
6350
|
+
if (target) {
|
|
6351
|
+
adapter.sendText(target.chatId, text, { ...target.opts, ...extraOpts })
|
|
5872
6352
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5873
6353
|
return true;
|
|
5874
6354
|
}
|
|
5875
|
-
this.logger.warn({ instanceName }, "No
|
|
6355
|
+
this.logger.warn({ instanceName }, "No postable target — instance topic notification not sent");
|
|
5876
6356
|
return false;
|
|
5877
6357
|
}
|
|
5878
6358
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
@@ -6433,6 +6913,41 @@ export class FleetManager {
|
|
|
6433
6913
|
extra: { generalName, promptKind: kind },
|
|
6434
6914
|
});
|
|
6435
6915
|
}
|
|
6916
|
+
/**
|
|
6917
|
+
* Offer a one-tap re-login next to an auth alert.
|
|
6918
|
+
*
|
|
6919
|
+
* The alert already names the remedy in words (`/login <backend>`), which
|
|
6920
|
+
* still leaves the user to retype it somewhere. The button routes into the
|
|
6921
|
+
* same chooser `/login` uses, so pressing it starts the flow in place.
|
|
6922
|
+
*
|
|
6923
|
+
* Backends with no remote login flow (opencode logs in from a terminal) get
|
|
6924
|
+
* no button — the alert's own wording already tells them what to run.
|
|
6925
|
+
*/
|
|
6926
|
+
async offerBackendLogin(targetInstance, backend) {
|
|
6927
|
+
if (!LOGIN_FLOWS[backend])
|
|
6928
|
+
return;
|
|
6929
|
+
const adapterId = this.getInstanceAdapterId(targetInstance);
|
|
6930
|
+
const adapter = this.getAdapterForInstance(targetInstance);
|
|
6931
|
+
const chatId = this.getGroupIdForInstance(targetInstance);
|
|
6932
|
+
const topicId = this.fleetConfig?.instances[targetInstance]?.topic_id;
|
|
6933
|
+
const threadId = topicId != null ? String(topicId) : undefined;
|
|
6934
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6935
|
+
this.logger.warn({ targetInstance, backend, adapterId, chatId }, "Cannot address the re-login button — the alert text still names the command");
|
|
6936
|
+
return;
|
|
6937
|
+
}
|
|
6938
|
+
await this.postNonceButtonPrompt({
|
|
6939
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6940
|
+
alertType: "login",
|
|
6941
|
+
instanceName: "login",
|
|
6942
|
+
adapter,
|
|
6943
|
+
adapterId,
|
|
6944
|
+
chatId,
|
|
6945
|
+
threadId,
|
|
6946
|
+
message: t("login.offer", backend),
|
|
6947
|
+
choices: [{ action: backend, label: t("login.offer_action", backend) }],
|
|
6948
|
+
expiredText: t("buttons.stale"),
|
|
6949
|
+
});
|
|
6950
|
+
}
|
|
6436
6951
|
/** Consume a General assist button exactly once. */
|
|
6437
6952
|
async handleInteractivePromptAssist(data, callbackAdapterId, receivingAdapter) {
|
|
6438
6953
|
const claimed = this.consumeNonceCallback(INTERACTIVE_ASSIST_CALLBACK_PREFIX, /^interactive-assist:([0-9a-f]+):(confirm|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
@@ -7421,11 +7936,15 @@ export class FleetManager {
|
|
|
7421
7936
|
if (cleanupFailed) {
|
|
7422
7937
|
await chat.adapter.sendText(chat.chatId, t("login.web_cleanup_failed", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7423
7938
|
}
|
|
7939
|
+
const send = (text) => chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
7424
7940
|
let text;
|
|
7425
7941
|
if (ok) {
|
|
7426
|
-
|
|
7427
|
-
|
|
7428
|
-
|
|
7942
|
+
// Same order as the web-login path: the login result goes out first,
|
|
7943
|
+
// then the recovery reports its own outcome. Waiting for recovery to
|
|
7944
|
+
// build this message is what made a successful login look hung.
|
|
7945
|
+
await send(t("login.completed", backend));
|
|
7946
|
+
await announcePostLoginRecovery(backend, () => this.recoverBackendInstances(backend), send);
|
|
7947
|
+
return;
|
|
7429
7948
|
}
|
|
7430
7949
|
else if (detail === "cancelled") {
|
|
7431
7950
|
// The cancel command's own reply already announced this — a second
|
|
@@ -7549,31 +8068,53 @@ export class FleetManager {
|
|
|
7549
8068
|
* only re-reads credentials on process start (a paused instance's CLI is
|
|
7550
8069
|
* already dead, so waking it respawns with the new token for free).
|
|
7551
8070
|
*/
|
|
7552
|
-
|
|
8071
|
+
/**
|
|
8072
|
+
* Wake/restart every instance of a backend after a successful re-login.
|
|
8073
|
+
*
|
|
8074
|
+
* Bounded by a wall-clock deadline. This used to be an unbounded sequential
|
|
8075
|
+
* loop, and the caller only built its "login completed" message AFTER it
|
|
8076
|
+
* returned — so with several instances (or one slow restart) the user was
|
|
8077
|
+
* told nothing at all for minutes, concluded the login had hung, and
|
|
8078
|
+
* restarted things by hand. Instances that do not finish in time are NOT
|
|
8079
|
+
* cancelled: they are still coming back, and are reported as pending so the
|
|
8080
|
+
* message can say so instead of implying failure.
|
|
8081
|
+
*/
|
|
8082
|
+
async recoverBackendInstances(backend, deadlineMs = POST_LOGIN_RECOVERY_DEADLINE_MS) {
|
|
7553
8083
|
const woken = [];
|
|
7554
8084
|
const restarted = [];
|
|
8085
|
+
const pending = [];
|
|
8086
|
+
const deadline = Date.now() + deadlineMs;
|
|
7555
8087
|
for (const name of this.configuredBackendInstanceNames()) {
|
|
7556
8088
|
if (this.backendNameOf(name) !== backend)
|
|
7557
8089
|
continue;
|
|
7558
8090
|
const status = this.getInstanceStatus(name);
|
|
7559
|
-
|
|
8091
|
+
if (status !== "paused" && status !== "running")
|
|
8092
|
+
continue;
|
|
8093
|
+
const result = await runBeforeDeadline(async () => {
|
|
7560
8094
|
if (status === "paused") {
|
|
7561
8095
|
if (this.daemons.has(name))
|
|
7562
8096
|
await this.lifecycle.wake(name, 30_000);
|
|
7563
8097
|
else
|
|
7564
8098
|
await this.startPersistedPausedInstance(name);
|
|
7565
|
-
woken.push(name);
|
|
7566
8099
|
}
|
|
7567
|
-
else
|
|
8100
|
+
else {
|
|
7568
8101
|
await this.restartSingleInstance(name);
|
|
7569
|
-
restarted.push(name);
|
|
7570
8102
|
}
|
|
8103
|
+
}, deadline);
|
|
8104
|
+
if (result.status === "fulfilled") {
|
|
8105
|
+
(status === "paused" ? woken : restarted).push(name);
|
|
7571
8106
|
}
|
|
7572
|
-
|
|
7573
|
-
|
|
8107
|
+
else if (result.status === "timeout") {
|
|
8108
|
+
// Out of time: record it and stop waiting, but keep walking the list —
|
|
8109
|
+
// the remaining instances are checked against the same deadline and
|
|
8110
|
+
// fall straight through, so the caller still learns about all of them.
|
|
8111
|
+
pending.push(name);
|
|
8112
|
+
}
|
|
8113
|
+
else {
|
|
8114
|
+
this.logger.warn({ err: result.reason?.message, name, status }, "Post-login recovery failed");
|
|
7574
8115
|
}
|
|
7575
8116
|
}
|
|
7576
|
-
return { woken, restarted };
|
|
8117
|
+
return { woken, restarted, pending };
|
|
7577
8118
|
}
|
|
7578
8119
|
/** Backend chooser button → start that backend's login session. */
|
|
7579
8120
|
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
@@ -7985,7 +8526,7 @@ You CAN write code snippets, explain code, and answer technical questions direct
|
|
|
7985
8526
|
- **Delegate to 1 instance**: scoped to one project/repo, needs file access or execution.
|
|
7986
8527
|
- **Coordinate multiple**: spans repos, outputs feed each other, or parallel helps (max 3 per task).
|
|
7987
8528
|
|
|
7988
|
-
Instance discovery
|
|
8529
|
+
Instance discovery: start with list_instances(); follow the guidance in its response. Prefer reuse; never duplicate a running instance.
|
|
7989
8530
|
|
|
7990
8531
|
## Reply Contract
|
|
7991
8532
|
|
|
@@ -9613,11 +10154,19 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9613
10154
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
9614
10155
|
ensureWorkspaceGit(workDir);
|
|
9615
10156
|
const classicIdentity = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
|
|
10157
|
+
const toolProgress = classicIdentity
|
|
10158
|
+
? this.classicChannels?.getToolProgress(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.tool_progress)
|
|
10159
|
+
: this.fleetConfig?.defaults?.tool_progress;
|
|
10160
|
+
const replyCompletionGuard = classicIdentity
|
|
10161
|
+
? this.classicChannels?.getReplyCompletionGuard(classicIdentity.channelId, classicIdentity.adapterId, this.fleetConfig?.defaults?.reply_completion_guard)
|
|
10162
|
+
: this.fleetConfig?.defaults?.reply_completion_guard;
|
|
9616
10163
|
const config = {
|
|
9617
10164
|
...DEFAULT_INSTANCE_CONFIG,
|
|
9618
10165
|
...this.fleetConfig?.defaults,
|
|
9619
10166
|
working_directory: workDir,
|
|
9620
10167
|
lightweight: true,
|
|
10168
|
+
tool_progress: toolProgress ?? "off",
|
|
10169
|
+
reply_completion_guard: replyCompletionGuard ?? true,
|
|
9621
10170
|
...(backend ? { backend } : {}),
|
|
9622
10171
|
...(model ? { model } : {}),
|
|
9623
10172
|
...(classicIdentity?.displayName ? { display_name: classicIdentity.displayName } : {}),
|
|
@@ -9757,6 +10306,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9757
10306
|
clearInterval(this.topicCleanupTimer);
|
|
9758
10307
|
this.topicCleanupTimer = null;
|
|
9759
10308
|
}
|
|
10309
|
+
this.topicCleanupGeneration++;
|
|
9760
10310
|
if (this.sessionPruneTimer) {
|
|
9761
10311
|
clearInterval(this.sessionPruneTimer);
|
|
9762
10312
|
this.sessionPruneTimer = null;
|
|
@@ -9899,10 +10449,17 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9899
10449
|
// Keep that single message; the new process will adopt and finish it.
|
|
9900
10450
|
setUpdateProgressStage(this.dataDir, "stopping");
|
|
9901
10451
|
}
|
|
9902
|
-
const
|
|
9903
|
-
if (!trackedFullRestart &&
|
|
9904
|
-
|
|
9905
|
-
|
|
10452
|
+
const restartTarget = this.fleetNoticeTarget();
|
|
10453
|
+
if (!trackedFullRestart && this.adapter) {
|
|
10454
|
+
if (restartTarget) {
|
|
10455
|
+
await this.adapter.sendText(restartTarget.chatId, t("restart.full_initiated"), restartTarget.opts)
|
|
10456
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
|
|
10457
|
+
}
|
|
10458
|
+
else {
|
|
10459
|
+
// Say why nothing was posted. A restart that announces itself nowhere,
|
|
10460
|
+
// for a reason nobody logged, is the harder version of this bug.
|
|
10461
|
+
this.logger.warn("Full restart notice has no postable target — set a General topic or channel.options.general_channel_id");
|
|
10462
|
+
}
|
|
9906
10463
|
}
|
|
9907
10464
|
// Wait for idle with 5-minute timeout
|
|
9908
10465
|
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
|
|
@@ -10002,6 +10559,30 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10002
10559
|
this.notifyFleetError(t("fleet.reload_rejected", why));
|
|
10003
10560
|
return;
|
|
10004
10561
|
}
|
|
10562
|
+
// Classic behavior settings share the fleet defaults but are not entries
|
|
10563
|
+
// in fleet.yaml. Snapshot the old effective chain before reloading the
|
|
10564
|
+
// Classic file so SIGHUP can hot-apply either source without waiting for
|
|
10565
|
+
// the 30-second Classic poller.
|
|
10566
|
+
const oldClassicBehavior = new Map();
|
|
10567
|
+
if (this.classicChannels) {
|
|
10568
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10569
|
+
const runtimeConfig = this.daemons.get(ch.instanceName)?.getConfigSnapshot?.();
|
|
10570
|
+
oldClassicBehavior.set(ch.instanceName, {
|
|
10571
|
+
backend: this.classicChannels.getBackend(ch.channelId, ch.adapterId, oldConfig?.defaults?.backend),
|
|
10572
|
+
model: this.classicChannels.getModel(ch.channelId, ch.adapterId, oldConfig?.defaults?.model),
|
|
10573
|
+
autoPauseAfter: this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, oldConfig?.defaults?.auto_pause_after),
|
|
10574
|
+
// Settings mutates FleetManager's in-memory defaults before SIGHUP.
|
|
10575
|
+
// The live daemon is therefore the authority for the previous hot
|
|
10576
|
+
// values, exactly as in the fleet-topic reconciliation below.
|
|
10577
|
+
toolProgress: runtimeConfig?.tool_progress
|
|
10578
|
+
?? this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, oldConfig?.defaults?.tool_progress),
|
|
10579
|
+
replyCompletionGuard: runtimeConfig?.reply_completion_guard
|
|
10580
|
+
?? this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, oldConfig?.defaults?.reply_completion_guard),
|
|
10581
|
+
});
|
|
10582
|
+
}
|
|
10583
|
+
if (this.classicChannels.checkReload())
|
|
10584
|
+
this.reportClassicUnrecoverableIds();
|
|
10585
|
+
}
|
|
10005
10586
|
this.routing.rebuild(this.fleetConfig);
|
|
10006
10587
|
this.reregisterClassicChannels();
|
|
10007
10588
|
this.scheduler?.reload();
|
|
@@ -10066,6 +10647,35 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10066
10647
|
}
|
|
10067
10648
|
}
|
|
10068
10649
|
}
|
|
10650
|
+
// A Classic channel inherits fleet defaults beneath its own two levels.
|
|
10651
|
+
// Recompute that complete chain on SIGHUP. Only the two behavior switches
|
|
10652
|
+
// are hot; changes to backend/model/auto-pause retain the existing restart
|
|
10653
|
+
// semantics.
|
|
10654
|
+
if (this.classicChannels) {
|
|
10655
|
+
for (const ch of this.classicChannels.getAll()) {
|
|
10656
|
+
const old = oldClassicBehavior.get(ch.instanceName);
|
|
10657
|
+
if (!old || !this.daemons.has(ch.instanceName))
|
|
10658
|
+
continue;
|
|
10659
|
+
const backend = this.classicChannels.getBackend(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.backend);
|
|
10660
|
+
const model = this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model);
|
|
10661
|
+
const autoPauseAfter = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
10662
|
+
if (old.backend !== backend || old.model !== model || old.autoPauseAfter !== autoPauseAfter) {
|
|
10663
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic cold config changed — restarting");
|
|
10664
|
+
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
10665
|
+
await this.startClassicInstanceUnattended(ch, "classic instance after fleet reload");
|
|
10666
|
+
continue;
|
|
10667
|
+
}
|
|
10668
|
+
const toolProgress = this.classicChannels.getToolProgress(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.tool_progress);
|
|
10669
|
+
const replyCompletionGuard = this.classicChannels.getReplyCompletionGuard(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.reply_completion_guard);
|
|
10670
|
+
if (old.toolProgress !== toolProgress || old.replyCompletionGuard !== replyCompletionGuard) {
|
|
10671
|
+
this.applyHotConfigUpdate(ch.instanceName, {
|
|
10672
|
+
tool_progress: toolProgress,
|
|
10673
|
+
reply_completion_guard: replyCompletionGuard,
|
|
10674
|
+
});
|
|
10675
|
+
this.logger.info({ instanceName: ch.instanceName }, "Classic inherited hot config reloaded");
|
|
10676
|
+
}
|
|
10677
|
+
}
|
|
10678
|
+
}
|
|
10069
10679
|
// warm_cap is fleet-owned; enforce the reloaded value immediately against
|
|
10070
10680
|
// currently idle instances instead of waiting for a future state edge.
|
|
10071
10681
|
this.enforceWarmCap();
|
|
@@ -10683,11 +11293,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10683
11293
|
const instances = names.map(name => {
|
|
10684
11294
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
10685
11295
|
let cost = 0;
|
|
10686
|
-
let model = "";
|
|
10687
11296
|
try {
|
|
10688
11297
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
10689
11298
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
10690
|
-
model = data.model?.display_name ?? "";
|
|
10691
11299
|
}
|
|
10692
11300
|
catch (err) {
|
|
10693
11301
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
@@ -10700,8 +11308,40 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10700
11308
|
?? this.fleetConfig?.defaults?.backend
|
|
10701
11309
|
?? "claude-code");
|
|
10702
11310
|
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
10703
|
-
|
|
10704
|
-
|
|
11311
|
+
// context_pct: null when unavailable, not 0
|
|
11312
|
+
const context_pct = context ?? null;
|
|
11313
|
+
// Model: Only Claude Code has live statusline; others use the effective resolver.
|
|
11314
|
+
// readStatuslineModel provides the /ctx-aligned display_name+id combo for Claude.
|
|
11315
|
+
// Non-Claude backends must NOT read statusline.json model (may be stale from previous Claude run).
|
|
11316
|
+
const resolved = this.resolveInstanceModel(name);
|
|
11317
|
+
const liveModel = backend === "claude-code" ? readStatuslineModel(this.dataDir, name) : null;
|
|
11318
|
+
// Display value: live model for Claude, resolved.display for others (includes "auto (default)" for Kiro)
|
|
11319
|
+
const model = liveModel ?? resolved.display;
|
|
11320
|
+
// model_source: "live" when Claude statusline succeeded, else the resolver's source
|
|
11321
|
+
const model_source = liveModel ? "live" : resolved.source;
|
|
11322
|
+
// Effort: aligned with /ctx's effortLineFor — unsupported and antigravity don't show effort.
|
|
11323
|
+
const effortStrategy = this.effortStrategyFor(name);
|
|
11324
|
+
const isAgy = backend === "antigravity" || backend === "agy";
|
|
11325
|
+
const effortResolved = this.resolveInstanceEffort(name);
|
|
11326
|
+
// Only show effort if backend supports it and it's not antigravity
|
|
11327
|
+
const effort = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.effort;
|
|
11328
|
+
const effort_source = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.source;
|
|
11329
|
+
// Display name: fleet config → classic channel → undefined
|
|
11330
|
+
const display_name = classic
|
|
11331
|
+
? this.classicChannels?.getAll().find(ch => ch.instanceName === name)?.displayName
|
|
11332
|
+
: this.fleetConfig?.instances[name]?.display_name;
|
|
11333
|
+
return {
|
|
11334
|
+
name,
|
|
11335
|
+
display_name: display_name || undefined,
|
|
11336
|
+
status: this.getInstanceStatus(name),
|
|
11337
|
+
context_pct,
|
|
11338
|
+
cost,
|
|
11339
|
+
model,
|
|
11340
|
+
model_source,
|
|
11341
|
+
backend,
|
|
11342
|
+
effort,
|
|
11343
|
+
effort_source,
|
|
11344
|
+
};
|
|
10705
11345
|
});
|
|
10706
11346
|
return {
|
|
10707
11347
|
instances,
|