@songsid/agend 2.1.5-beta.2 → 2.1.5-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/claude-code.d.ts +12 -0
- package/dist/backend/claude-code.js +14 -2
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +25 -1
- package/dist/backend/codex.js +76 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.js +30 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +70 -0
- package/dist/backend/types.js +23 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +17 -2
- package/dist/channel/adapters/discord.js +82 -8
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +2 -2
- package/dist/channel/adapters/telegram.js +8 -8
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/types.d.ts +22 -1
- package/dist/cli.js +37 -37
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +4 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +181 -21
- package/dist/daemon.js +1054 -225
- package/dist/daemon.js.map +1 -1
- package/dist/deadline.d.ts +23 -0
- package/dist/deadline.js +29 -0
- package/dist/deadline.js.map +1 -0
- package/dist/fleet-context.d.ts +17 -1
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +189 -2
- package/dist/fleet-manager.js +843 -124
- package/dist/fleet-manager.js.map +1 -1
- package/dist/full-restart.d.ts +20 -0
- package/dist/full-restart.js +39 -0
- package/dist/full-restart.js.map +1 -0
- package/dist/instance-lifecycle.d.ts +8 -2
- package/dist/instance-lifecycle.js +47 -3
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instance-removal.d.ts +13 -0
- package/dist/instance-removal.js +14 -0
- package/dist/instance-removal.js.map +1 -0
- package/dist/instructions.js +2 -2
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +54 -4
- package/dist/locale.js.map +1 -1
- package/dist/login-controller.d.ts +31 -4
- package/dist/login-controller.js +52 -4
- package/dist/login-controller.js.map +1 -1
- package/dist/login-flows.d.ts +29 -0
- package/dist/login-flows.js +48 -1
- package/dist/login-flows.js.map +1 -1
- package/dist/outbound-handlers.d.ts +1 -0
- package/dist/outbound-handlers.js +118 -29
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -0
- package/dist/outbound-schemas.js +3 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +25 -2
- package/dist/pane-input-residue.js +31 -2
- package/dist/pane-input-residue.js.map +1 -1
- package/dist/restart-progress.d.ts +35 -5
- package/dist/restart-progress.js +177 -34
- package/dist/restart-progress.js.map +1 -1
- package/dist/service-restart-selection.d.ts +27 -0
- package/dist/service-restart-selection.js +25 -0
- package/dist/service-restart-selection.js.map +1 -0
- package/dist/topic-commands.d.ts +27 -2
- package/dist/topic-commands.js +59 -13
- package/dist/topic-commands.js.map +1 -1
- package/dist/turn-reply-guard.d.ts +41 -0
- package/dist/turn-reply-guard.js +77 -0
- package/dist/turn-reply-guard.js.map +1 -0
- package/dist/ui/dashboard.html +34 -8
- package/dist/ui/view.html +32 -18
- package/dist/ui/web-terminal/terminal.html +1 -0
- package/dist/update-marker.d.ts +11 -1
- package/dist/update-marker.js +19 -4
- package/dist/update-marker.js.map +1 -1
- package/dist/update-progress.d.ts +1 -1
- package/dist/update-progress.js +12 -0
- package/dist/update-progress.js.map +1 -1
- package/dist/usage/providers.d.ts +39 -0
- package/dist/usage/providers.js +69 -3
- package/dist/usage/providers.js.map +1 -1
- package/dist/view-api.d.ts +1 -0
- package/dist/view-api.js +12 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.d.ts +22 -1
- package/dist/web-api.js +25 -1
- package/dist/web-api.js.map +1 -1
- package/dist/web-terminal-http.d.ts +8 -0
- package/dist/web-terminal-http.js +29 -3
- package/dist/web-terminal-http.js.map +1 -1
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { spawnSync } from "node:child_process";
|
|
4
|
-
import { freemem, totalmem } from "node:os";
|
|
4
|
+
import { freemem, totalmem, cpus } from "node:os";
|
|
5
5
|
import { createServer } from "node:http";
|
|
6
6
|
import { join, dirname, basename } from "node:path";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { isDeepStrictEqual } from "node:util";
|
|
9
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
10
|
-
import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
|
|
10
|
+
import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
|
|
11
11
|
import { formatUpdateProgress } from "./update-progress.js";
|
|
12
12
|
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
13
13
|
import { readFleetMemory } from "./process-memory.js";
|
|
@@ -33,7 +33,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
33
33
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
34
34
|
import { Scheduler } from "./scheduler/index.js";
|
|
35
35
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
36
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
|
|
36
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, resolveInstanceContext, forgetInstanceContext, readStatuslineModel } from "./topic-commands.js";
|
|
37
37
|
import { DailySummary } from "./daily-summary.js";
|
|
38
38
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
39
39
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
@@ -48,19 +48,22 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
48
48
|
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
49
|
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
50
|
import { LoginSession } from "./login-manager.js";
|
|
51
|
-
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX } from "./login-controller.js";
|
|
51
|
+
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX, POST_LOGIN_RECOVERY_DEADLINE_MS, announcePostLoginRecovery } from "./login-controller.js";
|
|
52
|
+
import { runBeforeDeadline } from "./deadline.js";
|
|
52
53
|
import { LoginWindowLock } from "./login-window-lock.js";
|
|
53
54
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
54
55
|
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
55
56
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
56
57
|
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
58
|
+
import { assertExplicitInstanceRemoval } from "./instance-removal.js";
|
|
57
59
|
import { validateFleetConfig } from "./config-validator.js";
|
|
58
60
|
import { readLastInboundAt } from "./daemon.js";
|
|
59
61
|
import { clearPausedMarker } from "./pause-marker.js";
|
|
60
62
|
import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
61
63
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
62
64
|
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
63
|
-
import { RestartProgress } from "./restart-progress.js";
|
|
65
|
+
import { formatRestartProgressCompletion, RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
|
|
66
|
+
import { launchFullRestartHelper } from "./full-restart.js";
|
|
64
67
|
import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
|
|
65
68
|
import { StormWindow } from "./storm-window.js";
|
|
66
69
|
import { SpawnGate } from "./spawn-gate.js";
|
|
@@ -218,7 +221,7 @@ const HANG_CALLBACK_PREFIX = "hang:";
|
|
|
218
221
|
const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
|
|
219
222
|
const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
|
|
220
223
|
const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
|
|
221
|
-
const LOGIN_CALLBACK_PREFIX = "login:";
|
|
224
|
+
export const LOGIN_CALLBACK_PREFIX = "login:";
|
|
222
225
|
const INSTALL_CALLBACK_PREFIX = "install-select:";
|
|
223
226
|
const CLASSIC_APPROVE_CALLBACK_PREFIX = "classic-approve:";
|
|
224
227
|
const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
|
|
@@ -228,7 +231,51 @@ const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
|
|
|
228
231
|
/** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
|
|
229
232
|
const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
|
|
230
233
|
const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
|
|
231
|
-
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; //
|
|
234
|
+
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // hard validity bound for the cached CLI env
|
|
235
|
+
/**
|
|
236
|
+
* How old the cached CLI env may be before `/model` re-probes it live.
|
|
237
|
+
*
|
|
238
|
+
* The cache is a file under AGEND_HOME, so it outlives the process, and the only
|
|
239
|
+
* thing that refreshed it was the startup probe. That is why a newly released
|
|
240
|
+
* model showed up only after `agend stop` + `agend start`: a bare `agend restart`
|
|
241
|
+
* signals SIGUSR2 and restarts the instances inside the *same* manager process,
|
|
242
|
+
* so nothing re-probed and `/model` kept serving a list up to 24h old.
|
|
243
|
+
*/
|
|
244
|
+
const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
|
|
245
|
+
/**
|
|
246
|
+
* Upper bound on a live probe driven by `/model`.
|
|
247
|
+
*
|
|
248
|
+
* A probe chains bounded leaves but is not itself bounded: claude-code runs
|
|
249
|
+
* `--version` (5s) then listModels then listApiModels (an 8s AbortController
|
|
250
|
+
* against api.anthropic.com), and several backends' probes carry no explicit
|
|
251
|
+
* timeout at all. Awaiting that inline would make `/model` the next thing to
|
|
252
|
+
* hang, so it races this deadline and falls back to the cached list.
|
|
253
|
+
*
|
|
254
|
+
* Must stay above CLI_PROBE_LONGEST_CHAIN_MS — the probe's bounded steps run
|
|
255
|
+
* back to back, so clearing only the longest single leaf would be false
|
|
256
|
+
* confidence: a deadline above 8s but below the 13s chain still truncates a
|
|
257
|
+
* probe that would have succeeded. Bounding the chain rather than the leaf is
|
|
258
|
+
* the lesson from the usage hang, and the tests assert against the derived
|
|
259
|
+
* chain constant so raising either step cannot silently break it.
|
|
260
|
+
*
|
|
261
|
+
* The deadline exists to stop a probe hanging forever, not to cut short one that
|
|
262
|
+
* would have finished: truncating a legitimate probe serves the previous list
|
|
263
|
+
* and hides exactly the newly released model the user opened `/model` to find.
|
|
264
|
+
* The wait is announced before it starts, so it reads as progress, not a stall.
|
|
265
|
+
*/
|
|
266
|
+
export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
|
|
267
|
+
/**
|
|
268
|
+
* How many CLIs may cold-start at once, from BOTH memory and cores.
|
|
269
|
+
*
|
|
270
|
+
* Memory alone said 10 on any host with roughly 3GB free, so a three-core box
|
|
271
|
+
* started ten CLIs together, saturated the CPU, and healthy starts then missed
|
|
272
|
+
* their startup budget — which used to cost the user their conversation. Cores
|
|
273
|
+
* bound how many can actually make progress; memory bounds how many fit.
|
|
274
|
+
*/
|
|
275
|
+
export function deriveSpawnConcurrency(freeMemMB, cores) {
|
|
276
|
+
const byMemory = Math.floor(freeMemMB / 300);
|
|
277
|
+
return Math.max(2, Math.min(10, byMemory, Math.max(1, cores)));
|
|
278
|
+
}
|
|
232
279
|
export class FleetManager {
|
|
233
280
|
dataDir;
|
|
234
281
|
static signalTarget = null;
|
|
@@ -270,6 +317,10 @@ export class FleetManager {
|
|
|
270
317
|
reloadPending = false;
|
|
271
318
|
/** A running reconciliation; only one may mutate lifecycle/config state at a time. */
|
|
272
319
|
reconcileInFlight = null;
|
|
320
|
+
/** Topology checks are serialized separately from config reconciliation. */
|
|
321
|
+
topicCleanupInFlight = null;
|
|
322
|
+
topicCleanupGeneration = 0;
|
|
323
|
+
topicProbeWarnings = new Map();
|
|
273
324
|
logger = createLogger("info");
|
|
274
325
|
topicCommands;
|
|
275
326
|
// sessionName → instanceName mapping for external sessions
|
|
@@ -379,6 +430,8 @@ export class FleetManager {
|
|
|
379
430
|
updateProgressEditRunning = false;
|
|
380
431
|
lastUpdateProgressText = null;
|
|
381
432
|
updateCompletionTipText = null;
|
|
433
|
+
/** Injectable only to keep the chat→CLI reload hand-off deterministic in tests. */
|
|
434
|
+
fullRestartLauncher = launchFullRestartHelper;
|
|
382
435
|
eventLogPruneTimer = null;
|
|
383
436
|
logRotateTimer = null;
|
|
384
437
|
/** Days of event/activity history to keep. */
|
|
@@ -417,8 +470,7 @@ export class FleetManager {
|
|
|
417
470
|
const explicit = this.fleetConfig?.defaults?.startup?.concurrency;
|
|
418
471
|
if (explicit != null)
|
|
419
472
|
return Math.max(1, Math.min(20, explicit));
|
|
420
|
-
|
|
421
|
-
return Math.max(2, Math.min(10, Math.floor(freeMemMB / 300)));
|
|
473
|
+
return deriveSpawnConcurrency(Math.round(freemem() / (1024 * 1024)), cpus().length);
|
|
422
474
|
}
|
|
423
475
|
/** Wire the one fleet-wide storm into notification and recovery surfaces. */
|
|
424
476
|
bindStormWindowEvents() {
|
|
@@ -765,6 +817,28 @@ export class FleetManager {
|
|
|
765
817
|
child.once("error", err => this.failUpdateProgress(err.message));
|
|
766
818
|
child.unref();
|
|
767
819
|
}
|
|
820
|
+
async handleRestartSlash(data, adapterId) {
|
|
821
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
822
|
+
await data.respond(t("not_authorized"));
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
if (data.options?.mode !== "full") {
|
|
826
|
+
await data.respond(t("restart.graceful"));
|
|
827
|
+
process.kill(process.pid, "SIGUSR2");
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
830
|
+
// Discord exposes only `full` as a choice. Keep a runtime check anyway: a
|
|
831
|
+
// briefly stale command schema must never turn an unknown value into SIGUSR1.
|
|
832
|
+
const messageId = await data.respond(t("restart.full_preparing"));
|
|
833
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
834
|
+
if (!messageId || !adapter) {
|
|
835
|
+
this.logger.error({ adapterId, hasMessageId: !!messageId }, "Full restart response could not be persisted — reload refused");
|
|
836
|
+
await data.respond(t("restart.full_launch_failed"));
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
const chatId = String(this.getChannelConfig(adapterId)?.group_id ?? data.channelId);
|
|
840
|
+
await this.requestFullRestart(adapter, chatId, data.channelId, messageId);
|
|
841
|
+
}
|
|
768
842
|
async handleTipsSlash(data, adapterId) {
|
|
769
843
|
if (!this.fleetConfig)
|
|
770
844
|
return;
|
|
@@ -859,6 +933,45 @@ export class FleetManager {
|
|
|
859
933
|
}
|
|
860
934
|
return channels[0];
|
|
861
935
|
}
|
|
936
|
+
/**
|
|
937
|
+
* The configured world that owns `chatId`, when that is provably NOT the
|
|
938
|
+
* world `target` lives in. Returns undefined when they agree, when there is
|
|
939
|
+
* nothing to check, or when no configured channel claims the id.
|
|
940
|
+
*
|
|
941
|
+
* Deliberately one-sided: only a POSITIVE match against another channel's
|
|
942
|
+
* group id counts as foreign. A chat id that matches nothing may still be
|
|
943
|
+
* legitimate for this world (a classic channel, a DM), and treating
|
|
944
|
+
* "unrecognised" as "wrong" would stop seeding for cases that work today.
|
|
945
|
+
*
|
|
946
|
+
* Read from config rather than the live worlds map on purpose: a channel
|
|
947
|
+
* whose adapter failed to start still owns its group id, and the coordinates
|
|
948
|
+
* are just as unusable by the target's adapter either way.
|
|
949
|
+
*
|
|
950
|
+
* This is the other half of what scheduleSourceAdapter fixed. That one stops
|
|
951
|
+
* the trigger NOTICE being sent through the wrong bot; this one stops the
|
|
952
|
+
* same coordinates being planted as the target instance's reply context,
|
|
953
|
+
* which is what made its own replies fail until someone spoke to it (#752).
|
|
954
|
+
*/
|
|
955
|
+
scheduleChatWorldMismatch(target, chatId) {
|
|
956
|
+
if (!chatId)
|
|
957
|
+
return undefined;
|
|
958
|
+
const targetWorld = this.getInstanceAdapterId(target);
|
|
959
|
+
if (!targetWorld)
|
|
960
|
+
return undefined;
|
|
961
|
+
const channels = this.fleetConfig?.channels
|
|
962
|
+
?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
|
|
963
|
+
// ALL owners, not the first: a persona bot shares the primary's guild
|
|
964
|
+
// (quickstart writes `group_id: primary.group_id`), so one group id is
|
|
965
|
+
// legitimately claimed by two channels. Taking the first match called a
|
|
966
|
+
// persona instance's own guild "another world" and stopped seeding a
|
|
967
|
+
// context it can address perfectly well.
|
|
968
|
+
const owners = channels.filter(ch => ch.group_id != null && String(ch.group_id) === String(chatId));
|
|
969
|
+
if (owners.length === 0)
|
|
970
|
+
return undefined;
|
|
971
|
+
if (owners.some(ch => (ch.id ?? ch.type) === targetWorld))
|
|
972
|
+
return undefined;
|
|
973
|
+
return owners[0].id ?? owners[0].type;
|
|
974
|
+
}
|
|
862
975
|
/** Get the group_id for an instance's bound adapter */
|
|
863
976
|
getGroupIdForInstance(name) {
|
|
864
977
|
const adapterId = this.getInstanceAdapterId(name);
|
|
@@ -1688,15 +1801,31 @@ export class FleetManager {
|
|
|
1688
1801
|
}
|
|
1689
1802
|
backendNameOf(name) {
|
|
1690
1803
|
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1691
|
-
const
|
|
1692
|
-
|
|
1693
|
-
|
|
1804
|
+
const fleetInstance = this.fleetConfig?.instances[name];
|
|
1805
|
+
// A malformed/manual config can give a fleet and Classic entry the same
|
|
1806
|
+
// instance name. Fleet ownership wins, including its inherited default;
|
|
1807
|
+
// otherwise a Classic override could make backend-scoped recovery restart
|
|
1808
|
+
// the shared process under the wrong login result.
|
|
1809
|
+
if (fleetInstance)
|
|
1810
|
+
return fleetInstance.backend ?? fleetDefault ?? "claude-code";
|
|
1694
1811
|
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1695
1812
|
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1696
1813
|
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1697
1814
|
}
|
|
1698
1815
|
return fleetDefault ?? "claude-code";
|
|
1699
1816
|
}
|
|
1817
|
+
/**
|
|
1818
|
+
* Every configured instance whose backend may share credentials. ClassicBot
|
|
1819
|
+
* rows live only in classicBot.yaml, so backend-wide operations must not use
|
|
1820
|
+
* fleetConfig.instances as their roster. Set keeps a malformed name collision
|
|
1821
|
+
* from restarting the same process twice; backendNameOf defines ownership.
|
|
1822
|
+
*/
|
|
1823
|
+
configuredBackendInstanceNames() {
|
|
1824
|
+
const names = new Set(Object.keys(this.fleetConfig?.instances ?? {}));
|
|
1825
|
+
for (const channel of this.classicChannels?.getAll() ?? [])
|
|
1826
|
+
names.add(channel.instanceName);
|
|
1827
|
+
return [...names];
|
|
1828
|
+
}
|
|
1700
1829
|
/**
|
|
1701
1830
|
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1702
1831
|
* fails many instances within the same second. Two notices per incident at
|
|
@@ -1751,16 +1880,60 @@ export class FleetManager {
|
|
|
1751
1880
|
if (!generalName)
|
|
1752
1881
|
return null;
|
|
1753
1882
|
const adapter = this.getAdapterForInstance(generalName);
|
|
1883
|
+
const adapterId = this.getInstanceAdapterId(generalName);
|
|
1754
1884
|
const chatId = this.getGroupIdForInstance(generalName);
|
|
1755
1885
|
if (!adapter || !chatId)
|
|
1756
1886
|
return null;
|
|
1757
1887
|
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
1758
1888
|
return {
|
|
1759
1889
|
adapter,
|
|
1890
|
+
resolveAdapter: adapterId ? () => this.readyProgressAdapter(adapterId) : undefined,
|
|
1760
1891
|
chatId,
|
|
1761
1892
|
threadId: topicId != null ? String(topicId) : undefined,
|
|
1762
1893
|
};
|
|
1763
1894
|
}
|
|
1895
|
+
/** Resolve only an adapter generation that can accept progress delivery.
|
|
1896
|
+
* Discord exposes direct gateway readiness; adapters without a health
|
|
1897
|
+
* snapshot use the fleet startup/retry state. */
|
|
1898
|
+
readyProgressAdapter(adapterId) {
|
|
1899
|
+
const adapter = this.adapters.get(adapterId);
|
|
1900
|
+
if (!adapter)
|
|
1901
|
+
return undefined;
|
|
1902
|
+
const health = adapter.getHealthSnapshot?.();
|
|
1903
|
+
if (health)
|
|
1904
|
+
return health.isReady ? adapter : undefined;
|
|
1905
|
+
return this.adapterState.get(adapterId)?.status === "connected" ? adapter : undefined;
|
|
1906
|
+
}
|
|
1907
|
+
/** Last-resort completion after RestartProgress could not deliver to its
|
|
1908
|
+
* adopted target. It stays bounded and never wakes a not-ready gateway. */
|
|
1909
|
+
async sendFleetStartCompletionFallback(chatId, text, threadId) {
|
|
1910
|
+
const adapterId = this.getPrimaryAdapterId();
|
|
1911
|
+
const adapter = adapterId ? this.readyProgressAdapter(adapterId) : undefined;
|
|
1912
|
+
if (!adapter) {
|
|
1913
|
+
this.logger.error({ adapterId }, "Fleet start completion fallback skipped because the primary adapter is not ready");
|
|
1914
|
+
return false;
|
|
1915
|
+
}
|
|
1916
|
+
let timer;
|
|
1917
|
+
const timeout = new Promise(resolve => {
|
|
1918
|
+
timer = setTimeout(() => resolve({ status: "timeout" }), RESTART_PROGRESS_TERMINAL_TIMEOUT_MS);
|
|
1919
|
+
timer.unref?.();
|
|
1920
|
+
});
|
|
1921
|
+
const delivery = Promise.resolve()
|
|
1922
|
+
.then(() => adapter.sendText(chatId, text, { threadId }))
|
|
1923
|
+
.then(() => ({ status: "sent" }), err => ({ status: "failed", err }));
|
|
1924
|
+
const result = await Promise.race([delivery, timeout]);
|
|
1925
|
+
if (timer)
|
|
1926
|
+
clearTimeout(timer);
|
|
1927
|
+
if (result.status === "sent")
|
|
1928
|
+
return true;
|
|
1929
|
+
if (result.status === "failed") {
|
|
1930
|
+
this.logger.error({ err: result.err }, "Failed to send fleet start completion fallback");
|
|
1931
|
+
}
|
|
1932
|
+
else {
|
|
1933
|
+
this.logger.error({ timeout_ms: RESTART_PROGRESS_TERMINAL_TIMEOUT_MS }, "Timed out sending fleet start completion fallback");
|
|
1934
|
+
}
|
|
1935
|
+
return false;
|
|
1936
|
+
}
|
|
1764
1937
|
async stopInstance(name) {
|
|
1765
1938
|
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
1766
1939
|
this.cancelStartupRetry(name);
|
|
@@ -1906,6 +2079,130 @@ export class FleetManager {
|
|
|
1906
2079
|
this.updateCompletionTipText = null;
|
|
1907
2080
|
this.startUpdateProgressMonitor(adapter);
|
|
1908
2081
|
}
|
|
2082
|
+
/** Persist the public response, wait for idle, then start the canonical service restart. */
|
|
2083
|
+
async requestFullRestart(adapter, chatId, threadId, messageId) {
|
|
2084
|
+
const target = {
|
|
2085
|
+
adapterId: adapter.id,
|
|
2086
|
+
chatId,
|
|
2087
|
+
...(threadId ? { threadId } : {}),
|
|
2088
|
+
messageId,
|
|
2089
|
+
};
|
|
2090
|
+
// This check is synchronous with the marker write below. A second command
|
|
2091
|
+
// cannot slip through between them on Node's event loop and replace the
|
|
2092
|
+
// first command's delivery target or launch a competing restart helper.
|
|
2093
|
+
if (this.shuttingDown || isUpdateInProgress(this.dataDir)) {
|
|
2094
|
+
this.logger.warn({ adapterId: adapter.id }, "Full restart refused because another planned restart is active");
|
|
2095
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
|
|
2096
|
+
return false;
|
|
2097
|
+
}
|
|
2098
|
+
if (!persistFullRestartProgress(this.dataDir, target)) {
|
|
2099
|
+
this.logger.error({ adapterId: adapter.id }, "Full restart marker could not be persisted — reload refused");
|
|
2100
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2101
|
+
return false;
|
|
2102
|
+
}
|
|
2103
|
+
const ownedMarker = readUpdateProgress(this.dataDir);
|
|
2104
|
+
if (!ownedMarker) {
|
|
2105
|
+
this.logger.error({ adapterId: adapter.id }, "Full restart marker disappeared after persistence — reload refused");
|
|
2106
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2107
|
+
return false;
|
|
2108
|
+
}
|
|
2109
|
+
this.lastUpdateProgressText = null;
|
|
2110
|
+
this.updateCompletionTipText = null;
|
|
2111
|
+
this.startUpdateProgressMonitor(adapter);
|
|
2112
|
+
await this.waitForFullRestartIdleGrace();
|
|
2113
|
+
// An update can begin while the idle wait yields. Never launch a second
|
|
2114
|
+
// process replacement against a marker we no longer own.
|
|
2115
|
+
if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target)) {
|
|
2116
|
+
this.logger.warn({ adapterId: adapter.id }, "Full restart superseded during idle wait — reload refused");
|
|
2117
|
+
if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2118
|
+
clearUpdateMarker(this.dataDir);
|
|
2119
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
|
|
2120
|
+
return false;
|
|
2121
|
+
}
|
|
2122
|
+
let helper;
|
|
2123
|
+
try {
|
|
2124
|
+
helper = await this.fullRestartLauncher();
|
|
2125
|
+
}
|
|
2126
|
+
catch (err) {
|
|
2127
|
+
this.logger.error({ err }, "Full restart helper failed to spawn — reload refused");
|
|
2128
|
+
if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2129
|
+
clearUpdateMarker(this.dataDir);
|
|
2130
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2131
|
+
return false;
|
|
2132
|
+
}
|
|
2133
|
+
void helper.completion.then(result => {
|
|
2134
|
+
// A zero exit means the service manager accepted the restart. launchd
|
|
2135
|
+
// may report that before its SIGTERM reaches us, so only an explicit
|
|
2136
|
+
// helper error/non-zero exit is evidence of failure. A signal is also
|
|
2137
|
+
// ambiguous under systemd because this helper shares the old cgroup.
|
|
2138
|
+
if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2139
|
+
return;
|
|
2140
|
+
if (!result.error && (result.code === 0 || result.code === null))
|
|
2141
|
+
return;
|
|
2142
|
+
const detail = result.error
|
|
2143
|
+
? "reload helper failed after launch"
|
|
2144
|
+
: `reload helper exited before process hand-off (code ${result.code ?? "null"}, signal ${result.signal ?? "none"})`;
|
|
2145
|
+
this.logger.error({ result }, "Full restart helper exited before the fleet began shutting down");
|
|
2146
|
+
setUpdateProgressStage(this.dataDir, "failed", { error: detail });
|
|
2147
|
+
});
|
|
2148
|
+
return true;
|
|
2149
|
+
}
|
|
2150
|
+
isOwnedFullRestartMarker(startedAt, target) {
|
|
2151
|
+
const marker = readUpdateProgress(this.dataDir);
|
|
2152
|
+
if (!marker || marker.startedAt !== startedAt || marker.pid !== process.pid)
|
|
2153
|
+
return false;
|
|
2154
|
+
if (updateProgressOperation(marker.progress) !== "full-restart")
|
|
2155
|
+
return false;
|
|
2156
|
+
const current = marker.progress.target;
|
|
2157
|
+
return current.adapterId === target.adapterId
|
|
2158
|
+
&& current.chatId === target.chatId
|
|
2159
|
+
&& current.threadId === target.threadId
|
|
2160
|
+
&& current.messageId === target.messageId;
|
|
2161
|
+
}
|
|
2162
|
+
/** Give current work the same bounded idle grace used by graceful reload. */
|
|
2163
|
+
async waitForFullRestartIdleGrace() {
|
|
2164
|
+
const instanceNames = [...this.daemons.keys()];
|
|
2165
|
+
if (instanceNames.length === 0)
|
|
2166
|
+
return;
|
|
2167
|
+
const IDLE_TIMEOUT_MS = 5 * 60_000;
|
|
2168
|
+
let timeoutHandle;
|
|
2169
|
+
const deadline = new Promise((_, reject) => {
|
|
2170
|
+
timeoutHandle = setTimeout(() => reject(new Error("Idle wait timed out after 5 minutes")), IDLE_TIMEOUT_MS);
|
|
2171
|
+
});
|
|
2172
|
+
try {
|
|
2173
|
+
await Promise.race([
|
|
2174
|
+
Promise.all(instanceNames.map(async (name) => {
|
|
2175
|
+
const daemon = this.daemons.get(name);
|
|
2176
|
+
if (!daemon)
|
|
2177
|
+
return;
|
|
2178
|
+
this.logger.info(`Full restart: waiting for ${name} to idle...`);
|
|
2179
|
+
await daemon.waitForIdle(10_000);
|
|
2180
|
+
})),
|
|
2181
|
+
deadline,
|
|
2182
|
+
]);
|
|
2183
|
+
}
|
|
2184
|
+
catch (err) {
|
|
2185
|
+
this.logger.warn({ err }, "Full restart idle wait timed out — continuing with service restart");
|
|
2186
|
+
}
|
|
2187
|
+
finally {
|
|
2188
|
+
clearTimeout(timeoutHandle);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
async reportFullRestartFailure(adapter, chatId, threadId, messageId, text) {
|
|
2192
|
+
try {
|
|
2193
|
+
await adapter.editMessage(chatId, messageId, text, threadId);
|
|
2194
|
+
return;
|
|
2195
|
+
}
|
|
2196
|
+
catch (err) {
|
|
2197
|
+
this.logger.warn({ err, adapterId: adapter.id }, "Failed to edit rejected full-restart request; posting a fresh notice");
|
|
2198
|
+
}
|
|
2199
|
+
try {
|
|
2200
|
+
await adapter.sendText(chatId, text, { threadId });
|
|
2201
|
+
}
|
|
2202
|
+
catch (err) {
|
|
2203
|
+
this.logger.error({ err, adapterId: adapter.id }, "Failed to deliver full-restart rejection");
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
1909
2206
|
failUpdateProgress(message) {
|
|
1910
2207
|
setUpdateProgressStage(this.dataDir, "failed", { error: message });
|
|
1911
2208
|
}
|
|
@@ -1988,6 +2285,9 @@ export class FleetManager {
|
|
|
1988
2285
|
&& savedUpdateProgress.progress.stage !== "complete"
|
|
1989
2286
|
? savedUpdateProgress
|
|
1990
2287
|
: null;
|
|
2288
|
+
const pendingProgressOperation = pendingUpdateProgress
|
|
2289
|
+
? updateProgressOperation(pendingUpdateProgress.progress)
|
|
2290
|
+
: null;
|
|
1991
2291
|
if (pendingUpdateProgress) {
|
|
1992
2292
|
setUpdateProgressStage(this.dataDir, "starting", { version: pendingUpdateProgress.progress.version });
|
|
1993
2293
|
}
|
|
@@ -2125,10 +2425,7 @@ export class FleetManager {
|
|
|
2125
2425
|
...fleet.defaults.daily_summary,
|
|
2126
2426
|
};
|
|
2127
2427
|
this.dailySummary = new DailySummary(summaryConfig, costGuardConfig.timezone, (text) => {
|
|
2128
|
-
|
|
2129
|
-
return;
|
|
2130
|
-
this.adapter.sendText(String(this.fleetConfig.channel.group_id), text)
|
|
2131
|
-
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
2428
|
+
this.postDailySummary(text);
|
|
2132
2429
|
// Rotate classic channel chat logs daily
|
|
2133
2430
|
this.classicChannels?.rotateLogs();
|
|
2134
2431
|
this.rotateInboxes();
|
|
@@ -2268,7 +2565,7 @@ export class FleetManager {
|
|
|
2268
2565
|
const allEntries = Object.entries(fleet.instances);
|
|
2269
2566
|
const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
2270
2567
|
const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
2271
|
-
const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingUpdateProgress ? "update" : "restart" });
|
|
2568
|
+
const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingProgressOperation === "full-restart" ? "reload" : pendingUpdateProgress ? "update" : "restart" });
|
|
2272
2569
|
if (generals.length > 0) {
|
|
2273
2570
|
for (const [name, cfg] of generals) {
|
|
2274
2571
|
try {
|
|
@@ -2317,12 +2614,17 @@ export class FleetManager {
|
|
|
2317
2614
|
progressStart = adapterStartup.then(() => {
|
|
2318
2615
|
if (pendingUpdateProgress) {
|
|
2319
2616
|
const saved = pendingUpdateProgress.progress.target;
|
|
2320
|
-
const
|
|
2321
|
-
|
|
2322
|
-
|
|
2617
|
+
const target = {
|
|
2618
|
+
adapter: this.adapters.get(saved.adapterId),
|
|
2619
|
+
// Adapter retries replace the failed object in this map. Resolve at
|
|
2620
|
+
// every progress delivery so the adopted update message follows the
|
|
2621
|
+
// live, ready generation instead of remaining pinned to a stopped
|
|
2622
|
+
// client. This also waits when the first generation failed before
|
|
2623
|
+
// any adapter object was registered.
|
|
2624
|
+
resolveAdapter: () => this.readyProgressAdapter(saved.adapterId),
|
|
2323
2625
|
chatId: saved.chatId,
|
|
2324
2626
|
threadId: saved.threadId,
|
|
2325
|
-
}
|
|
2627
|
+
};
|
|
2326
2628
|
return startupProgress.resume(target, saved.messageId);
|
|
2327
2629
|
}
|
|
2328
2630
|
return startupProgress.start(this.restartProgressTarget());
|
|
@@ -2432,16 +2734,25 @@ export class FleetManager {
|
|
|
2432
2734
|
version: agendVersion,
|
|
2433
2735
|
pausedNames,
|
|
2434
2736
|
failedNames,
|
|
2435
|
-
tipText:
|
|
2737
|
+
tipText: pendingProgressOperation === "update" && this.tipsEnabled()
|
|
2436
2738
|
? (() => {
|
|
2437
2739
|
const tip = this.pickAvailableTip();
|
|
2438
2740
|
return tip ? this.formatTip(tip) : undefined;
|
|
2439
2741
|
})()
|
|
2440
2742
|
: undefined,
|
|
2441
2743
|
});
|
|
2442
|
-
if (!progressCompleted &&
|
|
2744
|
+
if (!progressCompleted && fleet.channel?.group_id) {
|
|
2443
2745
|
let text;
|
|
2444
|
-
if (
|
|
2746
|
+
if (pendingProgressOperation === "full-restart") {
|
|
2747
|
+
text = formatRestartProgressCompletion("reload", {
|
|
2748
|
+
running: started,
|
|
2749
|
+
total,
|
|
2750
|
+
version: agendVersion,
|
|
2751
|
+
pausedNames,
|
|
2752
|
+
failedNames,
|
|
2753
|
+
}, pendingUpdateProgress.startedAt);
|
|
2754
|
+
}
|
|
2755
|
+
else if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
2445
2756
|
text = t("fleet.ready", started, total, agendVersion);
|
|
2446
2757
|
}
|
|
2447
2758
|
else if (failedNames.length === 0) {
|
|
@@ -2451,9 +2762,7 @@ export class FleetManager {
|
|
|
2451
2762
|
text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
|
|
2452
2763
|
+ (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
|
|
2453
2764
|
}
|
|
2454
|
-
this.
|
|
2455
|
-
threadId: generalThreadId != null ? String(generalThreadId) : undefined,
|
|
2456
|
-
}).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
|
|
2765
|
+
await this.sendFleetStartCompletionFallback(String(fleet.channel.group_id), text, generalThreadId != null ? String(generalThreadId) : undefined);
|
|
2457
2766
|
}
|
|
2458
2767
|
}
|
|
2459
2768
|
// Health HTTP endpoint
|
|
@@ -2714,10 +3023,11 @@ export class FleetManager {
|
|
|
2714
3023
|
accessManager,
|
|
2715
3024
|
inboxDir,
|
|
2716
3025
|
});
|
|
2717
|
-
const
|
|
3026
|
+
const adapter = this.adapter;
|
|
3027
|
+
const world = new AdapterWorld(adapterId, adapter, accessManager, channelConfig);
|
|
2718
3028
|
this.worlds.set(adapterId, world);
|
|
2719
|
-
this.adapters.set(adapterId,
|
|
2720
|
-
this.bindAdapterHealth(
|
|
3029
|
+
this.adapters.set(adapterId, adapter);
|
|
3030
|
+
this.bindAdapterHealth(adapter, adapterId);
|
|
2721
3031
|
this.adapter.on("message", safeHandler(async (msg) => {
|
|
2722
3032
|
await this.handleInboundMessage(msg);
|
|
2723
3033
|
}, this.logger, "adapter.message"));
|
|
@@ -2762,12 +3072,7 @@ export class FleetManager {
|
|
|
2762
3072
|
return;
|
|
2763
3073
|
}
|
|
2764
3074
|
}, this.logger, "adapter.callback_query"));
|
|
2765
|
-
this.adapter.
|
|
2766
|
-
// Skip unbind if we archived this topic ourselves
|
|
2767
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
2768
|
-
return;
|
|
2769
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
2770
|
-
}, this.logger, "adapter.topic_closed"));
|
|
3075
|
+
this.bindTopicClosedHandler(adapter, adapterId, "adapter.topic_closed");
|
|
2771
3076
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
2772
3077
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
2773
3078
|
if (data.command === "start") {
|
|
@@ -2989,13 +3294,7 @@ export class FleetManager {
|
|
|
2989
3294
|
await data.respond(this.topicCommands.getDashboardText());
|
|
2990
3295
|
}
|
|
2991
3296
|
else if (data.command === "restart") {
|
|
2992
|
-
|
|
2993
|
-
if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
|
|
2994
|
-
await data.respond(t("not_authorized"));
|
|
2995
|
-
return;
|
|
2996
|
-
}
|
|
2997
|
-
await data.respond(t("restart.graceful"));
|
|
2998
|
-
process.kill(process.pid, "SIGUSR2");
|
|
3297
|
+
await this.handleRestartSlash(data, adapterId);
|
|
2999
3298
|
}
|
|
3000
3299
|
else if (data.command === "compact") {
|
|
3001
3300
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
@@ -3164,11 +3463,7 @@ export class FleetManager {
|
|
|
3164
3463
|
return;
|
|
3165
3464
|
}
|
|
3166
3465
|
}, this.logger, `adapter[${adapterId}].callback_query`));
|
|
3167
|
-
|
|
3168
|
-
if (this.topicArchiver.isArchived(data.threadId))
|
|
3169
|
-
return;
|
|
3170
|
-
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
3171
|
-
}, this.logger, `adapter[${adapterId}].topic_closed`));
|
|
3466
|
+
this.bindTopicClosedHandler(adapter, adapterId, `adapter[${adapterId}].topic_closed`);
|
|
3172
3467
|
// Slash commands: classic bot + admin commands
|
|
3173
3468
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
3174
3469
|
if (data.command === "start") {
|
|
@@ -3338,13 +3633,7 @@ export class FleetManager {
|
|
|
3338
3633
|
await data.respond(this.topicCommands.getDashboardText());
|
|
3339
3634
|
}
|
|
3340
3635
|
else if (data.command === "restart") {
|
|
3341
|
-
|
|
3342
|
-
if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
|
|
3343
|
-
await data.respond(t("not_authorized"));
|
|
3344
|
-
return;
|
|
3345
|
-
}
|
|
3346
|
-
await data.respond(t("restart.graceful"));
|
|
3347
|
-
process.kill(process.pid, "SIGUSR2");
|
|
3636
|
+
await this.handleRestartSlash(data, adapterId);
|
|
3348
3637
|
}
|
|
3349
3638
|
else if (data.command === "compact") {
|
|
3350
3639
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
@@ -4533,7 +4822,9 @@ export class FleetManager {
|
|
|
4533
4822
|
// The adapter resolves only after the platform POST returns. Keep
|
|
4534
4823
|
// outward-facing logs/cancel state on the same confirmation boundary:
|
|
4535
4824
|
// a routed-but-failed reply is not a delivered reply.
|
|
4536
|
-
|
|
4825
|
+
// `statusOnly` is daemon-owned envelope metadata, not an MCP argument:
|
|
4826
|
+
// an agent cannot invent it to suppress the normal completion marker.
|
|
4827
|
+
if (!error && result != null && msg.statusOnly !== true) {
|
|
4537
4828
|
try {
|
|
4538
4829
|
this.afterReplyRouted(instanceName, args, senderSessionName);
|
|
4539
4830
|
}
|
|
@@ -4639,12 +4930,25 @@ export class FleetManager {
|
|
|
4639
4930
|
// the target instance's configured world so replies use its persona
|
|
4640
4931
|
// after a fresh start instead of falling back to channels[0].
|
|
4641
4932
|
const adapterId = this.getInstanceAdapterId(target);
|
|
4933
|
+
// ...but the stored reply coordinates belong to the chat the schedule
|
|
4934
|
+
// was CREATED in, which is not always the target's world. Pairing them
|
|
4935
|
+
// with the target's adapter hands one platform's ids to another's API:
|
|
4936
|
+
// a Telegram group + forum topic delivered to a Discord instance made
|
|
4937
|
+
// every reply on that turn fetch `/channels/<telegram topic>` and fail
|
|
4938
|
+
// with Unknown Channel, and because the seeding repeats on each trigger
|
|
4939
|
+
// it stayed broken until a real inbound overwrote the context (#752).
|
|
4940
|
+
const foreignWorld = this.scheduleChatWorldMismatch(target, reply_chat_id);
|
|
4941
|
+
if (foreignWorld) {
|
|
4942
|
+
this.logger.warn({ scheduleId: id, target, foreignWorld, targetWorld: adapterId, chatId: reply_chat_id }, "Schedule reply target belongs to another channel world — not seeding chat context; the instance keeps its own last known chat");
|
|
4943
|
+
}
|
|
4642
4944
|
await this.deliverToInstance(target, {
|
|
4643
4945
|
type: "fleet_schedule_trigger",
|
|
4644
4946
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
4645
4947
|
meta: {
|
|
4646
|
-
|
|
4647
|
-
|
|
4948
|
+
// Omitted on a mismatch: the daemon then keeps its persisted
|
|
4949
|
+
// last-chat, which is in the right world, instead of being
|
|
4950
|
+
// overwritten with coordinates the target's adapter cannot address.
|
|
4951
|
+
...(foreignWorld ? {} : { chat_id: reply_chat_id, thread_id: reply_thread_id }),
|
|
4648
4952
|
user: "scheduler",
|
|
4649
4953
|
...(adapterId ? { adapter_id: adapterId } : {}),
|
|
4650
4954
|
},
|
|
@@ -4676,19 +4980,69 @@ export class FleetManager {
|
|
|
4676
4980
|
this.scheduler.recordRun(id, "instance_offline", `retry ${retryCount}x failed`);
|
|
4677
4981
|
this.notifyScheduleFailure(schedule);
|
|
4678
4982
|
}
|
|
4983
|
+
/**
|
|
4984
|
+
* The adapter that can actually post into a chat, found by its group.
|
|
4985
|
+
*
|
|
4986
|
+
* A schedule records where it was created (reply_chat_id) separately from
|
|
4987
|
+
* what it triggers (target). Those need not share a platform: a Telegram
|
|
4988
|
+
* group can schedule a Discord-topic instance. Picking the adapter from the
|
|
4989
|
+
* target then sends a Telegram chat id through the Discord bot, which fails
|
|
4990
|
+
* with Unknown Channel — the source topic never hears that its schedule ran.
|
|
4991
|
+
*
|
|
4992
|
+
* When several bots share one guild, the primary wins: a persona should not
|
|
4993
|
+
* be the voice announcing fleet scheduling.
|
|
4994
|
+
*/
|
|
4995
|
+
adapterForChat(chatId) {
|
|
4996
|
+
const id = String(chatId);
|
|
4997
|
+
const matches = [...this.worlds.values()].filter(world => String(world.groupId) === id);
|
|
4998
|
+
if (matches.length === 0)
|
|
4999
|
+
return undefined;
|
|
5000
|
+
const primaryId = this.getPrimaryAdapterId();
|
|
5001
|
+
return (matches.find(world => world.id === primaryId) ?? matches[0]).adapter;
|
|
5002
|
+
}
|
|
5003
|
+
/**
|
|
5004
|
+
* The adapter that can answer a schedule in the chat it was created from.
|
|
5005
|
+
*
|
|
5006
|
+
* A schedule records its creator (source) and its trigger (target) separately,
|
|
5007
|
+
* and they need not share a platform — the live fleet has a Telegram group
|
|
5008
|
+
* scheduling a Discord-topic instance. Routing by target sends a Telegram chat
|
|
5009
|
+
* id through the Discord bot, which is one half of the Unknown Channel errors.
|
|
5010
|
+
*
|
|
5011
|
+
* The creator's own adapter comes first, and only when its world actually owns
|
|
5012
|
+
* that chat. Classic keeps its own identity, so a schedule made from a
|
|
5013
|
+
* persona-bound Classic channel is answered by that persona: the primary bot
|
|
5014
|
+
* may not even have access there, and would be the wrong voice if it did.
|
|
5015
|
+
* Falling back to the target's adapter is deliberately NOT an option — that is
|
|
5016
|
+
* the misroute itself; callers say why they stayed silent instead.
|
|
5017
|
+
*/
|
|
5018
|
+
scheduleSourceAdapter(schedule) {
|
|
5019
|
+
const chatId = String(schedule.reply_chat_id);
|
|
5020
|
+
const sourceAdapterId = schedule.source ? this.getInstanceAdapterId(schedule.source) : undefined;
|
|
5021
|
+
const sourceWorld = sourceAdapterId ? this.worlds.get(sourceAdapterId) : undefined;
|
|
5022
|
+
if (sourceWorld && String(sourceWorld.groupId) === chatId)
|
|
5023
|
+
return sourceWorld.adapter;
|
|
5024
|
+
return this.adapterForChat(chatId);
|
|
5025
|
+
}
|
|
4679
5026
|
notifySourceTopic(schedule) {
|
|
4680
|
-
const adapter = this.
|
|
4681
|
-
if (!adapter)
|
|
5027
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5028
|
+
if (!adapter) {
|
|
5029
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — trigger notice not sent");
|
|
4682
5030
|
return;
|
|
5031
|
+
}
|
|
4683
5032
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" triggered, target: ${schedule.target}`;
|
|
4684
5033
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4685
5034
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
4686
5035
|
}).catch((err) => this.logger.error({ err }, "Failed to send cross-instance notification"));
|
|
4687
5036
|
}
|
|
4688
5037
|
notifyScheduleFailure(schedule) {
|
|
4689
|
-
|
|
4690
|
-
|
|
5038
|
+
// Same resolver as the success path: a failure notice was still being sent
|
|
5039
|
+
// through the target's adapter, so a Telegram-created schedule for a
|
|
5040
|
+
// Discord target announced its failure into the wrong platform.
|
|
5041
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
5042
|
+
if (!adapter) {
|
|
5043
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — failure notice not sent");
|
|
4691
5044
|
return;
|
|
5045
|
+
}
|
|
4692
5046
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" trigger failed: instance ${schedule.target} is offline.`;
|
|
4693
5047
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4694
5048
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
@@ -5099,26 +5453,176 @@ export class FleetManager {
|
|
|
5099
5453
|
sessionPruneTimer = null;
|
|
5100
5454
|
classicReloadTimer = null;
|
|
5101
5455
|
botUserId;
|
|
5102
|
-
/** Periodically check if bound topics still exist */
|
|
5456
|
+
/** Periodically check if bound topics still exist. Never deletes user data. */
|
|
5103
5457
|
startTopicCleanupPoller() {
|
|
5104
|
-
this.topicCleanupTimer
|
|
5105
|
-
|
|
5458
|
+
if (this.topicCleanupTimer)
|
|
5459
|
+
clearInterval(this.topicCleanupTimer);
|
|
5460
|
+
const generation = ++this.topicCleanupGeneration;
|
|
5461
|
+
this.topicCleanupTimer = setInterval(() => { void this.scheduleTopicCleanup(generation); }, 5 * 60_000);
|
|
5462
|
+
}
|
|
5463
|
+
/** Coalesce timer ticks; an outage must not create overlapping destructive-looking scans. */
|
|
5464
|
+
scheduleTopicCleanup(generation = this.topicCleanupGeneration) {
|
|
5465
|
+
if (this.topicCleanupInFlight)
|
|
5466
|
+
return this.topicCleanupInFlight;
|
|
5467
|
+
const run = this.runTopicCleanup(generation).finally(() => {
|
|
5468
|
+
if (this.topicCleanupInFlight === run)
|
|
5469
|
+
this.topicCleanupInFlight = null;
|
|
5470
|
+
});
|
|
5471
|
+
this.topicCleanupInFlight = run;
|
|
5472
|
+
return run;
|
|
5473
|
+
}
|
|
5474
|
+
confirmedProbeFence(adapterId, adapter) {
|
|
5475
|
+
const health = adapter.getHealthSnapshot?.();
|
|
5476
|
+
if (health) {
|
|
5477
|
+
return health.status === "connected" && health.isReady
|
|
5478
|
+
? { generation: health.generation }
|
|
5479
|
+
: null;
|
|
5480
|
+
}
|
|
5481
|
+
return this.adapterState.get(adapterId)?.status === "connected" ? {} : null;
|
|
5482
|
+
}
|
|
5483
|
+
sameProbeFence(adapterId, adapter, before, result) {
|
|
5484
|
+
const after = this.confirmedProbeFence(adapterId, adapter);
|
|
5485
|
+
if (!after)
|
|
5486
|
+
return false;
|
|
5487
|
+
if (before.generation !== after.generation)
|
|
5488
|
+
return false;
|
|
5489
|
+
return result?.generation === undefined || result.generation === after.generation;
|
|
5490
|
+
}
|
|
5491
|
+
warnTopicProbeUnknown(instanceName, threadId, adapterId, reason) {
|
|
5492
|
+
const key = `${adapterId ?? "unbound"}:${reason}`;
|
|
5493
|
+
const now = Date.now();
|
|
5494
|
+
const last = this.topicProbeWarnings.get(key) ?? 0;
|
|
5495
|
+
if (now - last < FleetManager.FLEET_ERROR_THROTTLE_MS)
|
|
5496
|
+
return;
|
|
5497
|
+
this.topicProbeWarnings.set(key, now);
|
|
5498
|
+
this.logger.error({ instanceName, threadId, adapterId, reason }, "Topic presence could not be confirmed — retaining instance and all data");
|
|
5499
|
+
this.notifyFleetError(t("fleet.topic_probe_unknown", instanceName, adapterId ?? "unbound"));
|
|
5500
|
+
}
|
|
5501
|
+
/** One fixed-snapshot topology pass. Automatic evidence can only quarantine. */
|
|
5502
|
+
async runTopicCleanup(generation) {
|
|
5503
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5504
|
+
return;
|
|
5505
|
+
const snapshot = [...this.routing.entries()].filter(([, target]) => isProbeableRouteTarget(target));
|
|
5506
|
+
const missing = [];
|
|
5507
|
+
for (const [threadId, target] of snapshot) {
|
|
5508
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5106
5509
|
return;
|
|
5107
|
-
|
|
5108
|
-
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
}
|
|
5116
|
-
}
|
|
5117
|
-
catch (err) {
|
|
5118
|
-
this.logger.debug({ err, threadId }, "Topic existence check failed");
|
|
5119
|
-
}
|
|
5510
|
+
// The route may have been replaced while an earlier probe was in flight.
|
|
5511
|
+
if (this.routing.resolve(threadId) !== target)
|
|
5512
|
+
continue;
|
|
5513
|
+
const adapterId = this.getInstanceAdapterId(target.name);
|
|
5514
|
+
const adapter = adapterId ? this.adapters.get(adapterId) : undefined;
|
|
5515
|
+
if (!adapterId || !adapter?.probeTopicPresence) {
|
|
5516
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-unavailable");
|
|
5517
|
+
continue;
|
|
5120
5518
|
}
|
|
5121
|
-
|
|
5519
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5520
|
+
if (!before) {
|
|
5521
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-not-ready");
|
|
5522
|
+
continue;
|
|
5523
|
+
}
|
|
5524
|
+
let result;
|
|
5525
|
+
try {
|
|
5526
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5527
|
+
}
|
|
5528
|
+
catch {
|
|
5529
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5530
|
+
}
|
|
5531
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown)
|
|
5532
|
+
return;
|
|
5533
|
+
if (!this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5534
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "owner-adapter-generation-changed");
|
|
5535
|
+
continue;
|
|
5536
|
+
}
|
|
5537
|
+
if (result.status === "unknown") {
|
|
5538
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.reason);
|
|
5539
|
+
}
|
|
5540
|
+
else if (result.status === "missing") {
|
|
5541
|
+
missing.push({ threadId, target, adapterId, adapter, generation: result.generation });
|
|
5542
|
+
}
|
|
5543
|
+
}
|
|
5544
|
+
if (generation !== this.topicCleanupGeneration || this.shuttingDown || missing.length === 0)
|
|
5545
|
+
return;
|
|
5546
|
+
if (missing.length > 1) {
|
|
5547
|
+
this.logger.error({ missing: missing.map(item => ({ instanceName: item.target.name, threadId: item.threadId, adapterId: item.adapterId })) }, "Multiple topics appeared missing in one pass — treating topology evidence as untrusted and retaining all data");
|
|
5548
|
+
this.notifyFleetError(t("fleet.topic_probe_bulk", missing.length));
|
|
5549
|
+
return;
|
|
5550
|
+
}
|
|
5551
|
+
const item = missing[0];
|
|
5552
|
+
if (this.routing.resolve(item.threadId) !== item.target
|
|
5553
|
+
|| this.getInstanceAdapterId(item.target.name) !== item.adapterId
|
|
5554
|
+
|| this.adapters.get(item.adapterId) !== item.adapter)
|
|
5555
|
+
return;
|
|
5556
|
+
const current = this.confirmedProbeFence(item.adapterId, item.adapter);
|
|
5557
|
+
if (!current || (item.generation !== undefined && current.generation !== item.generation)) {
|
|
5558
|
+
this.warnTopicProbeUnknown(item.target.name, item.threadId, item.adapterId, "owner-adapter-changed-before-action");
|
|
5559
|
+
return;
|
|
5560
|
+
}
|
|
5561
|
+
this.topicCommands.handleTopicDeleted(item.threadId, {
|
|
5562
|
+
source: "provider-probe",
|
|
5563
|
+
adapterId: item.adapterId,
|
|
5564
|
+
generation: item.generation,
|
|
5565
|
+
});
|
|
5566
|
+
}
|
|
5567
|
+
/**
|
|
5568
|
+
* A gateway event is only a hint. Confirm it through the passive REST probe;
|
|
5569
|
+
* reconnecting gateways have emitted false channelDelete events in practice.
|
|
5570
|
+
*/
|
|
5571
|
+
async handleProviderTopicClosed(threadId, adapterId, adapter) {
|
|
5572
|
+
const target = this.routing.resolve(threadId);
|
|
5573
|
+
if (!target || !isProbeableRouteTarget(target))
|
|
5574
|
+
return;
|
|
5575
|
+
if (this.getInstanceAdapterId(target.name) !== adapterId)
|
|
5576
|
+
return;
|
|
5577
|
+
if (this.adapters.get(adapterId) !== adapter)
|
|
5578
|
+
return;
|
|
5579
|
+
const before = this.confirmedProbeFence(adapterId, adapter);
|
|
5580
|
+
if (!before || !adapter.probeTopicPresence) {
|
|
5581
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-from-unready-adapter");
|
|
5582
|
+
return;
|
|
5583
|
+
}
|
|
5584
|
+
let result;
|
|
5585
|
+
try {
|
|
5586
|
+
result = await adapter.probeTopicPresence(threadId);
|
|
5587
|
+
}
|
|
5588
|
+
catch {
|
|
5589
|
+
result = { status: "unknown", reason: "provider-probe-threw" };
|
|
5590
|
+
}
|
|
5591
|
+
if (this.routing.resolve(threadId) !== target
|
|
5592
|
+
|| this.getInstanceAdapterId(target.name) !== adapterId
|
|
5593
|
+
|| this.adapters.get(adapterId) !== adapter
|
|
5594
|
+
|| !this.sameProbeFence(adapterId, adapter, before, result)) {
|
|
5595
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, "topic-close-generation-changed");
|
|
5596
|
+
return;
|
|
5597
|
+
}
|
|
5598
|
+
if (result.status !== "missing") {
|
|
5599
|
+
this.warnTopicProbeUnknown(target.name, threadId, adapterId, result.status === "unknown" ? result.reason : "topic-close-not-confirmed-missing");
|
|
5600
|
+
return;
|
|
5601
|
+
}
|
|
5602
|
+
this.topicCommands.handleTopicDeleted(threadId, {
|
|
5603
|
+
source: "provider-event",
|
|
5604
|
+
adapterId,
|
|
5605
|
+
generation: result.generation,
|
|
5606
|
+
});
|
|
5607
|
+
}
|
|
5608
|
+
bindTopicClosedHandler(adapter, adapterId, label) {
|
|
5609
|
+
adapter.on("topic_closed", safeHandler(async (data) => {
|
|
5610
|
+
if (this.topicArchiver.isArchived(data.threadId))
|
|
5611
|
+
return;
|
|
5612
|
+
await this.handleProviderTopicClosed(data.threadId, adapterId, adapter);
|
|
5613
|
+
}, this.logger, label));
|
|
5614
|
+
}
|
|
5615
|
+
/**
|
|
5616
|
+
* Remove only the volatile route. The instance config, daemon, schedules,
|
|
5617
|
+
* teams, metadata directory, and working tree remain untouched until an
|
|
5618
|
+
* authenticated explicit deletion is requested.
|
|
5619
|
+
*/
|
|
5620
|
+
quarantineMissingTopic(threadId, target, evidence) {
|
|
5621
|
+
if (!isProbeableRouteTarget(target) || this.routing.resolve(threadId) !== target)
|
|
5622
|
+
return;
|
|
5623
|
+
this.routing.unregister(threadId);
|
|
5624
|
+
this.logger.error({ instanceName: target.name, threadId, ...evidence }, "Topic is confirmed missing — route quarantined; instance configuration and user data were retained");
|
|
5625
|
+
this.notifyFleetError(t("fleet.topic_quarantined", target.name, threadId));
|
|
5122
5626
|
}
|
|
5123
5627
|
/**
|
|
5124
5628
|
* Patch only values changed in the effective config into the original YAML
|
|
@@ -5271,7 +5775,8 @@ export class FleetManager {
|
|
|
5271
5775
|
}
|
|
5272
5776
|
}
|
|
5273
5777
|
}
|
|
5274
|
-
async removeInstance(name) {
|
|
5778
|
+
async removeInstance(name, authorization) {
|
|
5779
|
+
assertExplicitInstanceRemoval(authorization);
|
|
5275
5780
|
// Drop cached pane context — the map is keyed by instance name and nothing
|
|
5276
5781
|
// else evicted deleted entries, so it grew for the life of the process.
|
|
5277
5782
|
forgetInstanceContext(name);
|
|
@@ -5297,7 +5802,7 @@ export class FleetManager {
|
|
|
5297
5802
|
}
|
|
5298
5803
|
}
|
|
5299
5804
|
}
|
|
5300
|
-
await this.lifecycle.remove(name);
|
|
5805
|
+
await this.lifecycle.remove(name, authorization);
|
|
5301
5806
|
// Clean up per-instance tracking maps so they don't grow unbounded
|
|
5302
5807
|
// as instances are created and deleted over the lifetime of the fleet.
|
|
5303
5808
|
this.lastActivity.delete(name);
|
|
@@ -5534,6 +6039,65 @@ export class FleetManager {
|
|
|
5534
6039
|
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5535
6040
|
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5536
6041
|
}
|
|
6042
|
+
/**
|
|
6043
|
+
* Where a fleet-wide notice can actually be posted, or null if nowhere.
|
|
6044
|
+
*
|
|
6045
|
+
* On Telegram a group id is itself a chat, so posting straight to it is
|
|
6046
|
+
* right. On Discord it is a *guild* id, and sending there makes the adapter
|
|
6047
|
+
* fetch a channel that does not exist — DiscordAPIError 10003 Unknown
|
|
6048
|
+
* Channel. That is why the daily summary never arrived on a Discord fleet:
|
|
6049
|
+
* it had been posting to the guild every night and only the catch handler
|
|
6050
|
+
* ever saw it.
|
|
6051
|
+
*
|
|
6052
|
+
* Discord therefore needs a real channel: the General topic (resolved from
|
|
6053
|
+
* config rather than findGeneralInstance, so a fleet-level fault can still be
|
|
6054
|
+
* reported while the General daemon is down), else the adapter's configured
|
|
6055
|
+
* general_channel_id. With neither, there is no safe target and the caller
|
|
6056
|
+
* should say so rather than send into a guaranteed failure.
|
|
6057
|
+
*/
|
|
6058
|
+
fleetNoticeTarget(adapterId) {
|
|
6059
|
+
const cfg = this.getChannelConfig(adapterId);
|
|
6060
|
+
const groupId = cfg?.group_id;
|
|
6061
|
+
if (groupId == null)
|
|
6062
|
+
return null;
|
|
6063
|
+
const chatId = String(groupId);
|
|
6064
|
+
// The General must belong to the SAME adapter as the group above. Taking
|
|
6065
|
+
// whichever General comes first in the instance map produced a mixed target
|
|
6066
|
+
// on a dual-platform fleet — a Telegram group id carrying a Discord channel
|
|
6067
|
+
// as its thread — and made the result depend on map insertion order.
|
|
6068
|
+
// Resolved from config, not from a live daemon, so a fleet-level fault is
|
|
6069
|
+
// still reportable while the General itself is down.
|
|
6070
|
+
const ownerId = cfg?.id ?? cfg?.type;
|
|
6071
|
+
const generalTopic = Object.entries(this.fleetConfig?.instances ?? {})
|
|
6072
|
+
.find(([name, instance]) => instance.general_topic === true
|
|
6073
|
+
&& this.getInstanceAdapterId(name) === ownerId)?.[1]?.topic_id;
|
|
6074
|
+
if (generalTopic != null)
|
|
6075
|
+
return { chatId, opts: { threadId: String(generalTopic) } };
|
|
6076
|
+
if (cfg?.type === "discord") {
|
|
6077
|
+
const configured = cfg.options?.general_channel_id;
|
|
6078
|
+
if (configured != null && String(configured)) {
|
|
6079
|
+
return { chatId, opts: { threadId: String(configured) } };
|
|
6080
|
+
}
|
|
6081
|
+
return null; // a guild id is not a channel; sending would always fail
|
|
6082
|
+
}
|
|
6083
|
+
return { chatId, opts: {} };
|
|
6084
|
+
}
|
|
6085
|
+
/**
|
|
6086
|
+
* Post the daily summary where fleet-wide notices go.
|
|
6087
|
+
*
|
|
6088
|
+
* A named method rather than an inline closure so a test can drive the real
|
|
6089
|
+
* thing: asserting on fleetNoticeTarget alone leaves the call site free to go
|
|
6090
|
+
* back to posting at the bare group id, which is the defect this replaced.
|
|
6091
|
+
*/
|
|
6092
|
+
postDailySummary(text) {
|
|
6093
|
+
const target = this.fleetNoticeTarget();
|
|
6094
|
+
if (!this.adapter || !target) {
|
|
6095
|
+
this.logger.warn("Daily summary has no postable target — set a General topic or channel.options.general_channel_id");
|
|
6096
|
+
return;
|
|
6097
|
+
}
|
|
6098
|
+
this.adapter.sendText(target.chatId, text, target.opts)
|
|
6099
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
6100
|
+
}
|
|
5537
6101
|
notifyFleetError(text) {
|
|
5538
6102
|
const now = Date.now();
|
|
5539
6103
|
const key = text.slice(0, 200);
|
|
@@ -5560,10 +6124,12 @@ export class FleetManager {
|
|
|
5560
6124
|
dispatched = this.notifyInstanceTopic(general, body);
|
|
5561
6125
|
}
|
|
5562
6126
|
else {
|
|
5563
|
-
// No General instance — fall back to the primary channel's
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
6127
|
+
// No General instance — fall back to the primary channel's own notice
|
|
6128
|
+
// target. Posting to the bare group id looked right but is a guild id on
|
|
6129
|
+
// Discord, so every such fallback failed inside the catch handler.
|
|
6130
|
+
const target = this.fleetNoticeTarget();
|
|
6131
|
+
if (this.adapter && target) {
|
|
6132
|
+
this.adapter.sendText(target.chatId, body, target.opts)
|
|
5567
6133
|
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5568
6134
|
dispatched = true;
|
|
5569
6135
|
}
|
|
@@ -5622,13 +6188,16 @@ export class FleetManager {
|
|
|
5622
6188
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
5623
6189
|
return true;
|
|
5624
6190
|
}
|
|
5625
|
-
// Fallback:
|
|
5626
|
-
|
|
5627
|
-
|
|
6191
|
+
// Fallback: the instance has neither a topic nor a classic channel, so post
|
|
6192
|
+
// where fleet-wide notices go. Not the bare group id: on Discord that is a
|
|
6193
|
+
// guild, and the send fails inside the catch handler.
|
|
6194
|
+
const target = this.fleetNoticeTarget(this.getInstanceAdapterId(instanceName));
|
|
6195
|
+
if (target) {
|
|
6196
|
+
adapter.sendText(target.chatId, text, { ...target.opts, ...extraOpts })
|
|
5628
6197
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5629
6198
|
return true;
|
|
5630
6199
|
}
|
|
5631
|
-
this.logger.warn({ instanceName }, "No
|
|
6200
|
+
this.logger.warn({ instanceName }, "No postable target — instance topic notification not sent");
|
|
5632
6201
|
return false;
|
|
5633
6202
|
}
|
|
5634
6203
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
@@ -6189,6 +6758,41 @@ export class FleetManager {
|
|
|
6189
6758
|
extra: { generalName, promptKind: kind },
|
|
6190
6759
|
});
|
|
6191
6760
|
}
|
|
6761
|
+
/**
|
|
6762
|
+
* Offer a one-tap re-login next to an auth alert.
|
|
6763
|
+
*
|
|
6764
|
+
* The alert already names the remedy in words (`/login <backend>`), which
|
|
6765
|
+
* still leaves the user to retype it somewhere. The button routes into the
|
|
6766
|
+
* same chooser `/login` uses, so pressing it starts the flow in place.
|
|
6767
|
+
*
|
|
6768
|
+
* Backends with no remote login flow (opencode logs in from a terminal) get
|
|
6769
|
+
* no button — the alert's own wording already tells them what to run.
|
|
6770
|
+
*/
|
|
6771
|
+
async offerBackendLogin(targetInstance, backend) {
|
|
6772
|
+
if (!LOGIN_FLOWS[backend])
|
|
6773
|
+
return;
|
|
6774
|
+
const adapterId = this.getInstanceAdapterId(targetInstance);
|
|
6775
|
+
const adapter = this.getAdapterForInstance(targetInstance);
|
|
6776
|
+
const chatId = this.getGroupIdForInstance(targetInstance);
|
|
6777
|
+
const topicId = this.fleetConfig?.instances[targetInstance]?.topic_id;
|
|
6778
|
+
const threadId = topicId != null ? String(topicId) : undefined;
|
|
6779
|
+
if (!adapter || !adapterId || !chatId) {
|
|
6780
|
+
this.logger.warn({ targetInstance, backend, adapterId, chatId }, "Cannot address the re-login button — the alert text still names the command");
|
|
6781
|
+
return;
|
|
6782
|
+
}
|
|
6783
|
+
await this.postNonceButtonPrompt({
|
|
6784
|
+
prefix: LOGIN_CALLBACK_PREFIX,
|
|
6785
|
+
alertType: "login",
|
|
6786
|
+
instanceName: "login",
|
|
6787
|
+
adapter,
|
|
6788
|
+
adapterId,
|
|
6789
|
+
chatId,
|
|
6790
|
+
threadId,
|
|
6791
|
+
message: t("login.offer", backend),
|
|
6792
|
+
choices: [{ action: backend, label: t("login.offer_action", backend) }],
|
|
6793
|
+
expiredText: t("buttons.stale"),
|
|
6794
|
+
});
|
|
6795
|
+
}
|
|
6192
6796
|
/** Consume a General assist button exactly once. */
|
|
6193
6797
|
async handleInteractivePromptAssist(data, callbackAdapterId, receivingAdapter) {
|
|
6194
6798
|
const claimed = this.consumeNonceCallback(INTERACTIVE_ASSIST_CALLBACK_PREFIX, /^interactive-assist:([0-9a-f]+):(confirm|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
@@ -6904,8 +7508,8 @@ export class FleetManager {
|
|
|
6904
7508
|
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
6905
7509
|
async promptLoginBackends(chat) {
|
|
6906
7510
|
const configured = new Set();
|
|
6907
|
-
for (const
|
|
6908
|
-
configured.add(
|
|
7511
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
7512
|
+
configured.add(this.backendNameOf(name));
|
|
6909
7513
|
}
|
|
6910
7514
|
const choices = Object.keys(LOGIN_FLOWS)
|
|
6911
7515
|
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
@@ -7177,11 +7781,15 @@ export class FleetManager {
|
|
|
7177
7781
|
if (cleanupFailed) {
|
|
7178
7782
|
await chat.adapter.sendText(chat.chatId, t("login.web_cleanup_failed", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7179
7783
|
}
|
|
7784
|
+
const send = (text) => chat.adapter.sendText(chat.chatId, text, { threadId: chat.threadId }).catch(() => { });
|
|
7180
7785
|
let text;
|
|
7181
7786
|
if (ok) {
|
|
7182
|
-
|
|
7183
|
-
|
|
7184
|
-
|
|
7787
|
+
// Same order as the web-login path: the login result goes out first,
|
|
7788
|
+
// then the recovery reports its own outcome. Waiting for recovery to
|
|
7789
|
+
// build this message is what made a successful login look hung.
|
|
7790
|
+
await send(t("login.completed", backend));
|
|
7791
|
+
await announcePostLoginRecovery(backend, () => this.recoverBackendInstances(backend), send);
|
|
7792
|
+
return;
|
|
7185
7793
|
}
|
|
7186
7794
|
else if (detail === "cancelled") {
|
|
7187
7795
|
// The cancel command's own reply already announced this — a second
|
|
@@ -7305,32 +7913,53 @@ export class FleetManager {
|
|
|
7305
7913
|
* only re-reads credentials on process start (a paused instance's CLI is
|
|
7306
7914
|
* already dead, so waking it respawns with the new token for free).
|
|
7307
7915
|
*/
|
|
7308
|
-
|
|
7916
|
+
/**
|
|
7917
|
+
* Wake/restart every instance of a backend after a successful re-login.
|
|
7918
|
+
*
|
|
7919
|
+
* Bounded by a wall-clock deadline. This used to be an unbounded sequential
|
|
7920
|
+
* loop, and the caller only built its "login completed" message AFTER it
|
|
7921
|
+
* returned — so with several instances (or one slow restart) the user was
|
|
7922
|
+
* told nothing at all for minutes, concluded the login had hung, and
|
|
7923
|
+
* restarted things by hand. Instances that do not finish in time are NOT
|
|
7924
|
+
* cancelled: they are still coming back, and are reported as pending so the
|
|
7925
|
+
* message can say so instead of implying failure.
|
|
7926
|
+
*/
|
|
7927
|
+
async recoverBackendInstances(backend, deadlineMs = POST_LOGIN_RECOVERY_DEADLINE_MS) {
|
|
7309
7928
|
const woken = [];
|
|
7310
7929
|
const restarted = [];
|
|
7311
|
-
|
|
7312
|
-
|
|
7313
|
-
|
|
7930
|
+
const pending = [];
|
|
7931
|
+
const deadline = Date.now() + deadlineMs;
|
|
7932
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
7933
|
+
if (this.backendNameOf(name) !== backend)
|
|
7314
7934
|
continue;
|
|
7315
7935
|
const status = this.getInstanceStatus(name);
|
|
7316
|
-
|
|
7936
|
+
if (status !== "paused" && status !== "running")
|
|
7937
|
+
continue;
|
|
7938
|
+
const result = await runBeforeDeadline(async () => {
|
|
7317
7939
|
if (status === "paused") {
|
|
7318
7940
|
if (this.daemons.has(name))
|
|
7319
7941
|
await this.lifecycle.wake(name, 30_000);
|
|
7320
7942
|
else
|
|
7321
7943
|
await this.startPersistedPausedInstance(name);
|
|
7322
|
-
woken.push(name);
|
|
7323
7944
|
}
|
|
7324
|
-
else
|
|
7945
|
+
else {
|
|
7325
7946
|
await this.restartSingleInstance(name);
|
|
7326
|
-
restarted.push(name);
|
|
7327
7947
|
}
|
|
7948
|
+
}, deadline);
|
|
7949
|
+
if (result.status === "fulfilled") {
|
|
7950
|
+
(status === "paused" ? woken : restarted).push(name);
|
|
7328
7951
|
}
|
|
7329
|
-
|
|
7330
|
-
|
|
7952
|
+
else if (result.status === "timeout") {
|
|
7953
|
+
// Out of time: record it and stop waiting, but keep walking the list —
|
|
7954
|
+
// the remaining instances are checked against the same deadline and
|
|
7955
|
+
// fall straight through, so the caller still learns about all of them.
|
|
7956
|
+
pending.push(name);
|
|
7957
|
+
}
|
|
7958
|
+
else {
|
|
7959
|
+
this.logger.warn({ err: result.reason?.message, name, status }, "Post-login recovery failed");
|
|
7331
7960
|
}
|
|
7332
7961
|
}
|
|
7333
|
-
return { woken, restarted };
|
|
7962
|
+
return { woken, restarted, pending };
|
|
7334
7963
|
}
|
|
7335
7964
|
/** Backend chooser button → start that backend's login session. */
|
|
7336
7965
|
async handleLoginBackendSelect(data, callbackAdapterId, receivingAdapter) {
|
|
@@ -7742,7 +8371,7 @@ You CAN write code snippets, explain code, and answer technical questions direct
|
|
|
7742
8371
|
- **Delegate to 1 instance**: scoped to one project/repo, needs file access or execution.
|
|
7743
8372
|
- **Coordinate multiple**: spans repos, outputs feed each other, or parallel helps (max 3 per task).
|
|
7744
8373
|
|
|
7745
|
-
Instance discovery
|
|
8374
|
+
Instance discovery: start with list_instances(); follow the guidance in its response. Prefer reuse; never duplicate a running instance.
|
|
7746
8375
|
|
|
7747
8376
|
## Reply Contract
|
|
7748
8377
|
|
|
@@ -8376,6 +9005,32 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8376
9005
|
catch { /* missing / stale / corrupt */ }
|
|
8377
9006
|
return null;
|
|
8378
9007
|
}
|
|
9008
|
+
/** True when a cached CLI env is old enough that `/model` should re-probe. */
|
|
9009
|
+
cliEnvNeedsRefresh(env) {
|
|
9010
|
+
return !env || typeof env.probedAt !== "number" || Date.now() - env.probedAt >= CLI_ENV_FRESH_MS;
|
|
9011
|
+
}
|
|
9012
|
+
/**
|
|
9013
|
+
* Run a live probe under a deadline, falling back to whatever the cache holds.
|
|
9014
|
+
* A model list is an aid: a vendor that stops answering must degrade to the
|
|
9015
|
+
* previous list, never stall the command that asked for it.
|
|
9016
|
+
*/
|
|
9017
|
+
async probeBackendBounded(backend) {
|
|
9018
|
+
const work = this.probeBackend(backend);
|
|
9019
|
+
work.catch(() => { });
|
|
9020
|
+
let timer;
|
|
9021
|
+
const deadline = new Promise(resolve => {
|
|
9022
|
+
timer = setTimeout(() => {
|
|
9023
|
+
this.logger.warn({ backend, deadlineMs: CLI_ENV_PROBE_DEADLINE_MS }, "CLI env live probe exceeded its deadline — serving the cached model list");
|
|
9024
|
+
resolve(null);
|
|
9025
|
+
}, CLI_ENV_PROBE_DEADLINE_MS);
|
|
9026
|
+
});
|
|
9027
|
+
try {
|
|
9028
|
+
return await Promise.race([work, deadline]);
|
|
9029
|
+
}
|
|
9030
|
+
finally {
|
|
9031
|
+
clearTimeout(timer);
|
|
9032
|
+
}
|
|
9033
|
+
}
|
|
8379
9034
|
/**
|
|
8380
9035
|
* Resolve the effective model for a fleet or ClassicBot instance, plus where it
|
|
8381
9036
|
* came from. Single source of truth for `/model` and `/ctx` — precedence:
|
|
@@ -8477,16 +9132,23 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8477
9132
|
void this.probeBackend(b);
|
|
8478
9133
|
}
|
|
8479
9134
|
/** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
|
|
8480
|
-
async getModelOptions(instanceName, refresh = false) {
|
|
9135
|
+
async getModelOptions(instanceName, refresh = false, onLiveProbe) {
|
|
8481
9136
|
const backendName = this.backendNameForInstance(instanceName);
|
|
8482
|
-
|
|
8483
|
-
|
|
8484
|
-
|
|
8485
|
-
|
|
8486
|
-
|
|
8487
|
-
//
|
|
8488
|
-
|
|
8489
|
-
|
|
9137
|
+
const cached = this.readCliEnv(backendName);
|
|
9138
|
+
if (!refresh && cached?.models.length && !this.cliEnvNeedsRefresh(cached))
|
|
9139
|
+
return cached.models;
|
|
9140
|
+
// About to go to the vendor: let the caller say so. A silent 1–10s pause on
|
|
9141
|
+
// an interactive command reads as another hang, which is the wrong lesson to
|
|
9142
|
+
// teach a user who has just been bitten by one.
|
|
9143
|
+
onLiveProbe?.();
|
|
9144
|
+
// Stale, missing, or a forced refresh → probe live (also refreshes the cache).
|
|
9145
|
+
// A newly released model is invisible until this runs, which is why staleness
|
|
9146
|
+
// triggers it rather than waiting for the 24h hard expiry or a cold start.
|
|
9147
|
+
const env = await this.probeBackendBounded(backendName);
|
|
9148
|
+
if (env?.models.length)
|
|
9149
|
+
return env.models;
|
|
9150
|
+
// Probe failed or timed out: the previous list is still the best answer.
|
|
9151
|
+
return cached?.models ?? [];
|
|
8490
9152
|
}
|
|
8491
9153
|
/**
|
|
8492
9154
|
* Model catalog behind the `list_models` tool.
|
|
@@ -8777,7 +9439,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8777
9439
|
await data.respond(t("model.usage"));
|
|
8778
9440
|
return;
|
|
8779
9441
|
}
|
|
8780
|
-
const options = await this.getModelOptions(name, isRefresh)
|
|
9442
|
+
const options = await this.getModelOptions(name, isRefresh, () => {
|
|
9443
|
+
void data.respond(t("model.refreshing")).catch(() => { });
|
|
9444
|
+
});
|
|
8781
9445
|
if (options.length === 0) {
|
|
8782
9446
|
await data.respond(t("model.list_unavailable", name));
|
|
8783
9447
|
return;
|
|
@@ -9479,6 +10143,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9479
10143
|
clearInterval(this.topicCleanupTimer);
|
|
9480
10144
|
this.topicCleanupTimer = null;
|
|
9481
10145
|
}
|
|
10146
|
+
this.topicCleanupGeneration++;
|
|
9482
10147
|
if (this.sessionPruneTimer) {
|
|
9483
10148
|
clearInterval(this.sessionPruneTimer);
|
|
9484
10149
|
this.sessionPruneTimer = null;
|
|
@@ -9611,10 +10276,27 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9611
10276
|
return;
|
|
9612
10277
|
}
|
|
9613
10278
|
this.logger.info(`Full restart: waiting for ${instanceNames.length} instances to idle...`);
|
|
9614
|
-
const
|
|
9615
|
-
|
|
9616
|
-
|
|
9617
|
-
|
|
10279
|
+
const trackedProgress = readUpdateProgress(this.dataDir);
|
|
10280
|
+
const trackedFullRestart = trackedProgress
|
|
10281
|
+
&& updateProgressOperation(trackedProgress.progress) === "full-restart"
|
|
10282
|
+
&& trackedProgress.progress.stage !== "failed"
|
|
10283
|
+
&& trackedProgress.progress.stage !== "complete";
|
|
10284
|
+
if (trackedFullRestart) {
|
|
10285
|
+
// `/restart full` already posted and persisted one public progress message.
|
|
10286
|
+
// Keep that single message; the new process will adopt and finish it.
|
|
10287
|
+
setUpdateProgressStage(this.dataDir, "stopping");
|
|
10288
|
+
}
|
|
10289
|
+
const restartTarget = this.fleetNoticeTarget();
|
|
10290
|
+
if (!trackedFullRestart && this.adapter) {
|
|
10291
|
+
if (restartTarget) {
|
|
10292
|
+
await this.adapter.sendText(restartTarget.chatId, t("restart.full_initiated"), restartTarget.opts)
|
|
10293
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
|
|
10294
|
+
}
|
|
10295
|
+
else {
|
|
10296
|
+
// Say why nothing was posted. A restart that announces itself nowhere,
|
|
10297
|
+
// for a reason nobody logged, is the harder version of this bug.
|
|
10298
|
+
this.logger.warn("Full restart notice has no postable target — set a General topic or channel.options.general_channel_id");
|
|
10299
|
+
}
|
|
9618
10300
|
}
|
|
9619
10301
|
// Wait for idle with 5-minute timeout
|
|
9620
10302
|
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
|
|
@@ -9788,6 +10470,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9788
10470
|
this.logger.error("Cannot restart: no config path (was startAll called?)");
|
|
9789
10471
|
return;
|
|
9790
10472
|
}
|
|
10473
|
+
// A graceful restart keeps this manager process, so the startup probe does
|
|
10474
|
+
// not run again. Without this, `agend restart` left the cached CLI env
|
|
10475
|
+
// untouched and `/model` kept serving an old list until a cold start —
|
|
10476
|
+
// exactly the "only stop+start works" report. Background, never blocking:
|
|
10477
|
+
// /model re-probes on staleness anyway, this just makes a restart do the
|
|
10478
|
+
// refreshing a user expects of it.
|
|
10479
|
+
this.probeCliEnvs();
|
|
9791
10480
|
const instanceNames = [...this.daemons.keys()];
|
|
9792
10481
|
if (instanceNames.length === 0) {
|
|
9793
10482
|
this.logger.info("No instances to restart");
|
|
@@ -10388,11 +11077,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10388
11077
|
const instances = names.map(name => {
|
|
10389
11078
|
const statusFile = join(this.getInstanceDir(name), "statusline.json");
|
|
10390
11079
|
let cost = 0;
|
|
10391
|
-
let model = "";
|
|
10392
11080
|
try {
|
|
10393
11081
|
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
10394
11082
|
cost = data.cost?.total_cost_usd ?? 0;
|
|
10395
|
-
model = data.model?.display_name ?? "";
|
|
10396
11083
|
}
|
|
10397
11084
|
catch (err) {
|
|
10398
11085
|
this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
|
|
@@ -10405,8 +11092,40 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
10405
11092
|
?? this.fleetConfig?.defaults?.backend
|
|
10406
11093
|
?? "claude-code");
|
|
10407
11094
|
const { context } = resolveInstanceContext(this.dataDir, name, backend);
|
|
10408
|
-
|
|
10409
|
-
|
|
11095
|
+
// context_pct: null when unavailable, not 0
|
|
11096
|
+
const context_pct = context ?? null;
|
|
11097
|
+
// Model: Only Claude Code has live statusline; others use the effective resolver.
|
|
11098
|
+
// readStatuslineModel provides the /ctx-aligned display_name+id combo for Claude.
|
|
11099
|
+
// Non-Claude backends must NOT read statusline.json model (may be stale from previous Claude run).
|
|
11100
|
+
const resolved = this.resolveInstanceModel(name);
|
|
11101
|
+
const liveModel = backend === "claude-code" ? readStatuslineModel(this.dataDir, name) : null;
|
|
11102
|
+
// Display value: live model for Claude, resolved.display for others (includes "auto (default)" for Kiro)
|
|
11103
|
+
const model = liveModel ?? resolved.display;
|
|
11104
|
+
// model_source: "live" when Claude statusline succeeded, else the resolver's source
|
|
11105
|
+
const model_source = liveModel ? "live" : resolved.source;
|
|
11106
|
+
// Effort: aligned with /ctx's effortLineFor — unsupported and antigravity don't show effort.
|
|
11107
|
+
const effortStrategy = this.effortStrategyFor(name);
|
|
11108
|
+
const isAgy = backend === "antigravity" || backend === "agy";
|
|
11109
|
+
const effortResolved = this.resolveInstanceEffort(name);
|
|
11110
|
+
// Only show effort if backend supports it and it's not antigravity
|
|
11111
|
+
const effort = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.effort;
|
|
11112
|
+
const effort_source = (effortStrategy === "unsupported" || isAgy) ? null : effortResolved.source;
|
|
11113
|
+
// Display name: fleet config → classic channel → undefined
|
|
11114
|
+
const display_name = classic
|
|
11115
|
+
? this.classicChannels?.getAll().find(ch => ch.instanceName === name)?.displayName
|
|
11116
|
+
: this.fleetConfig?.instances[name]?.display_name;
|
|
11117
|
+
return {
|
|
11118
|
+
name,
|
|
11119
|
+
display_name: display_name || undefined,
|
|
11120
|
+
status: this.getInstanceStatus(name),
|
|
11121
|
+
context_pct,
|
|
11122
|
+
cost,
|
|
11123
|
+
model,
|
|
11124
|
+
model_source,
|
|
11125
|
+
backend,
|
|
11126
|
+
effort,
|
|
11127
|
+
effort_source,
|
|
11128
|
+
};
|
|
10410
11129
|
});
|
|
10411
11130
|
return {
|
|
10412
11131
|
instances,
|