@songsid/agend 2.1.4 → 2.1.5-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/claude-code.js +2 -2
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/types.d.ts +22 -0
- package/dist/backend/types.js +23 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +2 -0
- package/dist/channel/adapters/discord.js +25 -3
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +5 -0
- package/dist/channel/adapters/telegram.js +11 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/types.d.ts +7 -0
- package/dist/cli.js +37 -37
- package/dist/cli.js.map +1 -1
- package/dist/config.js +38 -0
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts +38 -0
- package/dist/daemon.js +95 -43
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +5 -0
- package/dist/fleet-manager.d.ts +126 -0
- package/dist/fleet-manager.js +688 -95
- package/dist/fleet-manager.js.map +1 -1
- package/dist/full-restart.d.ts +20 -0
- package/dist/full-restart.js +39 -0
- package/dist/full-restart.js.map +1 -0
- package/dist/locale.js +72 -2
- package/dist/locale.js.map +1 -1
- package/dist/login-controller.d.ts +127 -0
- package/dist/login-controller.js +450 -0
- package/dist/login-controller.js.map +1 -0
- package/dist/login-flows.d.ts +41 -3
- package/dist/login-flows.js +37 -5
- package/dist/login-flows.js.map +1 -1
- package/dist/login-manager.d.ts +12 -1
- package/dist/login-manager.js +64 -7
- package/dist/login-manager.js.map +1 -1
- package/dist/login-window-lock.d.ts +26 -0
- package/dist/login-window-lock.js +64 -0
- package/dist/login-window-lock.js.map +1 -0
- package/dist/restart-progress.d.ts +35 -5
- package/dist/restart-progress.js +195 -34
- package/dist/restart-progress.js.map +1 -1
- package/dist/service-restart-selection.d.ts +27 -0
- package/dist/service-restart-selection.js +25 -0
- package/dist/service-restart-selection.js.map +1 -0
- package/dist/tmux-manager.d.ts +19 -0
- package/dist/tmux-manager.js +79 -12
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.js +23 -7
- package/dist/topic-commands.js.map +1 -1
- package/dist/types.d.ts +18 -0
- package/dist/ui/web-terminal/terminal.css +17 -0
- package/dist/ui/web-terminal/terminal.html +44 -0
- package/dist/ui/web-terminal/terminal.js +148 -0
- package/dist/ui/web-terminal/vendor/LICENSE.addon-fit +19 -0
- package/dist/ui/web-terminal/vendor/LICENSE.addon-web-links +19 -0
- package/dist/ui/web-terminal/vendor/LICENSE.xterm +21 -0
- package/dist/ui/web-terminal/vendor/VENDORED.md +22 -0
- package/dist/ui/web-terminal/vendor/addon-fit.js +1 -0
- package/dist/ui/web-terminal/vendor/addon-web-links.js +1 -0
- package/dist/ui/web-terminal/vendor/xterm.css +285 -0
- package/dist/ui/web-terminal/vendor/xterm.js +1 -0
- package/dist/update-marker.d.ts +11 -1
- package/dist/update-marker.js +19 -4
- package/dist/update-marker.js.map +1 -1
- package/dist/update-progress.d.ts +1 -1
- package/dist/update-progress.js +12 -0
- package/dist/update-progress.js.map +1 -1
- package/dist/usage/providers.d.ts +39 -0
- package/dist/usage/providers.js +69 -3
- package/dist/usage/providers.js.map +1 -1
- package/dist/web-terminal-http.d.ts +61 -0
- package/dist/web-terminal-http.js +422 -0
- package/dist/web-terminal-http.js.map +1 -0
- package/dist/web-terminal.d.ts +336 -0
- package/dist/web-terminal.js +888 -0
- package/dist/web-terminal.js.map +1 -0
- package/dist/ws-server.d.ts +69 -0
- package/dist/ws-server.js +392 -0
- package/dist/ws-server.js.map +1 -0
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -7,7 +7,7 @@ import { join, dirname, basename } from "node:path";
|
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { isDeepStrictEqual } from "node:util";
|
|
9
9
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
10
|
-
import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
|
|
10
|
+
import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
|
|
11
11
|
import { formatUpdateProgress } from "./update-progress.js";
|
|
12
12
|
import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
|
|
13
13
|
import { readFleetMemory } from "./process-memory.js";
|
|
@@ -48,6 +48,8 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
48
48
|
import { handleUsageRequest, isUsagePath, usageProviderIdForBackend } from "./usage/usage-api.js";
|
|
49
49
|
import { LOGIN_FLOWS, LOGIN_BACKEND_ALIASES, checkAuthStatus } from "./login-flows.js";
|
|
50
50
|
import { LoginSession } from "./login-manager.js";
|
|
51
|
+
import { LoginController, LOGIN_TOKEN_RESEND_PREFIX } from "./login-controller.js";
|
|
52
|
+
import { LoginWindowLock } from "./login-window-lock.js";
|
|
51
53
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
52
54
|
import { setLocale, detectLocale, getLocale, t } from "./locale.js";
|
|
53
55
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
@@ -58,7 +60,8 @@ import { clearPausedMarker } from "./pause-marker.js";
|
|
|
58
60
|
import { releaseProcessFleetLock } from "./fleet-lock.js";
|
|
59
61
|
import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
|
|
60
62
|
import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
|
|
61
|
-
import { RestartProgress } from "./restart-progress.js";
|
|
63
|
+
import { formatRestartProgressCompletion, RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
|
|
64
|
+
import { launchFullRestartHelper } from "./full-restart.js";
|
|
62
65
|
import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
|
|
63
66
|
import { StormWindow } from "./storm-window.js";
|
|
64
67
|
import { SpawnGate } from "./spawn-gate.js";
|
|
@@ -226,7 +229,39 @@ const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
|
|
|
226
229
|
/** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
|
|
227
230
|
const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
|
|
228
231
|
const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
|
|
229
|
-
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; //
|
|
232
|
+
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // hard validity bound for the cached CLI env
|
|
233
|
+
/**
|
|
234
|
+
* How old the cached CLI env may be before `/model` re-probes it live.
|
|
235
|
+
*
|
|
236
|
+
* The cache is a file under AGEND_HOME, so it outlives the process, and the only
|
|
237
|
+
* thing that refreshed it was the startup probe. That is why a newly released
|
|
238
|
+
* model showed up only after `agend stop` + `agend start`: a bare `agend restart`
|
|
239
|
+
* signals SIGUSR2 and restarts the instances inside the *same* manager process,
|
|
240
|
+
* so nothing re-probed and `/model` kept serving a list up to 24h old.
|
|
241
|
+
*/
|
|
242
|
+
const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
|
|
243
|
+
/**
|
|
244
|
+
* Upper bound on a live probe driven by `/model`.
|
|
245
|
+
*
|
|
246
|
+
* A probe chains bounded leaves but is not itself bounded: claude-code runs
|
|
247
|
+
* `--version` (5s) then listModels then listApiModels (an 8s AbortController
|
|
248
|
+
* against api.anthropic.com), and several backends' probes carry no explicit
|
|
249
|
+
* timeout at all. Awaiting that inline would make `/model` the next thing to
|
|
250
|
+
* hang, so it races this deadline and falls back to the cached list.
|
|
251
|
+
*
|
|
252
|
+
* Must stay above CLI_PROBE_LONGEST_CHAIN_MS — the probe's bounded steps run
|
|
253
|
+
* back to back, so clearing only the longest single leaf would be false
|
|
254
|
+
* confidence: a deadline above 8s but below the 13s chain still truncates a
|
|
255
|
+
* probe that would have succeeded. Bounding the chain rather than the leaf is
|
|
256
|
+
* the lesson from the usage hang, and the tests assert against the derived
|
|
257
|
+
* chain constant so raising either step cannot silently break it.
|
|
258
|
+
*
|
|
259
|
+
* The deadline exists to stop a probe hanging forever, not to cut short one that
|
|
260
|
+
* would have finished: truncating a legitimate probe serves the previous list
|
|
261
|
+
* and hides exactly the newly released model the user opened `/model` to find.
|
|
262
|
+
* The wait is announced before it starts, so it reads as progress, not a stall.
|
|
263
|
+
*/
|
|
264
|
+
export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
|
|
230
265
|
export class FleetManager {
|
|
231
266
|
dataDir;
|
|
232
267
|
static signalTarget = null;
|
|
@@ -377,6 +412,8 @@ export class FleetManager {
|
|
|
377
412
|
updateProgressEditRunning = false;
|
|
378
413
|
lastUpdateProgressText = null;
|
|
379
414
|
updateCompletionTipText = null;
|
|
415
|
+
/** Injectable only to keep the chat→CLI reload hand-off deterministic in tests. */
|
|
416
|
+
fullRestartLauncher = launchFullRestartHelper;
|
|
380
417
|
eventLogPruneTimer = null;
|
|
381
418
|
logRotateTimer = null;
|
|
382
419
|
/** Days of event/activity history to keep. */
|
|
@@ -763,6 +800,28 @@ export class FleetManager {
|
|
|
763
800
|
child.once("error", err => this.failUpdateProgress(err.message));
|
|
764
801
|
child.unref();
|
|
765
802
|
}
|
|
803
|
+
async handleRestartSlash(data, adapterId) {
|
|
804
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
805
|
+
await data.respond(t("not_authorized"));
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
if (data.options?.mode !== "full") {
|
|
809
|
+
await data.respond(t("restart.graceful"));
|
|
810
|
+
process.kill(process.pid, "SIGUSR2");
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
// Discord exposes only `full` as a choice. Keep a runtime check anyway: a
|
|
814
|
+
// briefly stale command schema must never turn an unknown value into SIGUSR1.
|
|
815
|
+
const messageId = await data.respond(t("restart.full_preparing"));
|
|
816
|
+
const adapter = this.adapters.get(adapterId) ?? this.adapter;
|
|
817
|
+
if (!messageId || !adapter) {
|
|
818
|
+
this.logger.error({ adapterId, hasMessageId: !!messageId }, "Full restart response could not be persisted — reload refused");
|
|
819
|
+
await data.respond(t("restart.full_launch_failed"));
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
const chatId = String(this.getChannelConfig(adapterId)?.group_id ?? data.channelId);
|
|
823
|
+
await this.requestFullRestart(adapter, chatId, data.channelId, messageId);
|
|
824
|
+
}
|
|
766
825
|
async handleTipsSlash(data, adapterId) {
|
|
767
826
|
if (!this.fleetConfig)
|
|
768
827
|
return;
|
|
@@ -1686,15 +1745,31 @@ export class FleetManager {
|
|
|
1686
1745
|
}
|
|
1687
1746
|
backendNameOf(name) {
|
|
1688
1747
|
const fleetDefault = this.fleetConfig?.defaults?.backend;
|
|
1689
|
-
const
|
|
1690
|
-
|
|
1691
|
-
|
|
1748
|
+
const fleetInstance = this.fleetConfig?.instances[name];
|
|
1749
|
+
// A malformed/manual config can give a fleet and Classic entry the same
|
|
1750
|
+
// instance name. Fleet ownership wins, including its inherited default;
|
|
1751
|
+
// otherwise a Classic override could make backend-scoped recovery restart
|
|
1752
|
+
// the shared process under the wrong login result.
|
|
1753
|
+
if (fleetInstance)
|
|
1754
|
+
return fleetInstance.backend ?? fleetDefault ?? "claude-code";
|
|
1692
1755
|
// ClassicBot channels pick their own backend; the fleet default is only the fallback.
|
|
1693
1756
|
if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
|
|
1694
1757
|
return this.classicChannels.getBackendByInstance(name, fleetDefault);
|
|
1695
1758
|
}
|
|
1696
1759
|
return fleetDefault ?? "claude-code";
|
|
1697
1760
|
}
|
|
1761
|
+
/**
|
|
1762
|
+
* Every configured instance whose backend may share credentials. ClassicBot
|
|
1763
|
+
* rows live only in classicBot.yaml, so backend-wide operations must not use
|
|
1764
|
+
* fleetConfig.instances as their roster. Set keeps a malformed name collision
|
|
1765
|
+
* from restarting the same process twice; backendNameOf defines ownership.
|
|
1766
|
+
*/
|
|
1767
|
+
configuredBackendInstanceNames() {
|
|
1768
|
+
const names = new Set(Object.keys(this.fleetConfig?.instances ?? {}));
|
|
1769
|
+
for (const channel of this.classicChannels?.getAll() ?? [])
|
|
1770
|
+
names.add(channel.instanceName);
|
|
1771
|
+
return [...names];
|
|
1772
|
+
}
|
|
1698
1773
|
/**
|
|
1699
1774
|
* One fleet-level notice per burst, not one per instance: a post-update herd
|
|
1700
1775
|
* fails many instances within the same second. Two notices per incident at
|
|
@@ -1749,16 +1824,60 @@ export class FleetManager {
|
|
|
1749
1824
|
if (!generalName)
|
|
1750
1825
|
return null;
|
|
1751
1826
|
const adapter = this.getAdapterForInstance(generalName);
|
|
1827
|
+
const adapterId = this.getInstanceAdapterId(generalName);
|
|
1752
1828
|
const chatId = this.getGroupIdForInstance(generalName);
|
|
1753
1829
|
if (!adapter || !chatId)
|
|
1754
1830
|
return null;
|
|
1755
1831
|
const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
|
|
1756
1832
|
return {
|
|
1757
1833
|
adapter,
|
|
1834
|
+
resolveAdapter: adapterId ? () => this.readyProgressAdapter(adapterId) : undefined,
|
|
1758
1835
|
chatId,
|
|
1759
1836
|
threadId: topicId != null ? String(topicId) : undefined,
|
|
1760
1837
|
};
|
|
1761
1838
|
}
|
|
1839
|
+
/** Resolve only an adapter generation that can accept progress delivery.
|
|
1840
|
+
* Discord exposes direct gateway readiness; adapters without a health
|
|
1841
|
+
* snapshot use the fleet startup/retry state. */
|
|
1842
|
+
readyProgressAdapter(adapterId) {
|
|
1843
|
+
const adapter = this.adapters.get(adapterId);
|
|
1844
|
+
if (!adapter)
|
|
1845
|
+
return undefined;
|
|
1846
|
+
const health = adapter.getHealthSnapshot?.();
|
|
1847
|
+
if (health)
|
|
1848
|
+
return health.isReady ? adapter : undefined;
|
|
1849
|
+
return this.adapterState.get(adapterId)?.status === "connected" ? adapter : undefined;
|
|
1850
|
+
}
|
|
1851
|
+
/** Last-resort completion after RestartProgress could not deliver to its
|
|
1852
|
+
* adopted target. It stays bounded and never wakes a not-ready gateway. */
|
|
1853
|
+
async sendFleetStartCompletionFallback(chatId, text, threadId) {
|
|
1854
|
+
const adapterId = this.getPrimaryAdapterId();
|
|
1855
|
+
const adapter = adapterId ? this.readyProgressAdapter(adapterId) : undefined;
|
|
1856
|
+
if (!adapter) {
|
|
1857
|
+
this.logger.error({ adapterId }, "Fleet start completion fallback skipped because the primary adapter is not ready");
|
|
1858
|
+
return false;
|
|
1859
|
+
}
|
|
1860
|
+
let timer;
|
|
1861
|
+
const timeout = new Promise(resolve => {
|
|
1862
|
+
timer = setTimeout(() => resolve({ status: "timeout" }), RESTART_PROGRESS_TERMINAL_TIMEOUT_MS);
|
|
1863
|
+
timer.unref?.();
|
|
1864
|
+
});
|
|
1865
|
+
const delivery = Promise.resolve()
|
|
1866
|
+
.then(() => adapter.sendText(chatId, text, { threadId }))
|
|
1867
|
+
.then(() => ({ status: "sent" }), err => ({ status: "failed", err }));
|
|
1868
|
+
const result = await Promise.race([delivery, timeout]);
|
|
1869
|
+
if (timer)
|
|
1870
|
+
clearTimeout(timer);
|
|
1871
|
+
if (result.status === "sent")
|
|
1872
|
+
return true;
|
|
1873
|
+
if (result.status === "failed") {
|
|
1874
|
+
this.logger.error({ err: result.err }, "Failed to send fleet start completion fallback");
|
|
1875
|
+
}
|
|
1876
|
+
else {
|
|
1877
|
+
this.logger.error({ timeout_ms: RESTART_PROGRESS_TERMINAL_TIMEOUT_MS }, "Timed out sending fleet start completion fallback");
|
|
1878
|
+
}
|
|
1879
|
+
return false;
|
|
1880
|
+
}
|
|
1762
1881
|
async stopInstance(name) {
|
|
1763
1882
|
this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
|
|
1764
1883
|
this.cancelStartupRetry(name);
|
|
@@ -1904,6 +2023,130 @@ export class FleetManager {
|
|
|
1904
2023
|
this.updateCompletionTipText = null;
|
|
1905
2024
|
this.startUpdateProgressMonitor(adapter);
|
|
1906
2025
|
}
|
|
2026
|
+
/** Persist the public response, wait for idle, then start the canonical service restart. */
|
|
2027
|
+
async requestFullRestart(adapter, chatId, threadId, messageId) {
|
|
2028
|
+
const target = {
|
|
2029
|
+
adapterId: adapter.id,
|
|
2030
|
+
chatId,
|
|
2031
|
+
...(threadId ? { threadId } : {}),
|
|
2032
|
+
messageId,
|
|
2033
|
+
};
|
|
2034
|
+
// This check is synchronous with the marker write below. A second command
|
|
2035
|
+
// cannot slip through between them on Node's event loop and replace the
|
|
2036
|
+
// first command's delivery target or launch a competing restart helper.
|
|
2037
|
+
if (this.shuttingDown || isUpdateInProgress(this.dataDir)) {
|
|
2038
|
+
this.logger.warn({ adapterId: adapter.id }, "Full restart refused because another planned restart is active");
|
|
2039
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
|
|
2040
|
+
return false;
|
|
2041
|
+
}
|
|
2042
|
+
if (!persistFullRestartProgress(this.dataDir, target)) {
|
|
2043
|
+
this.logger.error({ adapterId: adapter.id }, "Full restart marker could not be persisted — reload refused");
|
|
2044
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2045
|
+
return false;
|
|
2046
|
+
}
|
|
2047
|
+
const ownedMarker = readUpdateProgress(this.dataDir);
|
|
2048
|
+
if (!ownedMarker) {
|
|
2049
|
+
this.logger.error({ adapterId: adapter.id }, "Full restart marker disappeared after persistence — reload refused");
|
|
2050
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2051
|
+
return false;
|
|
2052
|
+
}
|
|
2053
|
+
this.lastUpdateProgressText = null;
|
|
2054
|
+
this.updateCompletionTipText = null;
|
|
2055
|
+
this.startUpdateProgressMonitor(adapter);
|
|
2056
|
+
await this.waitForFullRestartIdleGrace();
|
|
2057
|
+
// An update can begin while the idle wait yields. Never launch a second
|
|
2058
|
+
// process replacement against a marker we no longer own.
|
|
2059
|
+
if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target)) {
|
|
2060
|
+
this.logger.warn({ adapterId: adapter.id }, "Full restart superseded during idle wait — reload refused");
|
|
2061
|
+
if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2062
|
+
clearUpdateMarker(this.dataDir);
|
|
2063
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
|
|
2064
|
+
return false;
|
|
2065
|
+
}
|
|
2066
|
+
let helper;
|
|
2067
|
+
try {
|
|
2068
|
+
helper = await this.fullRestartLauncher();
|
|
2069
|
+
}
|
|
2070
|
+
catch (err) {
|
|
2071
|
+
this.logger.error({ err }, "Full restart helper failed to spawn — reload refused");
|
|
2072
|
+
if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2073
|
+
clearUpdateMarker(this.dataDir);
|
|
2074
|
+
await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
|
|
2075
|
+
return false;
|
|
2076
|
+
}
|
|
2077
|
+
void helper.completion.then(result => {
|
|
2078
|
+
// A zero exit means the service manager accepted the restart. launchd
|
|
2079
|
+
// may report that before its SIGTERM reaches us, so only an explicit
|
|
2080
|
+
// helper error/non-zero exit is evidence of failure. A signal is also
|
|
2081
|
+
// ambiguous under systemd because this helper shares the old cgroup.
|
|
2082
|
+
if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
|
|
2083
|
+
return;
|
|
2084
|
+
if (!result.error && (result.code === 0 || result.code === null))
|
|
2085
|
+
return;
|
|
2086
|
+
const detail = result.error
|
|
2087
|
+
? "reload helper failed after launch"
|
|
2088
|
+
: `reload helper exited before process hand-off (code ${result.code ?? "null"}, signal ${result.signal ?? "none"})`;
|
|
2089
|
+
this.logger.error({ result }, "Full restart helper exited before the fleet began shutting down");
|
|
2090
|
+
setUpdateProgressStage(this.dataDir, "failed", { error: detail });
|
|
2091
|
+
});
|
|
2092
|
+
return true;
|
|
2093
|
+
}
|
|
2094
|
+
isOwnedFullRestartMarker(startedAt, target) {
|
|
2095
|
+
const marker = readUpdateProgress(this.dataDir);
|
|
2096
|
+
if (!marker || marker.startedAt !== startedAt || marker.pid !== process.pid)
|
|
2097
|
+
return false;
|
|
2098
|
+
if (updateProgressOperation(marker.progress) !== "full-restart")
|
|
2099
|
+
return false;
|
|
2100
|
+
const current = marker.progress.target;
|
|
2101
|
+
return current.adapterId === target.adapterId
|
|
2102
|
+
&& current.chatId === target.chatId
|
|
2103
|
+
&& current.threadId === target.threadId
|
|
2104
|
+
&& current.messageId === target.messageId;
|
|
2105
|
+
}
|
|
2106
|
+
/** Give current work the same bounded idle grace used by graceful reload. */
|
|
2107
|
+
async waitForFullRestartIdleGrace() {
|
|
2108
|
+
const instanceNames = [...this.daemons.keys()];
|
|
2109
|
+
if (instanceNames.length === 0)
|
|
2110
|
+
return;
|
|
2111
|
+
const IDLE_TIMEOUT_MS = 5 * 60_000;
|
|
2112
|
+
let timeoutHandle;
|
|
2113
|
+
const deadline = new Promise((_, reject) => {
|
|
2114
|
+
timeoutHandle = setTimeout(() => reject(new Error("Idle wait timed out after 5 minutes")), IDLE_TIMEOUT_MS);
|
|
2115
|
+
});
|
|
2116
|
+
try {
|
|
2117
|
+
await Promise.race([
|
|
2118
|
+
Promise.all(instanceNames.map(async (name) => {
|
|
2119
|
+
const daemon = this.daemons.get(name);
|
|
2120
|
+
if (!daemon)
|
|
2121
|
+
return;
|
|
2122
|
+
this.logger.info(`Full restart: waiting for ${name} to idle...`);
|
|
2123
|
+
await daemon.waitForIdle(10_000);
|
|
2124
|
+
})),
|
|
2125
|
+
deadline,
|
|
2126
|
+
]);
|
|
2127
|
+
}
|
|
2128
|
+
catch (err) {
|
|
2129
|
+
this.logger.warn({ err }, "Full restart idle wait timed out — continuing with service restart");
|
|
2130
|
+
}
|
|
2131
|
+
finally {
|
|
2132
|
+
clearTimeout(timeoutHandle);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
async reportFullRestartFailure(adapter, chatId, threadId, messageId, text) {
|
|
2136
|
+
try {
|
|
2137
|
+
await adapter.editMessage(chatId, messageId, text, threadId);
|
|
2138
|
+
return;
|
|
2139
|
+
}
|
|
2140
|
+
catch (err) {
|
|
2141
|
+
this.logger.warn({ err, adapterId: adapter.id }, "Failed to edit rejected full-restart request; posting a fresh notice");
|
|
2142
|
+
}
|
|
2143
|
+
try {
|
|
2144
|
+
await adapter.sendText(chatId, text, { threadId });
|
|
2145
|
+
}
|
|
2146
|
+
catch (err) {
|
|
2147
|
+
this.logger.error({ err, adapterId: adapter.id }, "Failed to deliver full-restart rejection");
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
1907
2150
|
failUpdateProgress(message) {
|
|
1908
2151
|
setUpdateProgressStage(this.dataDir, "failed", { error: message });
|
|
1909
2152
|
}
|
|
@@ -1965,6 +2208,8 @@ export class FleetManager {
|
|
|
1965
2208
|
}
|
|
1966
2209
|
/** Start all instances from fleet config */
|
|
1967
2210
|
async startAll(configPath) {
|
|
2211
|
+
this.loginWindow.reopen(); // a stopAll → startAll restart must accept login windows again
|
|
2212
|
+
this.loginController?.reopen();
|
|
1968
2213
|
const startupStartedAt = Date.now();
|
|
1969
2214
|
FleetManager.signalTarget = this;
|
|
1970
2215
|
this.startupComplete = false;
|
|
@@ -1984,6 +2229,9 @@ export class FleetManager {
|
|
|
1984
2229
|
&& savedUpdateProgress.progress.stage !== "complete"
|
|
1985
2230
|
? savedUpdateProgress
|
|
1986
2231
|
: null;
|
|
2232
|
+
const pendingProgressOperation = pendingUpdateProgress
|
|
2233
|
+
? updateProgressOperation(pendingUpdateProgress.progress)
|
|
2234
|
+
: null;
|
|
1987
2235
|
if (pendingUpdateProgress) {
|
|
1988
2236
|
setUpdateProgressStage(this.dataDir, "starting", { version: pendingUpdateProgress.progress.version });
|
|
1989
2237
|
}
|
|
@@ -2121,10 +2369,7 @@ export class FleetManager {
|
|
|
2121
2369
|
...fleet.defaults.daily_summary,
|
|
2122
2370
|
};
|
|
2123
2371
|
this.dailySummary = new DailySummary(summaryConfig, costGuardConfig.timezone, (text) => {
|
|
2124
|
-
|
|
2125
|
-
return;
|
|
2126
|
-
this.adapter.sendText(String(this.fleetConfig.channel.group_id), text)
|
|
2127
|
-
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
2372
|
+
this.postDailySummary(text);
|
|
2128
2373
|
// Rotate classic channel chat logs daily
|
|
2129
2374
|
this.classicChannels?.rotateLogs();
|
|
2130
2375
|
this.rotateInboxes();
|
|
@@ -2264,7 +2509,7 @@ export class FleetManager {
|
|
|
2264
2509
|
const allEntries = Object.entries(fleet.instances);
|
|
2265
2510
|
const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
|
|
2266
2511
|
const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
|
|
2267
|
-
const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingUpdateProgress ? "update" : "restart" });
|
|
2512
|
+
const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingProgressOperation === "full-restart" ? "reload" : pendingUpdateProgress ? "update" : "restart" });
|
|
2268
2513
|
if (generals.length > 0) {
|
|
2269
2514
|
for (const [name, cfg] of generals) {
|
|
2270
2515
|
try {
|
|
@@ -2313,12 +2558,17 @@ export class FleetManager {
|
|
|
2313
2558
|
progressStart = adapterStartup.then(() => {
|
|
2314
2559
|
if (pendingUpdateProgress) {
|
|
2315
2560
|
const saved = pendingUpdateProgress.progress.target;
|
|
2316
|
-
const
|
|
2317
|
-
|
|
2318
|
-
|
|
2561
|
+
const target = {
|
|
2562
|
+
adapter: this.adapters.get(saved.adapterId),
|
|
2563
|
+
// Adapter retries replace the failed object in this map. Resolve at
|
|
2564
|
+
// every progress delivery so the adopted update message follows the
|
|
2565
|
+
// live, ready generation instead of remaining pinned to a stopped
|
|
2566
|
+
// client. This also waits when the first generation failed before
|
|
2567
|
+
// any adapter object was registered.
|
|
2568
|
+
resolveAdapter: () => this.readyProgressAdapter(saved.adapterId),
|
|
2319
2569
|
chatId: saved.chatId,
|
|
2320
2570
|
threadId: saved.threadId,
|
|
2321
|
-
}
|
|
2571
|
+
};
|
|
2322
2572
|
return startupProgress.resume(target, saved.messageId);
|
|
2323
2573
|
}
|
|
2324
2574
|
return startupProgress.start(this.restartProgressTarget());
|
|
@@ -2428,16 +2678,25 @@ export class FleetManager {
|
|
|
2428
2678
|
version: agendVersion,
|
|
2429
2679
|
pausedNames,
|
|
2430
2680
|
failedNames,
|
|
2431
|
-
tipText:
|
|
2681
|
+
tipText: pendingProgressOperation === "update" && this.tipsEnabled()
|
|
2432
2682
|
? (() => {
|
|
2433
2683
|
const tip = this.pickAvailableTip();
|
|
2434
2684
|
return tip ? this.formatTip(tip) : undefined;
|
|
2435
2685
|
})()
|
|
2436
2686
|
: undefined,
|
|
2437
2687
|
});
|
|
2438
|
-
if (!progressCompleted &&
|
|
2688
|
+
if (!progressCompleted && fleet.channel?.group_id) {
|
|
2439
2689
|
let text;
|
|
2440
|
-
if (
|
|
2690
|
+
if (pendingProgressOperation === "full-restart") {
|
|
2691
|
+
text = formatRestartProgressCompletion("reload", {
|
|
2692
|
+
running: started,
|
|
2693
|
+
total,
|
|
2694
|
+
version: agendVersion,
|
|
2695
|
+
pausedNames,
|
|
2696
|
+
failedNames,
|
|
2697
|
+
}, pendingUpdateProgress.startedAt);
|
|
2698
|
+
}
|
|
2699
|
+
else if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
2441
2700
|
text = t("fleet.ready", started, total, agendVersion);
|
|
2442
2701
|
}
|
|
2443
2702
|
else if (failedNames.length === 0) {
|
|
@@ -2447,9 +2706,7 @@ export class FleetManager {
|
|
|
2447
2706
|
text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
|
|
2448
2707
|
+ (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
|
|
2449
2708
|
}
|
|
2450
|
-
this.
|
|
2451
|
-
threadId: generalThreadId != null ? String(generalThreadId) : undefined,
|
|
2452
|
-
}).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
|
|
2709
|
+
await this.sendFleetStartCompletionFallback(String(fleet.channel.group_id), text, generalThreadId != null ? String(generalThreadId) : undefined);
|
|
2453
2710
|
}
|
|
2454
2711
|
}
|
|
2455
2712
|
// Health HTTP endpoint
|
|
@@ -2735,6 +2992,8 @@ export class FleetManager {
|
|
|
2735
2992
|
return;
|
|
2736
2993
|
if (await this.handleLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2737
2994
|
return;
|
|
2995
|
+
if (await this.handleLoginTokenResend(data, adapterId, this.adapter ?? undefined))
|
|
2996
|
+
return;
|
|
2738
2997
|
if (await this.handleInstallLoginConfirm(data, adapterId, this.adapter ?? undefined))
|
|
2739
2998
|
return;
|
|
2740
2999
|
if (await this.handleClearConfirmation(data, adapterId, this.adapter ?? undefined))
|
|
@@ -2983,13 +3242,7 @@ export class FleetManager {
|
|
|
2983
3242
|
await data.respond(this.topicCommands.getDashboardText());
|
|
2984
3243
|
}
|
|
2985
3244
|
else if (data.command === "restart") {
|
|
2986
|
-
|
|
2987
|
-
if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
|
|
2988
|
-
await data.respond(t("not_authorized"));
|
|
2989
|
-
return;
|
|
2990
|
-
}
|
|
2991
|
-
await data.respond(t("restart.graceful"));
|
|
2992
|
-
process.kill(process.pid, "SIGUSR2");
|
|
3245
|
+
await this.handleRestartSlash(data, adapterId);
|
|
2993
3246
|
}
|
|
2994
3247
|
else if (data.command === "compact") {
|
|
2995
3248
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
@@ -3135,6 +3388,8 @@ export class FleetManager {
|
|
|
3135
3388
|
return;
|
|
3136
3389
|
if (await this.handleLoginConfirm(data, adapterId, adapter))
|
|
3137
3390
|
return;
|
|
3391
|
+
if (await this.handleLoginTokenResend(data, adapterId, adapter))
|
|
3392
|
+
return;
|
|
3138
3393
|
if (await this.handleInstallLoginConfirm(data, adapterId, adapter))
|
|
3139
3394
|
return;
|
|
3140
3395
|
if (await this.handleClearConfirmation(data, adapterId, adapter))
|
|
@@ -3330,13 +3585,7 @@ export class FleetManager {
|
|
|
3330
3585
|
await data.respond(this.topicCommands.getDashboardText());
|
|
3331
3586
|
}
|
|
3332
3587
|
else if (data.command === "restart") {
|
|
3333
|
-
|
|
3334
|
-
if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
|
|
3335
|
-
await data.respond(t("not_authorized"));
|
|
3336
|
-
return;
|
|
3337
|
-
}
|
|
3338
|
-
await data.respond(t("restart.graceful"));
|
|
3339
|
-
process.kill(process.pid, "SIGUSR2");
|
|
3588
|
+
await this.handleRestartSlash(data, adapterId);
|
|
3340
3589
|
}
|
|
3341
3590
|
else if (data.command === "compact") {
|
|
3342
3591
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
@@ -4668,19 +4917,69 @@ export class FleetManager {
|
|
|
4668
4917
|
this.scheduler.recordRun(id, "instance_offline", `retry ${retryCount}x failed`);
|
|
4669
4918
|
this.notifyScheduleFailure(schedule);
|
|
4670
4919
|
}
|
|
4920
|
+
/**
|
|
4921
|
+
* The adapter that can actually post into a chat, found by its group.
|
|
4922
|
+
*
|
|
4923
|
+
* A schedule records where it was created (reply_chat_id) separately from
|
|
4924
|
+
* what it triggers (target). Those need not share a platform: a Telegram
|
|
4925
|
+
* group can schedule a Discord-topic instance. Picking the adapter from the
|
|
4926
|
+
* target then sends a Telegram chat id through the Discord bot, which fails
|
|
4927
|
+
* with Unknown Channel — the source topic never hears that its schedule ran.
|
|
4928
|
+
*
|
|
4929
|
+
* When several bots share one guild, the primary wins: a persona should not
|
|
4930
|
+
* be the voice announcing fleet scheduling.
|
|
4931
|
+
*/
|
|
4932
|
+
adapterForChat(chatId) {
|
|
4933
|
+
const id = String(chatId);
|
|
4934
|
+
const matches = [...this.worlds.values()].filter(world => String(world.groupId) === id);
|
|
4935
|
+
if (matches.length === 0)
|
|
4936
|
+
return undefined;
|
|
4937
|
+
const primaryId = this.getPrimaryAdapterId();
|
|
4938
|
+
return (matches.find(world => world.id === primaryId) ?? matches[0]).adapter;
|
|
4939
|
+
}
|
|
4940
|
+
/**
|
|
4941
|
+
* The adapter that can answer a schedule in the chat it was created from.
|
|
4942
|
+
*
|
|
4943
|
+
* A schedule records its creator (source) and its trigger (target) separately,
|
|
4944
|
+
* and they need not share a platform — the live fleet has a Telegram group
|
|
4945
|
+
* scheduling a Discord-topic instance. Routing by target sends a Telegram chat
|
|
4946
|
+
* id through the Discord bot, which is one half of the Unknown Channel errors.
|
|
4947
|
+
*
|
|
4948
|
+
* The creator's own adapter comes first, and only when its world actually owns
|
|
4949
|
+
* that chat. Classic keeps its own identity, so a schedule made from a
|
|
4950
|
+
* persona-bound Classic channel is answered by that persona: the primary bot
|
|
4951
|
+
* may not even have access there, and would be the wrong voice if it did.
|
|
4952
|
+
* Falling back to the target's adapter is deliberately NOT an option — that is
|
|
4953
|
+
* the misroute itself; callers say why they stayed silent instead.
|
|
4954
|
+
*/
|
|
4955
|
+
scheduleSourceAdapter(schedule) {
|
|
4956
|
+
const chatId = String(schedule.reply_chat_id);
|
|
4957
|
+
const sourceAdapterId = schedule.source ? this.getInstanceAdapterId(schedule.source) : undefined;
|
|
4958
|
+
const sourceWorld = sourceAdapterId ? this.worlds.get(sourceAdapterId) : undefined;
|
|
4959
|
+
if (sourceWorld && String(sourceWorld.groupId) === chatId)
|
|
4960
|
+
return sourceWorld.adapter;
|
|
4961
|
+
return this.adapterForChat(chatId);
|
|
4962
|
+
}
|
|
4671
4963
|
notifySourceTopic(schedule) {
|
|
4672
|
-
const adapter = this.
|
|
4673
|
-
if (!adapter)
|
|
4964
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
4965
|
+
if (!adapter) {
|
|
4966
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — trigger notice not sent");
|
|
4674
4967
|
return;
|
|
4968
|
+
}
|
|
4675
4969
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" triggered, target: ${schedule.target}`;
|
|
4676
4970
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4677
4971
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
4678
4972
|
}).catch((err) => this.logger.error({ err }, "Failed to send cross-instance notification"));
|
|
4679
4973
|
}
|
|
4680
4974
|
notifyScheduleFailure(schedule) {
|
|
4681
|
-
|
|
4682
|
-
|
|
4975
|
+
// Same resolver as the success path: a failure notice was still being sent
|
|
4976
|
+
// through the target's adapter, so a Telegram-created schedule for a
|
|
4977
|
+
// Discord target announced its failure into the wrong platform.
|
|
4978
|
+
const adapter = this.scheduleSourceAdapter(schedule);
|
|
4979
|
+
if (!adapter) {
|
|
4980
|
+
this.logger.warn({ scheduleId: schedule.id, chatId: schedule.reply_chat_id, source: schedule.source }, "No adapter can reach the schedule's source chat — failure notice not sent");
|
|
4683
4981
|
return;
|
|
4982
|
+
}
|
|
4684
4983
|
const text = `⏰ Schedule "${schedule.label ?? schedule.id}" trigger failed: instance ${schedule.target} is offline.`;
|
|
4685
4984
|
adapter.sendText(schedule.reply_chat_id, text, {
|
|
4686
4985
|
threadId: schedule.reply_thread_id ?? undefined,
|
|
@@ -5526,6 +5825,65 @@ export class FleetManager {
|
|
|
5526
5825
|
this.logger.error({ ids: bad }, "classicBot.yaml holds ids that can never match");
|
|
5527
5826
|
this.notifyFleetError(t("classic.unrecoverable_ids", list));
|
|
5528
5827
|
}
|
|
5828
|
+
/**
|
|
5829
|
+
* Where a fleet-wide notice can actually be posted, or null if nowhere.
|
|
5830
|
+
*
|
|
5831
|
+
* On Telegram a group id is itself a chat, so posting straight to it is
|
|
5832
|
+
* right. On Discord it is a *guild* id, and sending there makes the adapter
|
|
5833
|
+
* fetch a channel that does not exist — DiscordAPIError 10003 Unknown
|
|
5834
|
+
* Channel. That is why the daily summary never arrived on a Discord fleet:
|
|
5835
|
+
* it had been posting to the guild every night and only the catch handler
|
|
5836
|
+
* ever saw it.
|
|
5837
|
+
*
|
|
5838
|
+
* Discord therefore needs a real channel: the General topic (resolved from
|
|
5839
|
+
* config rather than findGeneralInstance, so a fleet-level fault can still be
|
|
5840
|
+
* reported while the General daemon is down), else the adapter's configured
|
|
5841
|
+
* general_channel_id. With neither, there is no safe target and the caller
|
|
5842
|
+
* should say so rather than send into a guaranteed failure.
|
|
5843
|
+
*/
|
|
5844
|
+
fleetNoticeTarget(adapterId) {
|
|
5845
|
+
const cfg = this.getChannelConfig(adapterId);
|
|
5846
|
+
const groupId = cfg?.group_id;
|
|
5847
|
+
if (groupId == null)
|
|
5848
|
+
return null;
|
|
5849
|
+
const chatId = String(groupId);
|
|
5850
|
+
// The General must belong to the SAME adapter as the group above. Taking
|
|
5851
|
+
// whichever General comes first in the instance map produced a mixed target
|
|
5852
|
+
// on a dual-platform fleet — a Telegram group id carrying a Discord channel
|
|
5853
|
+
// as its thread — and made the result depend on map insertion order.
|
|
5854
|
+
// Resolved from config, not from a live daemon, so a fleet-level fault is
|
|
5855
|
+
// still reportable while the General itself is down.
|
|
5856
|
+
const ownerId = cfg?.id ?? cfg?.type;
|
|
5857
|
+
const generalTopic = Object.entries(this.fleetConfig?.instances ?? {})
|
|
5858
|
+
.find(([name, instance]) => instance.general_topic === true
|
|
5859
|
+
&& this.getInstanceAdapterId(name) === ownerId)?.[1]?.topic_id;
|
|
5860
|
+
if (generalTopic != null)
|
|
5861
|
+
return { chatId, opts: { threadId: String(generalTopic) } };
|
|
5862
|
+
if (cfg?.type === "discord") {
|
|
5863
|
+
const configured = cfg.options?.general_channel_id;
|
|
5864
|
+
if (configured != null && String(configured)) {
|
|
5865
|
+
return { chatId, opts: { threadId: String(configured) } };
|
|
5866
|
+
}
|
|
5867
|
+
return null; // a guild id is not a channel; sending would always fail
|
|
5868
|
+
}
|
|
5869
|
+
return { chatId, opts: {} };
|
|
5870
|
+
}
|
|
5871
|
+
/**
|
|
5872
|
+
* Post the daily summary where fleet-wide notices go.
|
|
5873
|
+
*
|
|
5874
|
+
* A named method rather than an inline closure so a test can drive the real
|
|
5875
|
+
* thing: asserting on fleetNoticeTarget alone leaves the call site free to go
|
|
5876
|
+
* back to posting at the bare group id, which is the defect this replaced.
|
|
5877
|
+
*/
|
|
5878
|
+
postDailySummary(text) {
|
|
5879
|
+
const target = this.fleetNoticeTarget();
|
|
5880
|
+
if (!this.adapter || !target) {
|
|
5881
|
+
this.logger.warn("Daily summary has no postable target — set a General topic or channel.options.general_channel_id");
|
|
5882
|
+
return;
|
|
5883
|
+
}
|
|
5884
|
+
this.adapter.sendText(target.chatId, text, target.opts)
|
|
5885
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to send daily summary"));
|
|
5886
|
+
}
|
|
5529
5887
|
notifyFleetError(text) {
|
|
5530
5888
|
const now = Date.now();
|
|
5531
5889
|
const key = text.slice(0, 200);
|
|
@@ -5552,10 +5910,12 @@ export class FleetManager {
|
|
|
5552
5910
|
dispatched = this.notifyInstanceTopic(general, body);
|
|
5553
5911
|
}
|
|
5554
5912
|
else {
|
|
5555
|
-
// No General instance — fall back to the primary channel's
|
|
5556
|
-
|
|
5557
|
-
|
|
5558
|
-
|
|
5913
|
+
// No General instance — fall back to the primary channel's own notice
|
|
5914
|
+
// target. Posting to the bare group id looked right but is a guild id on
|
|
5915
|
+
// Discord, so every such fallback failed inside the catch handler.
|
|
5916
|
+
const target = this.fleetNoticeTarget();
|
|
5917
|
+
if (this.adapter && target) {
|
|
5918
|
+
this.adapter.sendText(target.chatId, body, target.opts)
|
|
5559
5919
|
.catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
|
|
5560
5920
|
dispatched = true;
|
|
5561
5921
|
}
|
|
@@ -5614,13 +5974,16 @@ export class FleetManager {
|
|
|
5614
5974
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
|
|
5615
5975
|
return true;
|
|
5616
5976
|
}
|
|
5617
|
-
// Fallback:
|
|
5618
|
-
|
|
5619
|
-
|
|
5977
|
+
// Fallback: the instance has neither a topic nor a classic channel, so post
|
|
5978
|
+
// where fleet-wide notices go. Not the bare group id: on Discord that is a
|
|
5979
|
+
// guild, and the send fails inside the catch handler.
|
|
5980
|
+
const target = this.fleetNoticeTarget(this.getInstanceAdapterId(instanceName));
|
|
5981
|
+
if (target) {
|
|
5982
|
+
adapter.sendText(target.chatId, text, { ...target.opts, ...extraOpts })
|
|
5620
5983
|
.catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
|
|
5621
5984
|
return true;
|
|
5622
5985
|
}
|
|
5623
|
-
this.logger.warn({ instanceName }, "No
|
|
5986
|
+
this.logger.warn({ instanceName }, "No postable target — instance topic notification not sent");
|
|
5624
5987
|
return false;
|
|
5625
5988
|
}
|
|
5626
5989
|
// ── Nonce-armed button prompts (hang / assist / exit / clear) ──
|
|
@@ -6859,11 +7222,45 @@ export class FleetManager {
|
|
|
6859
7222
|
// repairs every instance of that backend, and instance delivery, pane-state
|
|
6860
7223
|
// detection, tool progress, and mcp_proxy_reply never observe login output.
|
|
6861
7224
|
activeLogin = null;
|
|
7225
|
+
/**
|
|
7226
|
+
* Web-terminal login (v2.1.5 default, `login.mode: web`). The relay code
|
|
7227
|
+
* below (activeLogin/LoginSession) is `login.mode: relay`, kept for one
|
|
7228
|
+
* release as the rollback lever and removed in 2.1.6.
|
|
7229
|
+
*/
|
|
7230
|
+
loginController = null;
|
|
7231
|
+
/**
|
|
7232
|
+
* One login/install window fleet-wide. Claimed synchronously before the
|
|
7233
|
+
* first await by web login, relay login and install alike (sol B1).
|
|
7234
|
+
*/
|
|
7235
|
+
loginWindow = new LoginWindowLock();
|
|
7236
|
+
get webLogin() {
|
|
7237
|
+
if (!this.loginController) {
|
|
7238
|
+
this.loginController = new LoginController({
|
|
7239
|
+
logger: this.logger,
|
|
7240
|
+
fleetConfig: () => this.fleetConfig,
|
|
7241
|
+
isFleetAdmin: (userId, adapterId) => this.isFleetAdmin(userId, adapterId),
|
|
7242
|
+
eventLog: () => this.eventLog,
|
|
7243
|
+
recoverBackendInstances: backend => this.recoverBackendInstances(backend),
|
|
7244
|
+
claimWindow: backend => this.loginWindow.tryClaim("web", backend),
|
|
7245
|
+
releaseWindow: claim => { this.loginWindow.release(claim); },
|
|
7246
|
+
isClaimCurrent: claim => this.loginWindow.isCurrent(claim),
|
|
7247
|
+
windowBusyMessage: () => this.loginWindow.busyMessage(),
|
|
7248
|
+
postButtons: async ({ prefix, instanceName, chat, message, choices, expiredText }) => {
|
|
7249
|
+
await this.postNonceButtonPrompt({
|
|
7250
|
+
prefix, alertType: "login", instanceName,
|
|
7251
|
+
adapter: chat.adapter, adapterId: chat.adapterId, chatId: chat.chatId, threadId: chat.threadId,
|
|
7252
|
+
message, choices, expiredText,
|
|
7253
|
+
});
|
|
7254
|
+
},
|
|
7255
|
+
});
|
|
7256
|
+
}
|
|
7257
|
+
return this.loginController;
|
|
7258
|
+
}
|
|
6862
7259
|
/** Post the backend chooser for a bare `/login`. Caller enforces admin. */
|
|
6863
7260
|
async promptLoginBackends(chat) {
|
|
6864
7261
|
const configured = new Set();
|
|
6865
|
-
for (const
|
|
6866
|
-
configured.add(
|
|
7262
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
7263
|
+
configured.add(this.backendNameOf(name));
|
|
6867
7264
|
}
|
|
6868
7265
|
const choices = Object.keys(LOGIN_FLOWS)
|
|
6869
7266
|
.filter(backend => configured.size === 0 || configured.has(backend))
|
|
@@ -7034,21 +7431,40 @@ export class FleetManager {
|
|
|
7034
7431
|
* posted instead (auth still valid — see the pre-check below).
|
|
7035
7432
|
*/
|
|
7036
7433
|
async startLoginSession(backendArg, chat, opts = {}) {
|
|
7434
|
+
if (this.webLogin.mode() === "web")
|
|
7435
|
+
return this.webLogin.start(backendArg, chat, opts);
|
|
7436
|
+
// ── legacy relay mode (login.mode: relay) — removed in 2.1.6 ──
|
|
7037
7437
|
const backend = LOGIN_BACKEND_ALIASES[backendArg.toLowerCase()] ?? backendArg.toLowerCase();
|
|
7038
7438
|
const flow = LOGIN_FLOWS[backend];
|
|
7039
7439
|
if (!flow)
|
|
7040
7440
|
return t("login.unsupported", backendArg);
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7441
|
+
// Declined in every mode — never a silent relay fallback (e.g. Antigravity).
|
|
7442
|
+
if (flow.remoteLogin === "unsupported")
|
|
7443
|
+
return t("login.remote_unsupported_agent_cli", backend, flow.command);
|
|
7444
|
+
// Reserve the window before the pre-check await. The claim is owned by this
|
|
7445
|
+
// region until it is transferred to launchLoginSession; any other exit
|
|
7446
|
+
// (buttons only, throw, shutdown) releases it.
|
|
7447
|
+
const claim = this.loginWindow.tryClaim("relay", backend);
|
|
7448
|
+
if (!claim)
|
|
7449
|
+
return this.loginWindow.busyMessage();
|
|
7450
|
+
let transferred = false;
|
|
7451
|
+
try {
|
|
7452
|
+
return await this.startRelayClaimed(flow, backend, chat, opts, claim, () => { transferred = true; });
|
|
7453
|
+
}
|
|
7454
|
+
finally {
|
|
7455
|
+
if (!transferred)
|
|
7456
|
+
this.loginWindow.release(claim);
|
|
7457
|
+
}
|
|
7458
|
+
}
|
|
7459
|
+
async startRelayClaimed(flow, backend, chat, opts, claim, markTransferred) {
|
|
7046
7460
|
// Token-free pre-check (5s cap): re-login while auth still works is
|
|
7047
7461
|
// usually a mistake, so it needs a confirmed click. An invalid OR
|
|
7048
7462
|
// uncertain result (timeout, missing binary) proceeds straight to login —
|
|
7049
7463
|
// an unreliable probe must never block the re-login the admin asked for.
|
|
7050
7464
|
if (!opts.skipAuthCheck && flow.authCheck) {
|
|
7051
7465
|
const status = await checkAuthStatus(flow.authCheck);
|
|
7466
|
+
if (!this.loginWindow.isCurrent(claim))
|
|
7467
|
+
return t("login.web_shutting_down"); // fleet shut down while we probed
|
|
7052
7468
|
if (status === "valid") {
|
|
7053
7469
|
await this.postNonceButtonPrompt({
|
|
7054
7470
|
prefix: LOGIN_CONFIRM_CALLBACK_PREFIX,
|
|
@@ -7068,18 +7484,27 @@ export class FleetManager {
|
|
|
7068
7484
|
return null;
|
|
7069
7485
|
}
|
|
7070
7486
|
}
|
|
7071
|
-
|
|
7487
|
+
markTransferred(); // launchLoginSession owns the claim from here
|
|
7488
|
+
return this.launchLoginSession(flow, backend, chat, claim);
|
|
7072
7489
|
}
|
|
7073
7490
|
/** Create the login window and session (pre-check already settled). */
|
|
7074
|
-
async launchLoginSession(flow, backend, chat) {
|
|
7075
|
-
|
|
7076
|
-
|
|
7077
|
-
|
|
7078
|
-
|
|
7079
|
-
|
|
7080
|
-
|
|
7081
|
-
|
|
7082
|
-
|
|
7491
|
+
async launchLoginSession(flow, backend, chat, claim) {
|
|
7492
|
+
// Owns `claim`: released on any failure before the session is published,
|
|
7493
|
+
// and by onDone afterwards. A shutdown during ensureSession stops us.
|
|
7494
|
+
let tmux;
|
|
7495
|
+
try {
|
|
7496
|
+
const sessionName = getTmuxSession();
|
|
7497
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7498
|
+
if (!this.loginWindow.isCurrent(claim)) {
|
|
7499
|
+
this.loginWindow.release(claim);
|
|
7500
|
+
return t("login.web_shutting_down");
|
|
7501
|
+
}
|
|
7502
|
+
tmux = new TmuxManager(sessionName, "");
|
|
7503
|
+
}
|
|
7504
|
+
catch (err) {
|
|
7505
|
+
this.loginWindow.release(claim);
|
|
7506
|
+
return t("login.failed", backend, err.message);
|
|
7507
|
+
}
|
|
7083
7508
|
const session = new LoginSession(flow, tmux, {
|
|
7084
7509
|
onMenu: async (options) => {
|
|
7085
7510
|
await this.postNonceButtonPrompt({
|
|
@@ -7101,8 +7526,12 @@ export class FleetManager {
|
|
|
7101
7526
|
onNeedInput: async (promptExcerpt) => {
|
|
7102
7527
|
await chat.adapter.sendText(chat.chatId, t("login.need_input", backend, promptExcerpt), { threadId: chat.threadId }).catch(() => { });
|
|
7103
7528
|
},
|
|
7104
|
-
onDone: async ({ ok, detail }) => {
|
|
7529
|
+
onDone: async ({ ok, detail, cleanupFailed }) => {
|
|
7105
7530
|
this.activeLogin = null;
|
|
7531
|
+
this.loginWindow.release(claim);
|
|
7532
|
+
if (cleanupFailed) {
|
|
7533
|
+
await chat.adapter.sendText(chat.chatId, t("login.web_cleanup_failed", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7534
|
+
}
|
|
7106
7535
|
let text;
|
|
7107
7536
|
if (ok) {
|
|
7108
7537
|
const { woken, restarted } = await this.recoverBackendInstances(backend);
|
|
@@ -7128,12 +7557,68 @@ export class FleetManager {
|
|
|
7128
7557
|
}
|
|
7129
7558
|
catch (err) {
|
|
7130
7559
|
this.activeLogin = null;
|
|
7131
|
-
|
|
7560
|
+
this.loginWindow.release(claim);
|
|
7561
|
+
const text = t("login.failed", backend, err.message);
|
|
7562
|
+
return err.cleanupFailed ? `${text}\n${t("login.web_cleanup_failed", backend)}` : text;
|
|
7563
|
+
}
|
|
7564
|
+
if (session.state === "done") {
|
|
7565
|
+
// Cancelled (user or shutdown) while starting: start() joined the
|
|
7566
|
+
// teardown, so nothing is left. onDone already released the claim.
|
|
7567
|
+
this.activeLogin = null;
|
|
7568
|
+
this.loginWindow.release(claim);
|
|
7569
|
+
return this.loginWindow.isClosed ? t("login.web_shutting_down") : t("login.cancelled", backend);
|
|
7570
|
+
}
|
|
7571
|
+
if (!this.loginWindow.isCurrent(claim)) {
|
|
7572
|
+
await session.cancel("cancelled").catch(() => { });
|
|
7573
|
+
this.activeLogin = null;
|
|
7574
|
+
this.loginWindow.release(claim);
|
|
7575
|
+
return t("login.web_shutting_down");
|
|
7132
7576
|
}
|
|
7133
7577
|
return t("login.started", backend);
|
|
7134
7578
|
}
|
|
7579
|
+
/** Fleet shutdown: end any web/relay login or install window and wait for its confirmed teardown. */
|
|
7580
|
+
async shutdownLoginWindows() {
|
|
7581
|
+
// Close the lock FIRST: in-flight starts parked in a pre-check or
|
|
7582
|
+
// ensureSession observe !isCurrent when they resume and stop; no new
|
|
7583
|
+
// window can be claimed while we stop.
|
|
7584
|
+
this.loginWindow.close();
|
|
7585
|
+
// Completion semantics live INSIDE the sessions: each awaits its own
|
|
7586
|
+
// confirmed cleanup, every tmux op on that path has a hard per-op bound
|
|
7587
|
+
// (web: abort ≤10 s + kill ≤31 s; legacy: create/list/kill ≤10 s each,
|
|
7588
|
+
// duplicates killed in parallel). This outer deadline is only a loud last
|
|
7589
|
+
// resort so `agend stop` cannot hang forever on a wedged tmux; reaching it
|
|
7590
|
+
// is an ERROR, logged and recorded, and it releases nothing re-claimable
|
|
7591
|
+
// (the lock stays closed, the controller keeps its entry). The timer is
|
|
7592
|
+
// cleared on success so a long-lived process never fires it spuriously.
|
|
7593
|
+
const SHUTDOWN_DEADLINE_MS = 120_000;
|
|
7594
|
+
const bounded = (p, what) => {
|
|
7595
|
+
if (!p)
|
|
7596
|
+
return Promise.resolve();
|
|
7597
|
+
let timer;
|
|
7598
|
+
const deadline = new Promise(r => {
|
|
7599
|
+
timer = setTimeout(() => {
|
|
7600
|
+
this.logger.error({ what, deadlineMs: SHUTDOWN_DEADLINE_MS }, "login window teardown still in flight at the shutdown deadline — a dedicated tmux server may survive; check `tmux -L agend-term-* ls`");
|
|
7601
|
+
this.eventLog?.insert("login", "login_window_shutdown_deadline", { what });
|
|
7602
|
+
r();
|
|
7603
|
+
}, SHUTDOWN_DEADLINE_MS);
|
|
7604
|
+
timer.unref?.();
|
|
7605
|
+
});
|
|
7606
|
+
return Promise.race([p.then(() => undefined, () => this.logger.warn({ what }, "login window shutdown failed")), deadline])
|
|
7607
|
+
.finally(() => { if (timer)
|
|
7608
|
+
clearTimeout(timer); });
|
|
7609
|
+
};
|
|
7610
|
+
// "cancelled" is the detail both legacy onDone handlers keep quiet about —
|
|
7611
|
+
// a stopping fleet must not announce "login failed — fleet shutdown".
|
|
7612
|
+
await Promise.all([
|
|
7613
|
+
bounded(this.loginController?.shutdown(), "web-login"),
|
|
7614
|
+
bounded(this.activeLogin?.session.cancel("cancelled"), "relay-login"),
|
|
7615
|
+
bounded(this.activeInstall?.session.cancel("cancelled"), "install"),
|
|
7616
|
+
]);
|
|
7617
|
+
}
|
|
7135
7618
|
/** `/login code <text>` — paste admin-supplied text into the login window. */
|
|
7136
7619
|
async loginSubmitInput(text) {
|
|
7620
|
+
if (this.loginController?.isActive())
|
|
7621
|
+
return t("login.web_code_not_needed");
|
|
7137
7622
|
if (!this.activeLogin)
|
|
7138
7623
|
return t("login.no_session");
|
|
7139
7624
|
const ok = await this.activeLogin.session.submitInput(text);
|
|
@@ -7141,6 +7626,8 @@ export class FleetManager {
|
|
|
7141
7626
|
}
|
|
7142
7627
|
/** `/login cancel` — abort the active session and remove its window. */
|
|
7143
7628
|
async cancelLoginSession() {
|
|
7629
|
+
if (this.loginController?.isActive())
|
|
7630
|
+
return this.loginController.cancel();
|
|
7144
7631
|
if (!this.activeLogin)
|
|
7145
7632
|
return t("login.no_session");
|
|
7146
7633
|
const backend = this.activeLogin.backend;
|
|
@@ -7176,9 +7663,8 @@ export class FleetManager {
|
|
|
7176
7663
|
async recoverBackendInstances(backend) {
|
|
7177
7664
|
const woken = [];
|
|
7178
7665
|
const restarted = [];
|
|
7179
|
-
for (const
|
|
7180
|
-
|
|
7181
|
-
if (effective !== backend)
|
|
7666
|
+
for (const name of this.configuredBackendInstanceNames()) {
|
|
7667
|
+
if (this.backendNameOf(name) !== backend)
|
|
7182
7668
|
continue;
|
|
7183
7669
|
const status = this.getInstanceStatus(name);
|
|
7184
7670
|
try {
|
|
@@ -7214,6 +7700,7 @@ export class FleetManager {
|
|
|
7214
7700
|
adapterId: entry.adapterId,
|
|
7215
7701
|
chatId: entry.chatId,
|
|
7216
7702
|
threadId: entry.threadId,
|
|
7703
|
+
userId: data.userId,
|
|
7217
7704
|
});
|
|
7218
7705
|
if (text)
|
|
7219
7706
|
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
@@ -7221,7 +7708,7 @@ export class FleetManager {
|
|
|
7221
7708
|
}
|
|
7222
7709
|
/** Re-login confirmation (auth pre-check said credentials still work). */
|
|
7223
7710
|
async handleLoginConfirm(data, callbackAdapterId, receivingAdapter) {
|
|
7224
|
-
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
7711
|
+
const claimed = this.consumeNonceCallback(LOGIN_CONFIRM_CALLBACK_PREFIX, /^login-confirm:([0-9a-f]+):(go|go-relogin|cancel)$/, data, callbackAdapterId, receivingAdapter);
|
|
7225
7712
|
if (claimed === null)
|
|
7226
7713
|
return false;
|
|
7227
7714
|
if (claimed === "consumed")
|
|
@@ -7238,11 +7725,24 @@ export class FleetManager {
|
|
|
7238
7725
|
adapterId: entry.adapterId,
|
|
7239
7726
|
chatId: entry.chatId,
|
|
7240
7727
|
threadId: entry.threadId,
|
|
7241
|
-
|
|
7728
|
+
userId: data.userId,
|
|
7729
|
+
}, { skipAuthCheck: true, tokenPresent: action === "go-relogin" });
|
|
7242
7730
|
if (text)
|
|
7243
7731
|
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
7244
7732
|
return true;
|
|
7245
7733
|
}
|
|
7734
|
+
/** "Resend token" button (web login): only the requester may press it; the token never enters the channel. */
|
|
7735
|
+
async handleLoginTokenResend(data, callbackAdapterId, receivingAdapter) {
|
|
7736
|
+
const claimed = this.consumeNonceCallback(LOGIN_TOKEN_RESEND_PREFIX, /^login-token:([0-9a-f]+):(resend)$/, data, callbackAdapterId, receivingAdapter);
|
|
7737
|
+
if (claimed === null)
|
|
7738
|
+
return false;
|
|
7739
|
+
if (claimed === "consumed")
|
|
7740
|
+
return true;
|
|
7741
|
+
const { entry } = claimed;
|
|
7742
|
+
const text = await this.webLogin.resendToken(data.userId);
|
|
7743
|
+
await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, text);
|
|
7744
|
+
return true;
|
|
7745
|
+
}
|
|
7246
7746
|
/** Kiro provider button → drive the CLI's arrow-key selector. */
|
|
7247
7747
|
async handleLoginMenuSelect(data, callbackAdapterId, receivingAdapter) {
|
|
7248
7748
|
const claimed = this.consumeNonceCallback(LOGIN_MENU_CALLBACK_PREFIX, /^login-menu:([0-9a-f]+):(\d)$/, data, callbackAdapterId, receivingAdapter);
|
|
@@ -7272,13 +7772,25 @@ export class FleetManager {
|
|
|
7272
7772
|
return t("install.unsupported", backendArg);
|
|
7273
7773
|
if (checkBinaryInstalled(info.binary))
|
|
7274
7774
|
return t("install.already", backend, info.binary);
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7775
|
+
// Reserve the fleet-wide window before the first await (shared with web/relay
|
|
7776
|
+
// login). Owned by this method until the session is published.
|
|
7777
|
+
const claim = this.loginWindow.tryClaim("install", backend);
|
|
7778
|
+
if (!claim)
|
|
7779
|
+
return this.loginWindow.busyMessage();
|
|
7780
|
+
let tmux;
|
|
7781
|
+
try {
|
|
7782
|
+
const sessionName = getTmuxSession();
|
|
7783
|
+
await TmuxManager.ensureSession(sessionName);
|
|
7784
|
+
if (!this.loginWindow.isCurrent(claim)) {
|
|
7785
|
+
this.loginWindow.release(claim);
|
|
7786
|
+
return t("login.web_shutting_down");
|
|
7787
|
+
}
|
|
7788
|
+
tmux = new TmuxManager(sessionName, "");
|
|
7789
|
+
}
|
|
7790
|
+
catch (err) {
|
|
7791
|
+
this.loginWindow.release(claim);
|
|
7792
|
+
return t("install.failed", backend, err.message);
|
|
7793
|
+
}
|
|
7282
7794
|
// A synthetic login flow: success is decided by the installer's exit code
|
|
7283
7795
|
// (LoginSession treats a clean exit as success), never by pane text — and
|
|
7284
7796
|
// installer output that happens to contain a URL must not be forwarded as
|
|
@@ -7293,8 +7805,12 @@ export class FleetManager {
|
|
|
7293
7805
|
onMenu: () => { },
|
|
7294
7806
|
onAuthHint: () => { },
|
|
7295
7807
|
onNeedInput: () => { },
|
|
7296
|
-
onDone: async ({ ok, detail }) => {
|
|
7808
|
+
onDone: async ({ ok, detail, cleanupFailed }) => {
|
|
7297
7809
|
this.activeInstall = null;
|
|
7810
|
+
this.loginWindow.release(claim);
|
|
7811
|
+
if (cleanupFailed) {
|
|
7812
|
+
await chat.adapter.sendText(chat.chatId, t("login.web_cleanup_failed", backend), { threadId: chat.threadId }).catch(() => { });
|
|
7813
|
+
}
|
|
7298
7814
|
if (!ok) {
|
|
7299
7815
|
// A cancel is user-initiated — the cancel command's own reply already
|
|
7300
7816
|
// said so; a second message here would be a duplicate.
|
|
@@ -7336,7 +7852,20 @@ export class FleetManager {
|
|
|
7336
7852
|
}
|
|
7337
7853
|
catch (err) {
|
|
7338
7854
|
this.activeInstall = null;
|
|
7339
|
-
|
|
7855
|
+
this.loginWindow.release(claim);
|
|
7856
|
+
const text = t("install.failed", backend, err.message);
|
|
7857
|
+
return err.cleanupFailed ? `${text}\n${t("login.web_cleanup_failed", backend)}` : text;
|
|
7858
|
+
}
|
|
7859
|
+
if (session.state === "done") {
|
|
7860
|
+
this.activeInstall = null;
|
|
7861
|
+
this.loginWindow.release(claim);
|
|
7862
|
+
return this.loginWindow.isClosed ? t("login.web_shutting_down") : t("install.cancelled", backend);
|
|
7863
|
+
}
|
|
7864
|
+
if (!this.loginWindow.isCurrent(claim)) {
|
|
7865
|
+
await session.cancel("cancelled").catch(() => { });
|
|
7866
|
+
this.activeInstall = null;
|
|
7867
|
+
this.loginWindow.release(claim);
|
|
7868
|
+
return t("login.web_shutting_down");
|
|
7340
7869
|
}
|
|
7341
7870
|
return t("install.started", backend);
|
|
7342
7871
|
}
|
|
@@ -7377,6 +7906,7 @@ export class FleetManager {
|
|
|
7377
7906
|
adapterId: entry.adapterId,
|
|
7378
7907
|
chatId: entry.chatId,
|
|
7379
7908
|
threadId: entry.threadId,
|
|
7909
|
+
userId: data.userId,
|
|
7380
7910
|
});
|
|
7381
7911
|
if (text)
|
|
7382
7912
|
await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
|
|
@@ -7388,7 +7918,7 @@ export class FleetManager {
|
|
|
7388
7918
|
await data.respond(t("permission.denied"));
|
|
7389
7919
|
return;
|
|
7390
7920
|
}
|
|
7391
|
-
const chat = { adapter, adapterId, chatId: data.channelId };
|
|
7921
|
+
const chat = { adapter, adapterId, chatId: data.channelId, userId: data.userId };
|
|
7392
7922
|
if (data.options?.cancel === true) {
|
|
7393
7923
|
await data.respond(await this.cancelLoginSession());
|
|
7394
7924
|
return;
|
|
@@ -8200,6 +8730,32 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8200
8730
|
catch { /* missing / stale / corrupt */ }
|
|
8201
8731
|
return null;
|
|
8202
8732
|
}
|
|
8733
|
+
/** True when a cached CLI env is old enough that `/model` should re-probe. */
|
|
8734
|
+
cliEnvNeedsRefresh(env) {
|
|
8735
|
+
return !env || typeof env.probedAt !== "number" || Date.now() - env.probedAt >= CLI_ENV_FRESH_MS;
|
|
8736
|
+
}
|
|
8737
|
+
/**
|
|
8738
|
+
* Run a live probe under a deadline, falling back to whatever the cache holds.
|
|
8739
|
+
* A model list is an aid: a vendor that stops answering must degrade to the
|
|
8740
|
+
* previous list, never stall the command that asked for it.
|
|
8741
|
+
*/
|
|
8742
|
+
async probeBackendBounded(backend) {
|
|
8743
|
+
const work = this.probeBackend(backend);
|
|
8744
|
+
work.catch(() => { });
|
|
8745
|
+
let timer;
|
|
8746
|
+
const deadline = new Promise(resolve => {
|
|
8747
|
+
timer = setTimeout(() => {
|
|
8748
|
+
this.logger.warn({ backend, deadlineMs: CLI_ENV_PROBE_DEADLINE_MS }, "CLI env live probe exceeded its deadline — serving the cached model list");
|
|
8749
|
+
resolve(null);
|
|
8750
|
+
}, CLI_ENV_PROBE_DEADLINE_MS);
|
|
8751
|
+
});
|
|
8752
|
+
try {
|
|
8753
|
+
return await Promise.race([work, deadline]);
|
|
8754
|
+
}
|
|
8755
|
+
finally {
|
|
8756
|
+
clearTimeout(timer);
|
|
8757
|
+
}
|
|
8758
|
+
}
|
|
8203
8759
|
/**
|
|
8204
8760
|
* Resolve the effective model for a fleet or ClassicBot instance, plus where it
|
|
8205
8761
|
* came from. Single source of truth for `/model` and `/ctx` — precedence:
|
|
@@ -8301,16 +8857,23 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8301
8857
|
void this.probeBackend(b);
|
|
8302
8858
|
}
|
|
8303
8859
|
/** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
|
|
8304
|
-
async getModelOptions(instanceName, refresh = false) {
|
|
8860
|
+
async getModelOptions(instanceName, refresh = false, onLiveProbe) {
|
|
8305
8861
|
const backendName = this.backendNameForInstance(instanceName);
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8311
|
-
//
|
|
8312
|
-
|
|
8313
|
-
|
|
8862
|
+
const cached = this.readCliEnv(backendName);
|
|
8863
|
+
if (!refresh && cached?.models.length && !this.cliEnvNeedsRefresh(cached))
|
|
8864
|
+
return cached.models;
|
|
8865
|
+
// About to go to the vendor: let the caller say so. A silent 1–10s pause on
|
|
8866
|
+
// an interactive command reads as another hang, which is the wrong lesson to
|
|
8867
|
+
// teach a user who has just been bitten by one.
|
|
8868
|
+
onLiveProbe?.();
|
|
8869
|
+
// Stale, missing, or a forced refresh → probe live (also refreshes the cache).
|
|
8870
|
+
// A newly released model is invisible until this runs, which is why staleness
|
|
8871
|
+
// triggers it rather than waiting for the 24h hard expiry or a cold start.
|
|
8872
|
+
const env = await this.probeBackendBounded(backendName);
|
|
8873
|
+
if (env?.models.length)
|
|
8874
|
+
return env.models;
|
|
8875
|
+
// Probe failed or timed out: the previous list is still the best answer.
|
|
8876
|
+
return cached?.models ?? [];
|
|
8314
8877
|
}
|
|
8315
8878
|
/**
|
|
8316
8879
|
* Model catalog behind the `list_models` tool.
|
|
@@ -8601,7 +9164,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
8601
9164
|
await data.respond(t("model.usage"));
|
|
8602
9165
|
return;
|
|
8603
9166
|
}
|
|
8604
|
-
const options = await this.getModelOptions(name, isRefresh)
|
|
9167
|
+
const options = await this.getModelOptions(name, isRefresh, () => {
|
|
9168
|
+
void data.respond(t("model.refreshing")).catch(() => { });
|
|
9169
|
+
});
|
|
8605
9170
|
if (options.length === 0) {
|
|
8606
9171
|
await data.respond(t("model.list_unavailable", name));
|
|
8607
9172
|
return;
|
|
@@ -9255,6 +9820,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9255
9820
|
clearInterval(this.watchdogTimer);
|
|
9256
9821
|
this.watchdogTimer = null;
|
|
9257
9822
|
}
|
|
9823
|
+
// A login/install window is a dedicated tmux server with its own TTL
|
|
9824
|
+
// timer and HTTP listener living in THIS process: without an explicit
|
|
9825
|
+
// shutdown it would outlive us as an owner-less login CLI (sol B3).
|
|
9826
|
+
await this.shutdownLoginWindows();
|
|
9258
9827
|
// Cancel adapter retry timers
|
|
9259
9828
|
for (const state of this.adapterState.values()) {
|
|
9260
9829
|
if (state.retryTimer) {
|
|
@@ -9431,10 +10000,27 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9431
10000
|
return;
|
|
9432
10001
|
}
|
|
9433
10002
|
this.logger.info(`Full restart: waiting for ${instanceNames.length} instances to idle...`);
|
|
9434
|
-
const
|
|
9435
|
-
|
|
9436
|
-
|
|
9437
|
-
|
|
10003
|
+
const trackedProgress = readUpdateProgress(this.dataDir);
|
|
10004
|
+
const trackedFullRestart = trackedProgress
|
|
10005
|
+
&& updateProgressOperation(trackedProgress.progress) === "full-restart"
|
|
10006
|
+
&& trackedProgress.progress.stage !== "failed"
|
|
10007
|
+
&& trackedProgress.progress.stage !== "complete";
|
|
10008
|
+
if (trackedFullRestart) {
|
|
10009
|
+
// `/restart full` already posted and persisted one public progress message.
|
|
10010
|
+
// Keep that single message; the new process will adopt and finish it.
|
|
10011
|
+
setUpdateProgressStage(this.dataDir, "stopping");
|
|
10012
|
+
}
|
|
10013
|
+
const restartTarget = this.fleetNoticeTarget();
|
|
10014
|
+
if (!trackedFullRestart && this.adapter) {
|
|
10015
|
+
if (restartTarget) {
|
|
10016
|
+
await this.adapter.sendText(restartTarget.chatId, t("restart.full_initiated"), restartTarget.opts)
|
|
10017
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
|
|
10018
|
+
}
|
|
10019
|
+
else {
|
|
10020
|
+
// Say why nothing was posted. A restart that announces itself nowhere,
|
|
10021
|
+
// for a reason nobody logged, is the harder version of this bug.
|
|
10022
|
+
this.logger.warn("Full restart notice has no postable target — set a General topic or channel.options.general_channel_id");
|
|
10023
|
+
}
|
|
9438
10024
|
}
|
|
9439
10025
|
// Wait for idle with 5-minute timeout
|
|
9440
10026
|
const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
|
|
@@ -9608,6 +10194,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
|
|
|
9608
10194
|
this.logger.error("Cannot restart: no config path (was startAll called?)");
|
|
9609
10195
|
return;
|
|
9610
10196
|
}
|
|
10197
|
+
// A graceful restart keeps this manager process, so the startup probe does
|
|
10198
|
+
// not run again. Without this, `agend restart` left the cached CLI env
|
|
10199
|
+
// untouched and `/model` kept serving an old list until a cold start —
|
|
10200
|
+
// exactly the "only stop+start works" report. Background, never blocking:
|
|
10201
|
+
// /model re-probes on staleness anyway, this just makes a restart do the
|
|
10202
|
+
// refreshing a user expects of it.
|
|
10203
|
+
this.probeCliEnvs();
|
|
9611
10204
|
const instanceNames = [...this.daemons.keys()];
|
|
9612
10205
|
if (instanceNames.length === 0) {
|
|
9613
10206
|
this.logger.info("No instances to restart");
|