@songsid/agend 2.1.0-beta.4 → 2.1.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +6 -0
- package/dist/backend/antigravity.js +34 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +6 -0
- package/dist/backend/claude-code.js +20 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +4 -0
- package/dist/backend/codex.js +5 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +57 -0
- package/dist/backend/grok.js +306 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.d.ts +6 -0
- package/dist/backend/kiro.js +36 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.d.ts +4 -0
- package/dist/backend/opencode.js +5 -0
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +59 -3
- package/dist/backend/types.js +11 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +89 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +18 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/chat-export.js +5 -2
- package/dist/chat-export.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +16 -1
- package/dist/classic-channel-manager.js +76 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +116 -19
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +70 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +14 -1
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +91 -6
- package/dist/daemon.js +602 -186
- package/dist/daemon.js.map +1 -1
- package/dist/daily-summary.js +7 -2
- package/dist/daily-summary.js.map +1 -1
- package/dist/fleet-context.d.ts +6 -0
- package/dist/fleet-manager.d.ts +120 -8
- package/dist/fleet-manager.js +1018 -142
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-system-prompt.js +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +19 -13
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +30 -51
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +19 -13
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -5
- package/dist/general-knowledge/skills/scheduling/SKILL.md +41 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +29 -116
- package/dist/instance-lifecycle.d.ts +15 -1
- package/dist/instance-lifecycle.js +71 -22
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +28 -9
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +224 -11
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +44 -2
- package/dist/outbound-schemas.js +50 -3
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/scheduler/db.d.ts +8 -0
- package/dist/scheduler/db.js +74 -5
- package/dist/scheduler/db.js.map +1 -1
- package/dist/scheduler/db.test.js +50 -0
- package/dist/scheduler/db.test.js.map +1 -1
- package/dist/scheduler/scheduler.d.ts +6 -0
- package/dist/scheduler/scheduler.js +99 -16
- package/dist/scheduler/scheduler.js.map +1 -1
- package/dist/scheduler/scheduler.test.js +131 -1
- package/dist/scheduler/scheduler.test.js.map +1 -1
- package/dist/scheduler/types.d.ts +9 -3
- package/dist/scheduler/types.js.map +1 -1
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +19 -1
- package/dist/topic-commands.js +148 -23
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +29 -1
- package/dist/tz-utils.d.ts +26 -0
- package/dist/tz-utils.js +46 -0
- package/dist/tz-utils.js.map +1 -0
- package/dist/ui/dashboard.html +3 -3
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.d.ts +1 -0
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -5,13 +5,13 @@ import { join, dirname, basename } from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
7
|
import { sdNotify } from "./sd-notify.js";
|
|
8
|
-
import
|
|
8
|
+
import { isScalar, parseDocument } from "yaml";
|
|
9
9
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
10
|
const __dirname = dirname(__filename);
|
|
11
11
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
12
12
|
const DEFAULT_OPEN_ACCESS = { mode: "open", allowed_users: [], max_pending_codes: 0, code_expiry_minutes: 0 };
|
|
13
13
|
import { isProbeableRouteTarget } from "./fleet-context.js";
|
|
14
|
-
import { loadFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
14
|
+
import { loadFleetConfig, loadRawFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
15
15
|
import { EventLog } from "./event-log.js";
|
|
16
16
|
import { AdapterWorld } from "./adapter-world.js";
|
|
17
17
|
import { CostGuard, formatCents } from "./cost-guard.js";
|
|
@@ -19,18 +19,20 @@ import { TmuxManager } from "./tmux-manager.js";
|
|
|
19
19
|
import { AccessManager } from "./channel/access-manager.js";
|
|
20
20
|
import { IpcClient } from "./channel/ipc-bridge.js";
|
|
21
21
|
import { createAdapter } from "./channel/factory.js";
|
|
22
|
+
import { createBackend } from "./backend/factory.js";
|
|
23
|
+
import { isModelCompatible } from "./backend/types.js";
|
|
22
24
|
import { createLogger } from "./logger.js";
|
|
23
25
|
import { processAttachments } from "./channel/attachment-handler.js";
|
|
24
26
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
25
27
|
import { Scheduler } from "./scheduler/index.js";
|
|
26
28
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
27
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
29
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
28
30
|
import { DailySummary } from "./daily-summary.js";
|
|
29
31
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
30
32
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
31
33
|
import { safeHandler } from "./safe-async.js";
|
|
32
34
|
import { RoutingEngine } from "./routing-engine.js";
|
|
33
|
-
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
35
|
+
import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
|
|
34
36
|
import { TopicArchiver } from "./topic-archiver.js";
|
|
35
37
|
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
36
38
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
@@ -39,7 +41,9 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
39
41
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
40
42
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
41
43
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
42
|
-
import { ClassicChannelManager } from "./classic-channel-manager.js";
|
|
44
|
+
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
45
|
+
import { readLastInboundAt } from "./daemon.js";
|
|
46
|
+
import { clearPausedMarker } from "./pause-marker.js";
|
|
43
47
|
import { getTmuxSession } from "./config.js";
|
|
44
48
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
45
49
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -50,6 +54,26 @@ export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
|
50
54
|
}
|
|
51
55
|
return instanceConfig?.topic_id != null ? String(instanceConfig.topic_id) : undefined;
|
|
52
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Pure warm-cap victim selection (extracted for testability). Given the current
|
|
59
|
+
* warm (running) instance names and a cap, return the LRU idle instances to evict
|
|
60
|
+
* so the running count returns to the cap. Skips: the `exclude` instance, any
|
|
61
|
+
* already-evicting, general instances (never evicted), and non-idle instances
|
|
62
|
+
* (working/stuck can't be evicted). Oldest last-inbound is evicted first; a
|
|
63
|
+
* missing timestamp (0) sorts oldest. cap <= 0 (or non-integer) = unlimited → [].
|
|
64
|
+
*/
|
|
65
|
+
export function selectLruEvictions(warm, cap, opts) {
|
|
66
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
67
|
+
return [];
|
|
68
|
+
if (warm.length <= cap)
|
|
69
|
+
return [];
|
|
70
|
+
const candidates = warm.filter(name => name !== opts.exclude
|
|
71
|
+
&& !opts.isEvicting(name)
|
|
72
|
+
&& !opts.isGeneral(name)
|
|
73
|
+
&& opts.isIdle(name));
|
|
74
|
+
candidates.sort((a, b) => opts.lastInboundAt(a) - opts.lastInboundAt(b));
|
|
75
|
+
return candidates.slice(0, warm.length - cap);
|
|
76
|
+
}
|
|
53
77
|
/** Retry cadence for retiring a cancel button whose delete failed (e.g. a DC
|
|
54
78
|
* forum thread the bot momentarily can't reach). 3 retries × 5min = 15min. */
|
|
55
79
|
const CANCEL_BTN_RETRY_INTERVAL_MS = 5 * 60_000;
|
|
@@ -58,6 +82,10 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
58
82
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
59
83
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
60
84
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
85
|
+
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
86
|
+
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
87
|
+
const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
|
|
88
|
+
const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
|
|
61
89
|
export class FleetManager {
|
|
62
90
|
dataDir;
|
|
63
91
|
children = new Map();
|
|
@@ -65,6 +93,9 @@ export class FleetManager {
|
|
|
65
93
|
/** @deprecated Use lifecycle.daemons — kept for backward compat */
|
|
66
94
|
get daemons() { return this.lifecycle.daemons; }
|
|
67
95
|
fleetConfig = null;
|
|
96
|
+
rawFleetConfig = {};
|
|
97
|
+
rawFleetDocument = null;
|
|
98
|
+
savedFleetConfigSnapshot = null;
|
|
68
99
|
adapter = null;
|
|
69
100
|
worlds = new Map();
|
|
70
101
|
adapters = new Map(); // derived view for backward compat
|
|
@@ -93,6 +124,18 @@ export class FleetManager {
|
|
|
93
124
|
// Topic icon + auto-archive state
|
|
94
125
|
topicIcons = {};
|
|
95
126
|
lastActivity = new Map();
|
|
127
|
+
/** Latest pane-derived execution snapshot reported by each daemon. */
|
|
128
|
+
instanceStateCache = new Map();
|
|
129
|
+
/** CLI pane status overrides; daemon.pid alone only proves FleetManager lives. */
|
|
130
|
+
instanceProcessStatus = new Map();
|
|
131
|
+
/** Instances currently being auto-paused by warm_cap, so concurrent checks don't double-evict. */
|
|
132
|
+
warmCapEvicting = new Set();
|
|
133
|
+
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
134
|
+
idleGatedDeliveryTails = new Map();
|
|
135
|
+
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
136
|
+
lastDeliveryAt = new Map();
|
|
137
|
+
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
138
|
+
instanceIdleWaiters = new Map();
|
|
96
139
|
lastInboundUser = new Map(); // instanceName → last username
|
|
97
140
|
// Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
|
|
98
141
|
// than one-per-instance. A button is retired (deleted, with bounded retry) on
|
|
@@ -104,6 +147,9 @@ export class FleetManager {
|
|
|
104
147
|
topicArchiver;
|
|
105
148
|
controlClient = null;
|
|
106
149
|
classicChannels = null;
|
|
150
|
+
pendingClassicStarts = new Map();
|
|
151
|
+
/** In-flight /model selections, keyed by nonce (see handleModelSelection). */
|
|
152
|
+
pendingModelSelects = new Map();
|
|
107
153
|
// Model failover state
|
|
108
154
|
failoverActive = new Map(); // instance → current failover model
|
|
109
155
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
@@ -160,6 +206,7 @@ export class FleetManager {
|
|
|
160
206
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
161
207
|
name,
|
|
162
208
|
status: this.getInstanceStatus(name),
|
|
209
|
+
state: this.getInstanceExecutionState(name),
|
|
163
210
|
ipc: this.instanceIpcClients.has(name),
|
|
164
211
|
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
165
212
|
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
@@ -174,9 +221,21 @@ export class FleetManager {
|
|
|
174
221
|
}
|
|
175
222
|
/** Load fleet.yaml and build routing table */
|
|
176
223
|
loadConfig(configPath) {
|
|
224
|
+
this.configPath = configPath;
|
|
225
|
+
const source = existsSync(configPath) ? readFileSync(configPath, "utf-8") : "{}\n";
|
|
226
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
227
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
228
|
+
throw new Error(`Invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
229
|
+
}
|
|
230
|
+
this.rawFleetConfig = loadRawFleetConfig(configPath);
|
|
177
231
|
this.fleetConfig = loadFleetConfig(configPath);
|
|
232
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
178
233
|
return this.fleetConfig;
|
|
179
234
|
}
|
|
235
|
+
/** User-authored fleet.yaml, before defaults are merged into instances. */
|
|
236
|
+
getRawFleetConfig() {
|
|
237
|
+
return structuredClone(this.rawFleetConfig);
|
|
238
|
+
}
|
|
180
239
|
/** Build topic routing table: { topicId -> RouteTarget } */
|
|
181
240
|
buildRoutingTable() {
|
|
182
241
|
if (this.fleetConfig) {
|
|
@@ -230,6 +289,41 @@ export class FleetManager {
|
|
|
230
289
|
return this.classicChannels?.getInstanceByChannel(channelId, adapterId)
|
|
231
290
|
?? this.routing.resolve(channelId)?.name;
|
|
232
291
|
}
|
|
292
|
+
async handlePauseWakeSlash(data, adapterId) {
|
|
293
|
+
const action = data.command;
|
|
294
|
+
const classicName = this.classicChannels?.getInstanceByChannel(data.channelId, adapterId);
|
|
295
|
+
if (classicName) {
|
|
296
|
+
if (!this.classicChannels?.isAdmin(data.userId)) {
|
|
297
|
+
await data.respond(t("permission.denied"));
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
await data.respond(await this.topicCommands.runPauseWake(classicName, action));
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
304
|
+
await data.respond(t("permission.denied"));
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
const route = this.routing.resolve(data.channelId);
|
|
308
|
+
if (!route) {
|
|
309
|
+
await data.respond(t("classic.no_agent"));
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
let target = route.name;
|
|
313
|
+
if (route.kind === "general") {
|
|
314
|
+
const requested = typeof data.options?.instance === "string" ? data.options.instance : undefined;
|
|
315
|
+
if (!requested) {
|
|
316
|
+
await data.respond(t(`${action}.usage`));
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
if (!this.fleetConfig?.instances[requested]) {
|
|
320
|
+
await data.respond(t("instance.not_found", requested));
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
target = requested;
|
|
324
|
+
}
|
|
325
|
+
await data.respond(await this.topicCommands.runPauseWake(target, action));
|
|
326
|
+
}
|
|
233
327
|
/** Get the adapter bound to an instance, falling back to primary adapter */
|
|
234
328
|
getAdapterForInstance(name) {
|
|
235
329
|
const worldId = this.instanceWorldBinding.get(name);
|
|
@@ -265,19 +359,27 @@ export class FleetManager {
|
|
|
265
359
|
*/
|
|
266
360
|
bindInstanceAdapter(name, adapterId, fromInbound = false) {
|
|
267
361
|
const cfg = this.fleetConfig?.instances[name];
|
|
268
|
-
if (fromInbound
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
362
|
+
if (fromInbound) {
|
|
363
|
+
// Skip inbound-derived binding for any instance that doesn't have an
|
|
364
|
+
// explicit channel_id — those default to primary adapter deterministically.
|
|
365
|
+
// This prevents a non-deterministic race where whichever adapter delivers
|
|
366
|
+
// first after restart wins the binding.
|
|
367
|
+
if (cfg?.general_topic || cfg?.channel_id)
|
|
368
|
+
return;
|
|
369
|
+
if (cfg && !cfg.channel_id)
|
|
370
|
+
return; // fleet instance without explicit binding → use primary
|
|
371
|
+
// Classic instance: don't override an existing binding (authoritative from /start)
|
|
372
|
+
if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
276
375
|
this.instanceWorldBinding.set(name, adapterId);
|
|
277
376
|
}
|
|
278
377
|
getInstanceStatus(name) {
|
|
279
378
|
if (this.lifecycle.isPaused(name))
|
|
280
379
|
return "paused";
|
|
380
|
+
const processStatus = this.instanceProcessStatus.get(name);
|
|
381
|
+
if (processStatus)
|
|
382
|
+
return processStatus;
|
|
281
383
|
const pidPath = join(this.getInstanceDir(name), "daemon.pid");
|
|
282
384
|
if (!existsSync(pidPath))
|
|
283
385
|
return "stopped";
|
|
@@ -290,24 +392,244 @@ export class FleetManager {
|
|
|
290
392
|
return "crashed";
|
|
291
393
|
}
|
|
292
394
|
}
|
|
293
|
-
|
|
294
|
-
|
|
395
|
+
getInstanceExecutionState(name) {
|
|
396
|
+
if (this.lifecycle.isPaused(name))
|
|
397
|
+
return null;
|
|
398
|
+
return this.instanceStateCache.get(name)?.state ?? null;
|
|
399
|
+
}
|
|
400
|
+
isClassicInstance(name) {
|
|
401
|
+
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
402
|
+
}
|
|
403
|
+
cacheInstanceExecutionState(name, msg) {
|
|
404
|
+
const state = msg.state;
|
|
405
|
+
if (state !== "idle" && state !== "working" && state !== "stuck")
|
|
406
|
+
return;
|
|
407
|
+
const previous = this.instanceStateCache.get(name);
|
|
408
|
+
const now = Date.now();
|
|
409
|
+
const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
410
|
+
this.instanceStateCache.set(name, {
|
|
411
|
+
state,
|
|
412
|
+
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
413
|
+
observedAt: numberOr(msg.observedAt, now),
|
|
414
|
+
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
415
|
+
});
|
|
416
|
+
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
417
|
+
check();
|
|
418
|
+
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
419
|
+
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
420
|
+
// edge into idle, not on every idle heartbeat.
|
|
421
|
+
if (state === "idle" && previous?.state !== "idle")
|
|
422
|
+
this.enforceWarmCap();
|
|
423
|
+
}
|
|
424
|
+
cacheInstanceProcessStatus(name, status) {
|
|
425
|
+
if (status === "running") {
|
|
426
|
+
this.instanceProcessStatus.delete(name);
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
if (status !== "crashed" && status !== "stopped")
|
|
430
|
+
return;
|
|
431
|
+
this.instanceProcessStatus.set(name, status);
|
|
432
|
+
// Never display the last ready prompt as current execution state after its
|
|
433
|
+
// owning CLI process has exited.
|
|
434
|
+
this.instanceStateCache.delete(name);
|
|
435
|
+
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
436
|
+
check();
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Fleet-wide warm cap: if more than `defaults.warm_cap` instances are running,
|
|
440
|
+
* auto-pause the least-recently-active idle instances until back at the cap.
|
|
441
|
+
* Never evicts general instances (must stay warm) or working/stuck instances
|
|
442
|
+
* (only idle). 0/unset = unlimited. wake-before-deliver re-warms any evicted
|
|
443
|
+
* instance when a message next arrives.
|
|
444
|
+
*
|
|
445
|
+
* @param exclude instance to spare (e.g. one just woken to receive a delivery).
|
|
446
|
+
*/
|
|
447
|
+
enforceWarmCap(exclude) {
|
|
448
|
+
const cap = this.fleetConfig?.defaults?.warm_cap ?? 0;
|
|
449
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
450
|
+
return; // 0/invalid = unlimited
|
|
451
|
+
const warm = [];
|
|
452
|
+
for (const name of this.daemons.keys()) {
|
|
453
|
+
if (this.getInstanceStatus(name) === "running")
|
|
454
|
+
warm.push(name);
|
|
455
|
+
}
|
|
456
|
+
if (warm.length <= cap)
|
|
457
|
+
return;
|
|
458
|
+
const victims = selectLruEvictions(warm, cap, {
|
|
459
|
+
exclude,
|
|
460
|
+
isEvicting: name => this.warmCapEvicting.has(name),
|
|
461
|
+
isGeneral: name => this.fleetConfig?.instances[name]?.general_topic === true,
|
|
462
|
+
isIdle: name => this.getInstanceExecutionState(name) === "idle",
|
|
463
|
+
lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
|
|
464
|
+
});
|
|
465
|
+
for (const victim of victims) {
|
|
466
|
+
this.warmCapEvicting.add(victim);
|
|
467
|
+
this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
|
|
468
|
+
this.lifecycle.pause(victim)
|
|
469
|
+
.catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
|
|
470
|
+
.finally(() => this.warmCapEvicting.delete(victim));
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
474
|
+
const isReady = () => {
|
|
475
|
+
const snapshot = this.instanceStateCache.get(instanceName);
|
|
476
|
+
return snapshot?.state === "idle"
|
|
477
|
+
&& (idleObservedAfter === 0 || snapshot.observedAt > idleObservedAfter);
|
|
478
|
+
};
|
|
479
|
+
if (isReady())
|
|
480
|
+
return Promise.resolve(true);
|
|
481
|
+
return new Promise(resolve => {
|
|
482
|
+
let settled = false;
|
|
483
|
+
const finish = (idle) => {
|
|
484
|
+
if (settled)
|
|
485
|
+
return;
|
|
486
|
+
settled = true;
|
|
487
|
+
clearTimeout(timeout);
|
|
488
|
+
clearInterval(queryTimer);
|
|
489
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
490
|
+
waiters?.delete(check);
|
|
491
|
+
if (waiters?.size === 0)
|
|
492
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
493
|
+
resolve(idle);
|
|
494
|
+
};
|
|
495
|
+
const check = () => { if (isReady())
|
|
496
|
+
finish(true); };
|
|
497
|
+
const query = () => {
|
|
498
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
499
|
+
if (ipc?.connected) {
|
|
500
|
+
ipc.send({ type: "query_instance_state", requestId: `idle-gate-${Date.now()}` });
|
|
501
|
+
}
|
|
502
|
+
check();
|
|
503
|
+
};
|
|
504
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
505
|
+
waiters.add(check);
|
|
506
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
507
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
508
|
+
const queryTimer = setInterval(query, 1_000);
|
|
509
|
+
query();
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs) {
|
|
513
|
+
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
295
514
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
515
|
+
const wakeStartedAt = Date.now();
|
|
296
516
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
517
|
+
// Waking added one to the warm count — make room by evicting a different
|
|
518
|
+
// LRU idle instance (never this one; it's about to work).
|
|
519
|
+
this.enforceWarmCap(instanceName);
|
|
520
|
+
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
521
|
+
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
522
|
+
}
|
|
523
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
524
|
+
if (!idle) {
|
|
525
|
+
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
297
526
|
}
|
|
298
527
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
299
528
|
if (!ipc?.connected)
|
|
300
529
|
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
301
530
|
ipc.send(payload);
|
|
531
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
532
|
+
}
|
|
533
|
+
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
534
|
+
async deliverToInstance(instanceName, payload, options = {}) {
|
|
535
|
+
const meta = payload.meta && typeof payload.meta === "object"
|
|
536
|
+
? payload.meta
|
|
537
|
+
: undefined;
|
|
538
|
+
const inferredCrossInstance = (typeof meta?.from_instance === "string" && meta.from_instance.length > 0)
|
|
539
|
+
|| meta?.is_cross_instance === true
|
|
540
|
+
|| payload.is_cross_instance === true;
|
|
541
|
+
const waitForIdle = options.waitForIdle
|
|
542
|
+
?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
|
|
543
|
+
if (!waitForIdle) {
|
|
544
|
+
if (this.lifecycle.isPaused(instanceName)) {
|
|
545
|
+
await this.lifecycle.wake(instanceName, 30_000);
|
|
546
|
+
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
547
|
+
}
|
|
548
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
549
|
+
if (!ipc?.connected)
|
|
550
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
551
|
+
ipc.send(payload);
|
|
552
|
+
// A cross-instance item arriving before the daemon observes this turn as
|
|
553
|
+
// working must not trust the stale idle snapshot from before the send.
|
|
554
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
558
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, payload, options.idleTimeoutMs ?? 60_000));
|
|
559
|
+
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
560
|
+
try {
|
|
561
|
+
await delivery;
|
|
562
|
+
}
|
|
563
|
+
finally {
|
|
564
|
+
if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
|
|
565
|
+
this.idleGatedDeliveryTails.delete(instanceName);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
/** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
|
|
570
|
+
isFleetAdmin(userId, adapterId) {
|
|
571
|
+
const allowed = this.getChannelConfig(adapterId)?.access?.allowed_users ?? [];
|
|
572
|
+
return allowed.some(entry => String(entry) === String(userId));
|
|
573
|
+
}
|
|
574
|
+
async changeInstancePauseState(name, action) {
|
|
575
|
+
if (action === "wake") {
|
|
576
|
+
await this.lifecycle.wake(name, 30_000);
|
|
577
|
+
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
578
|
+
return "awake";
|
|
579
|
+
}
|
|
580
|
+
await this.lifecycle.pause(name);
|
|
581
|
+
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
582
|
+
}
|
|
583
|
+
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
584
|
+
async restartClassicInstanceFromSettings(instanceName) {
|
|
585
|
+
if (!this.classicChannels)
|
|
586
|
+
throw new Error("Classic channel manager not initialized");
|
|
587
|
+
const wasRunning = this.daemons.has(instanceName);
|
|
588
|
+
this.classicChannels.reloadFromDisk();
|
|
589
|
+
this.reregisterClassicChannels();
|
|
590
|
+
const channel = this.classicChannels.getAll().find(item => item.instanceName === instanceName);
|
|
591
|
+
if (!channel)
|
|
592
|
+
throw new Error("Classic channel not found after reload");
|
|
593
|
+
if (!wasRunning)
|
|
594
|
+
return;
|
|
595
|
+
await this.stopInstance(instanceName);
|
|
596
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
597
|
+
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
302
598
|
}
|
|
303
599
|
async startInstance(name, config, topicMode) {
|
|
600
|
+
if (this.lifecycle.isPaused(name)) {
|
|
601
|
+
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
304
604
|
if (config.general_topic) {
|
|
605
|
+
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
606
|
+
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
607
|
+
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
608
|
+
if (backend === "antigravity") {
|
|
609
|
+
this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
|
|
610
|
+
this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
|
|
611
|
+
}
|
|
305
612
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
306
613
|
}
|
|
614
|
+
this.instanceProcessStatus.delete(name);
|
|
307
615
|
await this.lifecycle.start(name, config, topicMode);
|
|
308
616
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
309
617
|
await this.connectIpcToInstance(name);
|
|
310
618
|
}
|
|
619
|
+
/** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
|
|
620
|
+
async startPersistedPausedInstance(name) {
|
|
621
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
622
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
623
|
+
const fleetConfig = this.fleetConfig?.instances[name];
|
|
624
|
+
if (fleetConfig) {
|
|
625
|
+
await this.startInstance(name, fleetConfig, topicMode);
|
|
626
|
+
return;
|
|
627
|
+
}
|
|
628
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
629
|
+
if (!channel || !this.classicChannels)
|
|
630
|
+
throw new Error(`Paused instance '${name}' is no longer configured`);
|
|
631
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
632
|
+
}
|
|
311
633
|
/**
|
|
312
634
|
* Start instances with configurable concurrency and stagger delay.
|
|
313
635
|
* Instances sharing the same working_directory are serialized within a group
|
|
@@ -316,8 +638,21 @@ export class FleetManager {
|
|
|
316
638
|
*/
|
|
317
639
|
async startInstancesWithConcurrency(entries, topicMode) {
|
|
318
640
|
const raw = this.fleetConfig?.defaults?.startup;
|
|
319
|
-
const
|
|
641
|
+
const explicitConcurrency = raw?.concurrency;
|
|
320
642
|
const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
|
|
643
|
+
// Adaptive concurrency: if not explicitly set, estimate from available RAM.
|
|
644
|
+
// Each instance uses ~300MB (tmux + CLI process + model overhead).
|
|
645
|
+
const ESTIMATED_MB_PER_INSTANCE = 300;
|
|
646
|
+
const { freemem } = await import("node:os");
|
|
647
|
+
let concurrency;
|
|
648
|
+
if (explicitConcurrency != null) {
|
|
649
|
+
concurrency = Math.max(1, Math.min(20, explicitConcurrency));
|
|
650
|
+
}
|
|
651
|
+
else {
|
|
652
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
653
|
+
concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
|
|
654
|
+
this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
|
|
655
|
+
}
|
|
321
656
|
const byWorkDir = new Map();
|
|
322
657
|
for (const [name, config] of entries) {
|
|
323
658
|
const dir = config.working_directory;
|
|
@@ -339,6 +674,17 @@ export class FleetManager {
|
|
|
339
674
|
if (pendingTimer)
|
|
340
675
|
return;
|
|
341
676
|
while (running < concurrency && idx < groups.length) {
|
|
677
|
+
// Re-check memory if adaptive (no explicit concurrency set)
|
|
678
|
+
if (explicitConcurrency == null && running > 0) {
|
|
679
|
+
const nowFreeMB = Math.round(freemem() / (1024 * 1024));
|
|
680
|
+
if (nowFreeMB < ESTIMATED_MB_PER_INSTANCE) {
|
|
681
|
+
this.logger.warn({ freeMemMB: nowFreeMB, remaining: groups.length - idx }, "Low memory — pausing instance startup");
|
|
682
|
+
// Wait and retry in 5s
|
|
683
|
+
pendingTimer = true;
|
|
684
|
+
setTimeout(() => { pendingTimer = false; startNext(); }, 5000);
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
342
688
|
const now = Date.now();
|
|
343
689
|
const elapsed = now - lastStartAt;
|
|
344
690
|
if (lastStartAt > 0 && elapsed < staggerMs) {
|
|
@@ -367,21 +713,54 @@ export class FleetManager {
|
|
|
367
713
|
}
|
|
368
714
|
async stopInstance(name) {
|
|
369
715
|
this.failoverActive.delete(name);
|
|
716
|
+
this.instanceStateCache.delete(name);
|
|
717
|
+
this.instanceProcessStatus.delete(name);
|
|
718
|
+
this.lastDeliveryAt.delete(name);
|
|
370
719
|
return this.lifecycle.stop(name);
|
|
371
720
|
}
|
|
372
721
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
373
|
-
async restartSingleInstance(name) {
|
|
722
|
+
async restartSingleInstance(name, opts) {
|
|
374
723
|
if (this.configPath) {
|
|
375
724
|
this.loadConfig(this.configPath);
|
|
376
725
|
this.routing.rebuild(this.fleetConfig);
|
|
377
726
|
this.reregisterClassicChannels();
|
|
378
727
|
}
|
|
379
728
|
const config = this.fleetConfig?.instances[name];
|
|
380
|
-
if (
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
729
|
+
if (config) {
|
|
730
|
+
await this.stopInstance(name);
|
|
731
|
+
if (opts?.freshStart)
|
|
732
|
+
this.writeFreshStartMarker(name);
|
|
733
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
734
|
+
await this.startInstance(name, config, topicMode ?? false);
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
// Classic instance fallback
|
|
738
|
+
const channelId = this.classicChannels?.getChannelIdByInstance(name);
|
|
739
|
+
if (channelId) {
|
|
740
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
741
|
+
const adapterId = this.classicChannels.getAdapterIdByInstance(name);
|
|
742
|
+
await this.stopInstance(name);
|
|
743
|
+
await new Promise(r => setTimeout(r, 1000)); // let tmux clean up
|
|
744
|
+
if (opts?.freshStart)
|
|
745
|
+
this.writeFreshStartMarker(name);
|
|
746
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
throw new Error(`Instance not found: ${name}`);
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Mark an instance so its next daemon start skips session resume. Written AFTER
|
|
753
|
+
* stop (survives the old daemon's cleanup) and BEFORE start so the respawn reads
|
|
754
|
+
* it — reuses the crash-state → resumeDisabled path from crash-loop recovery.
|
|
755
|
+
* One-shot: the daemon deletes it on startup.
|
|
756
|
+
*/
|
|
757
|
+
writeFreshStartMarker(name) {
|
|
758
|
+
try {
|
|
759
|
+
writeFileSync(join(this.getInstanceDir(name), "crash-state.json"), JSON.stringify({ resumeDisabled: true, reason: "pty_error_restart" }));
|
|
760
|
+
}
|
|
761
|
+
catch (err) {
|
|
762
|
+
this.logger.warn({ err, name }, "freshStart: failed to write crash-state marker");
|
|
763
|
+
}
|
|
385
764
|
}
|
|
386
765
|
/** Load .env file from data dir into process.env */
|
|
387
766
|
loadEnvFile() {
|
|
@@ -477,9 +856,11 @@ export class FleetManager {
|
|
|
477
856
|
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
478
857
|
const oldBackends = new Map();
|
|
479
858
|
const oldModels = new Map();
|
|
859
|
+
const oldAutoPause = new Map();
|
|
480
860
|
for (const ch of this.classicChannels.getAll()) {
|
|
481
861
|
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
482
862
|
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
863
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
483
864
|
}
|
|
484
865
|
if (!this.classicChannels.checkReload())
|
|
485
866
|
return;
|
|
@@ -487,14 +868,16 @@ export class FleetManager {
|
|
|
487
868
|
for (const ch of this.classicChannels.getAll()) {
|
|
488
869
|
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
489
870
|
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
871
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
490
872
|
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
491
873
|
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
492
|
-
|
|
874
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
875
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
493
876
|
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
494
877
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
495
878
|
// Small delay to let tmux window clean up
|
|
496
879
|
await new Promise(r => setTimeout(r, 2000));
|
|
497
|
-
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
880
|
+
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel, newAutoPause).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
498
881
|
}
|
|
499
882
|
}
|
|
500
883
|
}
|
|
@@ -747,7 +1130,7 @@ export class FleetManager {
|
|
|
747
1130
|
let idx = 0;
|
|
748
1131
|
while (idx < channels.length) {
|
|
749
1132
|
const batch = channels.slice(idx, idx + concurrency);
|
|
750
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
1133
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
751
1134
|
idx += concurrency;
|
|
752
1135
|
}
|
|
753
1136
|
}
|
|
@@ -758,16 +1141,26 @@ export class FleetManager {
|
|
|
758
1141
|
const classicCount = this.classicChannels?.getAll().length ?? 0;
|
|
759
1142
|
const total = Object.keys(fleet.instances).length + classicCount;
|
|
760
1143
|
const started = this.daemons.size;
|
|
761
|
-
const
|
|
1144
|
+
const allNotRunning = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
1145
|
+
const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
|
|
1146
|
+
const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
|
|
762
1147
|
const generalName = this.findGeneralInstance();
|
|
763
1148
|
const generalThreadId = generalName ? fleet.instances[generalName]?.topic_id : undefined;
|
|
764
1149
|
const { createRequire } = await import("node:module");
|
|
765
1150
|
const _require = createRequire(import.meta.url);
|
|
766
1151
|
const agendVersion = _require("../package.json").version ?? "unknown";
|
|
767
1152
|
if (this.adapter && fleet.channel?.group_id) {
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1153
|
+
let text;
|
|
1154
|
+
if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
1155
|
+
text = t("fleet.ready", started, total, agendVersion);
|
|
1156
|
+
}
|
|
1157
|
+
else if (failedNames.length === 0) {
|
|
1158
|
+
text = t("fleet.ready", started, total, agendVersion) + `\n⏸ Paused: ${pausedNames.join(", ")}`;
|
|
1159
|
+
}
|
|
1160
|
+
else {
|
|
1161
|
+
text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
|
|
1162
|
+
+ (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
|
|
1163
|
+
}
|
|
771
1164
|
this.adapter.sendText(String(fleet.channel.group_id), text, {
|
|
772
1165
|
threadId: generalThreadId != null ? String(generalThreadId) : undefined,
|
|
773
1166
|
}).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
|
|
@@ -952,6 +1345,10 @@ export class FleetManager {
|
|
|
952
1345
|
await this.handleInboundMessage(msg);
|
|
953
1346
|
}, this.logger, "adapter.message"));
|
|
954
1347
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1348
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1349
|
+
return;
|
|
1350
|
+
if (await this.handleModelSelection(data))
|
|
1351
|
+
return;
|
|
955
1352
|
if (data.callbackData.startsWith("hang:")) {
|
|
956
1353
|
const parts = data.callbackData.split(":");
|
|
957
1354
|
const action = parts[1];
|
|
@@ -990,13 +1387,15 @@ export class FleetManager {
|
|
|
990
1387
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
991
1388
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
992
1389
|
if (data.command === "start") {
|
|
993
|
-
|
|
994
|
-
await data.respond(reply);
|
|
1390
|
+
await this.handleClassicStartSlash(data, adapterId);
|
|
995
1391
|
}
|
|
996
1392
|
else if (data.command === "stop") {
|
|
997
1393
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
998
1394
|
await data.respond(reply);
|
|
999
1395
|
}
|
|
1396
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1397
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1398
|
+
}
|
|
1000
1399
|
else if (data.command === "chat") {
|
|
1001
1400
|
const text = data.text ?? "";
|
|
1002
1401
|
if (!text) {
|
|
@@ -1052,6 +1451,9 @@ export class FleetManager {
|
|
|
1052
1451
|
const result = await this.topicCommands.sendCompact(name);
|
|
1053
1452
|
await data.respond(result);
|
|
1054
1453
|
}
|
|
1454
|
+
else if (data.command === "model") {
|
|
1455
|
+
await this.handleModelSlash(data, adapterId);
|
|
1456
|
+
}
|
|
1055
1457
|
else if (data.command === "cancel") {
|
|
1056
1458
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1057
1459
|
if (!name) {
|
|
@@ -1169,6 +1571,9 @@ export class FleetManager {
|
|
|
1169
1571
|
}
|
|
1170
1572
|
}, this.logger, "adapter.slash_command"));
|
|
1171
1573
|
await this.topicCommands.registerBotCommands().catch(e => this.logger.warn({ err: e }, "registerBotCommands failed (non-fatal)"));
|
|
1574
|
+
// Background-probe each backend's CLI env (version/models) → cli-env cache.
|
|
1575
|
+
// Non-blocking: /model & status views read the cache; never delays startup.
|
|
1576
|
+
this.probeCliEnvs();
|
|
1172
1577
|
this.adapter.on("started", safeHandler((username, userId) => {
|
|
1173
1578
|
this.logger.info(`Bot @${username} polling started. Ensure no other service is polling this bot token.`);
|
|
1174
1579
|
const w = this.worlds.values().next().value;
|
|
@@ -1235,6 +1640,10 @@ export class FleetManager {
|
|
|
1235
1640
|
await this.handleInboundMessage(msg);
|
|
1236
1641
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
1237
1642
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1643
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1644
|
+
return;
|
|
1645
|
+
if (await this.handleModelSelection(data))
|
|
1646
|
+
return;
|
|
1238
1647
|
if (data.callbackData.startsWith("hang:")) {
|
|
1239
1648
|
const parts = data.callbackData.split(":");
|
|
1240
1649
|
const action = parts[1];
|
|
@@ -1269,13 +1678,15 @@ export class FleetManager {
|
|
|
1269
1678
|
// Slash commands: classic bot + admin commands
|
|
1270
1679
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
1271
1680
|
if (data.command === "start") {
|
|
1272
|
-
|
|
1273
|
-
await data.respond(reply);
|
|
1681
|
+
await this.handleClassicStartSlash(data, adapterId);
|
|
1274
1682
|
}
|
|
1275
1683
|
else if (data.command === "stop") {
|
|
1276
1684
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
1277
1685
|
await data.respond(reply);
|
|
1278
1686
|
}
|
|
1687
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1688
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1689
|
+
}
|
|
1279
1690
|
else if (data.command === "chat") {
|
|
1280
1691
|
const text = data.text ?? "";
|
|
1281
1692
|
if (!text) {
|
|
@@ -1322,6 +1733,9 @@ export class FleetManager {
|
|
|
1322
1733
|
this.pasteRawToClassicInstance(name, `/chat load ${filename}`);
|
|
1323
1734
|
await data.respond(t("save.sent", `/chat load ${filename}`, name));
|
|
1324
1735
|
}
|
|
1736
|
+
else if (data.command === "model") {
|
|
1737
|
+
await this.handleModelSlash(data, adapterId);
|
|
1738
|
+
}
|
|
1325
1739
|
else if (data.command === "cancel") {
|
|
1326
1740
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1327
1741
|
if (!name) {
|
|
@@ -1532,10 +1946,22 @@ export class FleetManager {
|
|
|
1532
1946
|
else if (msg.type === "fleet_set_description") {
|
|
1533
1947
|
this.handleSetDescription(name, msg);
|
|
1534
1948
|
}
|
|
1949
|
+
else if (msg.type === "instance_process_state") {
|
|
1950
|
+
this.cacheInstanceProcessStatus(name, msg.status);
|
|
1951
|
+
}
|
|
1952
|
+
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
1953
|
+
this.cacheInstanceExecutionState(name, msg);
|
|
1954
|
+
if (msg.type === "instance_state_response") {
|
|
1955
|
+
this.cacheInstanceProcessStatus(name, msg.processStatus);
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1535
1958
|
}, this.logger, `ipc.message[${name}]`));
|
|
1536
1959
|
// Ask daemon for any sessions that registered before we connected
|
|
1537
1960
|
// (fixes race condition where mcp_ready was broadcast before fleet manager connected)
|
|
1538
1961
|
ipc.send({ type: "query_sessions" });
|
|
1962
|
+
// The initial state transition may have happened before FleetManager
|
|
1963
|
+
// connected, so seed the cache instead of waiting for another transition.
|
|
1964
|
+
ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
|
|
1539
1965
|
this.logger.debug({ name }, "Connected to instance IPC");
|
|
1540
1966
|
if (!this.statuslineWatcher.has(name)) {
|
|
1541
1967
|
this.statuslineWatcher.watch(name);
|
|
@@ -1798,9 +2224,21 @@ export class FleetManager {
|
|
|
1798
2224
|
}
|
|
1799
2225
|
}
|
|
1800
2226
|
const channelName = msg.username || chatId;
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
2227
|
+
const requestedBackend = text.slice("/start".length).trim().split(/\s+/, 1)[0] || undefined;
|
|
2228
|
+
if (requestedBackend) {
|
|
2229
|
+
// handleClassicStart binds the instance to this adapter authoritatively.
|
|
2230
|
+
const reply = await this.handleClassicStart(chatId, channelName, msg.userId, undefined, msg.adapterId, requestedBackend);
|
|
2231
|
+
await msgAdapter?.sendText(chatId, reply);
|
|
2232
|
+
}
|
|
2233
|
+
else if (msgAdapter) {
|
|
2234
|
+
await this.beginClassicBackendSelection({
|
|
2235
|
+
command: "start",
|
|
2236
|
+
channelId: chatId,
|
|
2237
|
+
channelName,
|
|
2238
|
+
userId: msg.userId,
|
|
2239
|
+
respond: async (reply) => (await msgAdapter.sendText(chatId, reply)).messageId,
|
|
2240
|
+
}, msgAdapter);
|
|
2241
|
+
}
|
|
1804
2242
|
return;
|
|
1805
2243
|
}
|
|
1806
2244
|
// Handle /stop command
|
|
@@ -1817,6 +2255,20 @@ export class FleetManager {
|
|
|
1817
2255
|
await msgAdapter?.sendText(chatId, reply);
|
|
1818
2256
|
return;
|
|
1819
2257
|
}
|
|
2258
|
+
const pauseWake = parsePauseWakeCommand(text);
|
|
2259
|
+
if (pauseWake) {
|
|
2260
|
+
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
2261
|
+
await msgAdapter?.sendText(chatId, t("permission.denied"));
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
const name = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
2265
|
+
if (!name) {
|
|
2266
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
2267
|
+
return;
|
|
2268
|
+
}
|
|
2269
|
+
await msgAdapter?.sendText(chatId, await this.topicCommands.runPauseWake(name, pauseWake.action));
|
|
2270
|
+
return;
|
|
2271
|
+
}
|
|
1820
2272
|
// Handle /compact command (admin only)
|
|
1821
2273
|
if (text === "/compact" || text.startsWith("/compact@")) {
|
|
1822
2274
|
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
@@ -1950,6 +2402,7 @@ export class FleetManager {
|
|
|
1950
2402
|
user_id: msg.userId,
|
|
1951
2403
|
ts: msg.timestamp.toISOString(),
|
|
1952
2404
|
thread_id: "",
|
|
2405
|
+
adapter_id: msg.adapterId,
|
|
1953
2406
|
source: msg.source,
|
|
1954
2407
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
1955
2408
|
...extraMeta,
|
|
@@ -2042,6 +2495,7 @@ export class FleetManager {
|
|
|
2042
2495
|
user_id: msg.userId,
|
|
2043
2496
|
ts: msg.timestamp.toISOString(),
|
|
2044
2497
|
thread_id: msg.threadId ?? "",
|
|
2498
|
+
adapter_id: msg.adapterId,
|
|
2045
2499
|
source: msg.source,
|
|
2046
2500
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2047
2501
|
...extraMeta,
|
|
@@ -2198,7 +2652,7 @@ export class FleetManager {
|
|
|
2198
2652
|
}
|
|
2199
2653
|
// ===================== Scheduler =====================
|
|
2200
2654
|
async handleScheduleTrigger(schedule) {
|
|
2201
|
-
const { target, reply_chat_id, reply_thread_id, message, label, id, source } = schedule;
|
|
2655
|
+
const { target, reply_chat_id, reply_thread_id, message, label, id, source, silent } = schedule;
|
|
2202
2656
|
const RATE_LIMIT_DEFER_THRESHOLD = 85;
|
|
2203
2657
|
const rl = this.statuslineWatcher.getRateLimits(target);
|
|
2204
2658
|
if (rl && rl.five_hour_pct > RATE_LIMIT_DEFER_THRESHOLD) {
|
|
@@ -2213,6 +2667,20 @@ export class FleetManager {
|
|
|
2213
2667
|
this.logger.info({ target, scheduleId: id, rateLimitPct: rl.five_hour_pct }, "Schedule deferred due to rate limit");
|
|
2214
2668
|
return;
|
|
2215
2669
|
}
|
|
2670
|
+
// Silent mode: paste directly to tmux pane — no channel message.
|
|
2671
|
+
if (silent) {
|
|
2672
|
+
const ipc = this.instanceIpcClients.get(target);
|
|
2673
|
+
if (ipc) {
|
|
2674
|
+
ipc.send({ type: "raw_paste", content: message });
|
|
2675
|
+
this.scheduler.recordRun(id, "delivered");
|
|
2676
|
+
this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
|
|
2677
|
+
}
|
|
2678
|
+
else {
|
|
2679
|
+
this.scheduler.recordRun(id, "instance_offline", "IPC not connected");
|
|
2680
|
+
this.logger.warn({ target, scheduleId: id }, "Silent schedule: IPC not connected, skipping");
|
|
2681
|
+
}
|
|
2682
|
+
return;
|
|
2683
|
+
}
|
|
2216
2684
|
const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
|
|
2217
2685
|
const retryCount = schedulerDefaults?.retry_count ?? 3;
|
|
2218
2686
|
const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
|
|
@@ -2222,7 +2690,7 @@ export class FleetManager {
|
|
|
2222
2690
|
type: "fleet_schedule_trigger",
|
|
2223
2691
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
2224
2692
|
meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
|
|
2225
|
-
});
|
|
2693
|
+
}, { waitForIdle: true });
|
|
2226
2694
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
2227
2695
|
void this.sendCancelButton(target);
|
|
2228
2696
|
return true;
|
|
@@ -2281,6 +2749,7 @@ export class FleetManager {
|
|
|
2281
2749
|
case "fleet_schedule_create": {
|
|
2282
2750
|
const params = {
|
|
2283
2751
|
cron: payload.cron,
|
|
2752
|
+
at: payload.at,
|
|
2284
2753
|
message: payload.message,
|
|
2285
2754
|
source: instanceName,
|
|
2286
2755
|
target: payload.target || instanceName,
|
|
@@ -2288,6 +2757,7 @@ export class FleetManager {
|
|
|
2288
2757
|
reply_thread_id: meta.thread_id || null,
|
|
2289
2758
|
label: payload.label,
|
|
2290
2759
|
timezone: payload.timezone,
|
|
2760
|
+
silent: !!(payload.silent),
|
|
2291
2761
|
};
|
|
2292
2762
|
result = this.scheduler.create(params);
|
|
2293
2763
|
break;
|
|
@@ -2408,7 +2878,9 @@ export class FleetManager {
|
|
|
2408
2878
|
switch (op) {
|
|
2409
2879
|
case "create":
|
|
2410
2880
|
return this.scheduler.create({
|
|
2411
|
-
cron: args.cron,
|
|
2881
|
+
cron: args.cron,
|
|
2882
|
+
at: args.at,
|
|
2883
|
+
message: args.message,
|
|
2412
2884
|
source: instance, target: args.target || instance,
|
|
2413
2885
|
reply_chat_id: "", reply_thread_id: null,
|
|
2414
2886
|
label: args.label,
|
|
@@ -2664,73 +3136,92 @@ export class FleetManager {
|
|
|
2664
3136
|
}
|
|
2665
3137
|
}, 5 * 60_000);
|
|
2666
3138
|
}
|
|
2667
|
-
/**
|
|
2668
|
-
|
|
3139
|
+
/**
|
|
3140
|
+
* Patch only values changed in the effective config into the original YAML
|
|
3141
|
+
* document. Unknown keys, explicit overrides and comments remain untouched.
|
|
3142
|
+
*/
|
|
3143
|
+
saveFleetConfig(explicitPatches = []) {
|
|
2669
3144
|
if (!this.fleetConfig || !this.configPath)
|
|
2670
3145
|
return;
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
}
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
3146
|
+
if (!this.savedFleetConfigSnapshot)
|
|
3147
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3148
|
+
// Re-read immediately before patching so an unrelated concurrent/manual
|
|
3149
|
+
// edit is retained. Invalid concurrent YAML is never overwritten.
|
|
3150
|
+
const source = existsSync(this.configPath) ? readFileSync(this.configPath, "utf-8") : "{}\n";
|
|
3151
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
3152
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
3153
|
+
throw new Error(`Refusing to overwrite invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
3154
|
+
}
|
|
3155
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3156
|
+
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
3157
|
+
// Settings edits are expressed against the raw config. Persist them even
|
|
3158
|
+
// when the chosen override equals the inherited effective value, a case
|
|
3159
|
+
// the before/after runtime diff cannot observe.
|
|
3160
|
+
for (const patch of explicitPatches) {
|
|
3161
|
+
if (patch.remove) {
|
|
3162
|
+
this.rawFleetDocument.deleteIn(patch.path);
|
|
3163
|
+
}
|
|
3164
|
+
else {
|
|
3165
|
+
const before = this.rawFleetDocument.getIn(patch.path);
|
|
3166
|
+
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
3167
|
+
}
|
|
3168
|
+
}
|
|
3169
|
+
const output = String(this.rawFleetDocument);
|
|
3170
|
+
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
3171
|
+
writeFileSync(tempPath, output, "utf-8");
|
|
3172
|
+
if (existsSync(this.configPath))
|
|
3173
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
3174
|
+
renameSync(tempPath, this.configPath);
|
|
3175
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3176
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3177
|
+
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
3178
|
+
}
|
|
3179
|
+
patchFleetDocument(document, path, before, after) {
|
|
3180
|
+
if (Object.is(before, after))
|
|
3181
|
+
return;
|
|
3182
|
+
if (Array.isArray(before) && Array.isArray(after)) {
|
|
3183
|
+
const shared = Math.min(before.length, after.length);
|
|
3184
|
+
for (let i = 0; i < shared; i++) {
|
|
3185
|
+
this.patchFleetDocument(document, [...path, i], before[i], after[i]);
|
|
3186
|
+
}
|
|
3187
|
+
// Remove from the end so YAML sequence indexes do not shift underneath us.
|
|
3188
|
+
for (let i = before.length - 1; i >= after.length; i--)
|
|
3189
|
+
document.deleteIn([...path, i]);
|
|
3190
|
+
for (let i = shared; i < after.length; i++)
|
|
3191
|
+
document.setIn([...path, i], after[i]);
|
|
3192
|
+
return;
|
|
3193
|
+
}
|
|
3194
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3195
|
+
if (isRecord(before) && isRecord(after)) {
|
|
3196
|
+
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
3197
|
+
for (const key of keys) {
|
|
3198
|
+
// `channel` is a derived alias when the raw file uses `channels`.
|
|
3199
|
+
if (path.length === 0 && key === "channel" && this.rawFleetConfig.channels)
|
|
3200
|
+
continue;
|
|
3201
|
+
// Conversely, `channels` is a normalized alias for a legacy `channel`.
|
|
3202
|
+
// Keep the user's original shape unless the caller explicitly removed
|
|
3203
|
+
// `channel` (the Settings channels endpoint intentionally migrates it).
|
|
3204
|
+
if (path.length === 0 && key === "channels" && this.rawFleetConfig.channel && !this.rawFleetConfig.channels && after.channel !== undefined)
|
|
3205
|
+
continue;
|
|
3206
|
+
this.patchFleetDocument(document, [...path, key], before[key], after[key]);
|
|
3207
|
+
}
|
|
3208
|
+
return;
|
|
3209
|
+
}
|
|
3210
|
+
if (after === undefined) {
|
|
3211
|
+
document.deleteIn(path);
|
|
3212
|
+
}
|
|
3213
|
+
else if (path.length === 0) {
|
|
3214
|
+
document.contents = document.createNode(after);
|
|
3215
|
+
}
|
|
3216
|
+
else {
|
|
3217
|
+
const currentNode = document.getIn(path, true);
|
|
3218
|
+
if (isScalar(currentNode) && (after === null || typeof after !== "object")) {
|
|
3219
|
+
currentNode.value = after;
|
|
3220
|
+
}
|
|
3221
|
+
else {
|
|
3222
|
+
document.setIn(path, after);
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
2734
3225
|
}
|
|
2735
3226
|
async removeInstance(name) {
|
|
2736
3227
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
@@ -2771,8 +3262,14 @@ export class FleetManager {
|
|
|
2771
3262
|
}
|
|
2772
3263
|
}
|
|
2773
3264
|
startStatuslineWatcher(name) {
|
|
3265
|
+
if (this.lifecycle.isPaused(name))
|
|
3266
|
+
return;
|
|
2774
3267
|
this.statuslineWatcher.watch(name);
|
|
2775
3268
|
}
|
|
3269
|
+
stopStatuslineWatcher(name) {
|
|
3270
|
+
// Pausing stops I/O but retains the last observed limits for status views.
|
|
3271
|
+
this.statuslineWatcher.unwatch(name, true);
|
|
3272
|
+
}
|
|
2776
3273
|
reactMessageStatus(instanceName, chatId, messageId, emoji) {
|
|
2777
3274
|
// React via the adapter BOUND to this instance — NOT the first discord world.
|
|
2778
3275
|
// Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
|
|
@@ -3123,7 +3620,7 @@ export class FleetManager {
|
|
|
3123
3620
|
return [];
|
|
3124
3621
|
}
|
|
3125
3622
|
}
|
|
3126
|
-
async sendHangNotification(instanceName) {
|
|
3623
|
+
async sendHangNotification(instanceName, unchangedForMs) {
|
|
3127
3624
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3128
3625
|
if (!adapter)
|
|
3129
3626
|
return;
|
|
@@ -3132,11 +3629,18 @@ export class FleetManager {
|
|
|
3132
3629
|
if (!groupId)
|
|
3133
3630
|
return;
|
|
3134
3631
|
const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3632
|
+
const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
|
|
3633
|
+
const configuredMinutes = instanceHangConfig?.timeout_minutes
|
|
3634
|
+
?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
|
|
3635
|
+
?? 15;
|
|
3636
|
+
const unchangedMinutes = unchangedForMs == null
|
|
3637
|
+
? configuredMinutes
|
|
3638
|
+
: Math.max(1, Math.floor(unchangedForMs / 60_000));
|
|
3135
3639
|
this.setTopicIcon(instanceName, "red");
|
|
3136
3640
|
await adapter.notifyAlert(String(groupId), {
|
|
3137
3641
|
type: "hang",
|
|
3138
3642
|
instanceName,
|
|
3139
|
-
message: `⚠️ ${instanceName}
|
|
3643
|
+
message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
|
|
3140
3644
|
choices: [
|
|
3141
3645
|
{ id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
|
|
3142
3646
|
{ id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
|
|
@@ -3605,13 +4109,25 @@ When users create specialized instances, suggest these configurations:
|
|
|
3605
4109
|
const inboxDir = join(getAgendHome(), "workspaces", instanceName, "inbox");
|
|
3606
4110
|
mkdirSync(inboxDir, { recursive: true });
|
|
3607
4111
|
const dest = join(inboxDir, basename(tmpPath));
|
|
4112
|
+
// Copy to destination — failure means this attachment is skipped
|
|
3608
4113
|
try {
|
|
3609
|
-
renameSync(tmpPath, dest);
|
|
3610
|
-
}
|
|
3611
|
-
catch {
|
|
3612
4114
|
copyFileSync(tmpPath, dest);
|
|
4115
|
+
}
|
|
4116
|
+
catch (copyErr) {
|
|
4117
|
+
try {
|
|
4118
|
+
unlinkSync(dest);
|
|
4119
|
+
}
|
|
4120
|
+
catch { } // clean partial
|
|
4121
|
+
this.logger.warn({ err: copyErr.message, instanceName, dest }, "Attachment copy failed — skipping");
|
|
4122
|
+
continue;
|
|
4123
|
+
}
|
|
4124
|
+
// Cleanup source — failure is non-fatal (dest already valid)
|
|
4125
|
+
try {
|
|
3613
4126
|
unlinkSync(tmpPath);
|
|
3614
4127
|
}
|
|
4128
|
+
catch (cleanupErr) {
|
|
4129
|
+
this.logger.debug({ tmpPath, err: cleanupErr.message }, "Orphan tmp not cleaned");
|
|
4130
|
+
}
|
|
3615
4131
|
const savedKind = att.kind === "sticker" ? "photo" : att.kind;
|
|
3616
4132
|
paths.push(dest);
|
|
3617
4133
|
if (paths.length === 1)
|
|
@@ -3644,6 +4160,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3644
4160
|
user_id: msg.userId,
|
|
3645
4161
|
ts: msg.timestamp.toISOString(),
|
|
3646
4162
|
thread_id: msg.threadId ?? "",
|
|
4163
|
+
...(msg.adapterId ? { adapter_id: msg.adapterId } : {}),
|
|
3647
4164
|
source: msg.source,
|
|
3648
4165
|
...extraMeta,
|
|
3649
4166
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
@@ -3685,23 +4202,349 @@ When users create specialized instances, suggest these configurations:
|
|
|
3685
4202
|
ipc.send({ type: "raw_paste", content: text });
|
|
3686
4203
|
this.logger.info({ instanceName, text: text.slice(0, 100) }, "Raw paste sent to classic instance");
|
|
3687
4204
|
}
|
|
4205
|
+
/** Resolve the backend name configured for an instance (fleet or classic). */
|
|
4206
|
+
backendNameForInstance(instanceName) {
|
|
4207
|
+
const fleetCfg = this.fleetConfig?.instances[instanceName];
|
|
4208
|
+
if (fleetCfg?.backend)
|
|
4209
|
+
return fleetCfg.backend;
|
|
4210
|
+
const classic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined
|
|
4211
|
+
? this.classicChannels?.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend)
|
|
4212
|
+
: undefined;
|
|
4213
|
+
return classic ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
4214
|
+
}
|
|
4215
|
+
cliEnvPath(backend) {
|
|
4216
|
+
return join(getAgendHome(), "cli-env", `${backend.replace(/[^A-Za-z0-9._-]/g, "_")}.json`);
|
|
4217
|
+
}
|
|
4218
|
+
/** Read the cached CLI env for a backend, or null if missing/stale/unparseable. */
|
|
4219
|
+
readCliEnv(backend) {
|
|
4220
|
+
try {
|
|
4221
|
+
const env = JSON.parse(readFileSync(this.cliEnvPath(backend), "utf-8"));
|
|
4222
|
+
if (typeof env?.probedAt === "number" && Date.now() - env.probedAt < CLI_ENV_TTL_MS)
|
|
4223
|
+
return env;
|
|
4224
|
+
}
|
|
4225
|
+
catch { /* missing / stale / corrupt */ }
|
|
4226
|
+
return null;
|
|
4227
|
+
}
|
|
4228
|
+
/** Probe one backend's CLI env and cache it. Best-effort; never throws. */
|
|
4229
|
+
async probeBackend(backend) {
|
|
4230
|
+
try {
|
|
4231
|
+
const be = createBackend(backend, join(getAgendHome(), "cli-env"));
|
|
4232
|
+
if (!be.probeCLIEnv)
|
|
4233
|
+
return null;
|
|
4234
|
+
const probed = await be.probeCLIEnv({ workingDirectory: "", instanceDir: join(getAgendHome(), "cli-env"), instanceName: `probe-${backend}`, mcpServers: {} });
|
|
4235
|
+
const env = { backend, probedAt: Date.now(), ...probed };
|
|
4236
|
+
const path = this.cliEnvPath(backend);
|
|
4237
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
4238
|
+
writeFileSync(path, JSON.stringify(env, null, 2));
|
|
4239
|
+
return env;
|
|
4240
|
+
}
|
|
4241
|
+
catch (err) {
|
|
4242
|
+
this.logger.warn({ err, backend }, "CLI env probe failed");
|
|
4243
|
+
return null;
|
|
4244
|
+
}
|
|
4245
|
+
}
|
|
4246
|
+
/** Background-probe every distinct backend in use at startup (non-blocking). */
|
|
4247
|
+
probeCliEnvs() {
|
|
4248
|
+
const backends = new Set();
|
|
4249
|
+
if (this.fleetConfig?.defaults?.backend)
|
|
4250
|
+
backends.add(this.fleetConfig.defaults.backend);
|
|
4251
|
+
for (const inst of Object.values(this.fleetConfig?.instances ?? {}))
|
|
4252
|
+
if (inst.backend)
|
|
4253
|
+
backends.add(inst.backend);
|
|
4254
|
+
for (const ch of this.classicChannels?.getAll() ?? [])
|
|
4255
|
+
if (ch.backend)
|
|
4256
|
+
backends.add(ch.backend);
|
|
4257
|
+
if (backends.size === 0)
|
|
4258
|
+
backends.add("claude-code");
|
|
4259
|
+
for (const b of backends)
|
|
4260
|
+
void this.probeBackend(b);
|
|
4261
|
+
}
|
|
4262
|
+
/** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
|
|
4263
|
+
async getModelOptions(instanceName, refresh = false) {
|
|
4264
|
+
const backendName = this.backendNameForInstance(instanceName);
|
|
4265
|
+
if (!refresh) {
|
|
4266
|
+
const cached = this.readCliEnv(backendName);
|
|
4267
|
+
if (cached && cached.models.length)
|
|
4268
|
+
return cached.models;
|
|
4269
|
+
}
|
|
4270
|
+
// Cache miss / stale / forced refresh → probe live (also refreshes the cache).
|
|
4271
|
+
const env = await this.probeBackend(backendName);
|
|
4272
|
+
return env?.models ?? [];
|
|
4273
|
+
}
|
|
4274
|
+
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
4275
|
+
async handleModelSlash(data, adapterId) {
|
|
4276
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
4277
|
+
await data.respond(t("admin.required"));
|
|
4278
|
+
return;
|
|
4279
|
+
}
|
|
4280
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
4281
|
+
if (!name) {
|
|
4282
|
+
await data.respond(t("classic.no_agent"));
|
|
4283
|
+
return;
|
|
4284
|
+
}
|
|
4285
|
+
const requested = (typeof data.options?.name === "string" ? data.options.name.trim() : "")
|
|
4286
|
+
|| (data.text?.trim() ?? "");
|
|
4287
|
+
const isRefresh = requested === "--refresh" || requested === "refresh";
|
|
4288
|
+
if (requested && !isRefresh) {
|
|
4289
|
+
await data.respond(await this.applyModel(name, requested));
|
|
4290
|
+
return;
|
|
4291
|
+
}
|
|
4292
|
+
// No arg (or --refresh) → menu. Menu is DC-only this round (respondChoices); TG uses `/model <name>`.
|
|
4293
|
+
if (!data.respondChoices) {
|
|
4294
|
+
await data.respond("Usage: /model <name> — e.g. /model sonnet");
|
|
4295
|
+
return;
|
|
4296
|
+
}
|
|
4297
|
+
const options = await this.getModelOptions(name, isRefresh);
|
|
4298
|
+
if (options.length === 0) {
|
|
4299
|
+
await data.respond(`No model list available for ${name}. Type \`/model <name>\` to set one directly.`);
|
|
4300
|
+
return;
|
|
4301
|
+
}
|
|
4302
|
+
const nonce = randomBytes(6).toString("hex");
|
|
4303
|
+
const choices = options.slice(0, 25).map(o => ({
|
|
4304
|
+
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
4305
|
+
label: o.description ? `${o.label} — ${o.description}` : o.label,
|
|
4306
|
+
}));
|
|
4307
|
+
const timer = setTimeout(() => this.pendingModelSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
4308
|
+
timer.unref?.();
|
|
4309
|
+
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
4310
|
+
try {
|
|
4311
|
+
await data.respondChoices(`Choose a model for ${name}:`, choices);
|
|
4312
|
+
}
|
|
4313
|
+
catch (err) {
|
|
4314
|
+
this.pendingModelSelects.delete(nonce);
|
|
4315
|
+
clearTimeout(timer);
|
|
4316
|
+
this.logger.warn({ err, instanceName: name }, "model menu failed");
|
|
4317
|
+
await data.respond("Usage: /model <name> — e.g. /model sonnet");
|
|
4318
|
+
}
|
|
4319
|
+
}
|
|
4320
|
+
/** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
|
|
4321
|
+
async handleModelSelection(data) {
|
|
4322
|
+
if (!data.callbackData.startsWith(MODEL_SELECT_CALLBACK_PREFIX))
|
|
4323
|
+
return false;
|
|
4324
|
+
const match = data.callbackData.match(/^model-select:([0-9a-f]+):(.+)$/);
|
|
4325
|
+
if (!match)
|
|
4326
|
+
return true;
|
|
4327
|
+
const pending = this.pendingModelSelects.get(match[1]);
|
|
4328
|
+
if (!pending)
|
|
4329
|
+
return true;
|
|
4330
|
+
// Only the admin who opened the menu, in the same channel, may consume it.
|
|
4331
|
+
if (data.userId && data.userId !== pending.userId)
|
|
4332
|
+
return true;
|
|
4333
|
+
const cbChannel = data.threadId ?? data.chatId;
|
|
4334
|
+
if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
|
|
4335
|
+
return true;
|
|
4336
|
+
this.pendingModelSelects.delete(match[1]);
|
|
4337
|
+
clearTimeout(pending.timer);
|
|
4338
|
+
const result = await this.applyModel(pending.instanceName, match[2]);
|
|
4339
|
+
await pending.respond(result).catch(() => { });
|
|
4340
|
+
return true;
|
|
4341
|
+
}
|
|
4342
|
+
/** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
|
|
4343
|
+
async applyModel(instanceName, model) {
|
|
4344
|
+
const backendName = this.backendNameForInstance(instanceName);
|
|
4345
|
+
let strategy = "restart";
|
|
4346
|
+
try {
|
|
4347
|
+
strategy = createBackend(backendName, this.getInstanceDir(instanceName)).getModelSwitchStrategy?.(model) ?? "restart";
|
|
4348
|
+
}
|
|
4349
|
+
catch { /* default restart */ }
|
|
4350
|
+
const warn = isModelCompatible(backendName, model) ? "" : `⚠️ "${model}" doesn't match ${backendName}'s usual pattern — passing through anyway.\n`;
|
|
4351
|
+
if (strategy === "runtime") {
|
|
4352
|
+
if (!this.instanceIpcClients.get(instanceName))
|
|
4353
|
+
return `${warn}❌ ${instanceName} is not running.`;
|
|
4354
|
+
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
4355
|
+
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).`;
|
|
4356
|
+
}
|
|
4357
|
+
// restart: persist the model so the respawned CLI launches with it.
|
|
4358
|
+
let persisted = false;
|
|
4359
|
+
if (this.fleetConfig?.instances[instanceName]) {
|
|
4360
|
+
this.fleetConfig.instances[instanceName].model = model;
|
|
4361
|
+
this.saveFleetConfig();
|
|
4362
|
+
persisted = true;
|
|
4363
|
+
}
|
|
4364
|
+
else if (this.classicChannels?.setModelByInstance(instanceName, model)) {
|
|
4365
|
+
persisted = true;
|
|
4366
|
+
}
|
|
4367
|
+
if (!persisted)
|
|
4368
|
+
return `${warn}❌ Could not set model for ${instanceName}.`;
|
|
4369
|
+
await this.restartSingleInstance(instanceName);
|
|
4370
|
+
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.`;
|
|
4371
|
+
}
|
|
3688
4372
|
/** Read recent chat log for agent context */
|
|
3689
4373
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
3690
4374
|
const logDir = ClassicChannelManager.chatLogDir(instanceName);
|
|
3691
|
-
|
|
4375
|
+
// Use local timezone for date — must match logMessage's write path
|
|
4376
|
+
const tz = process.env.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4377
|
+
const today = new Date().toLocaleString("sv-SE", { timeZone: tz, hour12: false }).slice(0, 10);
|
|
3692
4378
|
const logFile = join(logDir, `${today}.log`);
|
|
3693
4379
|
try {
|
|
3694
4380
|
if (!existsSync(logFile))
|
|
3695
4381
|
return undefined;
|
|
3696
4382
|
const lines = readFileSync(logFile, "utf-8").trim().split("\n");
|
|
3697
|
-
|
|
4383
|
+
// The triggering message is written before forwardToClassicInstance runs
|
|
4384
|
+
// and is included separately under [User message]. Exclude that newest
|
|
4385
|
+
// log entry so the agent does not receive the same message twice. A chat
|
|
4386
|
+
// message may span physical lines, so remove from its timestamped entry
|
|
4387
|
+
// header rather than blindly dropping only the final continuation line.
|
|
4388
|
+
const entryHeader = /^\[\d{4}-\d{2}-\d{2}T[^\]]+\] <.*> /;
|
|
4389
|
+
let currentEntryStart = lines.length - 1;
|
|
4390
|
+
while (currentEntryStart > 0 && !entryHeader.test(lines[currentEntryStart])) {
|
|
4391
|
+
currentEntryStart--;
|
|
4392
|
+
}
|
|
4393
|
+
lines.splice(currentEntryStart);
|
|
4394
|
+
if (lines.length === 0 || maxLines <= 0)
|
|
4395
|
+
return undefined;
|
|
4396
|
+
return lines.slice(-maxLines).join("\n") || undefined;
|
|
3698
4397
|
}
|
|
3699
4398
|
catch {
|
|
3700
4399
|
return undefined;
|
|
3701
4400
|
}
|
|
3702
4401
|
}
|
|
4402
|
+
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
4403
|
+
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
4404
|
+
if (!this.classicChannels)
|
|
4405
|
+
return "Classic channel manager not initialized.";
|
|
4406
|
+
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
4407
|
+
const generalId = this.findGeneralInstance(adapterId);
|
|
4408
|
+
if (generalId)
|
|
4409
|
+
this.notifyInstanceTopic(generalId, t("alert.unauth_guild", guildId, userId));
|
|
4410
|
+
return t("classic.not_authorized_guild");
|
|
4411
|
+
}
|
|
4412
|
+
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
4413
|
+
return t("classic.already_active");
|
|
4414
|
+
if (this.routing.resolve(channelId))
|
|
4415
|
+
return t("classic.topic_bound");
|
|
4416
|
+
return undefined;
|
|
4417
|
+
}
|
|
4418
|
+
isBackendInstalled(backend) {
|
|
4419
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4420
|
+
return !!installation && checkBinaryInstalled(installation.binary);
|
|
4421
|
+
}
|
|
4422
|
+
getMissingBackendWarning(backend) {
|
|
4423
|
+
if (!backend)
|
|
4424
|
+
return undefined;
|
|
4425
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4426
|
+
if (!installation || this.isBackendInstalled(backend))
|
|
4427
|
+
return undefined;
|
|
4428
|
+
return t("classic.backend_not_installed", backend, installation.binary, installation.install);
|
|
4429
|
+
}
|
|
4430
|
+
/** Handle Discord's required static slash choice, warning before a likely startup failure. */
|
|
4431
|
+
async handleClassicStartSlash(data, adapterId) {
|
|
4432
|
+
const requestedBackend = typeof data.options?.backend === "string" ? data.options.backend : undefined;
|
|
4433
|
+
if (!requestedBackend) {
|
|
4434
|
+
// beta.31 made this option required. Discord can briefly retain the old
|
|
4435
|
+
// command schema client-side, however, so stale clients may still submit
|
|
4436
|
+
// `/start` without it. Do not resurrect the legacy 60-second component
|
|
4437
|
+
// menu in that case: fail immediately and make the user invoke the newly
|
|
4438
|
+
// registered command, which guarantees an explicit backend choice.
|
|
4439
|
+
await data.respond(t("classic.backend_required"));
|
|
4440
|
+
return;
|
|
4441
|
+
}
|
|
4442
|
+
const warning = this.getMissingBackendWarning(requestedBackend);
|
|
4443
|
+
// Keep the deferred ephemeral response useful even if daemon startup later
|
|
4444
|
+
// fails because the executable is absent. This is advisory, not a gate.
|
|
4445
|
+
if (warning)
|
|
4446
|
+
await data.respond(warning);
|
|
4447
|
+
const reply = await this.handleClassicStart(data.channelId, data.channelName, data.userId, data.guildId, adapterId, requestedBackend);
|
|
4448
|
+
await data.respond(warning ? `${warning}\n\n${reply}` : reply);
|
|
4449
|
+
}
|
|
4450
|
+
/** Present platform-native backend choices, then start on selection or timeout. */
|
|
4451
|
+
async beginClassicBackendSelection(data, adapter) {
|
|
4452
|
+
const adapterId = adapter.id;
|
|
4453
|
+
const blocker = this.validateClassicStart(data.channelId, data.userId, data.guildId, adapterId);
|
|
4454
|
+
if (blocker) {
|
|
4455
|
+
await data.respond(blocker);
|
|
4456
|
+
return;
|
|
4457
|
+
}
|
|
4458
|
+
const nonce = randomBytes(6).toString("hex");
|
|
4459
|
+
const choices = getClassicBackendChoices().map(choice => ({
|
|
4460
|
+
id: `${CLASSIC_BACKEND_CALLBACK_PREFIX}${nonce}:${choice.id}`,
|
|
4461
|
+
label: `${this.isBackendInstalled(choice.id) ? "✅" : "❌"} ${choice.label}`,
|
|
4462
|
+
}));
|
|
4463
|
+
const complete = data.respondChoices
|
|
4464
|
+
? async (text) => { await data.respond(text); }
|
|
4465
|
+
: async (text, messageId) => {
|
|
4466
|
+
if (messageId && adapter.editMessageRemoveButtons) {
|
|
4467
|
+
try {
|
|
4468
|
+
await adapter.editMessageRemoveButtons(data.channelId, messageId, text);
|
|
4469
|
+
return;
|
|
4470
|
+
}
|
|
4471
|
+
catch { /* fall back to a new message */ }
|
|
4472
|
+
}
|
|
4473
|
+
await data.respond(text);
|
|
4474
|
+
};
|
|
4475
|
+
const timer = setTimeout(() => {
|
|
4476
|
+
// Timeout: cancel the selection — do NOT fall back to default.
|
|
4477
|
+
const p = this.pendingClassicStarts.get(nonce);
|
|
4478
|
+
if (p) {
|
|
4479
|
+
this.pendingClassicStarts.delete(nonce);
|
|
4480
|
+
p.complete(t("classic.selection_expired"), p.messageId).catch(() => { });
|
|
4481
|
+
}
|
|
4482
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
4483
|
+
timer.unref?.();
|
|
4484
|
+
const pending = {
|
|
4485
|
+
channelId: data.channelId,
|
|
4486
|
+
channelName: data.channelName,
|
|
4487
|
+
userId: data.userId,
|
|
4488
|
+
guildId: data.guildId,
|
|
4489
|
+
adapterId,
|
|
4490
|
+
timer,
|
|
4491
|
+
complete,
|
|
4492
|
+
};
|
|
4493
|
+
this.pendingClassicStarts.set(nonce, pending);
|
|
4494
|
+
try {
|
|
4495
|
+
pending.messageId = data.respondChoices
|
|
4496
|
+
? await data.respondChoices(t("classic.choose_backend"), choices)
|
|
4497
|
+
: await adapter.promptUser(data.channelId, t("classic.choose_backend"), choices);
|
|
4498
|
+
}
|
|
4499
|
+
catch (err) {
|
|
4500
|
+
// A menu transport failure should not make /start unusable: consume the
|
|
4501
|
+
// pending request and immediately use the configured default.
|
|
4502
|
+
this.logger.warn({ err, channelId: data.channelId, adapterId }, "Classic backend menu failed; using default");
|
|
4503
|
+
await this.finishClassicBackendSelection(nonce);
|
|
4504
|
+
}
|
|
4505
|
+
}
|
|
4506
|
+
/** Consume a selection callback. Returns true for all ClassicBot callback IDs, including stale ones. */
|
|
4507
|
+
async handleClassicBackendSelection(data) {
|
|
4508
|
+
if (!data.callbackData.startsWith(CLASSIC_BACKEND_CALLBACK_PREFIX))
|
|
4509
|
+
return false;
|
|
4510
|
+
const match = data.callbackData.match(/^classic-backend:([0-9a-f]+):(.+)$/);
|
|
4511
|
+
if (!match)
|
|
4512
|
+
return true;
|
|
4513
|
+
const pending = this.pendingClassicStarts.get(match[1]);
|
|
4514
|
+
if (!pending)
|
|
4515
|
+
return true;
|
|
4516
|
+
// Telegram keyboards are visible to everyone in a group. Only the user who
|
|
4517
|
+
// issued /start may consume the pending selection.
|
|
4518
|
+
if (data.userId && data.userId !== pending.userId)
|
|
4519
|
+
return true;
|
|
4520
|
+
const callbackChannelId = data.threadId ?? data.chatId;
|
|
4521
|
+
if (callbackChannelId !== pending.channelId && data.chatId !== pending.channelId)
|
|
4522
|
+
return true;
|
|
4523
|
+
await this.finishClassicBackendSelection(match[1], match[2]);
|
|
4524
|
+
return true;
|
|
4525
|
+
}
|
|
4526
|
+
/** Atomically claim one pending request so timeout/click races create at most one instance. */
|
|
4527
|
+
async finishClassicBackendSelection(nonce, backend) {
|
|
4528
|
+
const pending = this.pendingClassicStarts.get(nonce);
|
|
4529
|
+
if (!pending)
|
|
4530
|
+
return;
|
|
4531
|
+
this.pendingClassicStarts.delete(nonce);
|
|
4532
|
+
clearTimeout(pending.timer);
|
|
4533
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4534
|
+
const effectiveBackend = selectedBackend
|
|
4535
|
+
?? this.classicChannels?.getDefaults().backend
|
|
4536
|
+
?? this.fleetConfig?.defaults?.backend
|
|
4537
|
+
?? "claude-code";
|
|
4538
|
+
const warning = this.getMissingBackendWarning(effectiveBackend);
|
|
4539
|
+
// Show the warning before starting so it survives a missing-binary startup
|
|
4540
|
+
// failure. The selected backend is still attempted as requested.
|
|
4541
|
+
if (warning)
|
|
4542
|
+
await pending.complete(warning, pending.messageId);
|
|
4543
|
+
const reply = await this.handleClassicStart(pending.channelId, pending.channelName, pending.userId, pending.guildId, pending.adapterId, selectedBackend);
|
|
4544
|
+
await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
|
|
4545
|
+
}
|
|
3703
4546
|
/** Start a classic channel instance with lightweight config */
|
|
3704
|
-
async startClassicInstance(instanceName, backend, preTaskCommand, model) {
|
|
4547
|
+
async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
|
|
3705
4548
|
if (this.daemons.has(instanceName))
|
|
3706
4549
|
return;
|
|
3707
4550
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
@@ -3713,41 +4556,34 @@ When users create specialized instances, suggest these configurations:
|
|
|
3713
4556
|
lightweight: true,
|
|
3714
4557
|
...(backend ? { backend } : {}),
|
|
3715
4558
|
...(model ? { model } : {}),
|
|
4559
|
+
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
3716
4560
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
3717
4561
|
};
|
|
3718
4562
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
3719
4563
|
await this.startInstance(instanceName, config, topicMode);
|
|
3720
4564
|
}
|
|
3721
4565
|
/** Handle /start slash command — register classic channel */
|
|
3722
|
-
async handleClassicStart(channelId, channelName, userId, guildId, adapterId) {
|
|
3723
|
-
|
|
4566
|
+
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
4567
|
+
const blocker = this.validateClassicStart(channelId, userId, guildId, adapterId);
|
|
4568
|
+
if (blocker)
|
|
4569
|
+
return blocker;
|
|
4570
|
+
const classicChannels = this.classicChannels;
|
|
4571
|
+
if (!classicChannels)
|
|
3724
4572
|
return "Classic channel manager not initialized.";
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
}
|
|
3730
|
-
return t("classic.not_authorized_guild");
|
|
3731
|
-
}
|
|
3732
|
-
// Per-bot check: a second bot may /start in the same channel (own agent).
|
|
3733
|
-
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
3734
|
-
return t("classic.already_active");
|
|
3735
|
-
// Classic no longer lives in the routing engine, so this only guards against
|
|
3736
|
-
// a fleet topic-mode instance colliding with the channel.
|
|
3737
|
-
if (this.routing.resolve(channelId))
|
|
3738
|
-
return t("classic.topic_bound");
|
|
3739
|
-
const instanceName = this.classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
3740
|
-
this.classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId);
|
|
4573
|
+
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
4574
|
+
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
4575
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4576
|
+
classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId, selectedBackend);
|
|
3741
4577
|
// Bind this classic instance to the bot that started it (authoritative), so
|
|
3742
4578
|
// replies/cancel go out through that bot even though every same-guild bot
|
|
3743
4579
|
// also sees the channel's messages.
|
|
3744
4580
|
if (adapterId)
|
|
3745
4581
|
this.bindInstanceAdapter(instanceName, adapterId);
|
|
3746
|
-
await this.startClassicInstance(instanceName,
|
|
4582
|
+
await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
3747
4583
|
this.reregisterClassicChannels();
|
|
3748
4584
|
// Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
|
|
3749
|
-
if (guildId && !
|
|
3750
|
-
|
|
4585
|
+
if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
|
|
4586
|
+
classicChannels.toggleCollab(channelId, adapterId);
|
|
3751
4587
|
}
|
|
3752
4588
|
this.logger.info({ channelId, adapterId, instanceName, userId }, "Classic channel started");
|
|
3753
4589
|
return t("classic.started");
|
|
@@ -3761,6 +4597,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3761
4597
|
return t("classic.no_agent");
|
|
3762
4598
|
this.instanceWorldBinding.delete(ch.instanceName);
|
|
3763
4599
|
await this.stopInstance(ch.instanceName).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to stop classic instance"));
|
|
4600
|
+
clearPausedMarker(this.getInstanceDir(ch.instanceName));
|
|
3764
4601
|
this.reregisterClassicChannels();
|
|
3765
4602
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
3766
4603
|
return t("classic.stopped");
|
|
@@ -3804,6 +4641,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
3804
4641
|
clearInterval(this.classicReloadTimer);
|
|
3805
4642
|
this.classicReloadTimer = null;
|
|
3806
4643
|
}
|
|
4644
|
+
for (const pending of this.pendingClassicStarts.values())
|
|
4645
|
+
clearTimeout(pending.timer);
|
|
4646
|
+
this.pendingClassicStarts.clear();
|
|
3807
4647
|
this.topicArchiver.stop();
|
|
3808
4648
|
this.scheduler?.shutdown();
|
|
3809
4649
|
// Stop instances in parallel batches to avoid long sequential waits.
|
|
@@ -4098,7 +4938,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4098
4938
|
let idx = 0;
|
|
4099
4939
|
while (idx < channels.length) {
|
|
4100
4940
|
const batch = channels.slice(idx, idx + concurrency);
|
|
4101
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4941
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4102
4942
|
idx += concurrency;
|
|
4103
4943
|
}
|
|
4104
4944
|
}
|
|
@@ -4110,13 +4950,23 @@ When users create specialized instances, suggest these configurations:
|
|
|
4110
4950
|
if (groupId && this.adapter) {
|
|
4111
4951
|
const total = Object.keys(fleet.instances).length;
|
|
4112
4952
|
const started = this.daemons.size;
|
|
4113
|
-
const
|
|
4953
|
+
const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
4954
|
+
const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
|
|
4955
|
+
const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
|
|
4114
4956
|
const { createRequire } = await import("node:module");
|
|
4115
4957
|
const _require2 = createRequire(import.meta.url);
|
|
4116
4958
|
const agendVersion2 = _require2("../package.json").version ?? "unknown";
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4959
|
+
let restartText;
|
|
4960
|
+
if (failedNames.length === 0 && pausedNames2.length === 0) {
|
|
4961
|
+
restartText = t("fleet.ready", started, total, agendVersion2);
|
|
4962
|
+
}
|
|
4963
|
+
else if (failedNames.length === 0) {
|
|
4964
|
+
restartText = t("fleet.ready", started, total, agendVersion2) + `\n⏸ Paused: ${pausedNames2.join(", ")}`;
|
|
4965
|
+
}
|
|
4966
|
+
else {
|
|
4967
|
+
restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
|
|
4968
|
+
+ (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
|
|
4969
|
+
}
|
|
4120
4970
|
await this.adapter.sendText(String(groupId), restartText, notifyOpts)
|
|
4121
4971
|
.catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
|
|
4122
4972
|
// Notify each instance's channel — staggered to avoid rate limit storm
|
|
@@ -4328,8 +5178,31 @@ When users create specialized instances, suggest these configurations:
|
|
|
4328
5178
|
// Fleet API (enriched for agent board)
|
|
4329
5179
|
if (req.method === "GET" && req.url === "/api/fleet") {
|
|
4330
5180
|
const sysInfo = this.getSysInfo();
|
|
4331
|
-
const
|
|
5181
|
+
const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
|
|
5182
|
+
const fleetNames = new Set(fleetInstances.map(inst => inst.name));
|
|
5183
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
5184
|
+
.filter(channel => !fleetNames.has(channel.instanceName))
|
|
5185
|
+
.map(channel => ({
|
|
5186
|
+
name: channel.instanceName,
|
|
5187
|
+
status: this.getInstanceStatus(channel.instanceName),
|
|
5188
|
+
state: this.getInstanceExecutionState(channel.instanceName),
|
|
5189
|
+
ipc: this.instanceIpcClients.has(channel.instanceName),
|
|
5190
|
+
costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
|
|
5191
|
+
rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
|
|
5192
|
+
classic: true,
|
|
5193
|
+
classicName: channel.name,
|
|
5194
|
+
channelId: channel.channelId,
|
|
5195
|
+
adapterId: channel.adapterId ?? null,
|
|
5196
|
+
}));
|
|
5197
|
+
const enriched = [...fleetInstances, ...classicInstances].map(inst => {
|
|
4332
5198
|
const config = this.fleetConfig?.instances[inst.name];
|
|
5199
|
+
const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
|
|
5200
|
+
const lastActivity = inst.classic
|
|
5201
|
+
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
5202
|
+
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
5203
|
+
const backend = inst.classic
|
|
5204
|
+
? this.classicChannels?.getBackendByInstance(inst.name, this.fleetConfig?.defaults.backend) ?? "claude-code"
|
|
5205
|
+
: config?.backend ?? "claude-code";
|
|
4333
5206
|
// Find claimed tasks for this instance
|
|
4334
5207
|
let currentTask = null;
|
|
4335
5208
|
try {
|
|
@@ -4342,13 +5215,16 @@ When users create specialized instances, suggest these configurations:
|
|
|
4342
5215
|
}
|
|
4343
5216
|
return {
|
|
4344
5217
|
...inst,
|
|
4345
|
-
description: config?.description ?? null,
|
|
4346
|
-
backend
|
|
5218
|
+
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
5219
|
+
backend,
|
|
4347
5220
|
tool_set: config?.tool_set ?? "full",
|
|
4348
5221
|
general_topic: config?.general_topic ?? false,
|
|
4349
|
-
|
|
5222
|
+
// User activity is persisted by the daemon, so both the board and
|
|
5223
|
+
// auto-pause retain an accurate age across fleet restarts.
|
|
5224
|
+
lastActivity,
|
|
4350
5225
|
currentTask,
|
|
4351
5226
|
idle: this.getInstanceIdle(inst.name),
|
|
5227
|
+
state: this.getInstanceExecutionState(inst.name),
|
|
4352
5228
|
};
|
|
4353
5229
|
});
|
|
4354
5230
|
res.setHeader("Access-Control-Allow-Origin", "*");
|