@songsid/agend 2.1.0-beta.3 → 2.1.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.js +3 -2
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.js +4 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +51 -0
- package/dist/backend/grok.js +282 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.js +3 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +12 -2
- package/dist/backend/types.js +1 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +85 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +17 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +14 -1
- package/dist/classic-channel-manager.js +65 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +108 -16
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +68 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +13 -1
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +76 -6
- package/dist/daemon.js +451 -113
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +4 -0
- package/dist/fleet-manager.d.ts +88 -7
- package/dist/fleet-manager.js +707 -129
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -1
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -0
- package/dist/instance-lifecycle.d.ts +11 -1
- package/dist/instance-lifecycle.js +64 -22
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +24 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +29 -10
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +203 -2
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +40 -1
- package/dist/outbound-schemas.js +36 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +19 -1
- package/dist/topic-commands.js +132 -23
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +23 -1
- package/dist/ui/dashboard.html +2 -2
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.d.ts +1 -0
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -5,13 +5,13 @@ import { join, dirname, basename } from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
7
|
import { sdNotify } from "./sd-notify.js";
|
|
8
|
-
import
|
|
8
|
+
import { isScalar, parseDocument } from "yaml";
|
|
9
9
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
10
|
const __dirname = dirname(__filename);
|
|
11
11
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
12
12
|
const DEFAULT_OPEN_ACCESS = { mode: "open", allowed_users: [], max_pending_codes: 0, code_expiry_minutes: 0 };
|
|
13
13
|
import { isProbeableRouteTarget } from "./fleet-context.js";
|
|
14
|
-
import { loadFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
14
|
+
import { loadFleetConfig, loadRawFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
15
15
|
import { EventLog } from "./event-log.js";
|
|
16
16
|
import { AdapterWorld } from "./adapter-world.js";
|
|
17
17
|
import { CostGuard, formatCents } from "./cost-guard.js";
|
|
@@ -24,13 +24,13 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
24
24
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
25
25
|
import { Scheduler } from "./scheduler/index.js";
|
|
26
26
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
27
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
27
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
28
28
|
import { DailySummary } from "./daily-summary.js";
|
|
29
29
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
30
30
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
31
31
|
import { safeHandler } from "./safe-async.js";
|
|
32
32
|
import { RoutingEngine } from "./routing-engine.js";
|
|
33
|
-
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
33
|
+
import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
|
|
34
34
|
import { TopicArchiver } from "./topic-archiver.js";
|
|
35
35
|
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
36
36
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
@@ -39,7 +39,9 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
39
39
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
40
40
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
41
41
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
42
|
-
import { ClassicChannelManager } from "./classic-channel-manager.js";
|
|
42
|
+
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
43
|
+
import { readLastInboundAt } from "./daemon.js";
|
|
44
|
+
import { clearPausedMarker } from "./pause-marker.js";
|
|
43
45
|
import { getTmuxSession } from "./config.js";
|
|
44
46
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
45
47
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -50,6 +52,26 @@ export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
|
50
52
|
}
|
|
51
53
|
return instanceConfig?.topic_id != null ? String(instanceConfig.topic_id) : undefined;
|
|
52
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Pure warm-cap victim selection (extracted for testability). Given the current
|
|
57
|
+
* warm (running) instance names and a cap, return the LRU idle instances to evict
|
|
58
|
+
* so the running count returns to the cap. Skips: the `exclude` instance, any
|
|
59
|
+
* already-evicting, general instances (never evicted), and non-idle instances
|
|
60
|
+
* (working/stuck can't be evicted). Oldest last-inbound is evicted first; a
|
|
61
|
+
* missing timestamp (0) sorts oldest. cap <= 0 (or non-integer) = unlimited → [].
|
|
62
|
+
*/
|
|
63
|
+
export function selectLruEvictions(warm, cap, opts) {
|
|
64
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
65
|
+
return [];
|
|
66
|
+
if (warm.length <= cap)
|
|
67
|
+
return [];
|
|
68
|
+
const candidates = warm.filter(name => name !== opts.exclude
|
|
69
|
+
&& !opts.isEvicting(name)
|
|
70
|
+
&& !opts.isGeneral(name)
|
|
71
|
+
&& opts.isIdle(name));
|
|
72
|
+
candidates.sort((a, b) => opts.lastInboundAt(a) - opts.lastInboundAt(b));
|
|
73
|
+
return candidates.slice(0, warm.length - cap);
|
|
74
|
+
}
|
|
53
75
|
/** Retry cadence for retiring a cancel button whose delete failed (e.g. a DC
|
|
54
76
|
* forum thread the bot momentarily can't reach). 3 retries × 5min = 15min. */
|
|
55
77
|
const CANCEL_BTN_RETRY_INTERVAL_MS = 5 * 60_000;
|
|
@@ -58,6 +80,8 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
58
80
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
59
81
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
60
82
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
83
|
+
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
84
|
+
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
61
85
|
export class FleetManager {
|
|
62
86
|
dataDir;
|
|
63
87
|
children = new Map();
|
|
@@ -65,6 +89,9 @@ export class FleetManager {
|
|
|
65
89
|
/** @deprecated Use lifecycle.daemons — kept for backward compat */
|
|
66
90
|
get daemons() { return this.lifecycle.daemons; }
|
|
67
91
|
fleetConfig = null;
|
|
92
|
+
rawFleetConfig = {};
|
|
93
|
+
rawFleetDocument = null;
|
|
94
|
+
savedFleetConfigSnapshot = null;
|
|
68
95
|
adapter = null;
|
|
69
96
|
worlds = new Map();
|
|
70
97
|
adapters = new Map(); // derived view for backward compat
|
|
@@ -93,6 +120,16 @@ export class FleetManager {
|
|
|
93
120
|
// Topic icon + auto-archive state
|
|
94
121
|
topicIcons = {};
|
|
95
122
|
lastActivity = new Map();
|
|
123
|
+
/** Latest pane-derived execution snapshot reported by each daemon. */
|
|
124
|
+
instanceStateCache = new Map();
|
|
125
|
+
/** Instances currently being auto-paused by warm_cap, so concurrent checks don't double-evict. */
|
|
126
|
+
warmCapEvicting = new Set();
|
|
127
|
+
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
128
|
+
idleGatedDeliveryTails = new Map();
|
|
129
|
+
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
130
|
+
lastDeliveryAt = new Map();
|
|
131
|
+
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
132
|
+
instanceIdleWaiters = new Map();
|
|
96
133
|
lastInboundUser = new Map(); // instanceName → last username
|
|
97
134
|
// Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
|
|
98
135
|
// than one-per-instance. A button is retired (deleted, with bounded retry) on
|
|
@@ -104,6 +141,7 @@ export class FleetManager {
|
|
|
104
141
|
topicArchiver;
|
|
105
142
|
controlClient = null;
|
|
106
143
|
classicChannels = null;
|
|
144
|
+
pendingClassicStarts = new Map();
|
|
107
145
|
// Model failover state
|
|
108
146
|
failoverActive = new Map(); // instance → current failover model
|
|
109
147
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
@@ -160,6 +198,7 @@ export class FleetManager {
|
|
|
160
198
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
161
199
|
name,
|
|
162
200
|
status: this.getInstanceStatus(name),
|
|
201
|
+
state: this.getInstanceExecutionState(name),
|
|
163
202
|
ipc: this.instanceIpcClients.has(name),
|
|
164
203
|
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
165
204
|
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
@@ -174,9 +213,21 @@ export class FleetManager {
|
|
|
174
213
|
}
|
|
175
214
|
/** Load fleet.yaml and build routing table */
|
|
176
215
|
loadConfig(configPath) {
|
|
216
|
+
this.configPath = configPath;
|
|
217
|
+
const source = existsSync(configPath) ? readFileSync(configPath, "utf-8") : "{}\n";
|
|
218
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
219
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
220
|
+
throw new Error(`Invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
221
|
+
}
|
|
222
|
+
this.rawFleetConfig = loadRawFleetConfig(configPath);
|
|
177
223
|
this.fleetConfig = loadFleetConfig(configPath);
|
|
224
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
178
225
|
return this.fleetConfig;
|
|
179
226
|
}
|
|
227
|
+
/** User-authored fleet.yaml, before defaults are merged into instances. */
|
|
228
|
+
getRawFleetConfig() {
|
|
229
|
+
return structuredClone(this.rawFleetConfig);
|
|
230
|
+
}
|
|
180
231
|
/** Build topic routing table: { topicId -> RouteTarget } */
|
|
181
232
|
buildRoutingTable() {
|
|
182
233
|
if (this.fleetConfig) {
|
|
@@ -230,6 +281,41 @@ export class FleetManager {
|
|
|
230
281
|
return this.classicChannels?.getInstanceByChannel(channelId, adapterId)
|
|
231
282
|
?? this.routing.resolve(channelId)?.name;
|
|
232
283
|
}
|
|
284
|
+
async handlePauseWakeSlash(data, adapterId) {
|
|
285
|
+
const action = data.command;
|
|
286
|
+
const classicName = this.classicChannels?.getInstanceByChannel(data.channelId, adapterId);
|
|
287
|
+
if (classicName) {
|
|
288
|
+
if (!this.classicChannels?.isAdmin(data.userId)) {
|
|
289
|
+
await data.respond(t("permission.denied"));
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
await data.respond(await this.topicCommands.runPauseWake(classicName, action));
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
296
|
+
await data.respond(t("permission.denied"));
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const route = this.routing.resolve(data.channelId);
|
|
300
|
+
if (!route) {
|
|
301
|
+
await data.respond(t("classic.no_agent"));
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
let target = route.name;
|
|
305
|
+
if (route.kind === "general") {
|
|
306
|
+
const requested = typeof data.options?.instance === "string" ? data.options.instance : undefined;
|
|
307
|
+
if (!requested) {
|
|
308
|
+
await data.respond(t(`${action}.usage`));
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
if (!this.fleetConfig?.instances[requested]) {
|
|
312
|
+
await data.respond(t("instance.not_found", requested));
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
target = requested;
|
|
316
|
+
}
|
|
317
|
+
await data.respond(await this.topicCommands.runPauseWake(target, action));
|
|
318
|
+
}
|
|
233
319
|
/** Get the adapter bound to an instance, falling back to primary adapter */
|
|
234
320
|
getAdapterForInstance(name) {
|
|
235
321
|
const worldId = this.instanceWorldBinding.get(name);
|
|
@@ -265,14 +351,19 @@ export class FleetManager {
|
|
|
265
351
|
*/
|
|
266
352
|
bindInstanceAdapter(name, adapterId, fromInbound = false) {
|
|
267
353
|
const cfg = this.fleetConfig?.instances[name];
|
|
268
|
-
if (fromInbound
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
354
|
+
if (fromInbound) {
|
|
355
|
+
// Skip inbound-derived binding for any instance that doesn't have an
|
|
356
|
+
// explicit channel_id — those default to primary adapter deterministically.
|
|
357
|
+
// This prevents a non-deterministic race where whichever adapter delivers
|
|
358
|
+
// first after restart wins the binding.
|
|
359
|
+
if (cfg?.general_topic || cfg?.channel_id)
|
|
360
|
+
return;
|
|
361
|
+
if (cfg && !cfg.channel_id)
|
|
362
|
+
return; // fleet instance without explicit binding → use primary
|
|
363
|
+
// Classic instance: don't override an existing binding (authoritative from /start)
|
|
364
|
+
if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
276
367
|
this.instanceWorldBinding.set(name, adapterId);
|
|
277
368
|
}
|
|
278
369
|
getInstanceStatus(name) {
|
|
@@ -290,24 +381,229 @@ export class FleetManager {
|
|
|
290
381
|
return "crashed";
|
|
291
382
|
}
|
|
292
383
|
}
|
|
293
|
-
|
|
294
|
-
|
|
384
|
+
getInstanceExecutionState(name) {
|
|
385
|
+
if (this.lifecycle.isPaused(name))
|
|
386
|
+
return null;
|
|
387
|
+
return this.instanceStateCache.get(name)?.state ?? null;
|
|
388
|
+
}
|
|
389
|
+
isClassicInstance(name) {
|
|
390
|
+
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
391
|
+
}
|
|
392
|
+
cacheInstanceExecutionState(name, msg) {
|
|
393
|
+
const state = msg.state;
|
|
394
|
+
if (state !== "idle" && state !== "working" && state !== "stuck")
|
|
395
|
+
return;
|
|
396
|
+
const previous = this.instanceStateCache.get(name);
|
|
397
|
+
const now = Date.now();
|
|
398
|
+
const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
399
|
+
this.instanceStateCache.set(name, {
|
|
400
|
+
state,
|
|
401
|
+
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
402
|
+
observedAt: numberOr(msg.observedAt, now),
|
|
403
|
+
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
404
|
+
});
|
|
405
|
+
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
406
|
+
check();
|
|
407
|
+
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
408
|
+
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
409
|
+
// edge into idle, not on every idle heartbeat.
|
|
410
|
+
if (state === "idle" && previous?.state !== "idle")
|
|
411
|
+
this.enforceWarmCap();
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Fleet-wide warm cap: if more than `defaults.warm_cap` instances are running,
|
|
415
|
+
* auto-pause the least-recently-active idle instances until back at the cap.
|
|
416
|
+
* Never evicts general instances (must stay warm) or working/stuck instances
|
|
417
|
+
* (only idle). 0/unset = unlimited. wake-before-deliver re-warms any evicted
|
|
418
|
+
* instance when a message next arrives.
|
|
419
|
+
*
|
|
420
|
+
* @param exclude instance to spare (e.g. one just woken to receive a delivery).
|
|
421
|
+
*/
|
|
422
|
+
enforceWarmCap(exclude) {
|
|
423
|
+
const cap = this.fleetConfig?.defaults?.warm_cap ?? 0;
|
|
424
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
425
|
+
return; // 0/invalid = unlimited
|
|
426
|
+
const warm = [];
|
|
427
|
+
for (const name of this.daemons.keys()) {
|
|
428
|
+
if (this.getInstanceStatus(name) === "running")
|
|
429
|
+
warm.push(name);
|
|
430
|
+
}
|
|
431
|
+
if (warm.length <= cap)
|
|
432
|
+
return;
|
|
433
|
+
const victims = selectLruEvictions(warm, cap, {
|
|
434
|
+
exclude,
|
|
435
|
+
isEvicting: name => this.warmCapEvicting.has(name),
|
|
436
|
+
isGeneral: name => this.fleetConfig?.instances[name]?.general_topic === true,
|
|
437
|
+
isIdle: name => this.getInstanceExecutionState(name) === "idle",
|
|
438
|
+
lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
|
|
439
|
+
});
|
|
440
|
+
for (const victim of victims) {
|
|
441
|
+
this.warmCapEvicting.add(victim);
|
|
442
|
+
this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
|
|
443
|
+
this.lifecycle.pause(victim)
|
|
444
|
+
.catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
|
|
445
|
+
.finally(() => this.warmCapEvicting.delete(victim));
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
449
|
+
const isReady = () => {
|
|
450
|
+
const snapshot = this.instanceStateCache.get(instanceName);
|
|
451
|
+
return snapshot?.state === "idle"
|
|
452
|
+
&& (idleObservedAfter === 0 || snapshot.observedAt > idleObservedAfter);
|
|
453
|
+
};
|
|
454
|
+
if (isReady())
|
|
455
|
+
return Promise.resolve(true);
|
|
456
|
+
return new Promise(resolve => {
|
|
457
|
+
let settled = false;
|
|
458
|
+
const finish = (idle) => {
|
|
459
|
+
if (settled)
|
|
460
|
+
return;
|
|
461
|
+
settled = true;
|
|
462
|
+
clearTimeout(timeout);
|
|
463
|
+
clearInterval(queryTimer);
|
|
464
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
465
|
+
waiters?.delete(check);
|
|
466
|
+
if (waiters?.size === 0)
|
|
467
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
468
|
+
resolve(idle);
|
|
469
|
+
};
|
|
470
|
+
const check = () => { if (isReady())
|
|
471
|
+
finish(true); };
|
|
472
|
+
const query = () => {
|
|
473
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
474
|
+
if (ipc?.connected) {
|
|
475
|
+
ipc.send({ type: "query_instance_state", requestId: `idle-gate-${Date.now()}` });
|
|
476
|
+
}
|
|
477
|
+
check();
|
|
478
|
+
};
|
|
479
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
480
|
+
waiters.add(check);
|
|
481
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
482
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
483
|
+
const queryTimer = setInterval(query, 1_000);
|
|
484
|
+
query();
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs) {
|
|
488
|
+
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
295
489
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
490
|
+
const wakeStartedAt = Date.now();
|
|
296
491
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
492
|
+
// Waking added one to the warm count — make room by evicting a different
|
|
493
|
+
// LRU idle instance (never this one; it's about to work).
|
|
494
|
+
this.enforceWarmCap(instanceName);
|
|
495
|
+
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
496
|
+
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
497
|
+
}
|
|
498
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
499
|
+
if (!idle) {
|
|
500
|
+
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
297
501
|
}
|
|
298
502
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
299
503
|
if (!ipc?.connected)
|
|
300
504
|
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
301
505
|
ipc.send(payload);
|
|
506
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
507
|
+
}
|
|
508
|
+
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
509
|
+
async deliverToInstance(instanceName, payload, options = {}) {
|
|
510
|
+
const meta = payload.meta && typeof payload.meta === "object"
|
|
511
|
+
? payload.meta
|
|
512
|
+
: undefined;
|
|
513
|
+
const inferredCrossInstance = (typeof meta?.from_instance === "string" && meta.from_instance.length > 0)
|
|
514
|
+
|| meta?.is_cross_instance === true
|
|
515
|
+
|| payload.is_cross_instance === true;
|
|
516
|
+
const waitForIdle = options.waitForIdle
|
|
517
|
+
?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
|
|
518
|
+
if (!waitForIdle) {
|
|
519
|
+
if (this.lifecycle.isPaused(instanceName)) {
|
|
520
|
+
await this.lifecycle.wake(instanceName, 30_000);
|
|
521
|
+
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
522
|
+
}
|
|
523
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
524
|
+
if (!ipc?.connected)
|
|
525
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
526
|
+
ipc.send(payload);
|
|
527
|
+
// A cross-instance item arriving before the daemon observes this turn as
|
|
528
|
+
// working must not trust the stale idle snapshot from before the send.
|
|
529
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
533
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, payload, options.idleTimeoutMs ?? 60_000));
|
|
534
|
+
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
535
|
+
try {
|
|
536
|
+
await delivery;
|
|
537
|
+
}
|
|
538
|
+
finally {
|
|
539
|
+
if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
|
|
540
|
+
this.idleGatedDeliveryTails.delete(instanceName);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
/** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
|
|
545
|
+
isFleetAdmin(userId, adapterId) {
|
|
546
|
+
const allowed = this.getChannelConfig(adapterId)?.access?.allowed_users ?? [];
|
|
547
|
+
return allowed.some(entry => String(entry) === String(userId));
|
|
548
|
+
}
|
|
549
|
+
async changeInstancePauseState(name, action) {
|
|
550
|
+
if (action === "wake") {
|
|
551
|
+
await this.lifecycle.wake(name, 30_000);
|
|
552
|
+
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
553
|
+
return "awake";
|
|
554
|
+
}
|
|
555
|
+
await this.lifecycle.pause(name);
|
|
556
|
+
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
557
|
+
}
|
|
558
|
+
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
559
|
+
async restartClassicInstanceFromSettings(instanceName) {
|
|
560
|
+
if (!this.classicChannels)
|
|
561
|
+
throw new Error("Classic channel manager not initialized");
|
|
562
|
+
const wasRunning = this.daemons.has(instanceName);
|
|
563
|
+
this.classicChannels.reloadFromDisk();
|
|
564
|
+
this.reregisterClassicChannels();
|
|
565
|
+
const channel = this.classicChannels.getAll().find(item => item.instanceName === instanceName);
|
|
566
|
+
if (!channel)
|
|
567
|
+
throw new Error("Classic channel not found after reload");
|
|
568
|
+
if (!wasRunning)
|
|
569
|
+
return;
|
|
570
|
+
await this.stopInstance(instanceName);
|
|
571
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
572
|
+
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
302
573
|
}
|
|
303
574
|
async startInstance(name, config, topicMode) {
|
|
575
|
+
if (this.lifecycle.isPaused(name)) {
|
|
576
|
+
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
304
579
|
if (config.general_topic) {
|
|
580
|
+
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
581
|
+
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
582
|
+
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
583
|
+
if (backend === "antigravity") {
|
|
584
|
+
this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
|
|
585
|
+
this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
|
|
586
|
+
}
|
|
305
587
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
306
588
|
}
|
|
307
589
|
await this.lifecycle.start(name, config, topicMode);
|
|
308
590
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
309
591
|
await this.connectIpcToInstance(name);
|
|
310
592
|
}
|
|
593
|
+
/** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
|
|
594
|
+
async startPersistedPausedInstance(name) {
|
|
595
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
596
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
597
|
+
const fleetConfig = this.fleetConfig?.instances[name];
|
|
598
|
+
if (fleetConfig) {
|
|
599
|
+
await this.startInstance(name, fleetConfig, topicMode);
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
603
|
+
if (!channel || !this.classicChannels)
|
|
604
|
+
throw new Error(`Paused instance '${name}' is no longer configured`);
|
|
605
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
606
|
+
}
|
|
311
607
|
/**
|
|
312
608
|
* Start instances with configurable concurrency and stagger delay.
|
|
313
609
|
* Instances sharing the same working_directory are serialized within a group
|
|
@@ -316,8 +612,21 @@ export class FleetManager {
|
|
|
316
612
|
*/
|
|
317
613
|
async startInstancesWithConcurrency(entries, topicMode) {
|
|
318
614
|
const raw = this.fleetConfig?.defaults?.startup;
|
|
319
|
-
const
|
|
615
|
+
const explicitConcurrency = raw?.concurrency;
|
|
320
616
|
const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
|
|
617
|
+
// Adaptive concurrency: if not explicitly set, estimate from available RAM.
|
|
618
|
+
// Each instance uses ~300MB (tmux + CLI process + model overhead).
|
|
619
|
+
const ESTIMATED_MB_PER_INSTANCE = 300;
|
|
620
|
+
const { freemem } = await import("node:os");
|
|
621
|
+
let concurrency;
|
|
622
|
+
if (explicitConcurrency != null) {
|
|
623
|
+
concurrency = Math.max(1, Math.min(20, explicitConcurrency));
|
|
624
|
+
}
|
|
625
|
+
else {
|
|
626
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
627
|
+
concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
|
|
628
|
+
this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
|
|
629
|
+
}
|
|
321
630
|
const byWorkDir = new Map();
|
|
322
631
|
for (const [name, config] of entries) {
|
|
323
632
|
const dir = config.working_directory;
|
|
@@ -339,6 +648,17 @@ export class FleetManager {
|
|
|
339
648
|
if (pendingTimer)
|
|
340
649
|
return;
|
|
341
650
|
while (running < concurrency && idx < groups.length) {
|
|
651
|
+
// Re-check memory if adaptive (no explicit concurrency set)
|
|
652
|
+
if (explicitConcurrency == null && running > 0) {
|
|
653
|
+
const nowFreeMB = Math.round(freemem() / (1024 * 1024));
|
|
654
|
+
if (nowFreeMB < ESTIMATED_MB_PER_INSTANCE) {
|
|
655
|
+
this.logger.warn({ freeMemMB: nowFreeMB, remaining: groups.length - idx }, "Low memory — pausing instance startup");
|
|
656
|
+
// Wait and retry in 5s
|
|
657
|
+
pendingTimer = true;
|
|
658
|
+
setTimeout(() => { pendingTimer = false; startNext(); }, 5000);
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
342
662
|
const now = Date.now();
|
|
343
663
|
const elapsed = now - lastStartAt;
|
|
344
664
|
if (lastStartAt > 0 && elapsed < staggerMs) {
|
|
@@ -367,6 +687,8 @@ export class FleetManager {
|
|
|
367
687
|
}
|
|
368
688
|
async stopInstance(name) {
|
|
369
689
|
this.failoverActive.delete(name);
|
|
690
|
+
this.instanceStateCache.delete(name);
|
|
691
|
+
this.lastDeliveryAt.delete(name);
|
|
370
692
|
return this.lifecycle.stop(name);
|
|
371
693
|
}
|
|
372
694
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
@@ -477,9 +799,11 @@ export class FleetManager {
|
|
|
477
799
|
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
478
800
|
const oldBackends = new Map();
|
|
479
801
|
const oldModels = new Map();
|
|
802
|
+
const oldAutoPause = new Map();
|
|
480
803
|
for (const ch of this.classicChannels.getAll()) {
|
|
481
804
|
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
482
805
|
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
806
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
483
807
|
}
|
|
484
808
|
if (!this.classicChannels.checkReload())
|
|
485
809
|
return;
|
|
@@ -487,14 +811,16 @@ export class FleetManager {
|
|
|
487
811
|
for (const ch of this.classicChannels.getAll()) {
|
|
488
812
|
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
489
813
|
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
814
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
490
815
|
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
491
816
|
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
492
|
-
|
|
817
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
818
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
493
819
|
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
494
820
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
495
821
|
// Small delay to let tmux window clean up
|
|
496
822
|
await new Promise(r => setTimeout(r, 2000));
|
|
497
|
-
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
823
|
+
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel, newAutoPause).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
498
824
|
}
|
|
499
825
|
}
|
|
500
826
|
}
|
|
@@ -747,7 +1073,7 @@ export class FleetManager {
|
|
|
747
1073
|
let idx = 0;
|
|
748
1074
|
while (idx < channels.length) {
|
|
749
1075
|
const batch = channels.slice(idx, idx + concurrency);
|
|
750
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
1076
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
751
1077
|
idx += concurrency;
|
|
752
1078
|
}
|
|
753
1079
|
}
|
|
@@ -758,16 +1084,26 @@ export class FleetManager {
|
|
|
758
1084
|
const classicCount = this.classicChannels?.getAll().length ?? 0;
|
|
759
1085
|
const total = Object.keys(fleet.instances).length + classicCount;
|
|
760
1086
|
const started = this.daemons.size;
|
|
761
|
-
const
|
|
1087
|
+
const allNotRunning = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
1088
|
+
const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
|
|
1089
|
+
const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
|
|
762
1090
|
const generalName = this.findGeneralInstance();
|
|
763
1091
|
const generalThreadId = generalName ? fleet.instances[generalName]?.topic_id : undefined;
|
|
764
1092
|
const { createRequire } = await import("node:module");
|
|
765
1093
|
const _require = createRequire(import.meta.url);
|
|
766
1094
|
const agendVersion = _require("../package.json").version ?? "unknown";
|
|
767
1095
|
if (this.adapter && fleet.channel?.group_id) {
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1096
|
+
let text;
|
|
1097
|
+
if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
1098
|
+
text = t("fleet.ready", started, total, agendVersion);
|
|
1099
|
+
}
|
|
1100
|
+
else if (failedNames.length === 0) {
|
|
1101
|
+
text = t("fleet.ready", started, total, agendVersion) + `\n⏸ Paused: ${pausedNames.join(", ")}`;
|
|
1102
|
+
}
|
|
1103
|
+
else {
|
|
1104
|
+
text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
|
|
1105
|
+
+ (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
|
|
1106
|
+
}
|
|
771
1107
|
this.adapter.sendText(String(fleet.channel.group_id), text, {
|
|
772
1108
|
threadId: generalThreadId != null ? String(generalThreadId) : undefined,
|
|
773
1109
|
}).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
|
|
@@ -952,6 +1288,8 @@ export class FleetManager {
|
|
|
952
1288
|
await this.handleInboundMessage(msg);
|
|
953
1289
|
}, this.logger, "adapter.message"));
|
|
954
1290
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1291
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1292
|
+
return;
|
|
955
1293
|
if (data.callbackData.startsWith("hang:")) {
|
|
956
1294
|
const parts = data.callbackData.split(":");
|
|
957
1295
|
const action = parts[1];
|
|
@@ -990,13 +1328,15 @@ export class FleetManager {
|
|
|
990
1328
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
991
1329
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
992
1330
|
if (data.command === "start") {
|
|
993
|
-
|
|
994
|
-
await data.respond(reply);
|
|
1331
|
+
await this.handleClassicStartSlash(data, adapterId, this.adapter);
|
|
995
1332
|
}
|
|
996
1333
|
else if (data.command === "stop") {
|
|
997
1334
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
998
1335
|
await data.respond(reply);
|
|
999
1336
|
}
|
|
1337
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1338
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1339
|
+
}
|
|
1000
1340
|
else if (data.command === "chat") {
|
|
1001
1341
|
const text = data.text ?? "";
|
|
1002
1342
|
if (!text) {
|
|
@@ -1235,6 +1575,8 @@ export class FleetManager {
|
|
|
1235
1575
|
await this.handleInboundMessage(msg);
|
|
1236
1576
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
1237
1577
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1578
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1579
|
+
return;
|
|
1238
1580
|
if (data.callbackData.startsWith("hang:")) {
|
|
1239
1581
|
const parts = data.callbackData.split(":");
|
|
1240
1582
|
const action = parts[1];
|
|
@@ -1269,13 +1611,15 @@ export class FleetManager {
|
|
|
1269
1611
|
// Slash commands: classic bot + admin commands
|
|
1270
1612
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
1271
1613
|
if (data.command === "start") {
|
|
1272
|
-
|
|
1273
|
-
await data.respond(reply);
|
|
1614
|
+
await this.handleClassicStartSlash(data, adapterId, adapter);
|
|
1274
1615
|
}
|
|
1275
1616
|
else if (data.command === "stop") {
|
|
1276
1617
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
1277
1618
|
await data.respond(reply);
|
|
1278
1619
|
}
|
|
1620
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1621
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1622
|
+
}
|
|
1279
1623
|
else if (data.command === "chat") {
|
|
1280
1624
|
const text = data.text ?? "";
|
|
1281
1625
|
if (!text) {
|
|
@@ -1532,10 +1876,16 @@ export class FleetManager {
|
|
|
1532
1876
|
else if (msg.type === "fleet_set_description") {
|
|
1533
1877
|
this.handleSetDescription(name, msg);
|
|
1534
1878
|
}
|
|
1879
|
+
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
1880
|
+
this.cacheInstanceExecutionState(name, msg);
|
|
1881
|
+
}
|
|
1535
1882
|
}, this.logger, `ipc.message[${name}]`));
|
|
1536
1883
|
// Ask daemon for any sessions that registered before we connected
|
|
1537
1884
|
// (fixes race condition where mcp_ready was broadcast before fleet manager connected)
|
|
1538
1885
|
ipc.send({ type: "query_sessions" });
|
|
1886
|
+
// The initial state transition may have happened before FleetManager
|
|
1887
|
+
// connected, so seed the cache instead of waiting for another transition.
|
|
1888
|
+
ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
|
|
1539
1889
|
this.logger.debug({ name }, "Connected to instance IPC");
|
|
1540
1890
|
if (!this.statuslineWatcher.has(name)) {
|
|
1541
1891
|
this.statuslineWatcher.watch(name);
|
|
@@ -1798,9 +2148,21 @@ export class FleetManager {
|
|
|
1798
2148
|
}
|
|
1799
2149
|
}
|
|
1800
2150
|
const channelName = msg.username || chatId;
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
2151
|
+
const requestedBackend = text.slice("/start".length).trim().split(/\s+/, 1)[0] || undefined;
|
|
2152
|
+
if (requestedBackend) {
|
|
2153
|
+
// handleClassicStart binds the instance to this adapter authoritatively.
|
|
2154
|
+
const reply = await this.handleClassicStart(chatId, channelName, msg.userId, undefined, msg.adapterId, requestedBackend);
|
|
2155
|
+
await msgAdapter?.sendText(chatId, reply);
|
|
2156
|
+
}
|
|
2157
|
+
else if (msgAdapter) {
|
|
2158
|
+
await this.beginClassicBackendSelection({
|
|
2159
|
+
command: "start",
|
|
2160
|
+
channelId: chatId,
|
|
2161
|
+
channelName,
|
|
2162
|
+
userId: msg.userId,
|
|
2163
|
+
respond: async (reply) => (await msgAdapter.sendText(chatId, reply)).messageId,
|
|
2164
|
+
}, msgAdapter);
|
|
2165
|
+
}
|
|
1804
2166
|
return;
|
|
1805
2167
|
}
|
|
1806
2168
|
// Handle /stop command
|
|
@@ -1817,6 +2179,20 @@ export class FleetManager {
|
|
|
1817
2179
|
await msgAdapter?.sendText(chatId, reply);
|
|
1818
2180
|
return;
|
|
1819
2181
|
}
|
|
2182
|
+
const pauseWake = parsePauseWakeCommand(text);
|
|
2183
|
+
if (pauseWake) {
|
|
2184
|
+
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
2185
|
+
await msgAdapter?.sendText(chatId, t("permission.denied"));
|
|
2186
|
+
return;
|
|
2187
|
+
}
|
|
2188
|
+
const name = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
2189
|
+
if (!name) {
|
|
2190
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
2191
|
+
return;
|
|
2192
|
+
}
|
|
2193
|
+
await msgAdapter?.sendText(chatId, await this.topicCommands.runPauseWake(name, pauseWake.action));
|
|
2194
|
+
return;
|
|
2195
|
+
}
|
|
1820
2196
|
// Handle /compact command (admin only)
|
|
1821
2197
|
if (text === "/compact" || text.startsWith("/compact@")) {
|
|
1822
2198
|
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
@@ -1950,6 +2326,7 @@ export class FleetManager {
|
|
|
1950
2326
|
user_id: msg.userId,
|
|
1951
2327
|
ts: msg.timestamp.toISOString(),
|
|
1952
2328
|
thread_id: "",
|
|
2329
|
+
adapter_id: msg.adapterId,
|
|
1953
2330
|
source: msg.source,
|
|
1954
2331
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
1955
2332
|
...extraMeta,
|
|
@@ -2042,6 +2419,7 @@ export class FleetManager {
|
|
|
2042
2419
|
user_id: msg.userId,
|
|
2043
2420
|
ts: msg.timestamp.toISOString(),
|
|
2044
2421
|
thread_id: msg.threadId ?? "",
|
|
2422
|
+
adapter_id: msg.adapterId,
|
|
2045
2423
|
source: msg.source,
|
|
2046
2424
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2047
2425
|
...extraMeta,
|
|
@@ -2222,7 +2600,7 @@ export class FleetManager {
|
|
|
2222
2600
|
type: "fleet_schedule_trigger",
|
|
2223
2601
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
2224
2602
|
meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
|
|
2225
|
-
});
|
|
2603
|
+
}, { waitForIdle: true });
|
|
2226
2604
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
2227
2605
|
void this.sendCancelButton(target);
|
|
2228
2606
|
return true;
|
|
@@ -2664,73 +3042,92 @@ export class FleetManager {
|
|
|
2664
3042
|
}
|
|
2665
3043
|
}, 5 * 60_000);
|
|
2666
3044
|
}
|
|
2667
|
-
/**
|
|
2668
|
-
|
|
3045
|
+
/**
|
|
3046
|
+
* Patch only values changed in the effective config into the original YAML
|
|
3047
|
+
* document. Unknown keys, explicit overrides and comments remain untouched.
|
|
3048
|
+
*/
|
|
3049
|
+
saveFleetConfig(explicitPatches = []) {
|
|
2669
3050
|
if (!this.fleetConfig || !this.configPath)
|
|
2670
3051
|
return;
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
}
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
3052
|
+
if (!this.savedFleetConfigSnapshot)
|
|
3053
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3054
|
+
// Re-read immediately before patching so an unrelated concurrent/manual
|
|
3055
|
+
// edit is retained. Invalid concurrent YAML is never overwritten.
|
|
3056
|
+
const source = existsSync(this.configPath) ? readFileSync(this.configPath, "utf-8") : "{}\n";
|
|
3057
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
3058
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
3059
|
+
throw new Error(`Refusing to overwrite invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
3060
|
+
}
|
|
3061
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3062
|
+
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
3063
|
+
// Settings edits are expressed against the raw config. Persist them even
|
|
3064
|
+
// when the chosen override equals the inherited effective value, a case
|
|
3065
|
+
// the before/after runtime diff cannot observe.
|
|
3066
|
+
for (const patch of explicitPatches) {
|
|
3067
|
+
if (patch.remove) {
|
|
3068
|
+
this.rawFleetDocument.deleteIn(patch.path);
|
|
3069
|
+
}
|
|
3070
|
+
else {
|
|
3071
|
+
const before = this.rawFleetDocument.getIn(patch.path);
|
|
3072
|
+
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
const output = String(this.rawFleetDocument);
|
|
3076
|
+
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
3077
|
+
writeFileSync(tempPath, output, "utf-8");
|
|
3078
|
+
if (existsSync(this.configPath))
|
|
3079
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
3080
|
+
renameSync(tempPath, this.configPath);
|
|
3081
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3082
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3083
|
+
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
3084
|
+
}
|
|
3085
|
+
patchFleetDocument(document, path, before, after) {
|
|
3086
|
+
if (Object.is(before, after))
|
|
3087
|
+
return;
|
|
3088
|
+
if (Array.isArray(before) && Array.isArray(after)) {
|
|
3089
|
+
const shared = Math.min(before.length, after.length);
|
|
3090
|
+
for (let i = 0; i < shared; i++) {
|
|
3091
|
+
this.patchFleetDocument(document, [...path, i], before[i], after[i]);
|
|
3092
|
+
}
|
|
3093
|
+
// Remove from the end so YAML sequence indexes do not shift underneath us.
|
|
3094
|
+
for (let i = before.length - 1; i >= after.length; i--)
|
|
3095
|
+
document.deleteIn([...path, i]);
|
|
3096
|
+
for (let i = shared; i < after.length; i++)
|
|
3097
|
+
document.setIn([...path, i], after[i]);
|
|
3098
|
+
return;
|
|
3099
|
+
}
|
|
3100
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3101
|
+
if (isRecord(before) && isRecord(after)) {
|
|
3102
|
+
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
3103
|
+
for (const key of keys) {
|
|
3104
|
+
// `channel` is a derived alias when the raw file uses `channels`.
|
|
3105
|
+
if (path.length === 0 && key === "channel" && this.rawFleetConfig.channels)
|
|
3106
|
+
continue;
|
|
3107
|
+
// Conversely, `channels` is a normalized alias for a legacy `channel`.
|
|
3108
|
+
// Keep the user's original shape unless the caller explicitly removed
|
|
3109
|
+
// `channel` (the Settings channels endpoint intentionally migrates it).
|
|
3110
|
+
if (path.length === 0 && key === "channels" && this.rawFleetConfig.channel && !this.rawFleetConfig.channels && after.channel !== undefined)
|
|
3111
|
+
continue;
|
|
3112
|
+
this.patchFleetDocument(document, [...path, key], before[key], after[key]);
|
|
3113
|
+
}
|
|
3114
|
+
return;
|
|
3115
|
+
}
|
|
3116
|
+
if (after === undefined) {
|
|
3117
|
+
document.deleteIn(path);
|
|
3118
|
+
}
|
|
3119
|
+
else if (path.length === 0) {
|
|
3120
|
+
document.contents = document.createNode(after);
|
|
3121
|
+
}
|
|
3122
|
+
else {
|
|
3123
|
+
const currentNode = document.getIn(path, true);
|
|
3124
|
+
if (isScalar(currentNode) && (after === null || typeof after !== "object")) {
|
|
3125
|
+
currentNode.value = after;
|
|
3126
|
+
}
|
|
3127
|
+
else {
|
|
3128
|
+
document.setIn(path, after);
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
2734
3131
|
}
|
|
2735
3132
|
async removeInstance(name) {
|
|
2736
3133
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
@@ -2771,8 +3168,14 @@ export class FleetManager {
|
|
|
2771
3168
|
}
|
|
2772
3169
|
}
|
|
2773
3170
|
startStatuslineWatcher(name) {
|
|
3171
|
+
if (this.lifecycle.isPaused(name))
|
|
3172
|
+
return;
|
|
2774
3173
|
this.statuslineWatcher.watch(name);
|
|
2775
3174
|
}
|
|
3175
|
+
stopStatuslineWatcher(name) {
|
|
3176
|
+
// Pausing stops I/O but retains the last observed limits for status views.
|
|
3177
|
+
this.statuslineWatcher.unwatch(name, true);
|
|
3178
|
+
}
|
|
2776
3179
|
reactMessageStatus(instanceName, chatId, messageId, emoji) {
|
|
2777
3180
|
// React via the adapter BOUND to this instance — NOT the first discord world.
|
|
2778
3181
|
// Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
|
|
@@ -3123,7 +3526,7 @@ export class FleetManager {
|
|
|
3123
3526
|
return [];
|
|
3124
3527
|
}
|
|
3125
3528
|
}
|
|
3126
|
-
async sendHangNotification(instanceName) {
|
|
3529
|
+
async sendHangNotification(instanceName, unchangedForMs) {
|
|
3127
3530
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3128
3531
|
if (!adapter)
|
|
3129
3532
|
return;
|
|
@@ -3132,11 +3535,18 @@ export class FleetManager {
|
|
|
3132
3535
|
if (!groupId)
|
|
3133
3536
|
return;
|
|
3134
3537
|
const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3538
|
+
const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
|
|
3539
|
+
const configuredMinutes = instanceHangConfig?.timeout_minutes
|
|
3540
|
+
?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
|
|
3541
|
+
?? 15;
|
|
3542
|
+
const unchangedMinutes = unchangedForMs == null
|
|
3543
|
+
? configuredMinutes
|
|
3544
|
+
: Math.max(1, Math.floor(unchangedForMs / 60_000));
|
|
3135
3545
|
this.setTopicIcon(instanceName, "red");
|
|
3136
3546
|
await adapter.notifyAlert(String(groupId), {
|
|
3137
3547
|
type: "hang",
|
|
3138
3548
|
instanceName,
|
|
3139
|
-
message: `⚠️ ${instanceName}
|
|
3549
|
+
message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
|
|
3140
3550
|
choices: [
|
|
3141
3551
|
{ id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
|
|
3142
3552
|
{ id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
|
|
@@ -3644,6 +4054,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3644
4054
|
user_id: msg.userId,
|
|
3645
4055
|
ts: msg.timestamp.toISOString(),
|
|
3646
4056
|
thread_id: msg.threadId ?? "",
|
|
4057
|
+
...(msg.adapterId ? { adapter_id: msg.adapterId } : {}),
|
|
3647
4058
|
source: msg.source,
|
|
3648
4059
|
...extraMeta,
|
|
3649
4060
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
@@ -3700,8 +4111,142 @@ When users create specialized instances, suggest these configurations:
|
|
|
3700
4111
|
return undefined;
|
|
3701
4112
|
}
|
|
3702
4113
|
}
|
|
4114
|
+
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
4115
|
+
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
4116
|
+
if (!this.classicChannels)
|
|
4117
|
+
return "Classic channel manager not initialized.";
|
|
4118
|
+
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
4119
|
+
const generalId = this.findGeneralInstance(adapterId);
|
|
4120
|
+
if (generalId)
|
|
4121
|
+
this.notifyInstanceTopic(generalId, t("alert.unauth_guild", guildId, userId));
|
|
4122
|
+
return t("classic.not_authorized_guild");
|
|
4123
|
+
}
|
|
4124
|
+
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
4125
|
+
return t("classic.already_active");
|
|
4126
|
+
if (this.routing.resolve(channelId))
|
|
4127
|
+
return t("classic.topic_bound");
|
|
4128
|
+
return undefined;
|
|
4129
|
+
}
|
|
4130
|
+
isBackendInstalled(backend) {
|
|
4131
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4132
|
+
return !!installation && checkBinaryInstalled(installation.binary);
|
|
4133
|
+
}
|
|
4134
|
+
getMissingBackendWarning(backend) {
|
|
4135
|
+
if (!backend)
|
|
4136
|
+
return undefined;
|
|
4137
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4138
|
+
if (!installation || this.isBackendInstalled(backend))
|
|
4139
|
+
return undefined;
|
|
4140
|
+
return t("classic.backend_not_installed", backend, installation.binary, installation.install);
|
|
4141
|
+
}
|
|
4142
|
+
/** Handle Discord's optional static slash choice, warning before a likely startup failure. */
|
|
4143
|
+
async handleClassicStartSlash(data, adapterId, adapter) {
|
|
4144
|
+
const requestedBackend = typeof data.options?.backend === "string" ? data.options.backend : undefined;
|
|
4145
|
+
if (!requestedBackend) {
|
|
4146
|
+
await this.beginClassicBackendSelection(data, adapter);
|
|
4147
|
+
return;
|
|
4148
|
+
}
|
|
4149
|
+
const warning = this.getMissingBackendWarning(requestedBackend);
|
|
4150
|
+
// Keep the deferred ephemeral response useful even if daemon startup later
|
|
4151
|
+
// fails because the executable is absent. This is advisory, not a gate.
|
|
4152
|
+
if (warning)
|
|
4153
|
+
await data.respond(warning);
|
|
4154
|
+
const reply = await this.handleClassicStart(data.channelId, data.channelName, data.userId, data.guildId, adapterId, requestedBackend);
|
|
4155
|
+
await data.respond(warning ? `${warning}\n\n${reply}` : reply);
|
|
4156
|
+
}
|
|
4157
|
+
/** Present platform-native backend choices, then start on selection or timeout. */
|
|
4158
|
+
async beginClassicBackendSelection(data, adapter) {
|
|
4159
|
+
const adapterId = adapter.id;
|
|
4160
|
+
const blocker = this.validateClassicStart(data.channelId, data.userId, data.guildId, adapterId);
|
|
4161
|
+
if (blocker) {
|
|
4162
|
+
await data.respond(blocker);
|
|
4163
|
+
return;
|
|
4164
|
+
}
|
|
4165
|
+
const nonce = randomBytes(6).toString("hex");
|
|
4166
|
+
const choices = getClassicBackendChoices().map(choice => ({
|
|
4167
|
+
id: `${CLASSIC_BACKEND_CALLBACK_PREFIX}${nonce}:${choice.id}`,
|
|
4168
|
+
label: `${this.isBackendInstalled(choice.id) ? "✅" : "❌"} ${choice.label}`,
|
|
4169
|
+
}));
|
|
4170
|
+
const complete = data.respondChoices
|
|
4171
|
+
? async (text) => { await data.respond(text); }
|
|
4172
|
+
: async (text, messageId) => {
|
|
4173
|
+
if (messageId && adapter.editMessageRemoveButtons) {
|
|
4174
|
+
try {
|
|
4175
|
+
await adapter.editMessageRemoveButtons(data.channelId, messageId, text);
|
|
4176
|
+
return;
|
|
4177
|
+
}
|
|
4178
|
+
catch { /* fall back to a new message */ }
|
|
4179
|
+
}
|
|
4180
|
+
await data.respond(text);
|
|
4181
|
+
};
|
|
4182
|
+
const timer = setTimeout(() => {
|
|
4183
|
+
void this.finishClassicBackendSelection(nonce).catch(err => this.logger.warn({ err, channelId: data.channelId, adapterId }, "Classic backend selection timeout fallback failed"));
|
|
4184
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
4185
|
+
timer.unref?.();
|
|
4186
|
+
const pending = {
|
|
4187
|
+
channelId: data.channelId,
|
|
4188
|
+
channelName: data.channelName,
|
|
4189
|
+
userId: data.userId,
|
|
4190
|
+
guildId: data.guildId,
|
|
4191
|
+
adapterId,
|
|
4192
|
+
timer,
|
|
4193
|
+
complete,
|
|
4194
|
+
};
|
|
4195
|
+
this.pendingClassicStarts.set(nonce, pending);
|
|
4196
|
+
try {
|
|
4197
|
+
pending.messageId = data.respondChoices
|
|
4198
|
+
? await data.respondChoices(t("classic.choose_backend"), choices)
|
|
4199
|
+
: await adapter.promptUser(data.channelId, t("classic.choose_backend"), choices);
|
|
4200
|
+
}
|
|
4201
|
+
catch (err) {
|
|
4202
|
+
// A menu transport failure should not make /start unusable: consume the
|
|
4203
|
+
// pending request and immediately use the configured default.
|
|
4204
|
+
this.logger.warn({ err, channelId: data.channelId, adapterId }, "Classic backend menu failed; using default");
|
|
4205
|
+
await this.finishClassicBackendSelection(nonce);
|
|
4206
|
+
}
|
|
4207
|
+
}
|
|
4208
|
+
/** Consume a selection callback. Returns true for all ClassicBot callback IDs, including stale ones. */
|
|
4209
|
+
async handleClassicBackendSelection(data) {
|
|
4210
|
+
if (!data.callbackData.startsWith(CLASSIC_BACKEND_CALLBACK_PREFIX))
|
|
4211
|
+
return false;
|
|
4212
|
+
const match = data.callbackData.match(/^classic-backend:([0-9a-f]+):(.+)$/);
|
|
4213
|
+
if (!match)
|
|
4214
|
+
return true;
|
|
4215
|
+
const pending = this.pendingClassicStarts.get(match[1]);
|
|
4216
|
+
if (!pending)
|
|
4217
|
+
return true;
|
|
4218
|
+
// Telegram keyboards are visible to everyone in a group. Only the user who
|
|
4219
|
+
// issued /start may consume the pending selection.
|
|
4220
|
+
if (data.userId && data.userId !== pending.userId)
|
|
4221
|
+
return true;
|
|
4222
|
+
const callbackChannelId = data.threadId ?? data.chatId;
|
|
4223
|
+
if (callbackChannelId !== pending.channelId && data.chatId !== pending.channelId)
|
|
4224
|
+
return true;
|
|
4225
|
+
await this.finishClassicBackendSelection(match[1], match[2]);
|
|
4226
|
+
return true;
|
|
4227
|
+
}
|
|
4228
|
+
/** Atomically claim one pending request so timeout/click races create at most one instance. */
|
|
4229
|
+
async finishClassicBackendSelection(nonce, backend) {
|
|
4230
|
+
const pending = this.pendingClassicStarts.get(nonce);
|
|
4231
|
+
if (!pending)
|
|
4232
|
+
return;
|
|
4233
|
+
this.pendingClassicStarts.delete(nonce);
|
|
4234
|
+
clearTimeout(pending.timer);
|
|
4235
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4236
|
+
const effectiveBackend = selectedBackend
|
|
4237
|
+
?? this.classicChannels?.getDefaults().backend
|
|
4238
|
+
?? this.fleetConfig?.defaults?.backend
|
|
4239
|
+
?? "claude-code";
|
|
4240
|
+
const warning = this.getMissingBackendWarning(effectiveBackend);
|
|
4241
|
+
// Show the warning before starting so it survives a missing-binary startup
|
|
4242
|
+
// failure. The selected backend is still attempted as requested.
|
|
4243
|
+
if (warning)
|
|
4244
|
+
await pending.complete(warning, pending.messageId);
|
|
4245
|
+
const reply = await this.handleClassicStart(pending.channelId, pending.channelName, pending.userId, pending.guildId, pending.adapterId, selectedBackend);
|
|
4246
|
+
await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
|
|
4247
|
+
}
|
|
3703
4248
|
/** Start a classic channel instance with lightweight config */
|
|
3704
|
-
async startClassicInstance(instanceName, backend, preTaskCommand, model) {
|
|
4249
|
+
async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
|
|
3705
4250
|
if (this.daemons.has(instanceName))
|
|
3706
4251
|
return;
|
|
3707
4252
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
@@ -3713,41 +4258,34 @@ When users create specialized instances, suggest these configurations:
|
|
|
3713
4258
|
lightweight: true,
|
|
3714
4259
|
...(backend ? { backend } : {}),
|
|
3715
4260
|
...(model ? { model } : {}),
|
|
4261
|
+
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
3716
4262
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
3717
4263
|
};
|
|
3718
4264
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
3719
4265
|
await this.startInstance(instanceName, config, topicMode);
|
|
3720
4266
|
}
|
|
3721
4267
|
/** Handle /start slash command — register classic channel */
|
|
3722
|
-
async handleClassicStart(channelId, channelName, userId, guildId, adapterId) {
|
|
3723
|
-
|
|
4268
|
+
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
4269
|
+
const blocker = this.validateClassicStart(channelId, userId, guildId, adapterId);
|
|
4270
|
+
if (blocker)
|
|
4271
|
+
return blocker;
|
|
4272
|
+
const classicChannels = this.classicChannels;
|
|
4273
|
+
if (!classicChannels)
|
|
3724
4274
|
return "Classic channel manager not initialized.";
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
}
|
|
3730
|
-
return t("classic.not_authorized_guild");
|
|
3731
|
-
}
|
|
3732
|
-
// Per-bot check: a second bot may /start in the same channel (own agent).
|
|
3733
|
-
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
3734
|
-
return t("classic.already_active");
|
|
3735
|
-
// Classic no longer lives in the routing engine, so this only guards against
|
|
3736
|
-
// a fleet topic-mode instance colliding with the channel.
|
|
3737
|
-
if (this.routing.resolve(channelId))
|
|
3738
|
-
return t("classic.topic_bound");
|
|
3739
|
-
const instanceName = this.classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
3740
|
-
this.classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId);
|
|
4275
|
+
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
4276
|
+
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
4277
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4278
|
+
classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId, selectedBackend);
|
|
3741
4279
|
// Bind this classic instance to the bot that started it (authoritative), so
|
|
3742
4280
|
// replies/cancel go out through that bot even though every same-guild bot
|
|
3743
4281
|
// also sees the channel's messages.
|
|
3744
4282
|
if (adapterId)
|
|
3745
4283
|
this.bindInstanceAdapter(instanceName, adapterId);
|
|
3746
|
-
await this.startClassicInstance(instanceName,
|
|
4284
|
+
await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
3747
4285
|
this.reregisterClassicChannels();
|
|
3748
4286
|
// Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
|
|
3749
|
-
if (guildId && !
|
|
3750
|
-
|
|
4287
|
+
if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
|
|
4288
|
+
classicChannels.toggleCollab(channelId, adapterId);
|
|
3751
4289
|
}
|
|
3752
4290
|
this.logger.info({ channelId, adapterId, instanceName, userId }, "Classic channel started");
|
|
3753
4291
|
return t("classic.started");
|
|
@@ -3761,6 +4299,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3761
4299
|
return t("classic.no_agent");
|
|
3762
4300
|
this.instanceWorldBinding.delete(ch.instanceName);
|
|
3763
4301
|
await this.stopInstance(ch.instanceName).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to stop classic instance"));
|
|
4302
|
+
clearPausedMarker(this.getInstanceDir(ch.instanceName));
|
|
3764
4303
|
this.reregisterClassicChannels();
|
|
3765
4304
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
3766
4305
|
return t("classic.stopped");
|
|
@@ -3804,6 +4343,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
3804
4343
|
clearInterval(this.classicReloadTimer);
|
|
3805
4344
|
this.classicReloadTimer = null;
|
|
3806
4345
|
}
|
|
4346
|
+
for (const pending of this.pendingClassicStarts.values())
|
|
4347
|
+
clearTimeout(pending.timer);
|
|
4348
|
+
this.pendingClassicStarts.clear();
|
|
3807
4349
|
this.topicArchiver.stop();
|
|
3808
4350
|
this.scheduler?.shutdown();
|
|
3809
4351
|
// Stop instances in parallel batches to avoid long sequential waits.
|
|
@@ -4098,7 +4640,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4098
4640
|
let idx = 0;
|
|
4099
4641
|
while (idx < channels.length) {
|
|
4100
4642
|
const batch = channels.slice(idx, idx + concurrency);
|
|
4101
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4643
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4102
4644
|
idx += concurrency;
|
|
4103
4645
|
}
|
|
4104
4646
|
}
|
|
@@ -4110,13 +4652,23 @@ When users create specialized instances, suggest these configurations:
|
|
|
4110
4652
|
if (groupId && this.adapter) {
|
|
4111
4653
|
const total = Object.keys(fleet.instances).length;
|
|
4112
4654
|
const started = this.daemons.size;
|
|
4113
|
-
const
|
|
4655
|
+
const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
4656
|
+
const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
|
|
4657
|
+
const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
|
|
4114
4658
|
const { createRequire } = await import("node:module");
|
|
4115
4659
|
const _require2 = createRequire(import.meta.url);
|
|
4116
4660
|
const agendVersion2 = _require2("../package.json").version ?? "unknown";
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4661
|
+
let restartText;
|
|
4662
|
+
if (failedNames.length === 0 && pausedNames2.length === 0) {
|
|
4663
|
+
restartText = t("fleet.ready", started, total, agendVersion2);
|
|
4664
|
+
}
|
|
4665
|
+
else if (failedNames.length === 0) {
|
|
4666
|
+
restartText = t("fleet.ready", started, total, agendVersion2) + `\n⏸ Paused: ${pausedNames2.join(", ")}`;
|
|
4667
|
+
}
|
|
4668
|
+
else {
|
|
4669
|
+
restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
|
|
4670
|
+
+ (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
|
|
4671
|
+
}
|
|
4120
4672
|
await this.adapter.sendText(String(groupId), restartText, notifyOpts)
|
|
4121
4673
|
.catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
|
|
4122
4674
|
// Notify each instance's channel — staggered to avoid rate limit storm
|
|
@@ -4328,8 +4880,31 @@ When users create specialized instances, suggest these configurations:
|
|
|
4328
4880
|
// Fleet API (enriched for agent board)
|
|
4329
4881
|
if (req.method === "GET" && req.url === "/api/fleet") {
|
|
4330
4882
|
const sysInfo = this.getSysInfo();
|
|
4331
|
-
const
|
|
4883
|
+
const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
|
|
4884
|
+
const fleetNames = new Set(fleetInstances.map(inst => inst.name));
|
|
4885
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
4886
|
+
.filter(channel => !fleetNames.has(channel.instanceName))
|
|
4887
|
+
.map(channel => ({
|
|
4888
|
+
name: channel.instanceName,
|
|
4889
|
+
status: this.getInstanceStatus(channel.instanceName),
|
|
4890
|
+
state: this.getInstanceExecutionState(channel.instanceName),
|
|
4891
|
+
ipc: this.instanceIpcClients.has(channel.instanceName),
|
|
4892
|
+
costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
|
|
4893
|
+
rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
|
|
4894
|
+
classic: true,
|
|
4895
|
+
classicName: channel.name,
|
|
4896
|
+
channelId: channel.channelId,
|
|
4897
|
+
adapterId: channel.adapterId ?? null,
|
|
4898
|
+
}));
|
|
4899
|
+
const enriched = [...fleetInstances, ...classicInstances].map(inst => {
|
|
4332
4900
|
const config = this.fleetConfig?.instances[inst.name];
|
|
4901
|
+
const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
|
|
4902
|
+
const lastActivity = inst.classic
|
|
4903
|
+
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
4904
|
+
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
4905
|
+
const backend = inst.classic
|
|
4906
|
+
? this.classicChannels?.getBackendByInstance(inst.name, this.fleetConfig?.defaults.backend) ?? "claude-code"
|
|
4907
|
+
: config?.backend ?? "claude-code";
|
|
4333
4908
|
// Find claimed tasks for this instance
|
|
4334
4909
|
let currentTask = null;
|
|
4335
4910
|
try {
|
|
@@ -4342,13 +4917,16 @@ When users create specialized instances, suggest these configurations:
|
|
|
4342
4917
|
}
|
|
4343
4918
|
return {
|
|
4344
4919
|
...inst,
|
|
4345
|
-
description: config?.description ?? null,
|
|
4346
|
-
backend
|
|
4920
|
+
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
4921
|
+
backend,
|
|
4347
4922
|
tool_set: config?.tool_set ?? "full",
|
|
4348
4923
|
general_topic: config?.general_topic ?? false,
|
|
4349
|
-
|
|
4924
|
+
// User activity is persisted by the daemon, so both the board and
|
|
4925
|
+
// auto-pause retain an accurate age across fleet restarts.
|
|
4926
|
+
lastActivity,
|
|
4350
4927
|
currentTask,
|
|
4351
4928
|
idle: this.getInstanceIdle(inst.name),
|
|
4929
|
+
state: this.getInstanceExecutionState(inst.name),
|
|
4352
4930
|
};
|
|
4353
4931
|
});
|
|
4354
4932
|
res.setHeader("Access-Control-Allow-Origin", "*");
|