@songsid/agend 2.1.0-beta.4 → 2.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +2 -1
- package/dist/access-path.js +3 -3
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli.js +1 -1
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +8 -0
- package/dist/agent-endpoint.js +36 -8
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +2 -0
- package/dist/backend/antigravity.js +30 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +2 -0
- package/dist/backend/claude-code.js +16 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/factory.js +4 -1
- package/dist/backend/factory.js.map +1 -1
- package/dist/backend/grok.d.ts +53 -0
- package/dist/backend/grok.js +302 -0
- package/dist/backend/grok.js.map +1 -0
- package/dist/backend/kiro.d.ts +2 -0
- package/dist/backend/kiro.js +32 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +36 -3
- package/dist/backend/types.js +1 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +20 -0
- package/dist/channel/adapters/discord.js +89 -11
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.js +4 -1
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +18 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/chat-export.js +5 -2
- package/dist/chat-export.js.map +1 -1
- package/dist/classic-channel-manager.d.ts +16 -1
- package/dist/classic-channel-manager.js +76 -4
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +116 -19
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +70 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +3 -1
- package/dist/config.js +14 -1
- package/dist/config.js.map +1 -1
- package/dist/context-guardian.d.ts +2 -0
- package/dist/context-guardian.js +10 -2
- package/dist/context-guardian.js.map +1 -1
- package/dist/daemon-entry.js +2 -1
- package/dist/daemon-entry.js.map +1 -1
- package/dist/daemon.d.ts +91 -6
- package/dist/daemon.js +602 -186
- package/dist/daemon.js.map +1 -1
- package/dist/daily-summary.js +7 -2
- package/dist/daily-summary.js.map +1 -1
- package/dist/fleet-context.d.ts +6 -0
- package/dist/fleet-manager.d.ts +113 -8
- package/dist/fleet-manager.js +972 -142
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-system-prompt.js +1 -1
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +19 -13
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +30 -51
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +19 -13
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +1 -5
- package/dist/general-knowledge/skills/scheduling/SKILL.md +41 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +29 -116
- package/dist/instance-lifecycle.d.ts +15 -1
- package/dist/instance-lifecycle.js +71 -22
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +28 -2
- package/dist/locale.js.map +1 -1
- package/dist/logger.d.ts +6 -0
- package/dist/logger.js +28 -9
- package/dist/logger.js.map +1 -1
- package/dist/outbound-handlers.d.ts +3 -0
- package/dist/outbound-handlers.js +224 -11
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +44 -2
- package/dist/outbound-schemas.js +50 -3
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pause-marker.d.ts +5 -0
- package/dist/pause-marker.js +36 -0
- package/dist/pause-marker.js.map +1 -0
- package/dist/scheduler/db.d.ts +8 -0
- package/dist/scheduler/db.js +74 -5
- package/dist/scheduler/db.js.map +1 -1
- package/dist/scheduler/db.test.js +50 -0
- package/dist/scheduler/db.test.js.map +1 -1
- package/dist/scheduler/scheduler.d.ts +6 -0
- package/dist/scheduler/scheduler.js +99 -16
- package/dist/scheduler/scheduler.js.map +1 -1
- package/dist/scheduler/scheduler.test.js +131 -1
- package/dist/scheduler/scheduler.test.js.map +1 -1
- package/dist/scheduler/types.d.ts +9 -3
- package/dist/scheduler/types.js.map +1 -1
- package/dist/settings-api.d.ts +20 -2
- package/dist/settings-api.js +213 -9
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +8 -0
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +1 -1
- package/dist/statusline-watcher.js +3 -2
- package/dist/statusline-watcher.js.map +1 -1
- package/dist/tmux-control.d.ts +7 -0
- package/dist/tmux-control.js +22 -1
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +19 -1
- package/dist/topic-commands.js +148 -23
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.js +2 -0
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/types.d.ts +29 -1
- package/dist/tz-utils.d.ts +26 -0
- package/dist/tz-utils.js +46 -0
- package/dist/tz-utils.js.map +1 -0
- package/dist/ui/dashboard.html +3 -3
- package/dist/ui/settings.html +453 -74
- package/dist/web-api.d.ts +1 -0
- package/dist/web-api.js +24 -15
- package/dist/web-api.js.map +1 -1
- package/dist/workflow-templates/default.md +0 -4
- package/package.json +2 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -5,13 +5,13 @@ import { join, dirname, basename } from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
|
|
7
7
|
import { sdNotify } from "./sd-notify.js";
|
|
8
|
-
import
|
|
8
|
+
import { isScalar, parseDocument } from "yaml";
|
|
9
9
|
const __filename = fileURLToPath(import.meta.url);
|
|
10
10
|
const __dirname = dirname(__filename);
|
|
11
11
|
/** Fallback access policy for a channel with no `access:` block — open (no gate). */
|
|
12
12
|
const DEFAULT_OPEN_ACCESS = { mode: "open", allowed_users: [], max_pending_codes: 0, code_expiry_minutes: 0 };
|
|
13
13
|
import { isProbeableRouteTarget } from "./fleet-context.js";
|
|
14
|
-
import { loadFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
14
|
+
import { loadFleetConfig, loadRawFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
|
|
15
15
|
import { EventLog } from "./event-log.js";
|
|
16
16
|
import { AdapterWorld } from "./adapter-world.js";
|
|
17
17
|
import { CostGuard, formatCents } from "./cost-guard.js";
|
|
@@ -19,18 +19,20 @@ import { TmuxManager } from "./tmux-manager.js";
|
|
|
19
19
|
import { AccessManager } from "./channel/access-manager.js";
|
|
20
20
|
import { IpcClient } from "./channel/ipc-bridge.js";
|
|
21
21
|
import { createAdapter } from "./channel/factory.js";
|
|
22
|
+
import { createBackend } from "./backend/factory.js";
|
|
23
|
+
import { isModelCompatible } from "./backend/types.js";
|
|
22
24
|
import { createLogger } from "./logger.js";
|
|
23
25
|
import { processAttachments } from "./channel/attachment-handler.js";
|
|
24
26
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
25
27
|
import { Scheduler } from "./scheduler/index.js";
|
|
26
28
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
27
|
-
import { TopicCommands, saveCommandForBackend, parseSaveFilename, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
29
|
+
import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
|
|
28
30
|
import { DailySummary } from "./daily-summary.js";
|
|
29
31
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
30
32
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
31
33
|
import { safeHandler } from "./safe-async.js";
|
|
32
34
|
import { RoutingEngine } from "./routing-engine.js";
|
|
33
|
-
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
35
|
+
import { InstanceLifecycle, BACKEND_INSTALLATION_INFO, checkBinaryInstalled, } from "./instance-lifecycle.js";
|
|
34
36
|
import { TopicArchiver } from "./topic-archiver.js";
|
|
35
37
|
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
36
38
|
import { outboundHandlers } from "./outbound-handlers.js";
|
|
@@ -39,7 +41,9 @@ import { handleViewRequest, isViewPath } from "./view-api.js";
|
|
|
39
41
|
import { handleSettingsRequest } from "./settings-api.js";
|
|
40
42
|
import { setLocale, detectLocale, t } from "./locale.js";
|
|
41
43
|
import { handleAgentRequest } from "./agent-endpoint.js";
|
|
42
|
-
import { ClassicChannelManager } from "./classic-channel-manager.js";
|
|
44
|
+
import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBackend, readClassicLastActivityAt } from "./classic-channel-manager.js";
|
|
45
|
+
import { readLastInboundAt } from "./daemon.js";
|
|
46
|
+
import { clearPausedMarker } from "./pause-marker.js";
|
|
43
47
|
import { getTmuxSession } from "./config.js";
|
|
44
48
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
45
49
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -50,6 +54,26 @@ export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
|
50
54
|
}
|
|
51
55
|
return instanceConfig?.topic_id != null ? String(instanceConfig.topic_id) : undefined;
|
|
52
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Pure warm-cap victim selection (extracted for testability). Given the current
|
|
59
|
+
* warm (running) instance names and a cap, return the LRU idle instances to evict
|
|
60
|
+
* so the running count returns to the cap. Skips: the `exclude` instance, any
|
|
61
|
+
* already-evicting, general instances (never evicted), and non-idle instances
|
|
62
|
+
* (working/stuck can't be evicted). Oldest last-inbound is evicted first; a
|
|
63
|
+
* missing timestamp (0) sorts oldest. cap <= 0 (or non-integer) = unlimited → [].
|
|
64
|
+
*/
|
|
65
|
+
export function selectLruEvictions(warm, cap, opts) {
|
|
66
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
67
|
+
return [];
|
|
68
|
+
if (warm.length <= cap)
|
|
69
|
+
return [];
|
|
70
|
+
const candidates = warm.filter(name => name !== opts.exclude
|
|
71
|
+
&& !opts.isEvicting(name)
|
|
72
|
+
&& !opts.isGeneral(name)
|
|
73
|
+
&& opts.isIdle(name));
|
|
74
|
+
candidates.sort((a, b) => opts.lastInboundAt(a) - opts.lastInboundAt(b));
|
|
75
|
+
return candidates.slice(0, warm.length - cap);
|
|
76
|
+
}
|
|
53
77
|
/** Retry cadence for retiring a cancel button whose delete failed (e.g. a DC
|
|
54
78
|
* forum thread the bot momentarily can't reach). 3 retries × 5min = 15min. */
|
|
55
79
|
const CANCEL_BTN_RETRY_INTERVAL_MS = 5 * 60_000;
|
|
@@ -58,6 +82,9 @@ const CANCEL_BTN_MAX_RETRIES = 3;
|
|
|
58
82
|
* buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
|
|
59
83
|
* reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
|
|
60
84
|
const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
|
|
85
|
+
const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
|
|
86
|
+
const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
|
|
87
|
+
const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
|
|
61
88
|
export class FleetManager {
|
|
62
89
|
dataDir;
|
|
63
90
|
children = new Map();
|
|
@@ -65,6 +92,9 @@ export class FleetManager {
|
|
|
65
92
|
/** @deprecated Use lifecycle.daemons — kept for backward compat */
|
|
66
93
|
get daemons() { return this.lifecycle.daemons; }
|
|
67
94
|
fleetConfig = null;
|
|
95
|
+
rawFleetConfig = {};
|
|
96
|
+
rawFleetDocument = null;
|
|
97
|
+
savedFleetConfigSnapshot = null;
|
|
68
98
|
adapter = null;
|
|
69
99
|
worlds = new Map();
|
|
70
100
|
adapters = new Map(); // derived view for backward compat
|
|
@@ -93,6 +123,18 @@ export class FleetManager {
|
|
|
93
123
|
// Topic icon + auto-archive state
|
|
94
124
|
topicIcons = {};
|
|
95
125
|
lastActivity = new Map();
|
|
126
|
+
/** Latest pane-derived execution snapshot reported by each daemon. */
|
|
127
|
+
instanceStateCache = new Map();
|
|
128
|
+
/** CLI pane status overrides; daemon.pid alone only proves FleetManager lives. */
|
|
129
|
+
instanceProcessStatus = new Map();
|
|
130
|
+
/** Instances currently being auto-paused by warm_cap, so concurrent checks don't double-evict. */
|
|
131
|
+
warmCapEvicting = new Set();
|
|
132
|
+
/** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
|
|
133
|
+
idleGatedDeliveryTails = new Map();
|
|
134
|
+
/** Non-user work must observe a fresh idle snapshot after the latest delivery. */
|
|
135
|
+
lastDeliveryAt = new Map();
|
|
136
|
+
/** State-cache updates wake event-driven idle waiters without busy polling. */
|
|
137
|
+
instanceIdleWaiters = new Map();
|
|
96
138
|
lastInboundUser = new Map(); // instanceName → last username
|
|
97
139
|
// Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
|
|
98
140
|
// than one-per-instance. A button is retired (deleted, with bounded retry) on
|
|
@@ -104,6 +146,9 @@ export class FleetManager {
|
|
|
104
146
|
topicArchiver;
|
|
105
147
|
controlClient = null;
|
|
106
148
|
classicChannels = null;
|
|
149
|
+
pendingClassicStarts = new Map();
|
|
150
|
+
/** In-flight /model selections, keyed by nonce (see handleModelSelection). */
|
|
151
|
+
pendingModelSelects = new Map();
|
|
107
152
|
// Model failover state
|
|
108
153
|
failoverActive = new Map(); // instance → current failover model
|
|
109
154
|
// IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
|
|
@@ -160,6 +205,7 @@ export class FleetManager {
|
|
|
160
205
|
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
161
206
|
name,
|
|
162
207
|
status: this.getInstanceStatus(name),
|
|
208
|
+
state: this.getInstanceExecutionState(name),
|
|
163
209
|
ipc: this.instanceIpcClients.has(name),
|
|
164
210
|
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
165
211
|
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
@@ -174,9 +220,21 @@ export class FleetManager {
|
|
|
174
220
|
}
|
|
175
221
|
/** Load fleet.yaml and build routing table */
|
|
176
222
|
loadConfig(configPath) {
|
|
223
|
+
this.configPath = configPath;
|
|
224
|
+
const source = existsSync(configPath) ? readFileSync(configPath, "utf-8") : "{}\n";
|
|
225
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
226
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
227
|
+
throw new Error(`Invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
228
|
+
}
|
|
229
|
+
this.rawFleetConfig = loadRawFleetConfig(configPath);
|
|
177
230
|
this.fleetConfig = loadFleetConfig(configPath);
|
|
231
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
178
232
|
return this.fleetConfig;
|
|
179
233
|
}
|
|
234
|
+
/** User-authored fleet.yaml, before defaults are merged into instances. */
|
|
235
|
+
getRawFleetConfig() {
|
|
236
|
+
return structuredClone(this.rawFleetConfig);
|
|
237
|
+
}
|
|
180
238
|
/** Build topic routing table: { topicId -> RouteTarget } */
|
|
181
239
|
buildRoutingTable() {
|
|
182
240
|
if (this.fleetConfig) {
|
|
@@ -230,6 +288,41 @@ export class FleetManager {
|
|
|
230
288
|
return this.classicChannels?.getInstanceByChannel(channelId, adapterId)
|
|
231
289
|
?? this.routing.resolve(channelId)?.name;
|
|
232
290
|
}
|
|
291
|
+
async handlePauseWakeSlash(data, adapterId) {
|
|
292
|
+
const action = data.command;
|
|
293
|
+
const classicName = this.classicChannels?.getInstanceByChannel(data.channelId, adapterId);
|
|
294
|
+
if (classicName) {
|
|
295
|
+
if (!this.classicChannels?.isAdmin(data.userId)) {
|
|
296
|
+
await data.respond(t("permission.denied"));
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
await data.respond(await this.topicCommands.runPauseWake(classicName, action));
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
303
|
+
await data.respond(t("permission.denied"));
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
const route = this.routing.resolve(data.channelId);
|
|
307
|
+
if (!route) {
|
|
308
|
+
await data.respond(t("classic.no_agent"));
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
let target = route.name;
|
|
312
|
+
if (route.kind === "general") {
|
|
313
|
+
const requested = typeof data.options?.instance === "string" ? data.options.instance : undefined;
|
|
314
|
+
if (!requested) {
|
|
315
|
+
await data.respond(t(`${action}.usage`));
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (!this.fleetConfig?.instances[requested]) {
|
|
319
|
+
await data.respond(t("instance.not_found", requested));
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
target = requested;
|
|
323
|
+
}
|
|
324
|
+
await data.respond(await this.topicCommands.runPauseWake(target, action));
|
|
325
|
+
}
|
|
233
326
|
/** Get the adapter bound to an instance, falling back to primary adapter */
|
|
234
327
|
getAdapterForInstance(name) {
|
|
235
328
|
const worldId = this.instanceWorldBinding.get(name);
|
|
@@ -265,19 +358,27 @@ export class FleetManager {
|
|
|
265
358
|
*/
|
|
266
359
|
bindInstanceAdapter(name, adapterId, fromInbound = false) {
|
|
267
360
|
const cfg = this.fleetConfig?.instances[name];
|
|
268
|
-
if (fromInbound
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
361
|
+
if (fromInbound) {
|
|
362
|
+
// Skip inbound-derived binding for any instance that doesn't have an
|
|
363
|
+
// explicit channel_id — those default to primary adapter deterministically.
|
|
364
|
+
// This prevents a non-deterministic race where whichever adapter delivers
|
|
365
|
+
// first after restart wins the binding.
|
|
366
|
+
if (cfg?.general_topic || cfg?.channel_id)
|
|
367
|
+
return;
|
|
368
|
+
if (cfg && !cfg.channel_id)
|
|
369
|
+
return; // fleet instance without explicit binding → use primary
|
|
370
|
+
// Classic instance: don't override an existing binding (authoritative from /start)
|
|
371
|
+
if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
276
374
|
this.instanceWorldBinding.set(name, adapterId);
|
|
277
375
|
}
|
|
278
376
|
getInstanceStatus(name) {
|
|
279
377
|
if (this.lifecycle.isPaused(name))
|
|
280
378
|
return "paused";
|
|
379
|
+
const processStatus = this.instanceProcessStatus.get(name);
|
|
380
|
+
if (processStatus)
|
|
381
|
+
return processStatus;
|
|
281
382
|
const pidPath = join(this.getInstanceDir(name), "daemon.pid");
|
|
282
383
|
if (!existsSync(pidPath))
|
|
283
384
|
return "stopped";
|
|
@@ -290,24 +391,244 @@ export class FleetManager {
|
|
|
290
391
|
return "crashed";
|
|
291
392
|
}
|
|
292
393
|
}
|
|
293
|
-
|
|
294
|
-
|
|
394
|
+
getInstanceExecutionState(name) {
|
|
395
|
+
if (this.lifecycle.isPaused(name))
|
|
396
|
+
return null;
|
|
397
|
+
return this.instanceStateCache.get(name)?.state ?? null;
|
|
398
|
+
}
|
|
399
|
+
isClassicInstance(name) {
|
|
400
|
+
return this.classicChannels?.getAll().some(channel => channel.instanceName === name) ?? false;
|
|
401
|
+
}
|
|
402
|
+
cacheInstanceExecutionState(name, msg) {
|
|
403
|
+
const state = msg.state;
|
|
404
|
+
if (state !== "idle" && state !== "working" && state !== "stuck")
|
|
405
|
+
return;
|
|
406
|
+
const previous = this.instanceStateCache.get(name);
|
|
407
|
+
const now = Date.now();
|
|
408
|
+
const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
409
|
+
this.instanceStateCache.set(name, {
|
|
410
|
+
state,
|
|
411
|
+
unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
|
|
412
|
+
observedAt: numberOr(msg.observedAt, now),
|
|
413
|
+
stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
|
|
414
|
+
});
|
|
415
|
+
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
416
|
+
check();
|
|
417
|
+
// warm_cap: a fresh transition into idle may free this instance for eviction,
|
|
418
|
+
// or (more usefully) reveal that the fleet is now over cap. Only fire on the
|
|
419
|
+
// edge into idle, not on every idle heartbeat.
|
|
420
|
+
if (state === "idle" && previous?.state !== "idle")
|
|
421
|
+
this.enforceWarmCap();
|
|
422
|
+
}
|
|
423
|
+
cacheInstanceProcessStatus(name, status) {
|
|
424
|
+
if (status === "running") {
|
|
425
|
+
this.instanceProcessStatus.delete(name);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
if (status !== "crashed" && status !== "stopped")
|
|
429
|
+
return;
|
|
430
|
+
this.instanceProcessStatus.set(name, status);
|
|
431
|
+
// Never display the last ready prompt as current execution state after its
|
|
432
|
+
// owning CLI process has exited.
|
|
433
|
+
this.instanceStateCache.delete(name);
|
|
434
|
+
for (const check of this.instanceIdleWaiters.get(name) ?? [])
|
|
435
|
+
check();
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Fleet-wide warm cap: if more than `defaults.warm_cap` instances are running,
|
|
439
|
+
* auto-pause the least-recently-active idle instances until back at the cap.
|
|
440
|
+
* Never evicts general instances (must stay warm) or working/stuck instances
|
|
441
|
+
* (only idle). 0/unset = unlimited. wake-before-deliver re-warms any evicted
|
|
442
|
+
* instance when a message next arrives.
|
|
443
|
+
*
|
|
444
|
+
* @param exclude instance to spare (e.g. one just woken to receive a delivery).
|
|
445
|
+
*/
|
|
446
|
+
enforceWarmCap(exclude) {
|
|
447
|
+
const cap = this.fleetConfig?.defaults?.warm_cap ?? 0;
|
|
448
|
+
if (!Number.isInteger(cap) || cap <= 0)
|
|
449
|
+
return; // 0/invalid = unlimited
|
|
450
|
+
const warm = [];
|
|
451
|
+
for (const name of this.daemons.keys()) {
|
|
452
|
+
if (this.getInstanceStatus(name) === "running")
|
|
453
|
+
warm.push(name);
|
|
454
|
+
}
|
|
455
|
+
if (warm.length <= cap)
|
|
456
|
+
return;
|
|
457
|
+
const victims = selectLruEvictions(warm, cap, {
|
|
458
|
+
exclude,
|
|
459
|
+
isEvicting: name => this.warmCapEvicting.has(name),
|
|
460
|
+
isGeneral: name => this.fleetConfig?.instances[name]?.general_topic === true,
|
|
461
|
+
isIdle: name => this.getInstanceExecutionState(name) === "idle",
|
|
462
|
+
lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
|
|
463
|
+
});
|
|
464
|
+
for (const victim of victims) {
|
|
465
|
+
this.warmCapEvicting.add(victim);
|
|
466
|
+
this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
|
|
467
|
+
this.lifecycle.pause(victim)
|
|
468
|
+
.catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
|
|
469
|
+
.finally(() => this.warmCapEvicting.delete(victim));
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
|
|
473
|
+
const isReady = () => {
|
|
474
|
+
const snapshot = this.instanceStateCache.get(instanceName);
|
|
475
|
+
return snapshot?.state === "idle"
|
|
476
|
+
&& (idleObservedAfter === 0 || snapshot.observedAt > idleObservedAfter);
|
|
477
|
+
};
|
|
478
|
+
if (isReady())
|
|
479
|
+
return Promise.resolve(true);
|
|
480
|
+
return new Promise(resolve => {
|
|
481
|
+
let settled = false;
|
|
482
|
+
const finish = (idle) => {
|
|
483
|
+
if (settled)
|
|
484
|
+
return;
|
|
485
|
+
settled = true;
|
|
486
|
+
clearTimeout(timeout);
|
|
487
|
+
clearInterval(queryTimer);
|
|
488
|
+
const waiters = this.instanceIdleWaiters.get(instanceName);
|
|
489
|
+
waiters?.delete(check);
|
|
490
|
+
if (waiters?.size === 0)
|
|
491
|
+
this.instanceIdleWaiters.delete(instanceName);
|
|
492
|
+
resolve(idle);
|
|
493
|
+
};
|
|
494
|
+
const check = () => { if (isReady())
|
|
495
|
+
finish(true); };
|
|
496
|
+
const query = () => {
|
|
497
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
498
|
+
if (ipc?.connected) {
|
|
499
|
+
ipc.send({ type: "query_instance_state", requestId: `idle-gate-${Date.now()}` });
|
|
500
|
+
}
|
|
501
|
+
check();
|
|
502
|
+
};
|
|
503
|
+
const waiters = this.instanceIdleWaiters.get(instanceName) ?? new Set();
|
|
504
|
+
waiters.add(check);
|
|
505
|
+
this.instanceIdleWaiters.set(instanceName, waiters);
|
|
506
|
+
const timeout = setTimeout(() => finish(false), timeoutMs);
|
|
507
|
+
const queryTimer = setInterval(query, 1_000);
|
|
508
|
+
query();
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
async deliverWithIdleGate(instanceName, payload, timeoutMs) {
|
|
512
|
+
let idleObservedAfter = this.lastDeliveryAt.get(instanceName) ?? 0;
|
|
295
513
|
if (this.lifecycle.isPaused(instanceName)) {
|
|
514
|
+
const wakeStartedAt = Date.now();
|
|
296
515
|
await this.lifecycle.wake(instanceName, 30_000);
|
|
516
|
+
// Waking added one to the warm count — make room by evicting a different
|
|
517
|
+
// LRU idle instance (never this one; it's about to work).
|
|
518
|
+
this.enforceWarmCap(instanceName);
|
|
519
|
+
// Never satisfy a post-wake gate from a stale pre-pause cache entry.
|
|
520
|
+
idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
|
|
521
|
+
}
|
|
522
|
+
const idle = await this.waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter);
|
|
523
|
+
if (!idle) {
|
|
524
|
+
this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
|
|
297
525
|
}
|
|
298
526
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
299
527
|
if (!ipc?.connected)
|
|
300
528
|
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
301
529
|
ipc.send(payload);
|
|
530
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
531
|
+
}
|
|
532
|
+
/** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
|
|
533
|
+
async deliverToInstance(instanceName, payload, options = {}) {
|
|
534
|
+
const meta = payload.meta && typeof payload.meta === "object"
|
|
535
|
+
? payload.meta
|
|
536
|
+
: undefined;
|
|
537
|
+
const inferredCrossInstance = (typeof meta?.from_instance === "string" && meta.from_instance.length > 0)
|
|
538
|
+
|| meta?.is_cross_instance === true
|
|
539
|
+
|| payload.is_cross_instance === true;
|
|
540
|
+
const waitForIdle = options.waitForIdle
|
|
541
|
+
?? ((options.isCrossInstance ?? inferredCrossInstance) || payload.type === "fleet_schedule_trigger");
|
|
542
|
+
if (!waitForIdle) {
|
|
543
|
+
if (this.lifecycle.isPaused(instanceName)) {
|
|
544
|
+
await this.lifecycle.wake(instanceName, 30_000);
|
|
545
|
+
this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
|
|
546
|
+
}
|
|
547
|
+
const ipc = this.instanceIpcClients.get(instanceName);
|
|
548
|
+
if (!ipc?.connected)
|
|
549
|
+
throw new Error(`Instance '${instanceName}' IPC is unavailable`);
|
|
550
|
+
ipc.send(payload);
|
|
551
|
+
// A cross-instance item arriving before the daemon observes this turn as
|
|
552
|
+
// working must not trust the stale idle snapshot from before the send.
|
|
553
|
+
this.lastDeliveryAt.set(instanceName, Date.now());
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
|
|
557
|
+
const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, payload, options.idleTimeoutMs ?? 60_000));
|
|
558
|
+
this.idleGatedDeliveryTails.set(instanceName, delivery);
|
|
559
|
+
try {
|
|
560
|
+
await delivery;
|
|
561
|
+
}
|
|
562
|
+
finally {
|
|
563
|
+
if (this.idleGatedDeliveryTails.get(instanceName) === delivery) {
|
|
564
|
+
this.idleGatedDeliveryTails.delete(instanceName);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
/** Fleet admin is an explicit config allowlist entry, not merely an open/paired user. */
|
|
569
|
+
isFleetAdmin(userId, adapterId) {
|
|
570
|
+
const allowed = this.getChannelConfig(adapterId)?.access?.allowed_users ?? [];
|
|
571
|
+
return allowed.some(entry => String(entry) === String(userId));
|
|
572
|
+
}
|
|
573
|
+
async changeInstancePauseState(name, action) {
|
|
574
|
+
if (action === "wake") {
|
|
575
|
+
await this.lifecycle.wake(name, 30_000);
|
|
576
|
+
this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
|
|
577
|
+
return "awake";
|
|
578
|
+
}
|
|
579
|
+
await this.lifecycle.pause(name);
|
|
580
|
+
return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
|
|
581
|
+
}
|
|
582
|
+
/** Apply a Settings edit to a ClassicBot channel without waiting for the poller. */
|
|
583
|
+
async restartClassicInstanceFromSettings(instanceName) {
|
|
584
|
+
if (!this.classicChannels)
|
|
585
|
+
throw new Error("Classic channel manager not initialized");
|
|
586
|
+
const wasRunning = this.daemons.has(instanceName);
|
|
587
|
+
this.classicChannels.reloadFromDisk();
|
|
588
|
+
this.reregisterClassicChannels();
|
|
589
|
+
const channel = this.classicChannels.getAll().find(item => item.instanceName === instanceName);
|
|
590
|
+
if (!channel)
|
|
591
|
+
throw new Error("Classic channel not found after reload");
|
|
592
|
+
if (!wasRunning)
|
|
593
|
+
return;
|
|
594
|
+
await this.stopInstance(instanceName);
|
|
595
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
596
|
+
await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
302
597
|
}
|
|
303
598
|
async startInstance(name, config, topicMode) {
|
|
599
|
+
if (this.lifecycle.isPaused(name)) {
|
|
600
|
+
this.logger.info({ name }, "Persisted paused instance — skipping startup");
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
304
603
|
if (config.general_topic) {
|
|
604
|
+
// antigravity (agy) does not read MCP instructions — fleet context and
|
|
605
|
+
// routing instructions are not injected, so it cannot act as a dispatcher.
|
|
606
|
+
const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
607
|
+
if (backend === "antigravity") {
|
|
608
|
+
this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
|
|
609
|
+
this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
|
|
610
|
+
}
|
|
305
611
|
this.ensureGeneralInstructions(config.working_directory, config.backend);
|
|
306
612
|
}
|
|
613
|
+
this.instanceProcessStatus.delete(name);
|
|
307
614
|
await this.lifecycle.start(name, config, topicMode);
|
|
308
615
|
// Auto-connect IPC — daemon.start() ensures socket is ready before resolving
|
|
309
616
|
await this.connectIpcToInstance(name);
|
|
310
617
|
}
|
|
618
|
+
/** Recreate a daemon for a marker-only paused instance after an explicit wake/delivery. */
|
|
619
|
+
async startPersistedPausedInstance(name) {
|
|
620
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic"
|
|
621
|
+
|| !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
|
|
622
|
+
const fleetConfig = this.fleetConfig?.instances[name];
|
|
623
|
+
if (fleetConfig) {
|
|
624
|
+
await this.startInstance(name, fleetConfig, topicMode);
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
|
|
628
|
+
if (!channel || !this.classicChannels)
|
|
629
|
+
throw new Error(`Paused instance '${name}' is no longer configured`);
|
|
630
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
631
|
+
}
|
|
311
632
|
/**
|
|
312
633
|
* Start instances with configurable concurrency and stagger delay.
|
|
313
634
|
* Instances sharing the same working_directory are serialized within a group
|
|
@@ -316,8 +637,21 @@ export class FleetManager {
|
|
|
316
637
|
*/
|
|
317
638
|
async startInstancesWithConcurrency(entries, topicMode) {
|
|
318
639
|
const raw = this.fleetConfig?.defaults?.startup;
|
|
319
|
-
const
|
|
640
|
+
const explicitConcurrency = raw?.concurrency;
|
|
320
641
|
const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
|
|
642
|
+
// Adaptive concurrency: if not explicitly set, estimate from available RAM.
|
|
643
|
+
// Each instance uses ~300MB (tmux + CLI process + model overhead).
|
|
644
|
+
const ESTIMATED_MB_PER_INSTANCE = 300;
|
|
645
|
+
const { freemem } = await import("node:os");
|
|
646
|
+
let concurrency;
|
|
647
|
+
if (explicitConcurrency != null) {
|
|
648
|
+
concurrency = Math.max(1, Math.min(20, explicitConcurrency));
|
|
649
|
+
}
|
|
650
|
+
else {
|
|
651
|
+
const freeMemMB = Math.round(freemem() / (1024 * 1024));
|
|
652
|
+
concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
|
|
653
|
+
this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
|
|
654
|
+
}
|
|
321
655
|
const byWorkDir = new Map();
|
|
322
656
|
for (const [name, config] of entries) {
|
|
323
657
|
const dir = config.working_directory;
|
|
@@ -339,6 +673,17 @@ export class FleetManager {
|
|
|
339
673
|
if (pendingTimer)
|
|
340
674
|
return;
|
|
341
675
|
while (running < concurrency && idx < groups.length) {
|
|
676
|
+
// Re-check memory if adaptive (no explicit concurrency set)
|
|
677
|
+
if (explicitConcurrency == null && running > 0) {
|
|
678
|
+
const nowFreeMB = Math.round(freemem() / (1024 * 1024));
|
|
679
|
+
if (nowFreeMB < ESTIMATED_MB_PER_INSTANCE) {
|
|
680
|
+
this.logger.warn({ freeMemMB: nowFreeMB, remaining: groups.length - idx }, "Low memory — pausing instance startup");
|
|
681
|
+
// Wait and retry in 5s
|
|
682
|
+
pendingTimer = true;
|
|
683
|
+
setTimeout(() => { pendingTimer = false; startNext(); }, 5000);
|
|
684
|
+
return;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
342
687
|
const now = Date.now();
|
|
343
688
|
const elapsed = now - lastStartAt;
|
|
344
689
|
if (lastStartAt > 0 && elapsed < staggerMs) {
|
|
@@ -367,21 +712,54 @@ export class FleetManager {
|
|
|
367
712
|
}
|
|
368
713
|
async stopInstance(name) {
|
|
369
714
|
this.failoverActive.delete(name);
|
|
715
|
+
this.instanceStateCache.delete(name);
|
|
716
|
+
this.instanceProcessStatus.delete(name);
|
|
717
|
+
this.lastDeliveryAt.delete(name);
|
|
370
718
|
return this.lifecycle.stop(name);
|
|
371
719
|
}
|
|
372
720
|
/** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
|
|
373
|
-
async restartSingleInstance(name) {
|
|
721
|
+
async restartSingleInstance(name, opts) {
|
|
374
722
|
if (this.configPath) {
|
|
375
723
|
this.loadConfig(this.configPath);
|
|
376
724
|
this.routing.rebuild(this.fleetConfig);
|
|
377
725
|
this.reregisterClassicChannels();
|
|
378
726
|
}
|
|
379
727
|
const config = this.fleetConfig?.instances[name];
|
|
380
|
-
if (
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
728
|
+
if (config) {
|
|
729
|
+
await this.stopInstance(name);
|
|
730
|
+
if (opts?.freshStart)
|
|
731
|
+
this.writeFreshStartMarker(name);
|
|
732
|
+
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
733
|
+
await this.startInstance(name, config, topicMode ?? false);
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
// Classic instance fallback
|
|
737
|
+
const channelId = this.classicChannels?.getChannelIdByInstance(name);
|
|
738
|
+
if (channelId) {
|
|
739
|
+
const fleetBackend = this.fleetConfig?.defaults?.backend;
|
|
740
|
+
const adapterId = this.classicChannels.getAdapterIdByInstance(name);
|
|
741
|
+
await this.stopInstance(name);
|
|
742
|
+
await new Promise(r => setTimeout(r, 1000)); // let tmux clean up
|
|
743
|
+
if (opts?.freshStart)
|
|
744
|
+
this.writeFreshStartMarker(name);
|
|
745
|
+
await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, fleetBackend), this.classicChannels.getPreTaskCommand(channelId, adapterId), this.classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
throw new Error(`Instance not found: ${name}`);
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Mark an instance so its next daemon start skips session resume. Written AFTER
|
|
752
|
+
* stop (survives the old daemon's cleanup) and BEFORE start so the respawn reads
|
|
753
|
+
* it — reuses the crash-state → resumeDisabled path from crash-loop recovery.
|
|
754
|
+
* One-shot: the daemon deletes it on startup.
|
|
755
|
+
*/
|
|
756
|
+
writeFreshStartMarker(name) {
|
|
757
|
+
try {
|
|
758
|
+
writeFileSync(join(this.getInstanceDir(name), "crash-state.json"), JSON.stringify({ resumeDisabled: true, reason: "pty_error_restart" }));
|
|
759
|
+
}
|
|
760
|
+
catch (err) {
|
|
761
|
+
this.logger.warn({ err, name }, "freshStart: failed to write crash-state marker");
|
|
762
|
+
}
|
|
385
763
|
}
|
|
386
764
|
/** Load .env file from data dir into process.env */
|
|
387
765
|
loadEnvFile() {
|
|
@@ -477,9 +855,11 @@ export class FleetManager {
|
|
|
477
855
|
const fleetModel = this.fleetConfig?.defaults?.model;
|
|
478
856
|
const oldBackends = new Map();
|
|
479
857
|
const oldModels = new Map();
|
|
858
|
+
const oldAutoPause = new Map();
|
|
480
859
|
for (const ch of this.classicChannels.getAll()) {
|
|
481
860
|
oldBackends.set(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend));
|
|
482
861
|
oldModels.set(ch.instanceName, this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel));
|
|
862
|
+
oldAutoPause.set(ch.instanceName, this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
483
863
|
}
|
|
484
864
|
if (!this.classicChannels.checkReload())
|
|
485
865
|
return;
|
|
@@ -487,14 +867,16 @@ export class FleetManager {
|
|
|
487
867
|
for (const ch of this.classicChannels.getAll()) {
|
|
488
868
|
const newBackend = this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend);
|
|
489
869
|
const newModel = this.classicChannels.getModel(ch.channelId, ch.adapterId, fleetModel);
|
|
870
|
+
const newAutoPause = this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after);
|
|
490
871
|
const backendChanged = oldBackends.get(ch.instanceName) !== newBackend;
|
|
491
872
|
const modelChanged = oldModels.get(ch.instanceName) !== newModel;
|
|
492
|
-
|
|
873
|
+
const autoPauseChanged = oldAutoPause.get(ch.instanceName) !== newAutoPause;
|
|
874
|
+
if (this.daemons.has(ch.instanceName) && (backendChanged || modelChanged || autoPauseChanged)) {
|
|
493
875
|
this.logger.info({ instanceName: ch.instanceName, backendFrom: oldBackends.get(ch.instanceName), backendTo: newBackend, modelFrom: oldModels.get(ch.instanceName), modelTo: newModel }, "Backend/model changed — restarting");
|
|
494
876
|
await this.stopInstance(ch.instanceName).catch(() => { });
|
|
495
877
|
// Small delay to let tmux window clean up
|
|
496
878
|
await new Promise(r => setTimeout(r, 2000));
|
|
497
|
-
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
879
|
+
await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel, newAutoPause).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
|
|
498
880
|
}
|
|
499
881
|
}
|
|
500
882
|
}
|
|
@@ -747,7 +1129,7 @@ export class FleetManager {
|
|
|
747
1129
|
let idx = 0;
|
|
748
1130
|
while (idx < channels.length) {
|
|
749
1131
|
const batch = channels.slice(idx, idx + concurrency);
|
|
750
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
1132
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
751
1133
|
idx += concurrency;
|
|
752
1134
|
}
|
|
753
1135
|
}
|
|
@@ -758,16 +1140,26 @@ export class FleetManager {
|
|
|
758
1140
|
const classicCount = this.classicChannels?.getAll().length ?? 0;
|
|
759
1141
|
const total = Object.keys(fleet.instances).length + classicCount;
|
|
760
1142
|
const started = this.daemons.size;
|
|
761
|
-
const
|
|
1143
|
+
const allNotRunning = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
1144
|
+
const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
|
|
1145
|
+
const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
|
|
762
1146
|
const generalName = this.findGeneralInstance();
|
|
763
1147
|
const generalThreadId = generalName ? fleet.instances[generalName]?.topic_id : undefined;
|
|
764
1148
|
const { createRequire } = await import("node:module");
|
|
765
1149
|
const _require = createRequire(import.meta.url);
|
|
766
1150
|
const agendVersion = _require("../package.json").version ?? "unknown";
|
|
767
1151
|
if (this.adapter && fleet.channel?.group_id) {
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1152
|
+
let text;
|
|
1153
|
+
if (failedNames.length === 0 && pausedNames.length === 0) {
|
|
1154
|
+
text = t("fleet.ready", started, total, agendVersion);
|
|
1155
|
+
}
|
|
1156
|
+
else if (failedNames.length === 0) {
|
|
1157
|
+
text = t("fleet.ready", started, total, agendVersion) + `\n⏸ Paused: ${pausedNames.join(", ")}`;
|
|
1158
|
+
}
|
|
1159
|
+
else {
|
|
1160
|
+
text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
|
|
1161
|
+
+ (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
|
|
1162
|
+
}
|
|
771
1163
|
this.adapter.sendText(String(fleet.channel.group_id), text, {
|
|
772
1164
|
threadId: generalThreadId != null ? String(generalThreadId) : undefined,
|
|
773
1165
|
}).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
|
|
@@ -952,6 +1344,10 @@ export class FleetManager {
|
|
|
952
1344
|
await this.handleInboundMessage(msg);
|
|
953
1345
|
}, this.logger, "adapter.message"));
|
|
954
1346
|
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
1347
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1348
|
+
return;
|
|
1349
|
+
if (await this.handleModelSelection(data))
|
|
1350
|
+
return;
|
|
955
1351
|
if (data.callbackData.startsWith("hang:")) {
|
|
956
1352
|
const parts = data.callbackData.split(":");
|
|
957
1353
|
const action = parts[1];
|
|
@@ -990,13 +1386,15 @@ export class FleetManager {
|
|
|
990
1386
|
// Handle classic bot slash commands (/start, /stop, /chat, /compact, /save, /load)
|
|
991
1387
|
this.adapter.on("slash_command", safeHandler(async (data) => {
|
|
992
1388
|
if (data.command === "start") {
|
|
993
|
-
|
|
994
|
-
await data.respond(reply);
|
|
1389
|
+
await this.handleClassicStartSlash(data, adapterId);
|
|
995
1390
|
}
|
|
996
1391
|
else if (data.command === "stop") {
|
|
997
1392
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
998
1393
|
await data.respond(reply);
|
|
999
1394
|
}
|
|
1395
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1396
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1397
|
+
}
|
|
1000
1398
|
else if (data.command === "chat") {
|
|
1001
1399
|
const text = data.text ?? "";
|
|
1002
1400
|
if (!text) {
|
|
@@ -1052,6 +1450,9 @@ export class FleetManager {
|
|
|
1052
1450
|
const result = await this.topicCommands.sendCompact(name);
|
|
1053
1451
|
await data.respond(result);
|
|
1054
1452
|
}
|
|
1453
|
+
else if (data.command === "model") {
|
|
1454
|
+
await this.handleModelSlash(data, adapterId);
|
|
1455
|
+
}
|
|
1055
1456
|
else if (data.command === "cancel") {
|
|
1056
1457
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1057
1458
|
if (!name) {
|
|
@@ -1235,6 +1636,10 @@ export class FleetManager {
|
|
|
1235
1636
|
await this.handleInboundMessage(msg);
|
|
1236
1637
|
}, this.logger, `adapter[${adapterId}].message`));
|
|
1237
1638
|
adapter.on("callback_query", safeHandler(async (data) => {
|
|
1639
|
+
if (await this.handleClassicBackendSelection(data))
|
|
1640
|
+
return;
|
|
1641
|
+
if (await this.handleModelSelection(data))
|
|
1642
|
+
return;
|
|
1238
1643
|
if (data.callbackData.startsWith("hang:")) {
|
|
1239
1644
|
const parts = data.callbackData.split(":");
|
|
1240
1645
|
const action = parts[1];
|
|
@@ -1269,13 +1674,15 @@ export class FleetManager {
|
|
|
1269
1674
|
// Slash commands: classic bot + admin commands
|
|
1270
1675
|
adapter.on("slash_command", safeHandler(async (data) => {
|
|
1271
1676
|
if (data.command === "start") {
|
|
1272
|
-
|
|
1273
|
-
await data.respond(reply);
|
|
1677
|
+
await this.handleClassicStartSlash(data, adapterId);
|
|
1274
1678
|
}
|
|
1275
1679
|
else if (data.command === "stop") {
|
|
1276
1680
|
const reply = await this.handleClassicStop(data.channelId, adapterId);
|
|
1277
1681
|
await data.respond(reply);
|
|
1278
1682
|
}
|
|
1683
|
+
else if (data.command === "pause" || data.command === "wake") {
|
|
1684
|
+
await this.handlePauseWakeSlash(data, adapterId);
|
|
1685
|
+
}
|
|
1279
1686
|
else if (data.command === "chat") {
|
|
1280
1687
|
const text = data.text ?? "";
|
|
1281
1688
|
if (!text) {
|
|
@@ -1322,6 +1729,9 @@ export class FleetManager {
|
|
|
1322
1729
|
this.pasteRawToClassicInstance(name, `/chat load ${filename}`);
|
|
1323
1730
|
await data.respond(t("save.sent", `/chat load ${filename}`, name));
|
|
1324
1731
|
}
|
|
1732
|
+
else if (data.command === "model") {
|
|
1733
|
+
await this.handleModelSlash(data, adapterId);
|
|
1734
|
+
}
|
|
1325
1735
|
else if (data.command === "cancel") {
|
|
1326
1736
|
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
1327
1737
|
if (!name) {
|
|
@@ -1532,10 +1942,22 @@ export class FleetManager {
|
|
|
1532
1942
|
else if (msg.type === "fleet_set_description") {
|
|
1533
1943
|
this.handleSetDescription(name, msg);
|
|
1534
1944
|
}
|
|
1945
|
+
else if (msg.type === "instance_process_state") {
|
|
1946
|
+
this.cacheInstanceProcessStatus(name, msg.status);
|
|
1947
|
+
}
|
|
1948
|
+
else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
|
|
1949
|
+
this.cacheInstanceExecutionState(name, msg);
|
|
1950
|
+
if (msg.type === "instance_state_response") {
|
|
1951
|
+
this.cacheInstanceProcessStatus(name, msg.processStatus);
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1535
1954
|
}, this.logger, `ipc.message[${name}]`));
|
|
1536
1955
|
// Ask daemon for any sessions that registered before we connected
|
|
1537
1956
|
// (fixes race condition where mcp_ready was broadcast before fleet manager connected)
|
|
1538
1957
|
ipc.send({ type: "query_sessions" });
|
|
1958
|
+
// The initial state transition may have happened before FleetManager
|
|
1959
|
+
// connected, so seed the cache instead of waiting for another transition.
|
|
1960
|
+
ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
|
|
1539
1961
|
this.logger.debug({ name }, "Connected to instance IPC");
|
|
1540
1962
|
if (!this.statuslineWatcher.has(name)) {
|
|
1541
1963
|
this.statuslineWatcher.watch(name);
|
|
@@ -1798,9 +2220,21 @@ export class FleetManager {
|
|
|
1798
2220
|
}
|
|
1799
2221
|
}
|
|
1800
2222
|
const channelName = msg.username || chatId;
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
2223
|
+
const requestedBackend = text.slice("/start".length).trim().split(/\s+/, 1)[0] || undefined;
|
|
2224
|
+
if (requestedBackend) {
|
|
2225
|
+
// handleClassicStart binds the instance to this adapter authoritatively.
|
|
2226
|
+
const reply = await this.handleClassicStart(chatId, channelName, msg.userId, undefined, msg.adapterId, requestedBackend);
|
|
2227
|
+
await msgAdapter?.sendText(chatId, reply);
|
|
2228
|
+
}
|
|
2229
|
+
else if (msgAdapter) {
|
|
2230
|
+
await this.beginClassicBackendSelection({
|
|
2231
|
+
command: "start",
|
|
2232
|
+
channelId: chatId,
|
|
2233
|
+
channelName,
|
|
2234
|
+
userId: msg.userId,
|
|
2235
|
+
respond: async (reply) => (await msgAdapter.sendText(chatId, reply)).messageId,
|
|
2236
|
+
}, msgAdapter);
|
|
2237
|
+
}
|
|
1804
2238
|
return;
|
|
1805
2239
|
}
|
|
1806
2240
|
// Handle /stop command
|
|
@@ -1817,6 +2251,20 @@ export class FleetManager {
|
|
|
1817
2251
|
await msgAdapter?.sendText(chatId, reply);
|
|
1818
2252
|
return;
|
|
1819
2253
|
}
|
|
2254
|
+
const pauseWake = parsePauseWakeCommand(text);
|
|
2255
|
+
if (pauseWake) {
|
|
2256
|
+
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
2257
|
+
await msgAdapter?.sendText(chatId, t("permission.denied"));
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
2260
|
+
const name = this.classicChannels.getInstanceByChannel(chatId, msg.adapterId);
|
|
2261
|
+
if (!name) {
|
|
2262
|
+
await msgAdapter?.sendText(chatId, t("classic.no_agent_start"));
|
|
2263
|
+
return;
|
|
2264
|
+
}
|
|
2265
|
+
await msgAdapter?.sendText(chatId, await this.topicCommands.runPauseWake(name, pauseWake.action));
|
|
2266
|
+
return;
|
|
2267
|
+
}
|
|
1820
2268
|
// Handle /compact command (admin only)
|
|
1821
2269
|
if (text === "/compact" || text.startsWith("/compact@")) {
|
|
1822
2270
|
if (!this.classicChannels.isAdmin(msg.userId)) {
|
|
@@ -1950,6 +2398,7 @@ export class FleetManager {
|
|
|
1950
2398
|
user_id: msg.userId,
|
|
1951
2399
|
ts: msg.timestamp.toISOString(),
|
|
1952
2400
|
thread_id: "",
|
|
2401
|
+
adapter_id: msg.adapterId,
|
|
1953
2402
|
source: msg.source,
|
|
1954
2403
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
1955
2404
|
...extraMeta,
|
|
@@ -2042,6 +2491,7 @@ export class FleetManager {
|
|
|
2042
2491
|
user_id: msg.userId,
|
|
2043
2492
|
ts: msg.timestamp.toISOString(),
|
|
2044
2493
|
thread_id: msg.threadId ?? "",
|
|
2494
|
+
adapter_id: msg.adapterId,
|
|
2045
2495
|
source: msg.source,
|
|
2046
2496
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
2047
2497
|
...extraMeta,
|
|
@@ -2198,7 +2648,7 @@ export class FleetManager {
|
|
|
2198
2648
|
}
|
|
2199
2649
|
// ===================== Scheduler =====================
|
|
2200
2650
|
async handleScheduleTrigger(schedule) {
|
|
2201
|
-
const { target, reply_chat_id, reply_thread_id, message, label, id, source } = schedule;
|
|
2651
|
+
const { target, reply_chat_id, reply_thread_id, message, label, id, source, silent } = schedule;
|
|
2202
2652
|
const RATE_LIMIT_DEFER_THRESHOLD = 85;
|
|
2203
2653
|
const rl = this.statuslineWatcher.getRateLimits(target);
|
|
2204
2654
|
if (rl && rl.five_hour_pct > RATE_LIMIT_DEFER_THRESHOLD) {
|
|
@@ -2213,6 +2663,20 @@ export class FleetManager {
|
|
|
2213
2663
|
this.logger.info({ target, scheduleId: id, rateLimitPct: rl.five_hour_pct }, "Schedule deferred due to rate limit");
|
|
2214
2664
|
return;
|
|
2215
2665
|
}
|
|
2666
|
+
// Silent mode: paste directly to tmux pane — no channel message.
|
|
2667
|
+
if (silent) {
|
|
2668
|
+
const ipc = this.instanceIpcClients.get(target);
|
|
2669
|
+
if (ipc) {
|
|
2670
|
+
ipc.send({ type: "raw_paste", content: message });
|
|
2671
|
+
this.scheduler.recordRun(id, "delivered");
|
|
2672
|
+
this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
|
|
2673
|
+
}
|
|
2674
|
+
else {
|
|
2675
|
+
this.scheduler.recordRun(id, "instance_offline", "IPC not connected");
|
|
2676
|
+
this.logger.warn({ target, scheduleId: id }, "Silent schedule: IPC not connected, skipping");
|
|
2677
|
+
}
|
|
2678
|
+
return;
|
|
2679
|
+
}
|
|
2216
2680
|
const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
|
|
2217
2681
|
const retryCount = schedulerDefaults?.retry_count ?? 3;
|
|
2218
2682
|
const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
|
|
@@ -2222,7 +2686,7 @@ export class FleetManager {
|
|
|
2222
2686
|
type: "fleet_schedule_trigger",
|
|
2223
2687
|
payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
|
|
2224
2688
|
meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
|
|
2225
|
-
});
|
|
2689
|
+
}, { waitForIdle: true });
|
|
2226
2690
|
// A scheduled trigger also puts the instance to work — show a cancel button.
|
|
2227
2691
|
void this.sendCancelButton(target);
|
|
2228
2692
|
return true;
|
|
@@ -2281,6 +2745,7 @@ export class FleetManager {
|
|
|
2281
2745
|
case "fleet_schedule_create": {
|
|
2282
2746
|
const params = {
|
|
2283
2747
|
cron: payload.cron,
|
|
2748
|
+
at: payload.at,
|
|
2284
2749
|
message: payload.message,
|
|
2285
2750
|
source: instanceName,
|
|
2286
2751
|
target: payload.target || instanceName,
|
|
@@ -2288,6 +2753,7 @@ export class FleetManager {
|
|
|
2288
2753
|
reply_thread_id: meta.thread_id || null,
|
|
2289
2754
|
label: payload.label,
|
|
2290
2755
|
timezone: payload.timezone,
|
|
2756
|
+
silent: !!(payload.silent),
|
|
2291
2757
|
};
|
|
2292
2758
|
result = this.scheduler.create(params);
|
|
2293
2759
|
break;
|
|
@@ -2408,7 +2874,9 @@ export class FleetManager {
|
|
|
2408
2874
|
switch (op) {
|
|
2409
2875
|
case "create":
|
|
2410
2876
|
return this.scheduler.create({
|
|
2411
|
-
cron: args.cron,
|
|
2877
|
+
cron: args.cron,
|
|
2878
|
+
at: args.at,
|
|
2879
|
+
message: args.message,
|
|
2412
2880
|
source: instance, target: args.target || instance,
|
|
2413
2881
|
reply_chat_id: "", reply_thread_id: null,
|
|
2414
2882
|
label: args.label,
|
|
@@ -2664,73 +3132,92 @@ export class FleetManager {
|
|
|
2664
3132
|
}
|
|
2665
3133
|
}, 5 * 60_000);
|
|
2666
3134
|
}
|
|
2667
|
-
/**
|
|
2668
|
-
|
|
3135
|
+
/**
|
|
3136
|
+
* Patch only values changed in the effective config into the original YAML
|
|
3137
|
+
* document. Unknown keys, explicit overrides and comments remain untouched.
|
|
3138
|
+
*/
|
|
3139
|
+
saveFleetConfig(explicitPatches = []) {
|
|
2669
3140
|
if (!this.fleetConfig || !this.configPath)
|
|
2670
3141
|
return;
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
}
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
3142
|
+
if (!this.savedFleetConfigSnapshot)
|
|
3143
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3144
|
+
// Re-read immediately before patching so an unrelated concurrent/manual
|
|
3145
|
+
// edit is retained. Invalid concurrent YAML is never overwritten.
|
|
3146
|
+
const source = existsSync(this.configPath) ? readFileSync(this.configPath, "utf-8") : "{}\n";
|
|
3147
|
+
this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
|
|
3148
|
+
if (this.rawFleetDocument.errors.length > 0) {
|
|
3149
|
+
throw new Error(`Refusing to overwrite invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
|
|
3150
|
+
}
|
|
3151
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3152
|
+
this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
|
|
3153
|
+
// Settings edits are expressed against the raw config. Persist them even
|
|
3154
|
+
// when the chosen override equals the inherited effective value, a case
|
|
3155
|
+
// the before/after runtime diff cannot observe.
|
|
3156
|
+
for (const patch of explicitPatches) {
|
|
3157
|
+
if (patch.remove) {
|
|
3158
|
+
this.rawFleetDocument.deleteIn(patch.path);
|
|
3159
|
+
}
|
|
3160
|
+
else {
|
|
3161
|
+
const before = this.rawFleetDocument.getIn(patch.path);
|
|
3162
|
+
this.patchFleetDocument(this.rawFleetDocument, patch.path, before, patch.value);
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
const output = String(this.rawFleetDocument);
|
|
3166
|
+
const tempPath = `${this.configPath}.tmp-${process.pid}`;
|
|
3167
|
+
writeFileSync(tempPath, output, "utf-8");
|
|
3168
|
+
if (existsSync(this.configPath))
|
|
3169
|
+
chmodSync(tempPath, statSync(this.configPath).mode);
|
|
3170
|
+
renameSync(tempPath, this.configPath);
|
|
3171
|
+
this.rawFleetConfig = loadRawFleetConfig(this.configPath);
|
|
3172
|
+
this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
|
|
3173
|
+
this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
|
|
3174
|
+
}
|
|
3175
|
+
patchFleetDocument(document, path, before, after) {
|
|
3176
|
+
if (Object.is(before, after))
|
|
3177
|
+
return;
|
|
3178
|
+
if (Array.isArray(before) && Array.isArray(after)) {
|
|
3179
|
+
const shared = Math.min(before.length, after.length);
|
|
3180
|
+
for (let i = 0; i < shared; i++) {
|
|
3181
|
+
this.patchFleetDocument(document, [...path, i], before[i], after[i]);
|
|
3182
|
+
}
|
|
3183
|
+
// Remove from the end so YAML sequence indexes do not shift underneath us.
|
|
3184
|
+
for (let i = before.length - 1; i >= after.length; i--)
|
|
3185
|
+
document.deleteIn([...path, i]);
|
|
3186
|
+
for (let i = shared; i < after.length; i++)
|
|
3187
|
+
document.setIn([...path, i], after[i]);
|
|
3188
|
+
return;
|
|
3189
|
+
}
|
|
3190
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3191
|
+
if (isRecord(before) && isRecord(after)) {
|
|
3192
|
+
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
3193
|
+
for (const key of keys) {
|
|
3194
|
+
// `channel` is a derived alias when the raw file uses `channels`.
|
|
3195
|
+
if (path.length === 0 && key === "channel" && this.rawFleetConfig.channels)
|
|
3196
|
+
continue;
|
|
3197
|
+
// Conversely, `channels` is a normalized alias for a legacy `channel`.
|
|
3198
|
+
// Keep the user's original shape unless the caller explicitly removed
|
|
3199
|
+
// `channel` (the Settings channels endpoint intentionally migrates it).
|
|
3200
|
+
if (path.length === 0 && key === "channels" && this.rawFleetConfig.channel && !this.rawFleetConfig.channels && after.channel !== undefined)
|
|
3201
|
+
continue;
|
|
3202
|
+
this.patchFleetDocument(document, [...path, key], before[key], after[key]);
|
|
3203
|
+
}
|
|
3204
|
+
return;
|
|
3205
|
+
}
|
|
3206
|
+
if (after === undefined) {
|
|
3207
|
+
document.deleteIn(path);
|
|
3208
|
+
}
|
|
3209
|
+
else if (path.length === 0) {
|
|
3210
|
+
document.contents = document.createNode(after);
|
|
3211
|
+
}
|
|
3212
|
+
else {
|
|
3213
|
+
const currentNode = document.getIn(path, true);
|
|
3214
|
+
if (isScalar(currentNode) && (after === null || typeof after !== "object")) {
|
|
3215
|
+
currentNode.value = after;
|
|
3216
|
+
}
|
|
3217
|
+
else {
|
|
3218
|
+
document.setIn(path, after);
|
|
3219
|
+
}
|
|
3220
|
+
}
|
|
2734
3221
|
}
|
|
2735
3222
|
async removeInstance(name) {
|
|
2736
3223
|
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
@@ -2771,8 +3258,14 @@ export class FleetManager {
|
|
|
2771
3258
|
}
|
|
2772
3259
|
}
|
|
2773
3260
|
startStatuslineWatcher(name) {
|
|
3261
|
+
if (this.lifecycle.isPaused(name))
|
|
3262
|
+
return;
|
|
2774
3263
|
this.statuslineWatcher.watch(name);
|
|
2775
3264
|
}
|
|
3265
|
+
stopStatuslineWatcher(name) {
|
|
3266
|
+
// Pausing stops I/O but retains the last observed limits for status views.
|
|
3267
|
+
this.statuslineWatcher.unwatch(name, true);
|
|
3268
|
+
}
|
|
2776
3269
|
reactMessageStatus(instanceName, chatId, messageId, emoji) {
|
|
2777
3270
|
// React via the adapter BOUND to this instance — NOT the first discord world.
|
|
2778
3271
|
// Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
|
|
@@ -3123,7 +3616,7 @@ export class FleetManager {
|
|
|
3123
3616
|
return [];
|
|
3124
3617
|
}
|
|
3125
3618
|
}
|
|
3126
|
-
async sendHangNotification(instanceName) {
|
|
3619
|
+
async sendHangNotification(instanceName, unchangedForMs) {
|
|
3127
3620
|
const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
|
|
3128
3621
|
if (!adapter)
|
|
3129
3622
|
return;
|
|
@@ -3132,11 +3625,18 @@ export class FleetManager {
|
|
|
3132
3625
|
if (!groupId)
|
|
3133
3626
|
return;
|
|
3134
3627
|
const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
3628
|
+
const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
|
|
3629
|
+
const configuredMinutes = instanceHangConfig?.timeout_minutes
|
|
3630
|
+
?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
|
|
3631
|
+
?? 15;
|
|
3632
|
+
const unchangedMinutes = unchangedForMs == null
|
|
3633
|
+
? configuredMinutes
|
|
3634
|
+
: Math.max(1, Math.floor(unchangedForMs / 60_000));
|
|
3135
3635
|
this.setTopicIcon(instanceName, "red");
|
|
3136
3636
|
await adapter.notifyAlert(String(groupId), {
|
|
3137
3637
|
type: "hang",
|
|
3138
3638
|
instanceName,
|
|
3139
|
-
message: `⚠️ ${instanceName}
|
|
3639
|
+
message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
|
|
3140
3640
|
choices: [
|
|
3141
3641
|
{ id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
|
|
3142
3642
|
{ id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
|
|
@@ -3605,13 +4105,25 @@ When users create specialized instances, suggest these configurations:
|
|
|
3605
4105
|
const inboxDir = join(getAgendHome(), "workspaces", instanceName, "inbox");
|
|
3606
4106
|
mkdirSync(inboxDir, { recursive: true });
|
|
3607
4107
|
const dest = join(inboxDir, basename(tmpPath));
|
|
4108
|
+
// Copy to destination — failure means this attachment is skipped
|
|
3608
4109
|
try {
|
|
3609
|
-
renameSync(tmpPath, dest);
|
|
3610
|
-
}
|
|
3611
|
-
catch {
|
|
3612
4110
|
copyFileSync(tmpPath, dest);
|
|
4111
|
+
}
|
|
4112
|
+
catch (copyErr) {
|
|
4113
|
+
try {
|
|
4114
|
+
unlinkSync(dest);
|
|
4115
|
+
}
|
|
4116
|
+
catch { } // clean partial
|
|
4117
|
+
this.logger.warn({ err: copyErr.message, instanceName, dest }, "Attachment copy failed — skipping");
|
|
4118
|
+
continue;
|
|
4119
|
+
}
|
|
4120
|
+
// Cleanup source — failure is non-fatal (dest already valid)
|
|
4121
|
+
try {
|
|
3613
4122
|
unlinkSync(tmpPath);
|
|
3614
4123
|
}
|
|
4124
|
+
catch (cleanupErr) {
|
|
4125
|
+
this.logger.debug({ tmpPath, err: cleanupErr.message }, "Orphan tmp not cleaned");
|
|
4126
|
+
}
|
|
3615
4127
|
const savedKind = att.kind === "sticker" ? "photo" : att.kind;
|
|
3616
4128
|
paths.push(dest);
|
|
3617
4129
|
if (paths.length === 1)
|
|
@@ -3644,6 +4156,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3644
4156
|
user_id: msg.userId,
|
|
3645
4157
|
ts: msg.timestamp.toISOString(),
|
|
3646
4158
|
thread_id: msg.threadId ?? "",
|
|
4159
|
+
...(msg.adapterId ? { adapter_id: msg.adapterId } : {}),
|
|
3647
4160
|
source: msg.source,
|
|
3648
4161
|
...extraMeta,
|
|
3649
4162
|
...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
|
|
@@ -3685,23 +4198,307 @@ When users create specialized instances, suggest these configurations:
|
|
|
3685
4198
|
ipc.send({ type: "raw_paste", content: text });
|
|
3686
4199
|
this.logger.info({ instanceName, text: text.slice(0, 100) }, "Raw paste sent to classic instance");
|
|
3687
4200
|
}
|
|
4201
|
+
/** Resolve the backend name configured for an instance (fleet or classic). */
|
|
4202
|
+
backendNameForInstance(instanceName) {
|
|
4203
|
+
const fleetCfg = this.fleetConfig?.instances[instanceName];
|
|
4204
|
+
if (fleetCfg?.backend)
|
|
4205
|
+
return fleetCfg.backend;
|
|
4206
|
+
const classic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined
|
|
4207
|
+
? this.classicChannels?.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend)
|
|
4208
|
+
: undefined;
|
|
4209
|
+
return classic ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
4210
|
+
}
|
|
4211
|
+
/** Best-effort model list for `/model` (empty ⇒ caller falls back to free-text). Never throws. */
|
|
4212
|
+
async getModelOptions(instanceName) {
|
|
4213
|
+
try {
|
|
4214
|
+
const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
|
|
4215
|
+
if (!backend.listModels)
|
|
4216
|
+
return [];
|
|
4217
|
+
return (await backend.listModels({
|
|
4218
|
+
workingDirectory: this.fleetConfig?.instances[instanceName]?.working_directory ?? "",
|
|
4219
|
+
instanceDir: this.getInstanceDir(instanceName),
|
|
4220
|
+
instanceName,
|
|
4221
|
+
mcpServers: {},
|
|
4222
|
+
})) ?? [];
|
|
4223
|
+
}
|
|
4224
|
+
catch (err) {
|
|
4225
|
+
this.logger.warn({ err, instanceName }, "listModels failed");
|
|
4226
|
+
return [];
|
|
4227
|
+
}
|
|
4228
|
+
}
|
|
4229
|
+
/** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
|
|
4230
|
+
async handleModelSlash(data, adapterId) {
|
|
4231
|
+
if (!this.isFleetAdmin(data.userId, adapterId)) {
|
|
4232
|
+
await data.respond(t("admin.required"));
|
|
4233
|
+
return;
|
|
4234
|
+
}
|
|
4235
|
+
const name = this.resolveSlashTarget(data.channelId, adapterId);
|
|
4236
|
+
if (!name) {
|
|
4237
|
+
await data.respond(t("classic.no_agent"));
|
|
4238
|
+
return;
|
|
4239
|
+
}
|
|
4240
|
+
const requested = (typeof data.options?.name === "string" ? data.options.name.trim() : "")
|
|
4241
|
+
|| (data.text?.trim() ?? "");
|
|
4242
|
+
if (requested) {
|
|
4243
|
+
await data.respond(await this.applyModel(name, requested));
|
|
4244
|
+
return;
|
|
4245
|
+
}
|
|
4246
|
+
// No arg → menu. Menu is DC-only this round (respondChoices); TG uses `/model <name>`.
|
|
4247
|
+
if (!data.respondChoices) {
|
|
4248
|
+
await data.respond("Usage: /model <name> — e.g. /model sonnet");
|
|
4249
|
+
return;
|
|
4250
|
+
}
|
|
4251
|
+
const options = await this.getModelOptions(name);
|
|
4252
|
+
if (options.length === 0) {
|
|
4253
|
+
await data.respond(`No model list available for ${name}. Type \`/model <name>\` to set one directly.`);
|
|
4254
|
+
return;
|
|
4255
|
+
}
|
|
4256
|
+
const nonce = randomBytes(6).toString("hex");
|
|
4257
|
+
const choices = options.slice(0, 25).map(o => ({
|
|
4258
|
+
id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
|
|
4259
|
+
label: o.description ? `${o.label} — ${o.description}` : o.label,
|
|
4260
|
+
}));
|
|
4261
|
+
const timer = setTimeout(() => this.pendingModelSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
4262
|
+
timer.unref?.();
|
|
4263
|
+
this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
|
|
4264
|
+
try {
|
|
4265
|
+
await data.respondChoices(`Choose a model for ${name}:`, choices);
|
|
4266
|
+
}
|
|
4267
|
+
catch (err) {
|
|
4268
|
+
this.pendingModelSelects.delete(nonce);
|
|
4269
|
+
clearTimeout(timer);
|
|
4270
|
+
this.logger.warn({ err, instanceName: name }, "model menu failed");
|
|
4271
|
+
await data.respond("Usage: /model <name> — e.g. /model sonnet");
|
|
4272
|
+
}
|
|
4273
|
+
}
|
|
4274
|
+
/** Consume a `/model` selection callback. Returns true for all model-select ids (incl. stale). */
|
|
4275
|
+
async handleModelSelection(data) {
|
|
4276
|
+
if (!data.callbackData.startsWith(MODEL_SELECT_CALLBACK_PREFIX))
|
|
4277
|
+
return false;
|
|
4278
|
+
const match = data.callbackData.match(/^model-select:([0-9a-f]+):(.+)$/);
|
|
4279
|
+
if (!match)
|
|
4280
|
+
return true;
|
|
4281
|
+
const pending = this.pendingModelSelects.get(match[1]);
|
|
4282
|
+
if (!pending)
|
|
4283
|
+
return true;
|
|
4284
|
+
// Only the admin who opened the menu, in the same channel, may consume it.
|
|
4285
|
+
if (data.userId && data.userId !== pending.userId)
|
|
4286
|
+
return true;
|
|
4287
|
+
const cbChannel = data.threadId ?? data.chatId;
|
|
4288
|
+
if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
|
|
4289
|
+
return true;
|
|
4290
|
+
this.pendingModelSelects.delete(match[1]);
|
|
4291
|
+
clearTimeout(pending.timer);
|
|
4292
|
+
const result = await this.applyModel(pending.instanceName, match[2]);
|
|
4293
|
+
await pending.respond(result).catch(() => { });
|
|
4294
|
+
return true;
|
|
4295
|
+
}
|
|
4296
|
+
/** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
|
|
4297
|
+
async applyModel(instanceName, model) {
|
|
4298
|
+
const backendName = this.backendNameForInstance(instanceName);
|
|
4299
|
+
let strategy = "restart";
|
|
4300
|
+
try {
|
|
4301
|
+
strategy = createBackend(backendName, this.getInstanceDir(instanceName)).getModelSwitchStrategy?.(model) ?? "restart";
|
|
4302
|
+
}
|
|
4303
|
+
catch { /* default restart */ }
|
|
4304
|
+
const warn = isModelCompatible(backendName, model) ? "" : `⚠️ "${model}" doesn't match ${backendName}'s usual pattern — passing through anyway.\n`;
|
|
4305
|
+
if (strategy === "runtime") {
|
|
4306
|
+
if (!this.instanceIpcClients.get(instanceName))
|
|
4307
|
+
return `${warn}❌ ${instanceName} is not running.`;
|
|
4308
|
+
this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
|
|
4309
|
+
return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).`;
|
|
4310
|
+
}
|
|
4311
|
+
// restart: persist the model so the respawned CLI launches with it.
|
|
4312
|
+
let persisted = false;
|
|
4313
|
+
if (this.fleetConfig?.instances[instanceName]) {
|
|
4314
|
+
this.fleetConfig.instances[instanceName].model = model;
|
|
4315
|
+
this.saveFleetConfig();
|
|
4316
|
+
persisted = true;
|
|
4317
|
+
}
|
|
4318
|
+
else if (this.classicChannels?.setModelByInstance(instanceName, model)) {
|
|
4319
|
+
persisted = true;
|
|
4320
|
+
}
|
|
4321
|
+
if (!persisted)
|
|
4322
|
+
return `${warn}❌ Could not set model for ${instanceName}.`;
|
|
4323
|
+
await this.restartSingleInstance(instanceName);
|
|
4324
|
+
return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.`;
|
|
4325
|
+
}
|
|
3688
4326
|
/** Read recent chat log for agent context */
|
|
3689
4327
|
getRecentChatLog(instanceName, maxLines = 10) {
|
|
3690
4328
|
const logDir = ClassicChannelManager.chatLogDir(instanceName);
|
|
3691
|
-
|
|
4329
|
+
// Use local timezone for date — must match logMessage's write path
|
|
4330
|
+
const tz = process.env.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4331
|
+
const today = new Date().toLocaleString("sv-SE", { timeZone: tz, hour12: false }).slice(0, 10);
|
|
3692
4332
|
const logFile = join(logDir, `${today}.log`);
|
|
3693
4333
|
try {
|
|
3694
4334
|
if (!existsSync(logFile))
|
|
3695
4335
|
return undefined;
|
|
3696
4336
|
const lines = readFileSync(logFile, "utf-8").trim().split("\n");
|
|
3697
|
-
|
|
4337
|
+
// The triggering message is written before forwardToClassicInstance runs
|
|
4338
|
+
// and is included separately under [User message]. Exclude that newest
|
|
4339
|
+
// log entry so the agent does not receive the same message twice. A chat
|
|
4340
|
+
// message may span physical lines, so remove from its timestamped entry
|
|
4341
|
+
// header rather than blindly dropping only the final continuation line.
|
|
4342
|
+
const entryHeader = /^\[\d{4}-\d{2}-\d{2}T[^\]]+\] <.*> /;
|
|
4343
|
+
let currentEntryStart = lines.length - 1;
|
|
4344
|
+
while (currentEntryStart > 0 && !entryHeader.test(lines[currentEntryStart])) {
|
|
4345
|
+
currentEntryStart--;
|
|
4346
|
+
}
|
|
4347
|
+
lines.splice(currentEntryStart);
|
|
4348
|
+
if (lines.length === 0 || maxLines <= 0)
|
|
4349
|
+
return undefined;
|
|
4350
|
+
return lines.slice(-maxLines).join("\n") || undefined;
|
|
3698
4351
|
}
|
|
3699
4352
|
catch {
|
|
3700
4353
|
return undefined;
|
|
3701
4354
|
}
|
|
3702
4355
|
}
|
|
4356
|
+
/** Return a user-facing blocker without mutating ClassicBot state. */
|
|
4357
|
+
validateClassicStart(channelId, userId, guildId, adapterId) {
|
|
4358
|
+
if (!this.classicChannels)
|
|
4359
|
+
return "Classic channel manager not initialized.";
|
|
4360
|
+
if (guildId && !this.classicChannels.isGuildAllowed(guildId)) {
|
|
4361
|
+
const generalId = this.findGeneralInstance(adapterId);
|
|
4362
|
+
if (generalId)
|
|
4363
|
+
this.notifyInstanceTopic(generalId, t("alert.unauth_guild", guildId, userId));
|
|
4364
|
+
return t("classic.not_authorized_guild");
|
|
4365
|
+
}
|
|
4366
|
+
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
4367
|
+
return t("classic.already_active");
|
|
4368
|
+
if (this.routing.resolve(channelId))
|
|
4369
|
+
return t("classic.topic_bound");
|
|
4370
|
+
return undefined;
|
|
4371
|
+
}
|
|
4372
|
+
isBackendInstalled(backend) {
|
|
4373
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4374
|
+
return !!installation && checkBinaryInstalled(installation.binary);
|
|
4375
|
+
}
|
|
4376
|
+
getMissingBackendWarning(backend) {
|
|
4377
|
+
if (!backend)
|
|
4378
|
+
return undefined;
|
|
4379
|
+
const installation = BACKEND_INSTALLATION_INFO[backend];
|
|
4380
|
+
if (!installation || this.isBackendInstalled(backend))
|
|
4381
|
+
return undefined;
|
|
4382
|
+
return t("classic.backend_not_installed", backend, installation.binary, installation.install);
|
|
4383
|
+
}
|
|
4384
|
+
/** Handle Discord's required static slash choice, warning before a likely startup failure. */
|
|
4385
|
+
async handleClassicStartSlash(data, adapterId) {
|
|
4386
|
+
const requestedBackend = typeof data.options?.backend === "string" ? data.options.backend : undefined;
|
|
4387
|
+
if (!requestedBackend) {
|
|
4388
|
+
// beta.31 made this option required. Discord can briefly retain the old
|
|
4389
|
+
// command schema client-side, however, so stale clients may still submit
|
|
4390
|
+
// `/start` without it. Do not resurrect the legacy 60-second component
|
|
4391
|
+
// menu in that case: fail immediately and make the user invoke the newly
|
|
4392
|
+
// registered command, which guarantees an explicit backend choice.
|
|
4393
|
+
await data.respond(t("classic.backend_required"));
|
|
4394
|
+
return;
|
|
4395
|
+
}
|
|
4396
|
+
const warning = this.getMissingBackendWarning(requestedBackend);
|
|
4397
|
+
// Keep the deferred ephemeral response useful even if daemon startup later
|
|
4398
|
+
// fails because the executable is absent. This is advisory, not a gate.
|
|
4399
|
+
if (warning)
|
|
4400
|
+
await data.respond(warning);
|
|
4401
|
+
const reply = await this.handleClassicStart(data.channelId, data.channelName, data.userId, data.guildId, adapterId, requestedBackend);
|
|
4402
|
+
await data.respond(warning ? `${warning}\n\n${reply}` : reply);
|
|
4403
|
+
}
|
|
4404
|
+
/** Present platform-native backend choices, then start on selection or timeout. */
|
|
4405
|
+
async beginClassicBackendSelection(data, adapter) {
|
|
4406
|
+
const adapterId = adapter.id;
|
|
4407
|
+
const blocker = this.validateClassicStart(data.channelId, data.userId, data.guildId, adapterId);
|
|
4408
|
+
if (blocker) {
|
|
4409
|
+
await data.respond(blocker);
|
|
4410
|
+
return;
|
|
4411
|
+
}
|
|
4412
|
+
const nonce = randomBytes(6).toString("hex");
|
|
4413
|
+
const choices = getClassicBackendChoices().map(choice => ({
|
|
4414
|
+
id: `${CLASSIC_BACKEND_CALLBACK_PREFIX}${nonce}:${choice.id}`,
|
|
4415
|
+
label: `${this.isBackendInstalled(choice.id) ? "✅" : "❌"} ${choice.label}`,
|
|
4416
|
+
}));
|
|
4417
|
+
const complete = data.respondChoices
|
|
4418
|
+
? async (text) => { await data.respond(text); }
|
|
4419
|
+
: async (text, messageId) => {
|
|
4420
|
+
if (messageId && adapter.editMessageRemoveButtons) {
|
|
4421
|
+
try {
|
|
4422
|
+
await adapter.editMessageRemoveButtons(data.channelId, messageId, text);
|
|
4423
|
+
return;
|
|
4424
|
+
}
|
|
4425
|
+
catch { /* fall back to a new message */ }
|
|
4426
|
+
}
|
|
4427
|
+
await data.respond(text);
|
|
4428
|
+
};
|
|
4429
|
+
const timer = setTimeout(() => {
|
|
4430
|
+
// Timeout: cancel the selection — do NOT fall back to default.
|
|
4431
|
+
const p = this.pendingClassicStarts.get(nonce);
|
|
4432
|
+
if (p) {
|
|
4433
|
+
this.pendingClassicStarts.delete(nonce);
|
|
4434
|
+
p.complete(t("classic.selection_expired"), p.messageId).catch(() => { });
|
|
4435
|
+
}
|
|
4436
|
+
}, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
|
|
4437
|
+
timer.unref?.();
|
|
4438
|
+
const pending = {
|
|
4439
|
+
channelId: data.channelId,
|
|
4440
|
+
channelName: data.channelName,
|
|
4441
|
+
userId: data.userId,
|
|
4442
|
+
guildId: data.guildId,
|
|
4443
|
+
adapterId,
|
|
4444
|
+
timer,
|
|
4445
|
+
complete,
|
|
4446
|
+
};
|
|
4447
|
+
this.pendingClassicStarts.set(nonce, pending);
|
|
4448
|
+
try {
|
|
4449
|
+
pending.messageId = data.respondChoices
|
|
4450
|
+
? await data.respondChoices(t("classic.choose_backend"), choices)
|
|
4451
|
+
: await adapter.promptUser(data.channelId, t("classic.choose_backend"), choices);
|
|
4452
|
+
}
|
|
4453
|
+
catch (err) {
|
|
4454
|
+
// A menu transport failure should not make /start unusable: consume the
|
|
4455
|
+
// pending request and immediately use the configured default.
|
|
4456
|
+
this.logger.warn({ err, channelId: data.channelId, adapterId }, "Classic backend menu failed; using default");
|
|
4457
|
+
await this.finishClassicBackendSelection(nonce);
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
/** Consume a selection callback. Returns true for all ClassicBot callback IDs, including stale ones. */
|
|
4461
|
+
async handleClassicBackendSelection(data) {
|
|
4462
|
+
if (!data.callbackData.startsWith(CLASSIC_BACKEND_CALLBACK_PREFIX))
|
|
4463
|
+
return false;
|
|
4464
|
+
const match = data.callbackData.match(/^classic-backend:([0-9a-f]+):(.+)$/);
|
|
4465
|
+
if (!match)
|
|
4466
|
+
return true;
|
|
4467
|
+
const pending = this.pendingClassicStarts.get(match[1]);
|
|
4468
|
+
if (!pending)
|
|
4469
|
+
return true;
|
|
4470
|
+
// Telegram keyboards are visible to everyone in a group. Only the user who
|
|
4471
|
+
// issued /start may consume the pending selection.
|
|
4472
|
+
if (data.userId && data.userId !== pending.userId)
|
|
4473
|
+
return true;
|
|
4474
|
+
const callbackChannelId = data.threadId ?? data.chatId;
|
|
4475
|
+
if (callbackChannelId !== pending.channelId && data.chatId !== pending.channelId)
|
|
4476
|
+
return true;
|
|
4477
|
+
await this.finishClassicBackendSelection(match[1], match[2]);
|
|
4478
|
+
return true;
|
|
4479
|
+
}
|
|
4480
|
+
/** Atomically claim one pending request so timeout/click races create at most one instance. */
|
|
4481
|
+
async finishClassicBackendSelection(nonce, backend) {
|
|
4482
|
+
const pending = this.pendingClassicStarts.get(nonce);
|
|
4483
|
+
if (!pending)
|
|
4484
|
+
return;
|
|
4485
|
+
this.pendingClassicStarts.delete(nonce);
|
|
4486
|
+
clearTimeout(pending.timer);
|
|
4487
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4488
|
+
const effectiveBackend = selectedBackend
|
|
4489
|
+
?? this.classicChannels?.getDefaults().backend
|
|
4490
|
+
?? this.fleetConfig?.defaults?.backend
|
|
4491
|
+
?? "claude-code";
|
|
4492
|
+
const warning = this.getMissingBackendWarning(effectiveBackend);
|
|
4493
|
+
// Show the warning before starting so it survives a missing-binary startup
|
|
4494
|
+
// failure. The selected backend is still attempted as requested.
|
|
4495
|
+
if (warning)
|
|
4496
|
+
await pending.complete(warning, pending.messageId);
|
|
4497
|
+
const reply = await this.handleClassicStart(pending.channelId, pending.channelName, pending.userId, pending.guildId, pending.adapterId, selectedBackend);
|
|
4498
|
+
await pending.complete(warning ? `${warning}\n\n${reply}` : reply, pending.messageId);
|
|
4499
|
+
}
|
|
3703
4500
|
/** Start a classic channel instance with lightweight config */
|
|
3704
|
-
async startClassicInstance(instanceName, backend, preTaskCommand, model) {
|
|
4501
|
+
async startClassicInstance(instanceName, backend, preTaskCommand, model, autoPauseAfter) {
|
|
3705
4502
|
if (this.daemons.has(instanceName))
|
|
3706
4503
|
return;
|
|
3707
4504
|
const workDir = join(getAgendHome(), "workspaces", instanceName);
|
|
@@ -3713,41 +4510,34 @@ When users create specialized instances, suggest these configurations:
|
|
|
3713
4510
|
lightweight: true,
|
|
3714
4511
|
...(backend ? { backend } : {}),
|
|
3715
4512
|
...(model ? { model } : {}),
|
|
4513
|
+
...(autoPauseAfter !== undefined ? { auto_pause_after: autoPauseAfter } : {}),
|
|
3716
4514
|
...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
|
|
3717
4515
|
};
|
|
3718
4516
|
const topicMode = this.fleetConfig?.channel?.mode === "topic";
|
|
3719
4517
|
await this.startInstance(instanceName, config, topicMode);
|
|
3720
4518
|
}
|
|
3721
4519
|
/** Handle /start slash command — register classic channel */
|
|
3722
|
-
async handleClassicStart(channelId, channelName, userId, guildId, adapterId) {
|
|
3723
|
-
|
|
4520
|
+
async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
|
|
4521
|
+
const blocker = this.validateClassicStart(channelId, userId, guildId, adapterId);
|
|
4522
|
+
if (blocker)
|
|
4523
|
+
return blocker;
|
|
4524
|
+
const classicChannels = this.classicChannels;
|
|
4525
|
+
if (!classicChannels)
|
|
3724
4526
|
return "Classic channel manager not initialized.";
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
}
|
|
3730
|
-
return t("classic.not_authorized_guild");
|
|
3731
|
-
}
|
|
3732
|
-
// Per-bot check: a second bot may /start in the same channel (own agent).
|
|
3733
|
-
if (this.classicChannels.isClassicChannel(channelId, adapterId))
|
|
3734
|
-
return t("classic.already_active");
|
|
3735
|
-
// Classic no longer lives in the routing engine, so this only guards against
|
|
3736
|
-
// a fleet topic-mode instance colliding with the channel.
|
|
3737
|
-
if (this.routing.resolve(channelId))
|
|
3738
|
-
return t("classic.topic_bound");
|
|
3739
|
-
const instanceName = this.classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
3740
|
-
this.classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId);
|
|
4527
|
+
const instanceName = classicChannels.deriveInstanceName(channelName || channelId, channelId, adapterId);
|
|
4528
|
+
clearPausedMarker(this.getInstanceDir(instanceName));
|
|
4529
|
+
const selectedBackend = isSelectableClassicBackend(backend) ? backend : undefined;
|
|
4530
|
+
classicChannels.register(channelId, adapterId, instanceName, channelName || channelId, userId, selectedBackend);
|
|
3741
4531
|
// Bind this classic instance to the bot that started it (authoritative), so
|
|
3742
4532
|
// replies/cancel go out through that bot even though every same-guild bot
|
|
3743
4533
|
// also sees the channel's messages.
|
|
3744
4534
|
if (adapterId)
|
|
3745
4535
|
this.bindInstanceAdapter(instanceName, adapterId);
|
|
3746
|
-
await this.startClassicInstance(instanceName,
|
|
4536
|
+
await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
|
|
3747
4537
|
this.reregisterClassicChannels();
|
|
3748
4538
|
// Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
|
|
3749
|
-
if (guildId && !
|
|
3750
|
-
|
|
4539
|
+
if (guildId && !classicChannels.isCollab(channelId, adapterId)) {
|
|
4540
|
+
classicChannels.toggleCollab(channelId, adapterId);
|
|
3751
4541
|
}
|
|
3752
4542
|
this.logger.info({ channelId, adapterId, instanceName, userId }, "Classic channel started");
|
|
3753
4543
|
return t("classic.started");
|
|
@@ -3761,6 +4551,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
3761
4551
|
return t("classic.no_agent");
|
|
3762
4552
|
this.instanceWorldBinding.delete(ch.instanceName);
|
|
3763
4553
|
await this.stopInstance(ch.instanceName).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to stop classic instance"));
|
|
4554
|
+
clearPausedMarker(this.getInstanceDir(ch.instanceName));
|
|
3764
4555
|
this.reregisterClassicChannels();
|
|
3765
4556
|
this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
|
|
3766
4557
|
return t("classic.stopped");
|
|
@@ -3804,6 +4595,9 @@ When users create specialized instances, suggest these configurations:
|
|
|
3804
4595
|
clearInterval(this.classicReloadTimer);
|
|
3805
4596
|
this.classicReloadTimer = null;
|
|
3806
4597
|
}
|
|
4598
|
+
for (const pending of this.pendingClassicStarts.values())
|
|
4599
|
+
clearTimeout(pending.timer);
|
|
4600
|
+
this.pendingClassicStarts.clear();
|
|
3807
4601
|
this.topicArchiver.stop();
|
|
3808
4602
|
this.scheduler?.shutdown();
|
|
3809
4603
|
// Stop instances in parallel batches to avoid long sequential waits.
|
|
@@ -4098,7 +4892,7 @@ When users create specialized instances, suggest these configurations:
|
|
|
4098
4892
|
let idx = 0;
|
|
4099
4893
|
while (idx < channels.length) {
|
|
4100
4894
|
const batch = channels.slice(idx, idx + concurrency);
|
|
4101
|
-
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4895
|
+
await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
|
|
4102
4896
|
idx += concurrency;
|
|
4103
4897
|
}
|
|
4104
4898
|
}
|
|
@@ -4110,13 +4904,23 @@ When users create specialized instances, suggest these configurations:
|
|
|
4110
4904
|
if (groupId && this.adapter) {
|
|
4111
4905
|
const total = Object.keys(fleet.instances).length;
|
|
4112
4906
|
const started = this.daemons.size;
|
|
4113
|
-
const
|
|
4907
|
+
const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
|
|
4908
|
+
const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
|
|
4909
|
+
const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
|
|
4114
4910
|
const { createRequire } = await import("node:module");
|
|
4115
4911
|
const _require2 = createRequire(import.meta.url);
|
|
4116
4912
|
const agendVersion2 = _require2("../package.json").version ?? "unknown";
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4913
|
+
let restartText;
|
|
4914
|
+
if (failedNames.length === 0 && pausedNames2.length === 0) {
|
|
4915
|
+
restartText = t("fleet.ready", started, total, agendVersion2);
|
|
4916
|
+
}
|
|
4917
|
+
else if (failedNames.length === 0) {
|
|
4918
|
+
restartText = t("fleet.ready", started, total, agendVersion2) + `\n⏸ Paused: ${pausedNames2.join(", ")}`;
|
|
4919
|
+
}
|
|
4920
|
+
else {
|
|
4921
|
+
restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
|
|
4922
|
+
+ (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
|
|
4923
|
+
}
|
|
4120
4924
|
await this.adapter.sendText(String(groupId), restartText, notifyOpts)
|
|
4121
4925
|
.catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
|
|
4122
4926
|
// Notify each instance's channel — staggered to avoid rate limit storm
|
|
@@ -4328,8 +5132,31 @@ When users create specialized instances, suggest these configurations:
|
|
|
4328
5132
|
// Fleet API (enriched for agent board)
|
|
4329
5133
|
if (req.method === "GET" && req.url === "/api/fleet") {
|
|
4330
5134
|
const sysInfo = this.getSysInfo();
|
|
4331
|
-
const
|
|
5135
|
+
const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
|
|
5136
|
+
const fleetNames = new Set(fleetInstances.map(inst => inst.name));
|
|
5137
|
+
const classicInstances = (this.classicChannels?.getAll() ?? [])
|
|
5138
|
+
.filter(channel => !fleetNames.has(channel.instanceName))
|
|
5139
|
+
.map(channel => ({
|
|
5140
|
+
name: channel.instanceName,
|
|
5141
|
+
status: this.getInstanceStatus(channel.instanceName),
|
|
5142
|
+
state: this.getInstanceExecutionState(channel.instanceName),
|
|
5143
|
+
ipc: this.instanceIpcClients.has(channel.instanceName),
|
|
5144
|
+
costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
|
|
5145
|
+
rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
|
|
5146
|
+
classic: true,
|
|
5147
|
+
classicName: channel.name,
|
|
5148
|
+
channelId: channel.channelId,
|
|
5149
|
+
adapterId: channel.adapterId ?? null,
|
|
5150
|
+
}));
|
|
5151
|
+
const enriched = [...fleetInstances, ...classicInstances].map(inst => {
|
|
4332
5152
|
const config = this.fleetConfig?.instances[inst.name];
|
|
5153
|
+
const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
|
|
5154
|
+
const lastActivity = inst.classic
|
|
5155
|
+
? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
|
|
5156
|
+
: (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
|
|
5157
|
+
const backend = inst.classic
|
|
5158
|
+
? this.classicChannels?.getBackendByInstance(inst.name, this.fleetConfig?.defaults.backend) ?? "claude-code"
|
|
5159
|
+
: config?.backend ?? "claude-code";
|
|
4333
5160
|
// Find claimed tasks for this instance
|
|
4334
5161
|
let currentTask = null;
|
|
4335
5162
|
try {
|
|
@@ -4342,13 +5169,16 @@ When users create specialized instances, suggest these configurations:
|
|
|
4342
5169
|
}
|
|
4343
5170
|
return {
|
|
4344
5171
|
...inst,
|
|
4345
|
-
description: config?.description ?? null,
|
|
4346
|
-
backend
|
|
5172
|
+
description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
|
|
5173
|
+
backend,
|
|
4347
5174
|
tool_set: config?.tool_set ?? "full",
|
|
4348
5175
|
general_topic: config?.general_topic ?? false,
|
|
4349
|
-
|
|
5176
|
+
// User activity is persisted by the daemon, so both the board and
|
|
5177
|
+
// auto-pause retain an accurate age across fleet restarts.
|
|
5178
|
+
lastActivity,
|
|
4350
5179
|
currentTask,
|
|
4351
5180
|
idle: this.getInstanceIdle(inst.name),
|
|
5181
|
+
state: this.getInstanceExecutionState(inst.name),
|
|
4352
5182
|
};
|
|
4353
5183
|
});
|
|
4354
5184
|
res.setHeader("Access-Control-Allow-Origin", "*");
|