@suzuke/agend 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.d.ts +1 -1
- package/dist/access-path.js +2 -2
- package/dist/access-path.js.map +1 -1
- package/dist/backend/claude-code.d.ts +1 -0
- package/dist/backend/claude-code.js +6 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +1 -0
- package/dist/backend/codex.js +16 -6
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.d.ts +1 -0
- package/dist/backend/gemini-cli.js +13 -1
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/opencode.d.ts +1 -0
- package/dist/backend/opencode.js +12 -4
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +2 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/access-manager.d.ts +5 -5
- package/dist/channel/access-manager.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +3 -2
- package/dist/channel/adapters/discord.js +11 -4
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +4 -3
- package/dist/channel/adapters/telegram.js +11 -5
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.js +30 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/types.d.ts +8 -7
- package/dist/cli.js +3 -3
- package/dist/cli.js.map +1 -1
- package/dist/daemon.d.ts +1 -1
- package/dist/daemon.js +18 -11
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-context.d.ts +23 -2
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +29 -25
- package/dist/fleet-manager.js +110 -638
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +46 -0
- package/dist/instance-lifecycle.js +283 -0
- package/dist/instance-lifecycle.js.map +1 -0
- package/dist/outbound-handlers.d.ts +30 -0
- package/dist/outbound-handlers.js +206 -0
- package/dist/outbound-handlers.js.map +1 -0
- package/dist/routing-engine.d.ts +22 -0
- package/dist/routing-engine.js +44 -0
- package/dist/routing-engine.js.map +1 -0
- package/dist/safe-async.d.ts +6 -0
- package/dist/safe-async.js +20 -0
- package/dist/safe-async.js.map +1 -0
- package/dist/setup-wizard.js +3 -3
- package/dist/setup-wizard.js.map +1 -1
- package/dist/statusline-watcher.d.ts +32 -0
- package/dist/statusline-watcher.js +64 -0
- package/dist/statusline-watcher.js.map +1 -0
- package/dist/tmux-manager.js +1 -1
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-archiver.d.ts +33 -0
- package/dist/topic-archiver.js +69 -0
- package/dist/topic-archiver.js.map +1 -0
- package/dist/topic-commands.d.ts +3 -2
- package/dist/topic-commands.js +34 -1
- package/dist/topic-commands.js.map +1 -1
- package/dist/types.d.ts +2 -2
- package/package.json +1 -1
package/dist/fleet-manager.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync } from "node:fs";
|
|
2
|
-
import { access } from "node:fs/promises";
|
|
3
2
|
import { createServer } from "node:http";
|
|
4
|
-
import { join, dirname
|
|
3
|
+
import { join, dirname } from "node:path";
|
|
5
4
|
import { homedir } from "node:os";
|
|
6
5
|
import { fileURLToPath } from "node:url";
|
|
7
6
|
import yaml from "js-yaml";
|
|
@@ -20,10 +19,16 @@ import { processAttachments } from "./channel/attachment-handler.js";
|
|
|
20
19
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
21
20
|
import { Scheduler } from "./scheduler/index.js";
|
|
22
21
|
import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
|
|
23
|
-
import { TopicCommands
|
|
22
|
+
import { TopicCommands } from "./topic-commands.js";
|
|
24
23
|
import { DailySummary } from "./daily-summary.js";
|
|
25
24
|
import { WebhookEmitter } from "./webhook-emitter.js";
|
|
26
25
|
import { TmuxControlClient } from "./tmux-control.js";
|
|
26
|
+
import { safeHandler } from "./safe-async.js";
|
|
27
|
+
import { RoutingEngine } from "./routing-engine.js";
|
|
28
|
+
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
29
|
+
import { TopicArchiver } from "./topic-archiver.js";
|
|
30
|
+
import { StatuslineWatcher } from "./statusline-watcher.js";
|
|
31
|
+
import { outboundHandlers } from "./outbound-handlers.js";
|
|
27
32
|
const TMUX_SESSION = "agend";
|
|
28
33
|
export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
29
34
|
if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
|
|
@@ -37,10 +42,13 @@ export function resolveReplyThreadId(argsThreadId, instanceConfig) {
|
|
|
37
42
|
export class FleetManager {
|
|
38
43
|
dataDir;
|
|
39
44
|
children = new Map();
|
|
40
|
-
|
|
45
|
+
lifecycle;
|
|
46
|
+
/** @deprecated Use lifecycle.daemons — kept for backward compat */
|
|
47
|
+
get daemons() { return this.lifecycle.daemons; }
|
|
41
48
|
fleetConfig = null;
|
|
42
49
|
adapter = null;
|
|
43
|
-
|
|
50
|
+
routing = new RoutingEngine();
|
|
51
|
+
get routingTable() { return this.routing.map; }
|
|
44
52
|
instanceIpcClients = new Map();
|
|
45
53
|
scheduler = null;
|
|
46
54
|
configPath = "";
|
|
@@ -50,25 +58,52 @@ export class FleetManager {
|
|
|
50
58
|
sessionRegistry = new Map();
|
|
51
59
|
eventLog = null;
|
|
52
60
|
costGuard = null;
|
|
53
|
-
|
|
54
|
-
instanceRateLimits = new Map();
|
|
61
|
+
statuslineWatcher;
|
|
55
62
|
dailySummary = null;
|
|
56
63
|
webhookEmitter = null;
|
|
57
64
|
// Topic icon + auto-archive state
|
|
58
65
|
topicIcons = {};
|
|
59
66
|
lastActivity = new Map();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
static ARCHIVE_IDLE_MS = 24 * 60 * 60 * 1000; // 24 hours
|
|
67
|
+
topicArchiver;
|
|
68
|
+
controlClient = null;
|
|
63
69
|
// Model failover state
|
|
64
70
|
failoverActive = new Map(); // instance → current failover model
|
|
65
|
-
controlClient = null;
|
|
66
71
|
// Health endpoint
|
|
67
72
|
healthServer = null;
|
|
68
73
|
startedAt = 0;
|
|
69
74
|
constructor(dataDir) {
|
|
70
75
|
this.dataDir = dataDir;
|
|
76
|
+
this.lifecycle = new InstanceLifecycle(this);
|
|
71
77
|
this.topicCommands = new TopicCommands(this);
|
|
78
|
+
this.topicArchiver = new TopicArchiver(this);
|
|
79
|
+
this.statuslineWatcher = new StatuslineWatcher(this);
|
|
80
|
+
}
|
|
81
|
+
// ── ArchiverContext bridge ────────────────────────────────────────────
|
|
82
|
+
lastActivityMs(name) {
|
|
83
|
+
return this.lastActivity.get(name) ?? 0;
|
|
84
|
+
}
|
|
85
|
+
// ── LifecycleContext bridge methods ──────────────────────────────────────
|
|
86
|
+
webhookEmit(event, name) {
|
|
87
|
+
this.webhookEmitter?.emit(event, name);
|
|
88
|
+
}
|
|
89
|
+
// ── SysInfo ────────────────────────────────────────────────────────────
|
|
90
|
+
getSysInfo() {
|
|
91
|
+
const mem = process.memoryUsage();
|
|
92
|
+
const toMB = (b) => Math.round(b / 1024 / 1024 * 10) / 10;
|
|
93
|
+
const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
|
|
94
|
+
name,
|
|
95
|
+
status: this.getInstanceStatus(name),
|
|
96
|
+
ipc: this.instanceIpcClients.has(name),
|
|
97
|
+
costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
|
|
98
|
+
rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
|
|
99
|
+
}));
|
|
100
|
+
return {
|
|
101
|
+
uptime_seconds: Math.floor((Date.now() - this.startedAt) / 1000),
|
|
102
|
+
memory_mb: { rss: toMB(mem.rss), heapUsed: toMB(mem.heapUsed), heapTotal: toMB(mem.heapTotal) },
|
|
103
|
+
instances,
|
|
104
|
+
fleet_cost_cents: this.costGuard?.getFleetTotalCents() ?? 0,
|
|
105
|
+
fleet_cost_limit_cents: this.costGuard?.getLimitCents() ?? 0,
|
|
106
|
+
};
|
|
72
107
|
}
|
|
73
108
|
/** Load fleet.yaml and build routing table */
|
|
74
109
|
loadConfig(configPath) {
|
|
@@ -77,18 +112,10 @@ export class FleetManager {
|
|
|
77
112
|
}
|
|
78
113
|
/** Build topic routing table: { topicId -> RouteTarget } */
|
|
79
114
|
buildRoutingTable() {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
return table;
|
|
83
|
-
for (const [name, inst] of Object.entries(this.fleetConfig.instances)) {
|
|
84
|
-
if (inst.topic_id != null) {
|
|
85
|
-
table.set(inst.topic_id, {
|
|
86
|
-
kind: inst.general_topic ? "general" : "instance",
|
|
87
|
-
name,
|
|
88
|
-
});
|
|
89
|
-
}
|
|
115
|
+
if (this.fleetConfig) {
|
|
116
|
+
this.routing.rebuild(this.fleetConfig);
|
|
90
117
|
}
|
|
91
|
-
return
|
|
118
|
+
return this.routing.map;
|
|
92
119
|
}
|
|
93
120
|
getInstanceDir(name) {
|
|
94
121
|
return join(this.dataDir, "instances", name);
|
|
@@ -107,65 +134,11 @@ export class FleetManager {
|
|
|
107
134
|
}
|
|
108
135
|
}
|
|
109
136
|
async startInstance(name, config, topicMode) {
|
|
110
|
-
|
|
111
|
-
this.logger.info({ name }, "Instance already running, skipping");
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
if (!existsSync(config.working_directory)) {
|
|
115
|
-
this.logger.error({ name, working_directory: config.working_directory }, "Working directory does not exist — skipping instance");
|
|
116
|
-
return;
|
|
117
|
-
}
|
|
118
|
-
const instanceDir = this.getInstanceDir(name);
|
|
119
|
-
mkdirSync(instanceDir, { recursive: true });
|
|
120
|
-
const { Daemon } = await import("./daemon.js");
|
|
121
|
-
const { createBackend } = await import("./backend/factory.js");
|
|
122
|
-
const backendName = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
|
|
123
|
-
const backend = createBackend(backendName, instanceDir);
|
|
124
|
-
const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.controlClient ?? undefined);
|
|
125
|
-
await daemon.start();
|
|
126
|
-
this.daemons.set(name, daemon);
|
|
127
|
-
daemon.on("restart_complete", (data) => {
|
|
128
|
-
this.eventLog?.insert(name, "context_rotation", data);
|
|
129
|
-
this.logger.info({ name, ...data }, "Context restart completed");
|
|
130
|
-
});
|
|
131
|
-
const hangDetector = daemon.getHangDetector();
|
|
132
|
-
if (hangDetector) {
|
|
133
|
-
hangDetector.on("hang", () => {
|
|
134
|
-
this.eventLog?.insert(name, "hang_detected", {});
|
|
135
|
-
this.logger.warn({ name }, "Instance appears hung");
|
|
136
|
-
this.sendHangNotification(name);
|
|
137
|
-
this.webhookEmitter?.emit("hang", name);
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
daemon.on("crash_loop", () => {
|
|
141
|
-
this.eventLog?.insert(name, "crash_loop", {});
|
|
142
|
-
this.logger.error({ name }, "Instance in crash loop — respawn paused");
|
|
143
|
-
this.notifyInstanceTopic(name, `🔴 ${name} keeps crashing shortly after launch — respawn paused. Check rate limits or run \`agend fleet restart\`.`);
|
|
144
|
-
this.setTopicIcon(name, "red");
|
|
145
|
-
});
|
|
146
|
-
this.setTopicIcon(name, "green");
|
|
147
|
-
this.touchActivity(name);
|
|
137
|
+
return this.lifecycle.start(name, config, topicMode);
|
|
148
138
|
}
|
|
149
139
|
async stopInstance(name) {
|
|
150
|
-
this.setTopicIcon(name, "remove");
|
|
151
140
|
this.failoverActive.delete(name);
|
|
152
|
-
|
|
153
|
-
if (daemon) {
|
|
154
|
-
await daemon.stop();
|
|
155
|
-
this.daemons.delete(name);
|
|
156
|
-
}
|
|
157
|
-
else {
|
|
158
|
-
const pidPath = join(this.getInstanceDir(name), "daemon.pid");
|
|
159
|
-
if (existsSync(pidPath)) {
|
|
160
|
-
const pid = parseInt(readFileSync(pidPath, "utf-8").trim(), 10);
|
|
161
|
-
try {
|
|
162
|
-
process.kill(pid, "SIGTERM");
|
|
163
|
-
}
|
|
164
|
-
catch (e) {
|
|
165
|
-
this.logger.debug({ err: e, pid }, "SIGTERM failed for stale process");
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
141
|
+
return this.lifecycle.stop(name);
|
|
169
142
|
}
|
|
170
143
|
/** Load .env file from data dir into process.env */
|
|
171
144
|
loadEnvFile() {
|
|
@@ -226,16 +199,16 @@ export class FleetManager {
|
|
|
226
199
|
this.webhookEmitter = new WebhookEmitter(webhookConfigs, this.logger);
|
|
227
200
|
this.logger.info({ count: webhookConfigs.length }, "Webhook emitter initialized");
|
|
228
201
|
}
|
|
229
|
-
this.costGuard.on("warn", (instance, totalCents, limitCents) => {
|
|
202
|
+
this.costGuard.on("warn", safeHandler((instance, totalCents, limitCents) => {
|
|
230
203
|
this.notifyInstanceTopic(instance, `⚠️ ${instance} cost: ${formatCents(totalCents)} / ${formatCents(limitCents)} (${Math.round(totalCents / limitCents * 100)}%)`);
|
|
231
204
|
this.webhookEmitter?.emit("cost_warning", instance, { cost_cents: totalCents, limit_cents: limitCents });
|
|
232
|
-
});
|
|
233
|
-
this.costGuard.on("limit", (instance, totalCents, limitCents) => {
|
|
205
|
+
}, this.logger, "costGuard.warn"));
|
|
206
|
+
this.costGuard.on("limit", safeHandler(async (instance, totalCents, limitCents) => {
|
|
234
207
|
this.notifyInstanceTopic(instance, `🛑 ${instance} daily limit ${formatCents(limitCents)} reached — pausing instance.`);
|
|
235
208
|
this.eventLog?.insert(instance, "instance_paused", { reason: "cost_limit", cost_cents: totalCents });
|
|
236
209
|
this.webhookEmitter?.emit("cost_limit", instance, { cost_cents: totalCents, limit_cents: limitCents });
|
|
237
|
-
this.stopInstance(instance)
|
|
238
|
-
});
|
|
210
|
+
await this.stopInstance(instance);
|
|
211
|
+
}, this.logger, "costGuard.limit"));
|
|
239
212
|
const summaryConfig = {
|
|
240
213
|
...DEFAULT_DAILY_SUMMARY,
|
|
241
214
|
...fleet.defaults?.daily_summary ?? {},
|
|
@@ -309,12 +282,11 @@ export class FleetManager {
|
|
|
309
282
|
await this.startSharedAdapter(fleet);
|
|
310
283
|
// Auto-create topics AFTER adapter is ready (needs adapter.createTopic)
|
|
311
284
|
await this.topicCommands.autoCreateTopics();
|
|
312
|
-
|
|
313
|
-
const routeSummary = [...this.routingTable.entries()].map(([tid, target]) => `#${tid}→${target.name}`).join(", ");
|
|
285
|
+
const routeSummary = this.routing.rebuild(this.fleetConfig);
|
|
314
286
|
this.logger.info(`Routes: ${routeSummary}`);
|
|
315
287
|
// Resolve topic icon emoji IDs and start idle archive poller
|
|
316
288
|
await this.resolveTopicIcons();
|
|
317
|
-
this.
|
|
289
|
+
this.topicArchiver.startPoller();
|
|
318
290
|
await new Promise(r => setTimeout(r, 3000));
|
|
319
291
|
await this.connectToInstances(fleet);
|
|
320
292
|
for (const name of Object.keys(fleet.instances)) {
|
|
@@ -352,7 +324,7 @@ export class FleetManager {
|
|
|
352
324
|
};
|
|
353
325
|
process.once("SIGUSR1", onFullRestart);
|
|
354
326
|
}
|
|
355
|
-
/** Start the shared
|
|
327
|
+
/** Start the shared channel adapter for topic mode */
|
|
356
328
|
async startSharedAdapter(fleet) {
|
|
357
329
|
const channelConfig = fleet.channel;
|
|
358
330
|
const botToken = process.env[channelConfig.bot_token_env];
|
|
@@ -371,10 +343,10 @@ export class FleetManager {
|
|
|
371
343
|
accessManager,
|
|
372
344
|
inboxDir,
|
|
373
345
|
});
|
|
374
|
-
this.adapter.on("message", (msg) => {
|
|
375
|
-
this.handleInboundMessage(msg);
|
|
376
|
-
});
|
|
377
|
-
this.adapter.on("callback_query", async (data) => {
|
|
346
|
+
this.adapter.on("message", safeHandler(async (msg) => {
|
|
347
|
+
await this.handleInboundMessage(msg);
|
|
348
|
+
}, this.logger, "adapter.message"));
|
|
349
|
+
this.adapter.on("callback_query", safeHandler(async (data) => {
|
|
378
350
|
if (data.callbackData.startsWith("hang:")) {
|
|
379
351
|
const parts = data.callbackData.split(":");
|
|
380
352
|
const action = parts[1];
|
|
@@ -395,28 +367,27 @@ export class FleetManager {
|
|
|
395
367
|
}
|
|
396
368
|
return;
|
|
397
369
|
}
|
|
398
|
-
});
|
|
399
|
-
this.adapter.on("topic_closed", (data) => {
|
|
400
|
-
const tid = parseInt(data.threadId, 10);
|
|
370
|
+
}, this.logger, "adapter.callback_query"));
|
|
371
|
+
this.adapter.on("topic_closed", safeHandler(async (data) => {
|
|
401
372
|
// Skip unbind if we archived this topic ourselves
|
|
402
|
-
if (this.
|
|
373
|
+
if (this.topicArchiver.isArchived(data.threadId))
|
|
403
374
|
return;
|
|
404
|
-
this.topicCommands.handleTopicDeleted(
|
|
405
|
-
});
|
|
375
|
+
await this.topicCommands.handleTopicDeleted(data.threadId);
|
|
376
|
+
}, this.logger, "adapter.topic_closed"));
|
|
406
377
|
await this.topicCommands.registerBotCommands();
|
|
407
378
|
await this.adapter.start();
|
|
408
379
|
if (fleet.channel?.group_id) {
|
|
409
380
|
this.adapter.setChatId(String(fleet.channel.group_id));
|
|
410
381
|
}
|
|
411
|
-
this.adapter.on("started", (username) => {
|
|
412
|
-
this.logger.info(`
|
|
413
|
-
});
|
|
414
|
-
this.adapter.on("polling_conflict", ({ attempt, delay }) => {
|
|
382
|
+
this.adapter.on("started", safeHandler((username) => {
|
|
383
|
+
this.logger.info(`Bot @${username} polling`);
|
|
384
|
+
}, this.logger, "adapter.started"));
|
|
385
|
+
this.adapter.on("polling_conflict", safeHandler(({ attempt, delay }) => {
|
|
415
386
|
this.logger.warn(`409 Conflict (attempt ${attempt}), retry in ${delay / 1000}s`);
|
|
416
|
-
});
|
|
417
|
-
this.adapter.on("handler_error", (err) => {
|
|
418
|
-
this.logger.warn({ err: err instanceof Error ? err.message : String(err) }, "
|
|
419
|
-
});
|
|
387
|
+
}, this.logger, "adapter.polling_conflict"));
|
|
388
|
+
this.adapter.on("handler_error", safeHandler((err) => {
|
|
389
|
+
this.logger.warn({ err: err instanceof Error ? err.message : String(err) }, "Adapter handler error");
|
|
390
|
+
}, this.logger, "adapter.handler_error"));
|
|
420
391
|
this.startTopicCleanupPoller();
|
|
421
392
|
// Prune stale external sessions every 5 minutes
|
|
422
393
|
this.sessionPruneTimer = setInterval(() => {
|
|
@@ -438,7 +409,7 @@ export class FleetManager {
|
|
|
438
409
|
try {
|
|
439
410
|
await ipc.connect();
|
|
440
411
|
this.instanceIpcClients.set(name, ipc);
|
|
441
|
-
ipc.on("message", (msg) => {
|
|
412
|
+
ipc.on("message", safeHandler(async (msg) => {
|
|
442
413
|
if (msg.type === "mcp_ready") {
|
|
443
414
|
// Register external sessions (sessionName differs from instance name)
|
|
444
415
|
const sessionName = msg.sessionName;
|
|
@@ -463,7 +434,7 @@ export class FleetManager {
|
|
|
463
434
|
this.sessionRegistry.set(sender, name);
|
|
464
435
|
this.logger.info({ sessionName: sender, instanceName: name }, "Registered external session");
|
|
465
436
|
}
|
|
466
|
-
this.handleOutboundFromInstance(name, msg)
|
|
437
|
+
await this.handleOutboundFromInstance(name, msg);
|
|
467
438
|
}
|
|
468
439
|
else if (msg.type === "fleet_tool_status") {
|
|
469
440
|
this.handleToolStatusFromInstance(name, msg);
|
|
@@ -472,13 +443,13 @@ export class FleetManager {
|
|
|
472
443
|
msg.type === "fleet_schedule_update" || msg.type === "fleet_schedule_delete") {
|
|
473
444
|
this.handleScheduleCrud(name, msg);
|
|
474
445
|
}
|
|
475
|
-
});
|
|
446
|
+
}, this.logger, `ipc.message[${name}]`));
|
|
476
447
|
// Ask daemon for any sessions that registered before we connected
|
|
477
448
|
// (fixes race condition where mcp_ready was broadcast before fleet manager connected)
|
|
478
449
|
ipc.send({ type: "query_sessions" });
|
|
479
450
|
this.logger.debug({ name }, "Connected to instance IPC");
|
|
480
|
-
if (!this.
|
|
481
|
-
this.
|
|
451
|
+
if (!this.statuslineWatcher.has(name)) {
|
|
452
|
+
this.statuslineWatcher.watch(name);
|
|
482
453
|
}
|
|
483
454
|
}
|
|
484
455
|
catch (err) {
|
|
@@ -497,7 +468,7 @@ export class FleetManager {
|
|
|
497
468
|
return undefined;
|
|
498
469
|
}
|
|
499
470
|
async handleInboundMessage(msg) {
|
|
500
|
-
const threadId = msg.threadId
|
|
471
|
+
const threadId = msg.threadId || undefined;
|
|
501
472
|
if (threadId == null) {
|
|
502
473
|
// General topic: check for /status command
|
|
503
474
|
if (await this.topicCommands.handleGeneralCommand(msg))
|
|
@@ -534,15 +505,15 @@ export class FleetManager {
|
|
|
534
505
|
}
|
|
535
506
|
return;
|
|
536
507
|
}
|
|
537
|
-
const target = this.
|
|
508
|
+
const target = this.routing.resolve(threadId);
|
|
538
509
|
if (!target) {
|
|
539
510
|
this.topicCommands.handleUnboundTopic(msg);
|
|
540
511
|
return;
|
|
541
512
|
}
|
|
542
513
|
const instanceName = target.name;
|
|
543
514
|
// Reopen archived topic before routing
|
|
544
|
-
if (this.
|
|
545
|
-
await this.
|
|
515
|
+
if (this.topicArchiver.isArchived(threadId)) {
|
|
516
|
+
await this.topicArchiver.reopen(threadId, instanceName);
|
|
546
517
|
}
|
|
547
518
|
this.touchActivity(instanceName);
|
|
548
519
|
this.setTopicIcon(instanceName, "blue");
|
|
@@ -561,7 +532,7 @@ export class FleetManager {
|
|
|
561
532
|
ipc.send({
|
|
562
533
|
type: "fleet_inbound",
|
|
563
534
|
content: text,
|
|
564
|
-
targetSession: instanceName, //
|
|
535
|
+
targetSession: instanceName, // Channel messages → instance's own session
|
|
565
536
|
meta: {
|
|
566
537
|
chat_id: msg.chatId,
|
|
567
538
|
message_id: msg.messageId,
|
|
@@ -577,13 +548,13 @@ export class FleetManager {
|
|
|
577
548
|
}
|
|
578
549
|
/** Handle outbound tool calls from a daemon instance */
|
|
579
550
|
replyIfRateLimited(instanceName, msg) {
|
|
580
|
-
const rl = this.
|
|
551
|
+
const rl = this.statuslineWatcher.getRateLimits(instanceName);
|
|
581
552
|
if (!rl || rl.seven_day_pct < 100)
|
|
582
553
|
return false;
|
|
583
554
|
if (this.adapter && msg.chatId) {
|
|
584
555
|
const threadId = msg.threadId ?? undefined;
|
|
585
556
|
this.adapter.sendText(msg.chatId, `⏸ ${instanceName} has hit the weekly usage limit. Your message was not delivered. Limit resets automatically — check /status for details.`, { threadId })
|
|
586
|
-
.catch(e => this.logger.
|
|
557
|
+
.catch(e => this.logger.warn({ err: e }, "Failed to send rate limit notice"));
|
|
587
558
|
}
|
|
588
559
|
this.logger.info({ instanceName }, "Blocked inbound message — weekly rate limit at 100%");
|
|
589
560
|
return true;
|
|
@@ -618,389 +589,13 @@ export class FleetManager {
|
|
|
618
589
|
}
|
|
619
590
|
return;
|
|
620
591
|
}
|
|
621
|
-
//
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
break;
|
|
629
|
-
}
|
|
630
|
-
if (!message) {
|
|
631
|
-
respond(null, "send_to_instance: missing required argument 'message'");
|
|
632
|
-
break;
|
|
633
|
-
}
|
|
634
|
-
const senderLabel = senderSessionName ?? instanceName;
|
|
635
|
-
const isExternalSender = senderSessionName != null && senderSessionName !== instanceName;
|
|
636
|
-
// Resolve target: could be an instance name or an external session name
|
|
637
|
-
let targetIpc = this.instanceIpcClients.get(targetName);
|
|
638
|
-
let targetSession = targetName; // default: target is the instance itself
|
|
639
|
-
let targetInstanceName = targetName;
|
|
640
|
-
if (!targetIpc) {
|
|
641
|
-
// Check if target is an external session
|
|
642
|
-
const hostInstance = this.sessionRegistry.get(targetName);
|
|
643
|
-
if (hostInstance) {
|
|
644
|
-
targetIpc = this.instanceIpcClients.get(hostInstance);
|
|
645
|
-
targetSession = targetName; // deliver to the external session
|
|
646
|
-
targetInstanceName = hostInstance;
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
if (!targetIpc) {
|
|
650
|
-
// Check if instance exists in config but is stopped
|
|
651
|
-
const existsInConfig = targetName in (this.fleetConfig?.instances ?? {});
|
|
652
|
-
if (existsInConfig) {
|
|
653
|
-
respond(null, `Instance '${targetName}' is stopped. Use start_instance('${targetName}') to start it first.`);
|
|
654
|
-
}
|
|
655
|
-
else {
|
|
656
|
-
respond(null, `Instance or session not found: ${targetName}`);
|
|
657
|
-
}
|
|
658
|
-
break;
|
|
659
|
-
}
|
|
660
|
-
// Build structured metadata (Phase 2)
|
|
661
|
-
const correlationId = args.correlation_id || `cid-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
662
|
-
const meta = {
|
|
663
|
-
chat_id: "",
|
|
664
|
-
message_id: `xmsg-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
665
|
-
user: `instance:${senderLabel}`,
|
|
666
|
-
user_id: `instance:${senderLabel}`,
|
|
667
|
-
ts: new Date().toISOString(),
|
|
668
|
-
thread_id: "",
|
|
669
|
-
from_instance: senderLabel,
|
|
670
|
-
correlation_id: correlationId,
|
|
671
|
-
};
|
|
672
|
-
if (args.request_kind)
|
|
673
|
-
meta.request_kind = args.request_kind;
|
|
674
|
-
if (args.requires_reply != null)
|
|
675
|
-
meta.requires_reply = String(args.requires_reply);
|
|
676
|
-
if (args.task_summary)
|
|
677
|
-
meta.task_summary = args.task_summary;
|
|
678
|
-
if (args.working_directory)
|
|
679
|
-
meta.working_directory = args.working_directory;
|
|
680
|
-
if (args.branch)
|
|
681
|
-
meta.branch = args.branch;
|
|
682
|
-
targetIpc.send({
|
|
683
|
-
type: "fleet_inbound",
|
|
684
|
-
targetSession,
|
|
685
|
-
content: message,
|
|
686
|
-
meta,
|
|
687
|
-
});
|
|
688
|
-
// Post to Telegram topics for visibility
|
|
689
|
-
const groupId = this.fleetConfig?.channel?.group_id;
|
|
690
|
-
if (groupId && this.adapter) {
|
|
691
|
-
const senderTopicId = this.fleetConfig?.instances[instanceName]?.topic_id;
|
|
692
|
-
const targetTopicId = this.fleetConfig?.instances[targetInstanceName]?.topic_id;
|
|
693
|
-
// Post full message to topics — adapter handles 4096-char chunking
|
|
694
|
-
// Only post to sender topic if sender is the instance itself (not external)
|
|
695
|
-
if (senderTopicId && !isExternalSender) {
|
|
696
|
-
this.adapter.sendText(String(groupId), `→ ${targetName}:\n${message}`, {
|
|
697
|
-
threadId: String(senderTopicId),
|
|
698
|
-
}).catch(e => this.logger.debug({ err: e }, "Failed to post cross-instance notification"));
|
|
699
|
-
}
|
|
700
|
-
// Only post to target topic if target is an instance (not external session)
|
|
701
|
-
if (targetTopicId && !this.sessionRegistry.has(targetName)) {
|
|
702
|
-
this.adapter.sendText(String(groupId), `← ${senderLabel}:\n${message}`, {
|
|
703
|
-
threadId: String(targetTopicId),
|
|
704
|
-
}).catch(e => this.logger.debug({ err: e }, "Failed to post cross-instance notification"));
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
this.logger.info(`✉ ${senderLabel} → ${targetName}: ${(message ?? "").slice(0, 100)}`);
|
|
708
|
-
respond({ sent: true, target: targetName, correlation_id: correlationId });
|
|
709
|
-
break;
|
|
710
|
-
}
|
|
711
|
-
case "list_instances": {
|
|
712
|
-
const senderLabel = senderSessionName ?? instanceName;
|
|
713
|
-
const allInstances = Object.entries(this.fleetConfig?.instances ?? {})
|
|
714
|
-
.filter(([name]) => name !== instanceName && name !== senderLabel)
|
|
715
|
-
.map(([name, config]) => ({
|
|
716
|
-
name,
|
|
717
|
-
type: "instance",
|
|
718
|
-
status: this.daemons.has(name) ? "running" : "stopped",
|
|
719
|
-
working_directory: config.working_directory,
|
|
720
|
-
topic_id: config.topic_id ?? null,
|
|
721
|
-
description: config.description ?? null,
|
|
722
|
-
tags: config.tags ?? [],
|
|
723
|
-
last_activity: this.lastActivity.get(name) ? new Date(this.lastActivity.get(name)).toISOString() : null,
|
|
724
|
-
}));
|
|
725
|
-
// Include external sessions (excluding self)
|
|
726
|
-
const externalSessions = [...this.sessionRegistry.entries()]
|
|
727
|
-
.filter(([sessName]) => sessName !== senderLabel)
|
|
728
|
-
.map(([sessName, hostInstance]) => ({
|
|
729
|
-
name: sessName, type: "session", host: hostInstance,
|
|
730
|
-
}));
|
|
731
|
-
respond({ instances: allInstances, external_sessions: externalSessions });
|
|
732
|
-
break;
|
|
733
|
-
}
|
|
734
|
-
// Phase 3: High-level collaboration tools (wrappers around send_to_instance)
|
|
735
|
-
case "request_information": {
|
|
736
|
-
const targetName = args.target_instance;
|
|
737
|
-
const question = args.question;
|
|
738
|
-
const context = args.context;
|
|
739
|
-
const body = context ? `${question}\n\nContext: ${context}` : question;
|
|
740
|
-
// Re-dispatch as send_to_instance with structured metadata
|
|
741
|
-
args.instance_name = targetName;
|
|
742
|
-
args.message = body;
|
|
743
|
-
args.request_kind = "query";
|
|
744
|
-
args.requires_reply = true;
|
|
745
|
-
args.task_summary = question.slice(0, 120);
|
|
746
|
-
// Recursively handle via the same switch (will hit send_to_instance case above)
|
|
747
|
-
return this.handleOutboundFromInstance(instanceName, { tool: "send_to_instance", args, requestId, fleetRequestId, senderSessionName });
|
|
748
|
-
}
|
|
749
|
-
case "delegate_task": {
|
|
750
|
-
const targetName = args.target_instance;
|
|
751
|
-
const task = args.task;
|
|
752
|
-
const criteria = args.success_criteria;
|
|
753
|
-
const context = args.context;
|
|
754
|
-
let body = task;
|
|
755
|
-
if (criteria)
|
|
756
|
-
body += `\n\nSuccess criteria: ${criteria}`;
|
|
757
|
-
if (context)
|
|
758
|
-
body += `\n\nContext: ${context}`;
|
|
759
|
-
args.instance_name = targetName;
|
|
760
|
-
args.message = body;
|
|
761
|
-
args.request_kind = "task";
|
|
762
|
-
args.requires_reply = true;
|
|
763
|
-
args.task_summary = task.slice(0, 120);
|
|
764
|
-
return this.handleOutboundFromInstance(instanceName, { tool: "send_to_instance", args, requestId, fleetRequestId, senderSessionName });
|
|
765
|
-
}
|
|
766
|
-
case "report_result": {
|
|
767
|
-
const targetName = args.target_instance;
|
|
768
|
-
const summary = args.summary;
|
|
769
|
-
const artifacts = args.artifacts;
|
|
770
|
-
if (!args.correlation_id) {
|
|
771
|
-
this.logger.warn({ instanceName, targetName }, "report_result called without correlation_id — recipient cannot match this to an original request");
|
|
772
|
-
}
|
|
773
|
-
let body = summary;
|
|
774
|
-
if (artifacts)
|
|
775
|
-
body += `\n\nArtifacts: ${artifacts}`;
|
|
776
|
-
args.instance_name = targetName;
|
|
777
|
-
args.message = body;
|
|
778
|
-
args.request_kind = "report";
|
|
779
|
-
args.requires_reply = false;
|
|
780
|
-
args.task_summary = summary.slice(0, 120);
|
|
781
|
-
return this.handleOutboundFromInstance(instanceName, { tool: "send_to_instance", args, requestId, fleetRequestId, senderSessionName });
|
|
782
|
-
}
|
|
783
|
-
// Phase 4: Capability discovery
|
|
784
|
-
case "describe_instance": {
|
|
785
|
-
const targetName = args.name;
|
|
786
|
-
const config = this.fleetConfig?.instances[targetName];
|
|
787
|
-
if (config) {
|
|
788
|
-
respond({
|
|
789
|
-
name: targetName,
|
|
790
|
-
type: "instance",
|
|
791
|
-
description: config.description ?? null,
|
|
792
|
-
tags: config.tags ?? [],
|
|
793
|
-
working_directory: config.working_directory,
|
|
794
|
-
status: this.daemons.has(targetName) ? "running" : "stopped",
|
|
795
|
-
topic_id: config.topic_id ?? null,
|
|
796
|
-
model: config.model ?? null,
|
|
797
|
-
last_activity: this.lastActivity.get(targetName) ? new Date(this.lastActivity.get(targetName)).toISOString() : null,
|
|
798
|
-
worktree_source: config.worktree_source ?? null,
|
|
799
|
-
});
|
|
800
|
-
break;
|
|
801
|
-
}
|
|
802
|
-
// Check if it's a known external session
|
|
803
|
-
const hostInstance = this.sessionRegistry.get(targetName);
|
|
804
|
-
if (hostInstance) {
|
|
805
|
-
respond({
|
|
806
|
-
name: targetName,
|
|
807
|
-
type: "session",
|
|
808
|
-
host: hostInstance,
|
|
809
|
-
status: "running",
|
|
810
|
-
});
|
|
811
|
-
break;
|
|
812
|
-
}
|
|
813
|
-
respond(null, `Instance or session '${targetName}' not found`);
|
|
814
|
-
break;
|
|
815
|
-
}
|
|
816
|
-
case "start_instance": {
|
|
817
|
-
const targetName = args.name;
|
|
818
|
-
// Already running?
|
|
819
|
-
if (this.daemons.has(targetName)) {
|
|
820
|
-
respond({ success: true, status: "already_running" });
|
|
821
|
-
break;
|
|
822
|
-
}
|
|
823
|
-
// Exists in config?
|
|
824
|
-
const targetConfig = this.fleetConfig?.instances[targetName];
|
|
825
|
-
if (!targetConfig) {
|
|
826
|
-
respond(null, `Instance '${targetName}' not found in fleet config`);
|
|
827
|
-
break;
|
|
828
|
-
}
|
|
829
|
-
try {
|
|
830
|
-
await this.startInstance(targetName, targetConfig, true);
|
|
831
|
-
await this.connectIpcToInstance(targetName);
|
|
832
|
-
respond({ success: true, status: "started" });
|
|
833
|
-
}
|
|
834
|
-
catch (err) {
|
|
835
|
-
respond(null, `Failed to start instance '${targetName}': ${err.message}`);
|
|
836
|
-
}
|
|
837
|
-
break;
|
|
838
|
-
}
|
|
839
|
-
case "create_instance": {
|
|
840
|
-
const directory = args.directory.replace(/^~/, process.env.HOME || "~");
|
|
841
|
-
const topicName = args.topic_name || basename(directory);
|
|
842
|
-
const description = args.description;
|
|
843
|
-
const branch = args.branch;
|
|
844
|
-
// Validate directory exists
|
|
845
|
-
try {
|
|
846
|
-
await access(directory);
|
|
847
|
-
}
|
|
848
|
-
catch {
|
|
849
|
-
respond(null, `Directory does not exist: ${directory}`);
|
|
850
|
-
break;
|
|
851
|
-
}
|
|
852
|
-
// Check for duplicate early (before worktree creation) — only when no branch
|
|
853
|
-
if (!branch) {
|
|
854
|
-
const expandHome = (p) => p.replace(/^~/, process.env.HOME || "~");
|
|
855
|
-
const existingInstance = Object.entries(this.fleetConfig?.instances ?? {})
|
|
856
|
-
.find(([_, config]) => expandHome(config.working_directory) === directory);
|
|
857
|
-
if (existingInstance) {
|
|
858
|
-
const [eName, eConfig] = existingInstance;
|
|
859
|
-
respond({
|
|
860
|
-
success: true,
|
|
861
|
-
status: "already_exists",
|
|
862
|
-
name: eName,
|
|
863
|
-
topic_id: eConfig.topic_id,
|
|
864
|
-
running: this.daemons.has(eName),
|
|
865
|
-
});
|
|
866
|
-
break;
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
// If branch specified, create git worktree
|
|
870
|
-
let workDir = directory;
|
|
871
|
-
let worktreePath;
|
|
872
|
-
if (branch) {
|
|
873
|
-
try {
|
|
874
|
-
const { execFile: execFileCb } = await import("node:child_process");
|
|
875
|
-
const { promisify } = await import("node:util");
|
|
876
|
-
const execFileAsync = promisify(execFileCb);
|
|
877
|
-
// Verify it's a git repo
|
|
878
|
-
await execFileAsync("git", ["rev-parse", "--git-dir"], { cwd: directory });
|
|
879
|
-
// Determine worktree path: sibling directory named repo-branch
|
|
880
|
-
const repoName = basename(directory);
|
|
881
|
-
const safeBranch = branch.replace(/\//g, "-");
|
|
882
|
-
worktreePath = join(dirname(directory), `${repoName}-${safeBranch}`);
|
|
883
|
-
// Check if branch exists
|
|
884
|
-
let branchExists = false;
|
|
885
|
-
try {
|
|
886
|
-
await execFileAsync("git", ["rev-parse", "--verify", branch], { cwd: directory });
|
|
887
|
-
branchExists = true;
|
|
888
|
-
}
|
|
889
|
-
catch { /* branch doesn't exist */ }
|
|
890
|
-
if (branchExists) {
|
|
891
|
-
await execFileAsync("git", ["worktree", "add", worktreePath, branch], { cwd: directory });
|
|
892
|
-
}
|
|
893
|
-
else {
|
|
894
|
-
await execFileAsync("git", ["worktree", "add", worktreePath, "-b", branch], { cwd: directory });
|
|
895
|
-
}
|
|
896
|
-
this.logger.info({ worktreePath, branch, repo: directory }, "Created git worktree for instance");
|
|
897
|
-
workDir = worktreePath;
|
|
898
|
-
}
|
|
899
|
-
catch (err) {
|
|
900
|
-
respond(null, `Failed to create worktree: ${err.message}`);
|
|
901
|
-
break;
|
|
902
|
-
}
|
|
903
|
-
}
|
|
904
|
-
// Check worktree path for duplicates (branch case only — non-branch already checked above)
|
|
905
|
-
if (worktreePath) {
|
|
906
|
-
const expandHome = (p) => p.replace(/^~/, process.env.HOME || "~");
|
|
907
|
-
const existingInstance = Object.entries(this.fleetConfig?.instances ?? {})
|
|
908
|
-
.find(([_, config]) => expandHome(config.working_directory) === workDir);
|
|
909
|
-
if (existingInstance) {
|
|
910
|
-
const [eName, eConfig] = existingInstance;
|
|
911
|
-
respond({
|
|
912
|
-
success: true,
|
|
913
|
-
status: "already_exists",
|
|
914
|
-
name: eName,
|
|
915
|
-
topic_id: eConfig.topic_id,
|
|
916
|
-
running: this.daemons.has(eName),
|
|
917
|
-
});
|
|
918
|
-
break;
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
// Sequential steps with rollback
|
|
922
|
-
let createdTopicId;
|
|
923
|
-
let newInstanceName;
|
|
924
|
-
try {
|
|
925
|
-
// Step a: Create Telegram topic
|
|
926
|
-
createdTopicId = await this.createForumTopic(topicName);
|
|
927
|
-
// Step b: Register in config
|
|
928
|
-
// Use topicName for worktree instances to avoid long paths (Unix socket limit 104 bytes)
|
|
929
|
-
const nameBase = worktreePath ? topicName : basename(workDir);
|
|
930
|
-
newInstanceName = `${sanitizeInstanceName(nameBase)}-t${createdTopicId}`;
|
|
931
|
-
const instanceConfig = {
|
|
932
|
-
...DEFAULT_INSTANCE_CONFIG,
|
|
933
|
-
...this.fleetConfig.defaults,
|
|
934
|
-
working_directory: workDir,
|
|
935
|
-
topic_id: createdTopicId,
|
|
936
|
-
...(description ? { description } : {}),
|
|
937
|
-
...(args.model ? { model: args.model } : {}),
|
|
938
|
-
...(args.backend ? { backend: args.backend } : {}),
|
|
939
|
-
...(worktreePath ? { worktree_source: directory } : {}),
|
|
940
|
-
};
|
|
941
|
-
this.fleetConfig.instances[newInstanceName] = instanceConfig;
|
|
942
|
-
this.routingTable.set(createdTopicId, { kind: "instance", name: newInstanceName });
|
|
943
|
-
this.saveFleetConfig();
|
|
944
|
-
// Step c: Start instance
|
|
945
|
-
await this.startInstance(newInstanceName, instanceConfig, true);
|
|
946
|
-
await this.connectIpcToInstance(newInstanceName);
|
|
947
|
-
respond({
|
|
948
|
-
success: true,
|
|
949
|
-
name: newInstanceName,
|
|
950
|
-
topic_id: createdTopicId,
|
|
951
|
-
...(worktreePath ? { worktree_path: worktreePath, branch } : {}),
|
|
952
|
-
});
|
|
953
|
-
}
|
|
954
|
-
catch (err) {
|
|
955
|
-
// Rollback in reverse order
|
|
956
|
-
if (newInstanceName && this.daemons.has(newInstanceName)) {
|
|
957
|
-
await this.stopInstance(newInstanceName).catch(() => { });
|
|
958
|
-
}
|
|
959
|
-
if (newInstanceName && this.fleetConfig?.instances[newInstanceName]) {
|
|
960
|
-
delete this.fleetConfig.instances[newInstanceName];
|
|
961
|
-
if (createdTopicId)
|
|
962
|
-
this.routingTable.delete(createdTopicId);
|
|
963
|
-
this.saveFleetConfig();
|
|
964
|
-
}
|
|
965
|
-
if (createdTopicId) {
|
|
966
|
-
await this.deleteForumTopic(createdTopicId);
|
|
967
|
-
}
|
|
968
|
-
// Rollback worktree
|
|
969
|
-
if (worktreePath) {
|
|
970
|
-
try {
|
|
971
|
-
const { execFile: execFileCb } = await import("node:child_process");
|
|
972
|
-
const { promisify } = await import("node:util");
|
|
973
|
-
const execFileAsync = promisify(execFileCb);
|
|
974
|
-
await execFileAsync("git", ["worktree", "remove", "--force", worktreePath], { cwd: directory });
|
|
975
|
-
}
|
|
976
|
-
catch { /* best-effort worktree cleanup */ }
|
|
977
|
-
}
|
|
978
|
-
respond(null, `Failed to create instance: ${err.message}`);
|
|
979
|
-
}
|
|
980
|
-
break;
|
|
981
|
-
}
|
|
982
|
-
case "delete_instance": {
|
|
983
|
-
const instanceName = args.name;
|
|
984
|
-
const deleteTopic = args.delete_topic ?? false;
|
|
985
|
-
const instanceConfig = this.fleetConfig?.instances[instanceName];
|
|
986
|
-
if (!instanceConfig) {
|
|
987
|
-
respond(null, `Instance not found: ${instanceName}`);
|
|
988
|
-
break;
|
|
989
|
-
}
|
|
990
|
-
if (instanceConfig.general_topic) {
|
|
991
|
-
respond(null, "Cannot delete the General instance");
|
|
992
|
-
break;
|
|
993
|
-
}
|
|
994
|
-
// Delete Telegram topic if requested (before removeInstance clears config)
|
|
995
|
-
if (deleteTopic && instanceConfig.topic_id) {
|
|
996
|
-
await this.deleteForumTopic(instanceConfig.topic_id);
|
|
997
|
-
}
|
|
998
|
-
await this.removeInstance(instanceName);
|
|
999
|
-
respond({ success: true, name: instanceName, topic_deleted: deleteTopic });
|
|
1000
|
-
break;
|
|
1001
|
-
}
|
|
1002
|
-
default:
|
|
1003
|
-
respond(null, `Unknown tool: ${tool}`);
|
|
592
|
+
// Dispatch fleet-specific tools via handler map
|
|
593
|
+
const handler = outboundHandlers.get(tool);
|
|
594
|
+
if (handler) {
|
|
595
|
+
await handler(this, args, respond, { instanceName, requestId, fleetRequestId, senderSessionName });
|
|
596
|
+
}
|
|
597
|
+
else {
|
|
598
|
+
respond(null, `Unknown tool: ${tool}`);
|
|
1004
599
|
}
|
|
1005
600
|
}
|
|
1006
601
|
/** Handle tool status update from a daemon instance */
|
|
@@ -1021,14 +616,14 @@ export class FleetManager {
|
|
|
1021
616
|
this.adapter.sendText(chatId, text, { threadId }).then((sent) => {
|
|
1022
617
|
const ipc = this.instanceIpcClients.get(instanceName);
|
|
1023
618
|
ipc?.send({ type: "fleet_tool_status_ack", messageId: sent.messageId });
|
|
1024
|
-
}).catch(e => this.logger.
|
|
619
|
+
}).catch(e => this.logger.warn({ err: e }, "Failed to send tool status message"));
|
|
1025
620
|
}
|
|
1026
621
|
}
|
|
1027
622
|
// ===================== Scheduler =====================
|
|
1028
623
|
async handleScheduleTrigger(schedule) {
|
|
1029
624
|
const { target, reply_chat_id, reply_thread_id, message, label, id, source } = schedule;
|
|
1030
625
|
const RATE_LIMIT_DEFER_THRESHOLD = 85;
|
|
1031
|
-
const rl = this.
|
|
626
|
+
const rl = this.statuslineWatcher.getRateLimits(target);
|
|
1032
627
|
if (rl && rl.five_hour_pct > RATE_LIMIT_DEFER_THRESHOLD) {
|
|
1033
628
|
this.scheduler.recordRun(id, "deferred", `5hr rate limit at ${rl.five_hour_pct}%`);
|
|
1034
629
|
this.eventLog?.insert(target, "schedule_deferred", {
|
|
@@ -1141,18 +736,9 @@ export class FleetManager {
|
|
|
1141
736
|
}
|
|
1142
737
|
async deleteForumTopic(topicId) {
|
|
1143
738
|
try {
|
|
1144
|
-
|
|
1145
|
-
const botTokenEnv = this.fleetConfig?.channel?.bot_token_env;
|
|
1146
|
-
if (!groupId || !botTokenEnv)
|
|
739
|
+
if (!this.adapter?.deleteTopic)
|
|
1147
740
|
return;
|
|
1148
|
-
|
|
1149
|
-
if (!botToken)
|
|
1150
|
-
return;
|
|
1151
|
-
await fetch(`https://api.telegram.org/bot${botToken}/deleteForumTopic`, {
|
|
1152
|
-
method: "POST",
|
|
1153
|
-
headers: { "Content-Type": "application/json" },
|
|
1154
|
-
body: JSON.stringify({ chat_id: groupId, message_thread_id: topicId }),
|
|
1155
|
-
});
|
|
741
|
+
await this.adapter.deleteTopic(topicId);
|
|
1156
742
|
}
|
|
1157
743
|
catch (err) {
|
|
1158
744
|
this.logger.warn({ err, topicId }, "Failed to delete forum topic during rollback");
|
|
@@ -1165,7 +751,7 @@ export class FleetManager {
|
|
|
1165
751
|
this.topicCleanupTimer = setInterval(async () => {
|
|
1166
752
|
if (!this.fleetConfig?.channel?.group_id || !this.adapter?.topicExists)
|
|
1167
753
|
return;
|
|
1168
|
-
for (const [threadId, target] of this.
|
|
754
|
+
for (const [threadId, target] of this.routing.entries()) {
|
|
1169
755
|
try {
|
|
1170
756
|
if (!isProbeableRouteTarget(target)) {
|
|
1171
757
|
continue;
|
|
@@ -1229,85 +815,18 @@ export class FleetManager {
|
|
|
1229
815
|
this.logger.info({ path: this.configPath }, "Saved fleet config");
|
|
1230
816
|
}
|
|
1231
817
|
async removeInstance(name) {
|
|
818
|
+
// Clean up schedules (scheduler is fleet-level, not lifecycle-level)
|
|
1232
819
|
const config = this.fleetConfig?.instances[name];
|
|
1233
|
-
if (
|
|
1234
|
-
return;
|
|
1235
|
-
// Never remove the General instance
|
|
1236
|
-
if (config.general_topic) {
|
|
1237
|
-
this.logger.warn({ name }, "Refusing to remove General instance");
|
|
1238
|
-
return;
|
|
1239
|
-
}
|
|
1240
|
-
// Clean up schedules
|
|
1241
|
-
if (this.scheduler && config.topic_id) {
|
|
820
|
+
if (this.scheduler && config?.topic_id) {
|
|
1242
821
|
const count = this.scheduler.deleteByInstanceOrThread(name, String(config.topic_id));
|
|
1243
822
|
if (count > 0) {
|
|
1244
823
|
this.logger.info({ name, count }, "Cleaned up schedules for deleted instance");
|
|
1245
824
|
}
|
|
1246
825
|
}
|
|
1247
|
-
|
|
1248
|
-
if (this.daemons.has(name)) {
|
|
1249
|
-
await this.stopInstance(name);
|
|
1250
|
-
}
|
|
1251
|
-
// Clean up git worktree if applicable
|
|
1252
|
-
if (config.worktree_source && config.working_directory) {
|
|
1253
|
-
const { existsSync } = await import("node:fs");
|
|
1254
|
-
if (!existsSync(config.working_directory)) {
|
|
1255
|
-
this.logger.info({ worktree: config.working_directory }, "Worktree directory already gone, skipping removal");
|
|
1256
|
-
}
|
|
1257
|
-
else {
|
|
1258
|
-
try {
|
|
1259
|
-
const { execFile: execFileCb } = await import("node:child_process");
|
|
1260
|
-
const { promisify } = await import("node:util");
|
|
1261
|
-
const execFileAsync = promisify(execFileCb);
|
|
1262
|
-
await execFileAsync("git", ["worktree", "remove", "--force", config.working_directory], {
|
|
1263
|
-
cwd: config.worktree_source,
|
|
1264
|
-
});
|
|
1265
|
-
this.logger.info({ worktree: config.working_directory }, "Removed git worktree");
|
|
1266
|
-
}
|
|
1267
|
-
catch (err) {
|
|
1268
|
-
this.logger.warn({ err, worktree: config.working_directory }, "Failed to remove git worktree");
|
|
1269
|
-
}
|
|
1270
|
-
}
|
|
1271
|
-
}
|
|
1272
|
-
// Clean up IPC
|
|
1273
|
-
const ipc = this.instanceIpcClients.get(name);
|
|
1274
|
-
if (ipc) {
|
|
1275
|
-
await ipc.close();
|
|
1276
|
-
this.instanceIpcClients.delete(name);
|
|
1277
|
-
}
|
|
1278
|
-
// Remove from routing table
|
|
1279
|
-
if (config.topic_id) {
|
|
1280
|
-
this.routingTable.delete(config.topic_id);
|
|
1281
|
-
}
|
|
1282
|
-
// Remove from fleet config and save
|
|
1283
|
-
delete this.fleetConfig.instances[name];
|
|
1284
|
-
this.saveFleetConfig();
|
|
1285
|
-
this.logger.info({ name }, "Instance removed");
|
|
826
|
+
return this.lifecycle.remove(name);
|
|
1286
827
|
}
|
|
1287
828
|
startStatuslineWatcher(name) {
|
|
1288
|
-
|
|
1289
|
-
const timer = setInterval(() => {
|
|
1290
|
-
try {
|
|
1291
|
-
const data = JSON.parse(readFileSync(statusFile, "utf-8"));
|
|
1292
|
-
this.costGuard?.updateCost(name, data.cost?.total_cost_usd ?? 0);
|
|
1293
|
-
const rl = data.rate_limits;
|
|
1294
|
-
if (rl) {
|
|
1295
|
-
const prev = this.instanceRateLimits.get(name);
|
|
1296
|
-
const newSevenDay = rl.seven_day?.used_percentage ?? 0;
|
|
1297
|
-
if (prev?.seven_day_pct === 100 && newSevenDay < 100) {
|
|
1298
|
-
this.notifyInstanceTopic(name, `✅ ${name} weekly usage limit has reset — instance is available again.`);
|
|
1299
|
-
this.logger.info({ name }, "Weekly rate limit recovered");
|
|
1300
|
-
}
|
|
1301
|
-
this.instanceRateLimits.set(name, {
|
|
1302
|
-
five_hour_pct: rl.five_hour?.used_percentage ?? 0,
|
|
1303
|
-
seven_day_pct: newSevenDay,
|
|
1304
|
-
});
|
|
1305
|
-
this.checkModelFailover(name, rl.five_hour?.used_percentage ?? 0);
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
catch { /* file may not exist yet or be mid-write */ }
|
|
1309
|
-
}, 10_000);
|
|
1310
|
-
this.statuslineWatchers.set(name, timer);
|
|
829
|
+
this.statuslineWatcher.watch(name);
|
|
1311
830
|
}
|
|
1312
831
|
// ── Model failover ──────────────────────────────────────────────────────
|
|
1313
832
|
static FAILOVER_TRIGGER_PCT = 90;
|
|
@@ -1388,7 +907,7 @@ export class FleetManager {
|
|
|
1388
907
|
const stickers = await this.adapter.getTopicIconStickers();
|
|
1389
908
|
if (stickers.length === 0)
|
|
1390
909
|
return;
|
|
1391
|
-
//
|
|
910
|
+
// getForumTopicIconStickers returns a fixed set of available icons.
|
|
1392
911
|
// Try to match by emoji character, fall back to positional.
|
|
1393
912
|
const find = (targets) => stickers.find((s) => targets.some((t) => s.emoji.includes(t)));
|
|
1394
913
|
const green = find(["🟢", "✅", "💚"]);
|
|
@@ -1421,53 +940,9 @@ export class FleetManager {
|
|
|
1421
940
|
this.lastActivity.set(instanceName, Date.now());
|
|
1422
941
|
}
|
|
1423
942
|
/** Start periodic idle archive checker */
|
|
1424
|
-
|
|
1425
|
-
this.archiveTimer = setInterval(() => {
|
|
1426
|
-
this.archiveIdleTopics().catch((err) => this.logger.debug({ err }, "Archive idle check failed"));
|
|
1427
|
-
}, 30 * 60_000); // check every 30 minutes
|
|
1428
|
-
}
|
|
1429
|
-
/** Close topics that have been idle beyond threshold */
|
|
1430
|
-
async archiveIdleTopics() {
|
|
1431
|
-
if (!this.adapter?.closeForumTopic || !this.fleetConfig)
|
|
1432
|
-
return;
|
|
1433
|
-
const now = Date.now();
|
|
1434
|
-
for (const [name, config] of Object.entries(this.fleetConfig.instances)) {
|
|
1435
|
-
const topicId = config.topic_id;
|
|
1436
|
-
if (topicId == null || config.general_topic)
|
|
1437
|
-
continue;
|
|
1438
|
-
if (this.archivedTopics.has(topicId))
|
|
1439
|
-
continue;
|
|
1440
|
-
const status = this.getInstanceStatus(name);
|
|
1441
|
-
if (status !== "running")
|
|
1442
|
-
continue; // only archive running-but-idle
|
|
1443
|
-
const last = this.lastActivity.get(name) ?? 0;
|
|
1444
|
-
if (last === 0)
|
|
1445
|
-
continue; // never active → skip (just started)
|
|
1446
|
-
if (now - last < FleetManager.ARCHIVE_IDLE_MS)
|
|
1447
|
-
continue;
|
|
1448
|
-
this.logger.info({ name, topicId, idleHours: Math.round((now - last) / 3600000) }, "Archiving idle topic");
|
|
1449
|
-
this.archivedTopics.add(topicId);
|
|
1450
|
-
this.setTopicIcon(name, "remove");
|
|
1451
|
-
await this.adapter.closeForumTopic(topicId);
|
|
1452
|
-
}
|
|
1453
|
-
}
|
|
1454
|
-
/** Reopen an archived topic and restore icon */
|
|
1455
|
-
async reopenArchivedTopic(topicId, instanceName) {
|
|
1456
|
-
if (!this.archivedTopics.has(topicId))
|
|
1457
|
-
return;
|
|
1458
|
-
this.archivedTopics.delete(topicId);
|
|
1459
|
-
if (this.adapter?.reopenForumTopic) {
|
|
1460
|
-
await this.adapter.reopenForumTopic(topicId);
|
|
1461
|
-
}
|
|
1462
|
-
this.setTopicIcon(instanceName, "green");
|
|
1463
|
-
this.touchActivity(instanceName);
|
|
1464
|
-
this.logger.info({ instanceName, topicId }, "Reopened archived topic");
|
|
1465
|
-
}
|
|
943
|
+
// archiveIdleTopics / reopenArchivedTopic → delegated to TopicArchiver
|
|
1466
944
|
clearStatuslineWatchers() {
|
|
1467
|
-
|
|
1468
|
-
clearInterval(timer);
|
|
1469
|
-
this.statuslineWatchers.clear();
|
|
1470
|
-
this.instanceRateLimits.clear();
|
|
945
|
+
this.statuslineWatcher.stopAll();
|
|
1471
946
|
this.failoverActive.clear();
|
|
1472
947
|
}
|
|
1473
948
|
async stopAll() {
|
|
@@ -1482,10 +957,7 @@ export class FleetManager {
|
|
|
1482
957
|
clearInterval(this.sessionPruneTimer);
|
|
1483
958
|
this.sessionPruneTimer = null;
|
|
1484
959
|
}
|
|
1485
|
-
|
|
1486
|
-
clearInterval(this.archiveTimer);
|
|
1487
|
-
this.archiveTimer = null;
|
|
1488
|
-
}
|
|
960
|
+
this.topicArchiver.stop();
|
|
1489
961
|
this.scheduler?.shutdown();
|
|
1490
962
|
await Promise.allSettled([...this.daemons.entries()].map(async ([name, daemon]) => {
|
|
1491
963
|
try {
|
|
@@ -1661,7 +1133,7 @@ export class FleetManager {
|
|
|
1661
1133
|
await this.startInstance(name, config, topicMode);
|
|
1662
1134
|
}
|
|
1663
1135
|
if (topicMode) {
|
|
1664
|
-
this.
|
|
1136
|
+
this.routing.rebuild(this.fleetConfig);
|
|
1665
1137
|
await new Promise(r => setTimeout(r, 3000));
|
|
1666
1138
|
await this.connectToInstances(fleet);
|
|
1667
1139
|
for (const name of Object.keys(fleet.instances)) {
|
|
@@ -1677,7 +1149,7 @@ export class FleetManager {
|
|
|
1677
1149
|
this.logger.info({ count: instances.length }, "Sending restart notification to instances");
|
|
1678
1150
|
for (const [name, config] of instances) {
|
|
1679
1151
|
const threadId = config.topic_id != null ? String(config.topic_id) : undefined;
|
|
1680
|
-
// Send to
|
|
1152
|
+
// Send to topic so the message appears in the instance's channel
|
|
1681
1153
|
if (threadId) {
|
|
1682
1154
|
this.adapter.sendText(String(groupId), "Fleet restart complete. Continue from where you left off.", { threadId })
|
|
1683
1155
|
.catch(e => this.logger.warn({ err: e, name, threadId }, "Failed to post per-instance restart notification"));
|