@agentconnect.md/daemon 1.0.0-rc.37 → 1.0.0-rc.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +422 -22
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -9,7 +9,7 @@ import process$1 from "node:process";
|
|
|
9
9
|
import { stripVTControlCharacters } from "node:util";
|
|
10
10
|
import { fileURLToPath } from "node:url";
|
|
11
11
|
import { createInterface } from "node:readline";
|
|
12
|
-
import { freemem, homedir, hostname,
|
|
12
|
+
import { cpus, freemem, homedir, hostname, totalmem, type } from "node:os";
|
|
13
13
|
import { spawn as spawn$1 } from "child_process";
|
|
14
14
|
import { Readable, Writable } from "node:stream";
|
|
15
15
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
@@ -7376,11 +7376,17 @@ const ConfigSchema = object({
|
|
|
7376
7376
|
maxAgents: number().int().default(8),
|
|
7377
7377
|
maxConcurrentSessions: number().int().default(32),
|
|
7378
7378
|
agentIdleTimeoutMs: number().int().default(9e5),
|
|
7379
|
+
idleSweepMs: number().int().default(6e4),
|
|
7380
|
+
shutdownDrainMs: number().int().default(25e3),
|
|
7381
|
+
cancelBackstopMs: number().int().default(3e4),
|
|
7379
7382
|
maxAttachmentBytes: number().int().default(8 * 1024 * 1024)
|
|
7380
7383
|
}).default({
|
|
7381
7384
|
maxAgents: 8,
|
|
7382
7385
|
maxConcurrentSessions: 32,
|
|
7383
7386
|
agentIdleTimeoutMs: 9e5,
|
|
7387
|
+
idleSweepMs: 6e4,
|
|
7388
|
+
shutdownDrainMs: 25e3,
|
|
7389
|
+
cancelBackstopMs: 3e4,
|
|
7384
7390
|
maxAttachmentBytes: 8 * 1024 * 1024
|
|
7385
7391
|
})
|
|
7386
7392
|
});
|
|
@@ -7587,6 +7593,19 @@ function cachedRuntimeNames(root) {
|
|
|
7587
7593
|
for (const [id, entry] of Object.entries(doc.agents)) if (entry.name) out[id] = entry.name;
|
|
7588
7594
|
return out;
|
|
7589
7595
|
}
|
|
7596
|
+
/**
|
|
7597
|
+
* Runtime versions (registry id -> `version`) read from the local cache. Used to
|
|
7598
|
+
* stamp the `facts/runtime-profile` the daemon reports per installed runtime.
|
|
7599
|
+
* Entries with no `version` are omitted (callers fall back to ''). `{}` when no
|
|
7600
|
+
* cached registry doc exists.
|
|
7601
|
+
*/
|
|
7602
|
+
function cachedRuntimeVersions(root) {
|
|
7603
|
+
const doc = readCachedDoc(root);
|
|
7604
|
+
if (!doc) return {};
|
|
7605
|
+
const out = {};
|
|
7606
|
+
for (const [id, entry] of Object.entries(doc.agents)) if (entry.version) out[id] = entry.version;
|
|
7607
|
+
return out;
|
|
7608
|
+
}
|
|
7590
7609
|
async function defaultRuntimes(root, opts = {}) {
|
|
7591
7610
|
const cached = (opts.mode ?? "blocking") === "cache-first" ? readCachedDoc(root) : null;
|
|
7592
7611
|
let doc;
|
|
@@ -16975,10 +16994,29 @@ var AcpHost = class {
|
|
|
16975
16994
|
async cancel(sessionId) {
|
|
16976
16995
|
await this.conn.agent.notify(methods.agent.session.cancel, { sessionId });
|
|
16977
16996
|
}
|
|
16978
|
-
|
|
16979
|
-
|
|
16980
|
-
|
|
16981
|
-
|
|
16997
|
+
/** Stop the adapter child: SIGTERM, then escalate to SIGKILL if it hasn't exited
|
|
16998
|
+
* within `deadlineMs` (a buggy/hung agent must never block daemon shutdown or an
|
|
16999
|
+
* idle reap). Idempotent — the child handle is cleared up front so a concurrent
|
|
17000
|
+
* stop (drain + reconcile racing) is a no-op rather than a double-kill. */
|
|
17001
|
+
async stop(deadlineMs = 5e3) {
|
|
17002
|
+
const child = this.child;
|
|
17003
|
+
if (!child) return;
|
|
17004
|
+
this.child = void 0;
|
|
17005
|
+
child.kill("SIGTERM");
|
|
17006
|
+
await new Promise((resolve) => {
|
|
17007
|
+
let settled = false;
|
|
17008
|
+
const done = () => {
|
|
17009
|
+
if (settled) return;
|
|
17010
|
+
settled = true;
|
|
17011
|
+
clearTimeout(timer);
|
|
17012
|
+
resolve();
|
|
17013
|
+
};
|
|
17014
|
+
const timer = setTimeout(() => {
|
|
17015
|
+
this.opts.log?.warn(`acp: child ignored SIGTERM after ${deadlineMs}ms — sending SIGKILL`);
|
|
17016
|
+
child.kill("SIGKILL");
|
|
17017
|
+
}, deadlineMs);
|
|
17018
|
+
child.once("exit", done);
|
|
17019
|
+
});
|
|
16982
17020
|
}
|
|
16983
17021
|
};
|
|
16984
17022
|
//#endregion
|
|
@@ -23361,6 +23399,27 @@ var LocalStore = class {
|
|
|
23361
23399
|
acpSessionId=excluded.acpSessionId, state=excluded.state,
|
|
23362
23400
|
lastDeliveredTs=excluded.lastDeliveredTs, updatedAt=excluded.updatedAt`).run(rec);
|
|
23363
23401
|
}
|
|
23402
|
+
/** Targeted state transition for an existing session (§7.3), stamping `updatedAt`
|
|
23403
|
+
* so the change counts as activity for the TTL/idle clocks. No-op if the key is
|
|
23404
|
+
* unknown (the row is created by the SessionManager on first turn). */
|
|
23405
|
+
setSessionState(key, state, updatedAt) {
|
|
23406
|
+
this.db.prepare("UPDATE sessions SET state = ?, updatedAt = ? WHERE key = ?").run(state, updatedAt, key);
|
|
23407
|
+
}
|
|
23408
|
+
/** Most-recent activity across an agent's non-closed sessions (epoch ms), or null
|
|
23409
|
+
* if it has none. Drives idle-host reaping (#111): a host with no recent session
|
|
23410
|
+
* activity AND no in-flight turn is past its idle window. */
|
|
23411
|
+
agentLastActivityTs(agentId) {
|
|
23412
|
+
return this.db.prepare("SELECT MAX(updatedAt) AS ts FROM sessions WHERE agentId = ? AND state != 'closed'").get(agentId)?.ts ?? null;
|
|
23413
|
+
}
|
|
23414
|
+
/** §7.3 TTL close: move every `idle` session untouched since `now - ttlMs` to
|
|
23415
|
+
* `closed`, returning the rows closed (for logging). `prompting`/`cancelling`
|
|
23416
|
+
* sessions are never closed — a live turn keeps the thread open. */
|
|
23417
|
+
closeIdleSessions(now, ttlMs) {
|
|
23418
|
+
const cutoff = now - ttlMs;
|
|
23419
|
+
const rows = this.db.prepare("SELECT key, channel, thread, agentId FROM sessions WHERE state = 'idle' AND updatedAt < ?").all(cutoff);
|
|
23420
|
+
if (rows.length) this.db.prepare("UPDATE sessions SET state = 'closed' WHERE state = 'idle' AND updatedAt < ?").run(cutoff);
|
|
23421
|
+
return rows;
|
|
23422
|
+
}
|
|
23364
23423
|
appendTranscript(e) {
|
|
23365
23424
|
this.db.prepare("INSERT OR IGNORE INTO transcript (channel, thread, ts, sender, kind, text) VALUES (@channel, @thread, @ts, @sender, @kind, @text)").run(e);
|
|
23366
23425
|
}
|
|
@@ -23512,10 +23571,13 @@ var SessionManager = class {
|
|
|
23512
23571
|
} else if (host.hasSession?.(rec.acpSessionId) === false) {
|
|
23513
23572
|
const cwd = await prepareWorkspace(agent);
|
|
23514
23573
|
let resumed = false;
|
|
23515
|
-
if (host.loadSupported?.())
|
|
23516
|
-
|
|
23517
|
-
|
|
23518
|
-
|
|
23574
|
+
if (host.loadSupported?.()) {
|
|
23575
|
+
this.deps.store.setSessionState(key, "resuming", Date.now());
|
|
23576
|
+
try {
|
|
23577
|
+
await host.loadSession(rec.acpSessionId, cwd);
|
|
23578
|
+
resumed = true;
|
|
23579
|
+
} catch {}
|
|
23580
|
+
}
|
|
23519
23581
|
if (!resumed) {
|
|
23520
23582
|
const mcpServers = this.deps.mcpServersFor?.({
|
|
23521
23583
|
agent,
|
|
@@ -81643,6 +81705,7 @@ var CpClient = class {
|
|
|
81643
81705
|
this.heartbeatMs = ok.heartbeatSec > 0 ? ok.heartbeatSec * 1e3 : this.deps.heartbeatDefaultMs;
|
|
81644
81706
|
this.armHeartbeat();
|
|
81645
81707
|
this.deps.log.info(`cp: READY (epoch=${this.sessionEpoch}, routingEpoch=${this.routingEpoch})`);
|
|
81708
|
+
for (const profile of this.deps.runtimeProfiles()) this.transport?.send(encode(buildEnvelope("facts/runtime-profile", profile)));
|
|
81646
81709
|
}
|
|
81647
81710
|
request(type, payload) {
|
|
81648
81711
|
const frame = buildEnvelope(type, payload);
|
|
@@ -81697,14 +81760,13 @@ var CpClient = class {
|
|
|
81697
81760
|
}
|
|
81698
81761
|
armHeartbeat() {
|
|
81699
81762
|
this.heartbeatTimer = this.deps.clock.setTimeout(() => {
|
|
81700
|
-
if (this.state
|
|
81701
|
-
this.transport?.send(encode(buildEnvelope("heartbeat", {
|
|
81763
|
+
if (this.state === "READY") this.transport?.send(encode(buildEnvelope("heartbeat", {
|
|
81702
81764
|
load: this.deps.loadSnapshot(),
|
|
81703
81765
|
health: "ok",
|
|
81704
81766
|
activeSessions: this.deps.activeSessions(),
|
|
81705
81767
|
degradedScopes: this.deps.degradedScopes?.() ?? []
|
|
81706
81768
|
})));
|
|
81707
|
-
this.armHeartbeat();
|
|
81769
|
+
if (this.state === "READY" || this.state === "DRAINING") this.armHeartbeat();
|
|
81708
81770
|
}, this.heartbeatMs);
|
|
81709
81771
|
}
|
|
81710
81772
|
stopHeartbeat() {
|
|
@@ -81751,13 +81813,36 @@ var CpClient = class {
|
|
|
81751
81813
|
case "agent/remove":
|
|
81752
81814
|
this.deps.configApply.applyAgentRemove(frame.payload.agentId);
|
|
81753
81815
|
return;
|
|
81754
|
-
case "agent/launch":
|
|
81755
|
-
|
|
81756
|
-
|
|
81757
|
-
|
|
81816
|
+
case "agent/launch": {
|
|
81817
|
+
const launch = frame.payload;
|
|
81818
|
+
this.deps.configApply.applyAgentLaunch(launch).then((launched) => this.reply(frame, "agent/launched", launched)).catch((err) => this.sendError(frame.id, "INTERNAL", `agent/launch failed: ${err.message}`, false));
|
|
81819
|
+
return;
|
|
81820
|
+
}
|
|
81821
|
+
case "agent/stop": {
|
|
81822
|
+
const stop = frame.payload;
|
|
81823
|
+
this.deps.configApply.applyAgentStop(stop).then((ack) => this.reply(frame, "ack", ack)).catch((err) => this.sendError(frame.id, "INTERNAL", `agent/stop failed: ${err.message}`, false));
|
|
81824
|
+
return;
|
|
81825
|
+
}
|
|
81826
|
+
case "daemon/drain": {
|
|
81827
|
+
const drain = frame.payload;
|
|
81828
|
+
this.state = "DRAINING";
|
|
81829
|
+
this.deps.configApply.applyDaemonDrain(drain, (p) => this.emit("drain/progress", p)).then((done) => {
|
|
81830
|
+
this.reply(frame, "drain/done", done);
|
|
81831
|
+
if (this.state === "DRAINING") this.state = "READY";
|
|
81832
|
+
}).catch((err) => {
|
|
81833
|
+
this.sendError(frame.id, "INTERNAL", `drain failed: ${err.message}`, false);
|
|
81834
|
+
if (this.state === "DRAINING") this.state = "READY";
|
|
81835
|
+
});
|
|
81836
|
+
return;
|
|
81837
|
+
}
|
|
81758
81838
|
case "daemon/restart":
|
|
81839
|
+
this.reply(frame, "daemon/control/ack", this.deps.configApply.applyDaemonRestart(frame.payload));
|
|
81840
|
+
return;
|
|
81759
81841
|
case "daemon/upgrade":
|
|
81760
|
-
this.
|
|
81842
|
+
this.reply(frame, "daemon/control/ack", this.deps.configApply.applyDaemonUpgrade(frame.payload));
|
|
81843
|
+
return;
|
|
81844
|
+
case "agent/prompt":
|
|
81845
|
+
this.sendError(frame.id, "INTERNAL", "agent/prompt not implemented", false);
|
|
81761
81846
|
return;
|
|
81762
81847
|
default:
|
|
81763
81848
|
this.deps.log.debug(`cp: ignoring ${frame.type}`);
|
|
@@ -81767,6 +81852,10 @@ var CpClient = class {
|
|
|
81767
81852
|
reply(req, type, payload) {
|
|
81768
81853
|
this.transport?.send(encode(buildEnvelope(type, payload, { corr: req.id })));
|
|
81769
81854
|
}
|
|
81855
|
+
/** Emit an uncorrelated EVT (e.g. `drain/progress`). */
|
|
81856
|
+
emit(type, payload) {
|
|
81857
|
+
this.transport?.send(encode(buildEnvelope(type, payload)));
|
|
81858
|
+
}
|
|
81770
81859
|
};
|
|
81771
81860
|
//#endregion
|
|
81772
81861
|
//#region src/cp/cp-cron.ts
|
|
@@ -82047,6 +82136,37 @@ function formatErr(err) {
|
|
|
82047
82136
|
}
|
|
82048
82137
|
const MAX_QUEUED_PER_SESSION = 10;
|
|
82049
82138
|
const IDLE_FLUSH_MS = 2e3;
|
|
82139
|
+
/** Build the wire SessionKey (protocol §5) for a pending turn — what `drain/done`
|
|
82140
|
+
* reports as released so the CP may reassign it. Uses the real `thread` (absent for
|
|
82141
|
+
* a channel-root message), NOT `statusThread` (which falls back to msgId): the CP
|
|
82142
|
+
* keys assignments by `thread ?? "-"`, so reporting the msgId would miss the match. */
|
|
82143
|
+
function pendingSessionKey(p) {
|
|
82144
|
+
const platform = p.platform;
|
|
82145
|
+
return p.thread !== void 0 ? {
|
|
82146
|
+
platform,
|
|
82147
|
+
channel: p.channel,
|
|
82148
|
+
thread: p.thread
|
|
82149
|
+
} : {
|
|
82150
|
+
platform,
|
|
82151
|
+
channel: p.channel
|
|
82152
|
+
};
|
|
82153
|
+
}
|
|
82154
|
+
/** Aggregate CPU time across all cores — busy vs idle jiffies. CPU utilization is the
|
|
82155
|
+
* ratio of busy-time DELTAS between two of these samples (a single sample is meaningless;
|
|
82156
|
+
* it's cumulative since boot). This is real utilization, NOT the run-queue load average. */
|
|
82157
|
+
function cpuTotals() {
|
|
82158
|
+
let idle = 0;
|
|
82159
|
+
let busy = 0;
|
|
82160
|
+
for (const c of cpus()) {
|
|
82161
|
+
const t = c.times;
|
|
82162
|
+
idle += t.idle;
|
|
82163
|
+
busy += t.user + t.nice + t.sys + t.irq;
|
|
82164
|
+
}
|
|
82165
|
+
return {
|
|
82166
|
+
idle,
|
|
82167
|
+
busy
|
|
82168
|
+
};
|
|
82169
|
+
}
|
|
82050
82170
|
var Daemon = class {
|
|
82051
82171
|
opts;
|
|
82052
82172
|
store;
|
|
@@ -82065,15 +82185,26 @@ var Daemon = class {
|
|
|
82065
82185
|
cfg;
|
|
82066
82186
|
log = makeLogger("info");
|
|
82067
82187
|
root = "";
|
|
82188
|
+
lastCpu = cpuTotals();
|
|
82068
82189
|
runtimes = {};
|
|
82069
82190
|
runtimeNames = {};
|
|
82191
|
+
runtimeVersions = {};
|
|
82070
82192
|
cpClient;
|
|
82071
82193
|
cpCrons;
|
|
82072
82194
|
cpAgents;
|
|
82073
82195
|
botUserIds = {};
|
|
82074
82196
|
cpRouting;
|
|
82197
|
+
clock;
|
|
82198
|
+
requestExit;
|
|
82199
|
+
draining = false;
|
|
82200
|
+
drainingAgents = /* @__PURE__ */ new Set();
|
|
82201
|
+
hostStopping = /* @__PURE__ */ new Map();
|
|
82202
|
+
idleSweepTimer;
|
|
82203
|
+
cancelTimers = /* @__PURE__ */ new Map();
|
|
82075
82204
|
constructor(opts = {}) {
|
|
82076
82205
|
this.opts = opts;
|
|
82206
|
+
this.clock = opts.clock ?? systemClock;
|
|
82207
|
+
this.requestExit = opts.requestExit ?? ((code) => process.exit(code));
|
|
82077
82208
|
}
|
|
82078
82209
|
async start() {
|
|
82079
82210
|
const root = resolveRoot(this.opts.root);
|
|
@@ -82103,6 +82234,7 @@ var Daemon = class {
|
|
|
82103
82234
|
});
|
|
82104
82235
|
this.runtimes = installedRuntimes(resolvedRuntimes);
|
|
82105
82236
|
this.runtimeNames = cachedRuntimeNames(root);
|
|
82237
|
+
this.runtimeVersions = cachedRuntimeVersions(root);
|
|
82106
82238
|
this.log.info(`runtimes ready: ${Object.keys(this.runtimes).join(", ") || "(none)"}`);
|
|
82107
82239
|
const skipped = Object.keys(resolvedRuntimes).filter((id) => !this.runtimes[id]);
|
|
82108
82240
|
if (skipped.length) this.log.info(`runtimes not installed (skipped): ${skipped.join(", ")}`);
|
|
@@ -82213,6 +82345,7 @@ var Daemon = class {
|
|
|
82213
82345
|
this.watcher.on("add", debounced).on("change", debounced).on("unlink", debounced);
|
|
82214
82346
|
this.log.info(`watching ${this.agentsDir} for agent changes`);
|
|
82215
82347
|
this.startCpClient(root);
|
|
82348
|
+
this.armIdleSweep();
|
|
82216
82349
|
this.log.info("daemon ready");
|
|
82217
82350
|
}
|
|
82218
82351
|
loadAgentList() {
|
|
@@ -82385,6 +82518,10 @@ var Daemon = class {
|
|
|
82385
82518
|
}
|
|
82386
82519
|
seenMsgIds = /* @__PURE__ */ new Set();
|
|
82387
82520
|
onInbound(msg) {
|
|
82521
|
+
if (this.draining) {
|
|
82522
|
+
this.log.debug(`routing: dropping inbound ${msg.msgId} (daemon draining)`);
|
|
82523
|
+
return;
|
|
82524
|
+
}
|
|
82388
82525
|
if (this.seenMsgIds.has(msg.msgId)) {
|
|
82389
82526
|
this.log.debug(`routing: duplicate ${msg.msgId} ignored`);
|
|
82390
82527
|
return;
|
|
@@ -82402,6 +82539,10 @@ var Daemon = class {
|
|
|
82402
82539
|
this.log.debug(`routing: dropped message in ch=${msg.channel} (no agent matched — not a mention of a known bot, not a subscribed 'all' channel, not a thread/DM hit)`);
|
|
82403
82540
|
return;
|
|
82404
82541
|
}
|
|
82542
|
+
if (this.drainingAgents.has(result.agentId)) {
|
|
82543
|
+
this.log.debug(`routing: dropping ${msg.msgId} for agent "${result.agentId}" (draining)`);
|
|
82544
|
+
return;
|
|
82545
|
+
}
|
|
82405
82546
|
this.log.info(`routing: ch=${msg.channel} → agent "${result.agentId}" (integration ${result.integrationId})`);
|
|
82406
82547
|
this.dispatch(result.agentId, msg, result.integrationId).catch((err) => this.log.error(`dispatch failed for agent "${result.agentId}": ${formatErr(err)}`));
|
|
82407
82548
|
}
|
|
@@ -82471,7 +82612,8 @@ var Daemon = class {
|
|
|
82471
82612
|
}
|
|
82472
82613
|
const conn = this.replyConnFor(target.agentId, target.integrationId);
|
|
82473
82614
|
const thread = msg.thread ?? msg.msgId;
|
|
82474
|
-
const
|
|
82615
|
+
const key = sessionKey(msg.platform, msg.channel, thread, target.agentId);
|
|
82616
|
+
const acpSessionId = this.store.getSession(key)?.acpSessionId;
|
|
82475
82617
|
const inflight = !!(acpSessionId && this.pending.has(acpSessionId));
|
|
82476
82618
|
if (command.kind === "stop") {
|
|
82477
82619
|
if (!inflight) {
|
|
@@ -82480,7 +82622,9 @@ var Daemon = class {
|
|
|
82480
82622
|
}
|
|
82481
82623
|
this.queued.delete(acpSessionId);
|
|
82482
82624
|
this.log.info(`command: stop → agent "${target.agentId}" session ${acpSessionId}`);
|
|
82625
|
+
this.store.setSessionState(key, "cancelling", this.clock.now());
|
|
82483
82626
|
this.hosts.get(target.agentId)?.cancel(acpSessionId).catch((err) => this.log.error(`command stop: cancel failed: ${err.message}`));
|
|
82627
|
+
this.armCancelBackstop(target.agentId, acpSessionId, key);
|
|
82484
82628
|
conn?.postMessage(msg.channel, "🛑 Stopped.", thread);
|
|
82485
82629
|
return;
|
|
82486
82630
|
}
|
|
@@ -82543,22 +82687,40 @@ var Daemon = class {
|
|
|
82543
82687
|
return [...out];
|
|
82544
82688
|
}
|
|
82545
82689
|
async dispatch(agentId, msg, integrationId) {
|
|
82690
|
+
if (this.draining || this.drainingAgents.has(agentId)) {
|
|
82691
|
+
this.log.debug(`dispatch: skipped for agent "${agentId}" (draining)`);
|
|
82692
|
+
return;
|
|
82693
|
+
}
|
|
82546
82694
|
const conv = new OutputConverger(this.agents.get(agentId).output.mode);
|
|
82547
82695
|
const rec = new TranscriptRecorder();
|
|
82548
82696
|
const replyConn = this.replyConnFor(agentId, integrationId);
|
|
82549
82697
|
const wasRunning = this.hostStarts.has(agentId);
|
|
82550
82698
|
const statusThread = msg.thread ?? msg.msgId;
|
|
82699
|
+
const key = sessionKey(msg.platform, msg.channel, statusThread, agentId);
|
|
82551
82700
|
replyConn?.setStatus(msg.channel, statusThread, wasRunning ? "is thinking…" : "is starting up…");
|
|
82552
|
-
|
|
82701
|
+
let handled;
|
|
82702
|
+
try {
|
|
82703
|
+
handled = await this.sessions.handle(agentId, msg);
|
|
82704
|
+
} catch (err) {
|
|
82705
|
+
this.store.setSessionState(key, "idle", this.clock.now());
|
|
82706
|
+
throw err;
|
|
82707
|
+
}
|
|
82708
|
+
const { sessionId, blocks } = handled;
|
|
82709
|
+
let resolveDone;
|
|
82710
|
+
const done = new Promise((r) => resolveDone = r);
|
|
82553
82711
|
const p = {
|
|
82554
82712
|
conv,
|
|
82555
82713
|
rec,
|
|
82556
82714
|
agentId,
|
|
82715
|
+
platform: msg.platform,
|
|
82716
|
+
sessionKey: key,
|
|
82557
82717
|
channel: msg.channel,
|
|
82558
82718
|
thread: msg.thread,
|
|
82559
82719
|
statusThread,
|
|
82560
82720
|
conn: replyConn,
|
|
82561
|
-
applyChain: Promise.resolve()
|
|
82721
|
+
applyChain: Promise.resolve(),
|
|
82722
|
+
done,
|
|
82723
|
+
resolveDone
|
|
82562
82724
|
};
|
|
82563
82725
|
this.pending.set(sessionId, p);
|
|
82564
82726
|
try {
|
|
@@ -82573,9 +82735,35 @@ var Daemon = class {
|
|
|
82573
82735
|
} finally {
|
|
82574
82736
|
this.clearIdle(p);
|
|
82575
82737
|
this.pending.delete(sessionId);
|
|
82738
|
+
this.store.setSessionState(key, "idle", this.clock.now());
|
|
82739
|
+
this.clearCancelBackstop(sessionId);
|
|
82740
|
+
p.resolveDone();
|
|
82576
82741
|
}
|
|
82577
82742
|
this.flushQueued(agentId, sessionId, integrationId);
|
|
82578
82743
|
}
|
|
82744
|
+
/** After `!stop` sends session/cancel, give the agent `cancelBackstopMs` to yield.
|
|
82745
|
+
* If the turn is still in flight when the timer fires, the agent ignored the
|
|
82746
|
+
* cancel — force-stop its host (the only hard kill available) so the session
|
|
82747
|
+
* can't be stuck in `cancelling` forever. dispatch's finally clears this timer
|
|
82748
|
+
* the moment the turn yields on its own. */
|
|
82749
|
+
armCancelBackstop(agentId, acpSessionId, key) {
|
|
82750
|
+
this.clearCancelBackstop(acpSessionId);
|
|
82751
|
+
const ms = this.cfg.limits.cancelBackstopMs;
|
|
82752
|
+
this.cancelTimers.set(acpSessionId, this.clock.setTimeout(() => {
|
|
82753
|
+
this.cancelTimers.delete(acpSessionId);
|
|
82754
|
+
if (!this.pending.has(acpSessionId)) return;
|
|
82755
|
+
this.log.warn(`command stop: agent "${agentId}" ignored session/cancel for ${ms}ms — force-stopping host (session ${acpSessionId})`);
|
|
82756
|
+
this.stopHost(agentId, 0);
|
|
82757
|
+
this.store.setSessionState(key, "idle", this.clock.now());
|
|
82758
|
+
}, ms));
|
|
82759
|
+
}
|
|
82760
|
+
clearCancelBackstop(acpSessionId) {
|
|
82761
|
+
const t = this.cancelTimers.get(acpSessionId);
|
|
82762
|
+
if (t !== void 0) {
|
|
82763
|
+
this.clock.clearTimeout(t);
|
|
82764
|
+
this.cancelTimers.delete(acpSessionId);
|
|
82765
|
+
}
|
|
82766
|
+
}
|
|
82579
82767
|
/**
|
|
82580
82768
|
* Apply one converger action against the session's Slack connection:
|
|
82581
82769
|
* - set-status → assistant.threads.setStatus (best-effort; '' clears)
|
|
@@ -82664,6 +82852,8 @@ var Daemon = class {
|
|
|
82664
82852
|
});
|
|
82665
82853
|
}
|
|
82666
82854
|
async ensureHostAsync(agentId) {
|
|
82855
|
+
const stopping = this.hostStopping.get(agentId);
|
|
82856
|
+
if (stopping) await stopping;
|
|
82667
82857
|
const host = this.ensureHost(agentId, this.cfg);
|
|
82668
82858
|
let p = this.hostStarts.get(agentId);
|
|
82669
82859
|
if (!p) {
|
|
@@ -82677,6 +82867,171 @@ var Daemon = class {
|
|
|
82677
82867
|
const intId = integrationId ?? this.agents.get(agentId)?.integrations[0]?.id;
|
|
82678
82868
|
return intId ? this.connByIntegration.get(intId) : void 0;
|
|
82679
82869
|
}
|
|
82870
|
+
/** Stop and evict an ACP adapter child, returning the agent to `provisioned`
|
|
82871
|
+
* (config kept; the next message lazily re-spawns it). Idempotent. The teardown
|
|
82872
|
+
* is registered in `hostStopping` so a concurrent ensureHostAsync waits for it
|
|
82873
|
+
* instead of spawning a second live child. */
|
|
82874
|
+
async stopHost(agentId, deadlineMs) {
|
|
82875
|
+
const host = this.hosts.get(agentId);
|
|
82876
|
+
this.hosts.delete(agentId);
|
|
82877
|
+
this.hostStarts.delete(agentId);
|
|
82878
|
+
if (!host) return;
|
|
82879
|
+
const stop = host.stop(deadlineMs).finally(() => {
|
|
82880
|
+
if (this.hostStopping.get(agentId) === stop) this.hostStopping.delete(agentId);
|
|
82881
|
+
});
|
|
82882
|
+
this.hostStopping.set(agentId, stop);
|
|
82883
|
+
await stop;
|
|
82884
|
+
}
|
|
82885
|
+
/** Recurring idle sweep (§7.2/§7.3): reap idle adapter children and TTL-close
|
|
82886
|
+
* idle sessions. Driven by the injected Clock so a FakeClock advances it in tests. */
|
|
82887
|
+
armIdleSweep() {
|
|
82888
|
+
const interval = this.cfg.limits.idleSweepMs;
|
|
82889
|
+
if (interval <= 0) return;
|
|
82890
|
+
this.idleSweepTimer = this.clock.setTimeout(() => {
|
|
82891
|
+
this.idleSweepTimer = void 0;
|
|
82892
|
+
try {
|
|
82893
|
+
this.sweepIdle();
|
|
82894
|
+
} catch (err) {
|
|
82895
|
+
this.log.error(`idle sweep failed: ${formatErr(err)}`);
|
|
82896
|
+
}
|
|
82897
|
+
if (!this.draining) this.armIdleSweep();
|
|
82898
|
+
}, interval);
|
|
82899
|
+
}
|
|
82900
|
+
sweepIdle() {
|
|
82901
|
+
const now = this.clock.now();
|
|
82902
|
+
const ttl = this.cfg.limits.agentIdleTimeoutMs;
|
|
82903
|
+
const closed = this.store.closeIdleSessions(now, ttl);
|
|
82904
|
+
if (closed.length) this.log.info(`idle: TTL-closed ${closed.length} session(s) (>${Math.round(ttl / 1e3)}s)`);
|
|
82905
|
+
for (const [agentId] of [...this.hosts]) {
|
|
82906
|
+
if (this.drainingAgents.has(agentId)) continue;
|
|
82907
|
+
if ([...this.pending.values()].some((p) => p.agentId === agentId)) continue;
|
|
82908
|
+
const last = this.store.agentLastActivityTs(agentId) ?? 0;
|
|
82909
|
+
if (now - last <= ttl) continue;
|
|
82910
|
+
this.log.info(`idle: reclaiming host "${agentId}" (idle ${Math.round((now - last) / 1e3)}s) → provisioned`);
|
|
82911
|
+
this.stopHost(agentId).catch((err) => this.log.error(`idle: stop host "${agentId}" failed: ${formatErr(err)}`));
|
|
82912
|
+
}
|
|
82913
|
+
}
|
|
82914
|
+
/** Race `work` against a Clock-driven deadline, always clearing the timer so a
|
|
82915
|
+
* finished drain never leaves a dangling timer holding the process open. */
|
|
82916
|
+
async raceDeadline(work, ms) {
|
|
82917
|
+
const delay = Math.min(Math.max(0, ms), 2147483647);
|
|
82918
|
+
let handle;
|
|
82919
|
+
const timeout = new Promise((resolve) => {
|
|
82920
|
+
handle = this.clock.setTimeout(() => resolve("timeout"), delay);
|
|
82921
|
+
});
|
|
82922
|
+
try {
|
|
82923
|
+
return await Promise.race([work.then(() => "done"), timeout]);
|
|
82924
|
+
} finally {
|
|
82925
|
+
if (handle !== void 0) this.clock.clearTimeout(handle);
|
|
82926
|
+
}
|
|
82927
|
+
}
|
|
82928
|
+
/**
|
|
82929
|
+
* §5.3 drain: gate new turns for the scope, await in-flight turns up to
|
|
82930
|
+
* `deadlineMs` (emitting `drain/progress` as each yields), then cancel any
|
|
82931
|
+
* straggler past the deadline. Returns BOTH the in-scope set (`matched`) and the
|
|
82932
|
+
* subset that actually finished (`drained`). The caller decides which to report
|
|
82933
|
+
* as released: a session is only safe to release once it is genuinely no longer
|
|
82934
|
+
* being served — either it drained, or the caller force-stops its host. Reporting
|
|
82935
|
+
* a still-running straggler would let the CP reassign it and double-serve.
|
|
82936
|
+
*/
|
|
82937
|
+
async drainScope(scope, deadlineMs, onProgress) {
|
|
82938
|
+
const match = (p) => {
|
|
82939
|
+
if (scope.kind === "daemon") return true;
|
|
82940
|
+
if (scope.kind === "agent") return p.agentId === scope.agentId;
|
|
82941
|
+
const k = scope.sessionKey;
|
|
82942
|
+
return p.platform === k.platform && p.channel === k.channel && (k.thread === void 0 || p.statusThread === k.thread);
|
|
82943
|
+
};
|
|
82944
|
+
if (scope.kind === "daemon") this.draining = true;
|
|
82945
|
+
else if (scope.kind === "agent") this.drainingAgents.add(scope.agentId);
|
|
82946
|
+
const keyOf = (sk) => `${sk.platform}:${sk.channel}:${sk.thread ?? "-"}`;
|
|
82947
|
+
const targets = [...this.pending.entries()].filter(([, p]) => match(p));
|
|
82948
|
+
const matched = /* @__PURE__ */ new Map();
|
|
82949
|
+
for (const [, p] of targets) matched.set(keyOf(pendingSessionKey(p)), pendingSessionKey(p));
|
|
82950
|
+
if (targets.length) this.log.info(`drain[${scope.kind}]: awaiting ${targets.length} turn(s) (deadline ${Math.round(deadlineMs / 1e3)}s)`);
|
|
82951
|
+
let settled = false;
|
|
82952
|
+
const drained = /* @__PURE__ */ new Map();
|
|
82953
|
+
let remaining = targets.length;
|
|
82954
|
+
const work = Promise.all(targets.map(([, p]) => p.done.then(() => {
|
|
82955
|
+
const sk = pendingSessionKey(p);
|
|
82956
|
+
drained.set(keyOf(sk), sk);
|
|
82957
|
+
remaining--;
|
|
82958
|
+
if (!settled) onProgress?.({
|
|
82959
|
+
remaining,
|
|
82960
|
+
drained: [...drained.values()]
|
|
82961
|
+
});
|
|
82962
|
+
})));
|
|
82963
|
+
const res = await this.raceDeadline(work, deadlineMs);
|
|
82964
|
+
settled = true;
|
|
82965
|
+
if (res === "timeout") for (const [sid, p] of this.pending) {
|
|
82966
|
+
if (!match(p)) continue;
|
|
82967
|
+
this.log.warn(`drain[${scope.kind}]: cancelling straggler turn (session ${sid})`);
|
|
82968
|
+
await this.hosts.get(p.agentId)?.cancel(sid).catch(() => {});
|
|
82969
|
+
}
|
|
82970
|
+
return {
|
|
82971
|
+
matched: [...matched.values()],
|
|
82972
|
+
drained: [...drained.values()]
|
|
82973
|
+
};
|
|
82974
|
+
}
|
|
82975
|
+
/** Handle a CP `daemon/drain` (§5.3). A bare drain is a rebalance: after
|
|
82976
|
+
* releasing sessions the daemon reclaims hosts and re-opens its gate (a teardown
|
|
82977
|
+
* arrives separately via daemon/restart). */
|
|
82978
|
+
async runDrain(drain, onProgress) {
|
|
82979
|
+
const deadlineMs = Math.max(0, new Date(drain.deadline).getTime() - this.clock.now());
|
|
82980
|
+
const { matched, drained } = await this.drainScope(drain.scope, deadlineMs, onProgress);
|
|
82981
|
+
let released;
|
|
82982
|
+
if (drain.scope.kind === "daemon") {
|
|
82983
|
+
for (const id of [...this.hosts.keys()]) await this.stopHost(id);
|
|
82984
|
+
this.draining = false;
|
|
82985
|
+
released = matched;
|
|
82986
|
+
} else if (drain.scope.kind === "agent") {
|
|
82987
|
+
await this.stopHost(drain.scope.agentId);
|
|
82988
|
+
this.drainingAgents.delete(drain.scope.agentId);
|
|
82989
|
+
released = matched;
|
|
82990
|
+
} else released = drained;
|
|
82991
|
+
this.log.info(`drain[${drain.scope.kind}]: done — released ${released.length} session(s)`);
|
|
82992
|
+
return { released };
|
|
82993
|
+
}
|
|
82994
|
+
/** `agent/stop` (§8.2): drain the agent's in-flight turns, stop its host, and
|
|
82995
|
+
* leave it gated so it stays down until a matching `agent/launch` revives it. */
|
|
82996
|
+
async stopAgent(agentId) {
|
|
82997
|
+
await this.drainScope({
|
|
82998
|
+
kind: "agent",
|
|
82999
|
+
agentId
|
|
83000
|
+
}, this.cfg.limits.shutdownDrainMs);
|
|
83001
|
+
await this.stopHost(agentId);
|
|
83002
|
+
}
|
|
83003
|
+
/** §2.5 SIGTERM / daemon shutdown: gate new turns, then await in-flight turns up
|
|
83004
|
+
* to `shutdownDrainMs`, cancelling stragglers. Safe to call repeatedly. */
|
|
83005
|
+
async drainForShutdown() {
|
|
83006
|
+
this.draining = true;
|
|
83007
|
+
const live = [...this.pending.values()];
|
|
83008
|
+
if (live.length === 0) return;
|
|
83009
|
+
const deadlineMs = this.cfg.limits.shutdownDrainMs;
|
|
83010
|
+
this.log.info(`shutdown: draining ${live.length} in-flight turn(s) (deadline ${Math.round(deadlineMs / 1e3)}s)`);
|
|
83011
|
+
if (await this.raceDeadline(Promise.all(live.map((p) => p.done)), deadlineMs) === "timeout") {
|
|
83012
|
+
this.log.warn(`shutdown: deadline hit with ${this.pending.size} turn(s) still in flight — cancelling`);
|
|
83013
|
+
for (const [sid, p] of this.pending) await this.hosts.get(p.agentId)?.cancel(sid).catch(() => {});
|
|
83014
|
+
}
|
|
83015
|
+
}
|
|
83016
|
+
/** daemon/restart + daemon/upgrade (§8.3): ack now, then drain + stop + exit so
|
|
83017
|
+
* the supervisor relaunches (the new binary, for upgrade). */
|
|
83018
|
+
scheduleFleetExit(kind, targetVersion) {
|
|
83019
|
+
const willDrainUntil = new Date(this.clock.now() + this.cfg.limits.shutdownDrainMs).toISOString();
|
|
83020
|
+
this.log.info(`cp: ${kind}${targetVersion ? ` → ${targetVersion}` : ""} requested — draining then exiting`);
|
|
83021
|
+
(async () => {
|
|
83022
|
+
try {
|
|
83023
|
+
await this.stop();
|
|
83024
|
+
} catch (err) {
|
|
83025
|
+
this.log.error(`cp: ${kind} shutdown failed: ${formatErr(err)}`);
|
|
83026
|
+
} finally {
|
|
83027
|
+
this.requestExit(0);
|
|
83028
|
+
}
|
|
83029
|
+
})();
|
|
83030
|
+
return {
|
|
83031
|
+
accepted: true,
|
|
83032
|
+
willDrainUntil
|
|
83033
|
+
};
|
|
83034
|
+
}
|
|
82680
83035
|
cpConfigApply() {
|
|
82681
83036
|
return {
|
|
82682
83037
|
applyConfigPush: (keys) => {
|
|
@@ -82702,7 +83057,26 @@ var Daemon = class {
|
|
|
82702
83057
|
upsertCron: (cron) => this.cpCrons.upsert(cron),
|
|
82703
83058
|
removeCron: (cronId) => this.cpCrons.remove(cronId),
|
|
82704
83059
|
applyRouteAssign: (a) => this.cpRouting?.upsertAssign(a),
|
|
82705
|
-
applyRouteUpdate: (u) => this.cpRouting?.applyUpdate(u)
|
|
83060
|
+
applyRouteUpdate: (u) => this.cpRouting?.applyUpdate(u),
|
|
83061
|
+
applyAgentLaunch: async (launch) => {
|
|
83062
|
+
const agent = this.agents.get(launch.agentId);
|
|
83063
|
+
if (!agent) throw new Error(`agent/launch: unknown agent ${launch.agentId}`);
|
|
83064
|
+
this.drainingAgents.delete(launch.agentId);
|
|
83065
|
+
await this.ensureHostAsync(launch.agentId);
|
|
83066
|
+
return {
|
|
83067
|
+
agentId: launch.agentId,
|
|
83068
|
+
launchId: randomUUID(),
|
|
83069
|
+
startedAt: new Date(this.clock.now()).toISOString(),
|
|
83070
|
+
runtime: agent.runtime
|
|
83071
|
+
};
|
|
83072
|
+
},
|
|
83073
|
+
applyAgentStop: async (stop) => {
|
|
83074
|
+
await this.stopAgent(stop.agentId);
|
|
83075
|
+
return { ok: true };
|
|
83076
|
+
},
|
|
83077
|
+
applyDaemonDrain: (drain, onProgress) => this.runDrain(drain, onProgress),
|
|
83078
|
+
applyDaemonRestart: (_req) => this.scheduleFleetExit("restart"),
|
|
83079
|
+
applyDaemonUpgrade: (req) => this.scheduleFleetExit("upgrade", req.targetVersion)
|
|
82706
83080
|
};
|
|
82707
83081
|
}
|
|
82708
83082
|
/** A CP cron fired: build a synthetic message and run it through the normal routing path. */
|
|
@@ -82726,6 +83100,17 @@ var Daemon = class {
|
|
|
82726
83100
|
};
|
|
82727
83101
|
this.onInbound(msg);
|
|
82728
83102
|
}
|
|
83103
|
+
/** Real CPU utilization (0..1) across all cores since the previous call — the busy
|
|
83104
|
+
* fraction of the time delta. Called once per heartbeat, so it reflects load over the
|
|
83105
|
+
* heartbeat interval (like a system monitor), not the run-queue load average. */
|
|
83106
|
+
cpuUtilization() {
|
|
83107
|
+
const now = cpuTotals();
|
|
83108
|
+
const idle = now.idle - this.lastCpu.idle;
|
|
83109
|
+
const busy = now.busy - this.lastCpu.busy;
|
|
83110
|
+
this.lastCpu = now;
|
|
83111
|
+
const total = idle + busy;
|
|
83112
|
+
return total > 0 ? Math.min(1, Math.max(0, busy / total)) : 0;
|
|
83113
|
+
}
|
|
82729
83114
|
startCpClient(root) {
|
|
82730
83115
|
const cp = this.cfg.controlPlane;
|
|
82731
83116
|
if (!cp?.enabled || !cp.url || !cp.key) {
|
|
@@ -82754,13 +83139,20 @@ var Daemon = class {
|
|
|
82754
83139
|
acp: true,
|
|
82755
83140
|
features: []
|
|
82756
83141
|
}),
|
|
83142
|
+
runtimeProfiles: () => Object.keys(this.runtimes).map((id) => ({
|
|
83143
|
+
runtime: id,
|
|
83144
|
+
version: this.runtimeVersions[id] ?? "",
|
|
83145
|
+
models: [],
|
|
83146
|
+
acpSupport: "full",
|
|
83147
|
+
toolCalling: true
|
|
83148
|
+
})),
|
|
82757
83149
|
localState: () => ({
|
|
82758
83150
|
assignments: [],
|
|
82759
83151
|
crons: this.cpCrons?.ids() ?? [],
|
|
82760
83152
|
leases: []
|
|
82761
83153
|
}),
|
|
82762
83154
|
loadSnapshot: () => ({
|
|
82763
|
-
cpu:
|
|
83155
|
+
cpu: this.cpuUtilization(),
|
|
82764
83156
|
mem: totalmem() > 0 ? 1 - freemem() / totalmem() : 0,
|
|
82765
83157
|
agents: this.hosts.size
|
|
82766
83158
|
}),
|
|
@@ -82775,8 +83167,16 @@ var Daemon = class {
|
|
|
82775
83167
|
this.log.info(`cp: connecting to ${url}…`);
|
|
82776
83168
|
}
|
|
82777
83169
|
async stop() {
|
|
83170
|
+
this.draining = true;
|
|
82778
83171
|
clearTimeout(this.debounceTimer);
|
|
83172
|
+
if (this.idleSweepTimer !== void 0) {
|
|
83173
|
+
this.clock.clearTimeout(this.idleSweepTimer);
|
|
83174
|
+
this.idleSweepTimer = void 0;
|
|
83175
|
+
}
|
|
83176
|
+
for (const t of this.cancelTimers.values()) this.clock.clearTimeout(t);
|
|
83177
|
+
this.cancelTimers.clear();
|
|
82779
83178
|
await this.watcher?.close();
|
|
83179
|
+
await this.drainForShutdown();
|
|
82780
83180
|
const errors = [];
|
|
82781
83181
|
this.scheduler?.stop();
|
|
82782
83182
|
await Promise.resolve(this.cpClient?.stop()).catch((e) => errors.push(e));
|