@pinet/broker-core 0.2.2 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-messaging.d.ts +27 -4
- package/dist/hibernation-commands.d.ts +123 -0
- package/dist/hibernation-commands.js +287 -0
- package/dist/hibernation-orchestrator.d.ts +327 -0
- package/dist/hibernation-orchestrator.js +1096 -0
- package/dist/hibernation-projection.d.ts +20 -0
- package/dist/hibernation-projection.js +60 -0
- package/dist/hibernation-status.d.ts +141 -0
- package/dist/hibernation-status.js +390 -0
- package/dist/hibernation-telemetry.d.ts +54 -0
- package/dist/hibernation-telemetry.js +119 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/leader.d.ts +134 -5
- package/dist/leader.js +359 -26
- package/dist/lifecycle.d.ts +5 -0
- package/dist/lifecycle.js +59 -0
- package/dist/mail-classification.d.ts +6 -1
- package/dist/message-send.d.ts +4 -3
- package/dist/router.d.ts +14 -1
- package/dist/router.js +15 -0
- package/dist/schema.d.ts +181 -2
- package/dist/schema.js +1243 -17
- package/dist/types.d.ts +288 -1
- package/dist/types.js +6 -0
- package/package.json +5 -5
package/dist/schema.js
CHANGED
|
@@ -3,6 +3,7 @@ import * as crypto from "node:crypto";
|
|
|
3
3
|
import * as fs from "node:fs";
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
import { classifyPinetMail } from "./mail-classification.js";
|
|
6
|
+
import { assertLegalLifecycleTransition } from "./lifecycle.js";
|
|
6
7
|
import { getDefaultDbPath } from "./paths.js";
|
|
7
8
|
import { DEFAULT_EXTERNAL_THREAD_SOURCE } from "./types.js";
|
|
8
9
|
function getSqliteJournalMode(result) {
|
|
@@ -40,6 +41,16 @@ function rowToAgent(row) {
|
|
|
40
41
|
resumableUntil: row.resumable_until,
|
|
41
42
|
idleSince: row.idle_since,
|
|
42
43
|
lastActivity: row.last_activity,
|
|
44
|
+
lifecycleState: row.lifecycle_state,
|
|
45
|
+
lifecycleVersion: row.lifecycle_version,
|
|
46
|
+
graceUntil: row.grace_until,
|
|
47
|
+
idleEligibleAt: row.idle_eligible_at,
|
|
48
|
+
hibernatedAt: row.hibernated_at,
|
|
49
|
+
terminatedAt: row.terminated_at,
|
|
50
|
+
hibernatePolicy: row.hibernate_policy,
|
|
51
|
+
hibernateReason: row.hibernate_reason,
|
|
52
|
+
lastWakeReason: row.last_wake_reason,
|
|
53
|
+
runtimeGeneration: row.runtime_generation,
|
|
43
54
|
};
|
|
44
55
|
}
|
|
45
56
|
function rowToThread(row) {
|
|
@@ -76,6 +87,39 @@ function normalizePortLeaseStatus(value) {
|
|
|
76
87
|
}
|
|
77
88
|
return "active";
|
|
78
89
|
}
|
|
90
|
+
function rowToWakeQueueEntry(row) {
|
|
91
|
+
return {
|
|
92
|
+
id: row.id,
|
|
93
|
+
agentId: row.agent_id,
|
|
94
|
+
repoRoot: row.repo_root,
|
|
95
|
+
triggerKind: row.trigger_kind,
|
|
96
|
+
triggerMessageId: row.trigger_message_id,
|
|
97
|
+
priority: row.priority,
|
|
98
|
+
reason: row.reason,
|
|
99
|
+
correlationId: row.correlation_id,
|
|
100
|
+
status: row.status,
|
|
101
|
+
attempt: row.attempt,
|
|
102
|
+
enqueuedAt: row.enqueued_at,
|
|
103
|
+
updatedAt: row.updated_at,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/** Parse a JSON array-of-strings column, tolerating malformed/legacy values. */
|
|
107
|
+
function parseStringArray(value) {
|
|
108
|
+
try {
|
|
109
|
+
const parsed = [];
|
|
110
|
+
const raw = JSON.parse(value);
|
|
111
|
+
if (Array.isArray(raw)) {
|
|
112
|
+
for (const item of raw) {
|
|
113
|
+
if (typeof item === "string")
|
|
114
|
+
parsed.push(item);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return parsed;
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return [];
|
|
121
|
+
}
|
|
122
|
+
}
|
|
79
123
|
function getStringMetadataValue(metadata, keys) {
|
|
80
124
|
for (const key of keys) {
|
|
81
125
|
const value = metadata?.[key];
|
|
@@ -370,6 +414,7 @@ function getAgentSessionMatchedBy(input) {
|
|
|
370
414
|
const threadId = normalizeSessionSearchNeedle(input.options.threadId);
|
|
371
415
|
const repo = normalizeSessionSearchNeedle(input.options.repo);
|
|
372
416
|
const worktreePath = normalizeSessionSearchNeedle(input.options.worktreePath);
|
|
417
|
+
const runtimeLocator = normalizeSessionSearchNeedle(input.options.runtimeLocator);
|
|
373
418
|
const tmuxSession = normalizeSessionSearchNeedle(input.options.tmuxSession);
|
|
374
419
|
if (agentName && matchesSessionSearchNeedle(input.agent.name, agentName)) {
|
|
375
420
|
matchedBy.push("agent_name");
|
|
@@ -398,8 +443,12 @@ function getAgentSessionMatchedBy(input) {
|
|
|
398
443
|
worktreeValues.some((value) => matchesSessionSearchNeedle(value, worktreePath))) {
|
|
399
444
|
matchedBy.push("worktree_path");
|
|
400
445
|
}
|
|
401
|
-
|
|
402
|
-
|
|
446
|
+
if (runtimeLocator && matchesSessionSearchNeedle(input.runtimeLocator, runtimeLocator)) {
|
|
447
|
+
matchedBy.push("runtime_locator");
|
|
448
|
+
}
|
|
449
|
+
if (tmuxSession &&
|
|
450
|
+
input.runtimeKind === "tmux" &&
|
|
451
|
+
matchesSessionSearchNeedle(input.runtimeLocator, tmuxSession)) {
|
|
403
452
|
matchedBy.push("tmux_session");
|
|
404
453
|
}
|
|
405
454
|
if (input.options.since || input.options.until) {
|
|
@@ -553,7 +602,19 @@ export function defaultDbPath() {
|
|
|
553
602
|
}
|
|
554
603
|
export const DEFAULT_RESUMABLE_WINDOW_MS = 15_000;
|
|
555
604
|
export const DEFAULT_DISCONNECTED_PURGE_GRACE_MS = 60 * 60_000;
|
|
556
|
-
export const CURRENT_BROKER_SCHEMA_VERSION =
|
|
605
|
+
export const CURRENT_BROKER_SCHEMA_VERSION = 24;
|
|
606
|
+
/**
|
|
607
|
+
* Lifecycle states whose durable identity, inbox, thread ownership, and runtime
|
|
608
|
+
* mapping MUST survive routine maintenance. A hibernation identity is
|
|
609
|
+
* intentionally "disconnected" (no live socket) yet must be revivable by a later
|
|
610
|
+
* wake, so ordinary disconnect-driven prune/purge/ownership-repair would destroy
|
|
611
|
+
* exactly the state hibernation preserves. `reap-candidate` is quarantined
|
|
612
|
+
* pending manual review and must likewise not be auto-released or deleted (that
|
|
613
|
+
* would discard evidence). `terminated` is a closed identity and remains
|
|
614
|
+
* ordinarily purgeable. This is a fixed constant list — never interpolated with
|
|
615
|
+
* external input — so it is safe to embed directly in SQL predicates.
|
|
616
|
+
*/
|
|
617
|
+
const PRESERVED_LIFECYCLE_STATES_SQL = "'hibernating','hibernated','waking','reap-candidate'";
|
|
557
618
|
const REQUIRED_AGENT_LIFECYCLE_COLUMNS = [
|
|
558
619
|
"stable_id",
|
|
559
620
|
"metadata",
|
|
@@ -1170,6 +1231,218 @@ function createPinetLaneTables(db) {
|
|
|
1170
1231
|
ON pinet_lane_participants(agent_id, lane_role, updated_at DESC);
|
|
1171
1232
|
`);
|
|
1172
1233
|
}
|
|
1234
|
+
// agent-standards-ignore prefer-inline-single-use-helper: schema migrations stay isolated and auditable by version.
|
|
1235
|
+
function createAgentHibernationTables(db) {
|
|
1236
|
+
for (const [name, sql] of [
|
|
1237
|
+
[
|
|
1238
|
+
"lifecycle_state",
|
|
1239
|
+
"ALTER TABLE agents ADD COLUMN lifecycle_state TEXT NOT NULL DEFAULT 'live'",
|
|
1240
|
+
],
|
|
1241
|
+
[
|
|
1242
|
+
"lifecycle_version",
|
|
1243
|
+
"ALTER TABLE agents ADD COLUMN lifecycle_version INTEGER NOT NULL DEFAULT 0",
|
|
1244
|
+
],
|
|
1245
|
+
["grace_until", "ALTER TABLE agents ADD COLUMN grace_until TEXT"],
|
|
1246
|
+
["idle_eligible_at", "ALTER TABLE agents ADD COLUMN idle_eligible_at TEXT"],
|
|
1247
|
+
["hibernated_at", "ALTER TABLE agents ADD COLUMN hibernated_at TEXT"],
|
|
1248
|
+
["terminated_at", "ALTER TABLE agents ADD COLUMN terminated_at TEXT"],
|
|
1249
|
+
[
|
|
1250
|
+
"hibernate_policy",
|
|
1251
|
+
"ALTER TABLE agents ADD COLUMN hibernate_policy TEXT NOT NULL DEFAULT 'never'",
|
|
1252
|
+
],
|
|
1253
|
+
["hibernate_reason", "ALTER TABLE agents ADD COLUMN hibernate_reason TEXT"],
|
|
1254
|
+
["last_wake_reason", "ALTER TABLE agents ADD COLUMN last_wake_reason TEXT"],
|
|
1255
|
+
[
|
|
1256
|
+
"runtime_generation",
|
|
1257
|
+
"ALTER TABLE agents ADD COLUMN runtime_generation INTEGER NOT NULL DEFAULT 0",
|
|
1258
|
+
],
|
|
1259
|
+
])
|
|
1260
|
+
ensureColumn(db, "agents", name, sql);
|
|
1261
|
+
db.exec(`
|
|
1262
|
+
CREATE TABLE IF NOT EXISTS agent_runtime_specs (
|
|
1263
|
+
agent_id TEXT PRIMARY KEY NOT NULL, stable_id TEXT NOT NULL, broker_owner_id TEXT NOT NULL,
|
|
1264
|
+
cwd TEXT NOT NULL, repo_root TEXT NOT NULL, worktree_path TEXT NOT NULL,
|
|
1265
|
+
runtime_kind TEXT NOT NULL DEFAULT 'tmux' CHECK(runtime_kind IN ('tmux','herdr')),
|
|
1266
|
+
tmux_socket TEXT, tmux_session TEXT, tmux_target TEXT,
|
|
1267
|
+
herdr_session TEXT, herdr_config_dir TEXT, herdr_pane_id TEXT, herdr_shell_pid INTEGER,
|
|
1268
|
+
executable TEXT NOT NULL, argv_json TEXT NOT NULL, env_allowlist_json TEXT NOT NULL,
|
|
1269
|
+
session_resume_ref TEXT NOT NULL, config_fingerprint TEXT NOT NULL,
|
|
1270
|
+
expected_host TEXT NOT NULL, expected_user TEXT NOT NULL, launch_source TEXT NOT NULL,
|
|
1271
|
+
vcs_identity TEXT,
|
|
1272
|
+
created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
|
|
1273
|
+
CHECK(
|
|
1274
|
+
(runtime_kind = 'tmux'
|
|
1275
|
+
AND tmux_socket IS NOT NULL AND tmux_session IS NOT NULL AND tmux_target IS NOT NULL
|
|
1276
|
+
AND herdr_session IS NULL AND herdr_config_dir IS NULL
|
|
1277
|
+
AND herdr_pane_id IS NULL AND herdr_shell_pid IS NULL)
|
|
1278
|
+
OR
|
|
1279
|
+
(runtime_kind = 'herdr'
|
|
1280
|
+
AND tmux_socket IS NULL AND tmux_session IS NULL AND tmux_target IS NULL
|
|
1281
|
+
AND herdr_session IS NOT NULL AND herdr_config_dir IS NOT NULL
|
|
1282
|
+
AND herdr_pane_id IS NOT NULL AND herdr_shell_pid IS NOT NULL)
|
|
1283
|
+
)
|
|
1284
|
+
);
|
|
1285
|
+
CREATE TABLE IF NOT EXISTS agent_lifecycle_leases (
|
|
1286
|
+
agent_id TEXT PRIMARY KEY NOT NULL, operation TEXT NOT NULL CHECK(operation IN ('hibernate','wake')),
|
|
1287
|
+
fence_token INTEGER NOT NULL, owner_broker_instance_id TEXT NOT NULL, lease_id TEXT NOT NULL UNIQUE,
|
|
1288
|
+
acquired_at TEXT NOT NULL, expires_at TEXT NOT NULL, attempt INTEGER NOT NULL,
|
|
1289
|
+
trigger_message_id INTEGER
|
|
1290
|
+
);
|
|
1291
|
+
CREATE TABLE IF NOT EXISTS agent_lifecycle_events (
|
|
1292
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, correlation_id TEXT NOT NULL, agent_id TEXT NOT NULL,
|
|
1293
|
+
from_state TEXT NOT NULL, to_state TEXT NOT NULL, lifecycle_version INTEGER NOT NULL,
|
|
1294
|
+
fence_token INTEGER, reason TEXT NOT NULL, trigger_source TEXT, actor TEXT NOT NULL,
|
|
1295
|
+
outcome TEXT NOT NULL, error_code TEXT, queue_depth INTEGER, oldest_queue_age_ms INTEGER,
|
|
1296
|
+
duration_ms INTEGER, rss_bytes_before INTEGER, rss_bytes_after INTEGER, created_at TEXT NOT NULL
|
|
1297
|
+
);
|
|
1298
|
+
CREATE INDEX IF NOT EXISTS idx_agent_lifecycle_events_agent_created
|
|
1299
|
+
ON agent_lifecycle_events(agent_id, created_at DESC);
|
|
1300
|
+
CREATE INDEX IF NOT EXISTS idx_agent_lifecycle_events_created
|
|
1301
|
+
ON agent_lifecycle_events(created_at DESC);
|
|
1302
|
+
CREATE TABLE IF NOT EXISTS agent_lifecycle_retention (
|
|
1303
|
+
singleton INTEGER PRIMARY KEY NOT NULL CHECK(singleton = 1),
|
|
1304
|
+
pruned_count INTEGER NOT NULL DEFAULT 0,
|
|
1305
|
+
last_pruned_at TEXT
|
|
1306
|
+
);
|
|
1307
|
+
INSERT OR IGNORE INTO agent_lifecycle_retention (singleton, pruned_count) VALUES (1, 0);
|
|
1308
|
+
CREATE TABLE IF NOT EXISTS agent_checkpoint_receipts (
|
|
1309
|
+
agent_id TEXT NOT NULL, runtime_generation INTEGER NOT NULL, correlation_id TEXT NOT NULL,
|
|
1310
|
+
hibernate_safe INTEGER NOT NULL, reason TEXT, session_resume_ref TEXT,
|
|
1311
|
+
pending_inbox_count INTEGER NOT NULL DEFAULT 0, rss_bytes INTEGER, created_at TEXT NOT NULL,
|
|
1312
|
+
PRIMARY KEY (agent_id, runtime_generation)
|
|
1313
|
+
);
|
|
1314
|
+
CREATE INDEX IF NOT EXISTS idx_agent_checkpoint_receipts_agent
|
|
1315
|
+
ON agent_checkpoint_receipts(agent_id, runtime_generation DESC);
|
|
1316
|
+
CREATE TABLE IF NOT EXISTS agent_wake_reservations (
|
|
1317
|
+
agent_id TEXT PRIMARY KEY NOT NULL, wake_lease_id TEXT NOT NULL, fence_token INTEGER NOT NULL,
|
|
1318
|
+
reserved_generation INTEGER NOT NULL, reservation_nonce TEXT NOT NULL DEFAULT '',
|
|
1319
|
+
correlation_id TEXT NOT NULL, created_at TEXT NOT NULL
|
|
1320
|
+
);
|
|
1321
|
+
CREATE TABLE IF NOT EXISTS agent_wake_queue (
|
|
1322
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, repo_root TEXT,
|
|
1323
|
+
trigger_kind TEXT NOT NULL, trigger_message_id INTEGER, priority INTEGER NOT NULL DEFAULT 100,
|
|
1324
|
+
reason TEXT NOT NULL, correlation_id TEXT NOT NULL,
|
|
1325
|
+
status TEXT NOT NULL DEFAULT 'queued'
|
|
1326
|
+
CHECK(status IN ('queued','dispatching','done','cancelled')),
|
|
1327
|
+
attempt INTEGER NOT NULL DEFAULT 0, enqueued_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
|
1328
|
+
);
|
|
1329
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_wake_queue_active_agent
|
|
1330
|
+
ON agent_wake_queue(agent_id) WHERE status IN ('queued','dispatching');
|
|
1331
|
+
CREATE INDEX IF NOT EXISTS idx_agent_wake_queue_dispatch
|
|
1332
|
+
ON agent_wake_queue(status, priority, id);
|
|
1333
|
+
`);
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* Per-attempt wake nonce: a fresh, opaque token minted on every wake reservation
|
|
1337
|
+
* so that two wake attempts for the same identity (which necessarily reuse the
|
|
1338
|
+
* same lease id, fence token, and `runtime_generation + 1`) are distinguishable.
|
|
1339
|
+
* Without it a slow runtime from an earlier, timed-out attempt could satisfy a
|
|
1340
|
+
* later attempt's otherwise-identical reservation. Added for dogfood DBs already
|
|
1341
|
+
* at v19 (fresh DBs get the column from the CREATE TABLE above).
|
|
1342
|
+
*/
|
|
1343
|
+
// agent-standards-ignore prefer-inline-single-use-helper: one-function-per-
|
|
1344
|
+
// migration-case is the established schema-migration seam (mirrors case 19's
|
|
1345
|
+
// createAgentHibernationTables); keeps the version switch a readable index.
|
|
1346
|
+
function addWakeReservationNonceColumn(db) {
|
|
1347
|
+
ensureColumn(db, "agent_wake_reservations", "reservation_nonce", "ALTER TABLE agent_wake_reservations ADD COLUMN reservation_nonce TEXT NOT NULL DEFAULT ''");
|
|
1348
|
+
}
|
|
1349
|
+
/**
|
|
1350
|
+
* Acceptance receipt: a single-row-per-agent record of the EXACT wake fence that
|
|
1351
|
+
* accepted a generation. It lets a runtime whose registration was accepted but
|
|
1352
|
+
* whose register RPC response was lost to a broker crash (committed acceptance
|
|
1353
|
+
* but never bound the socket / returned) replay its single-use wake fence and be
|
|
1354
|
+
* re-bound idempotently instead of being rejected and stranded. Superseded by
|
|
1355
|
+
* the next wake reservation (cleared in `reserveWakeGeneration`) so a stale fence
|
|
1356
|
+
* can never rebind during a fresh wake window. Added for dogfood DBs already at
|
|
1357
|
+
* v20 (fresh DBs also create it via this migration).
|
|
1358
|
+
*/
|
|
1359
|
+
// agent-standards-ignore prefer-inline-single-use-helper: one-function-per-
|
|
1360
|
+
// migration-case is the established schema-migration seam; keeps the version
|
|
1361
|
+
// switch a readable index.
|
|
1362
|
+
function createWakeAcceptanceReceiptTable(db) {
|
|
1363
|
+
db.exec(`
|
|
1364
|
+
CREATE TABLE IF NOT EXISTS agent_wake_acceptance_receipts (
|
|
1365
|
+
agent_id TEXT PRIMARY KEY NOT NULL,
|
|
1366
|
+
stable_id TEXT NOT NULL,
|
|
1367
|
+
wake_lease_id TEXT NOT NULL,
|
|
1368
|
+
fence_token INTEGER NOT NULL,
|
|
1369
|
+
reserved_generation INTEGER NOT NULL,
|
|
1370
|
+
reservation_nonce TEXT NOT NULL,
|
|
1371
|
+
accepted_at TEXT NOT NULL
|
|
1372
|
+
);
|
|
1373
|
+
`);
|
|
1374
|
+
}
|
|
1375
|
+
/**
|
|
1376
|
+
* Canonical VCS identity column: a broker-derived `owner/repo` captured at spawn
|
|
1377
|
+
* from the runtime's git remote (never inferred from directory names). The repo
|
|
1378
|
+
* allowlist authorization matches this identity EXACTLY, so distinct filesystem
|
|
1379
|
+
* roots that happen to share their final path segments (e.g.
|
|
1380
|
+
* `/trusted/gugu91/pinet` vs `/tmp/impostor/gugu91/pinet`) do not
|
|
1381
|
+
* collapse onto one authorization identity, and a repo shares one identity with
|
|
1382
|
+
* all of its git worktrees. Nullable — a spec captured without a resolvable
|
|
1383
|
+
* remote leaves it null and the fail-closed gate refuses. Added for dogfood DBs
|
|
1384
|
+
* already at v21 (fresh DBs get the column from the CREATE TABLE above).
|
|
1385
|
+
*/
|
|
1386
|
+
// agent-standards-ignore prefer-inline-single-use-helper: one-function-per-
|
|
1387
|
+
// migration-case is the established schema-migration seam; keeps the version
|
|
1388
|
+
// switch a readable index.
|
|
1389
|
+
function addRuntimeSpecVcsIdentityColumn(db) {
|
|
1390
|
+
ensureColumn(db, "agent_runtime_specs", "vcs_identity", "ALTER TABLE agent_runtime_specs ADD COLUMN vcs_identity TEXT");
|
|
1391
|
+
}
|
|
1392
|
+
/** Runtime backend discriminant. Existing rows are tmux runtimes. */
|
|
1393
|
+
// agent-standards-ignore prefer-inline-single-use-helper: one-function-per-
|
|
1394
|
+
// migration-case is the established schema-migration seam; keeps the version
|
|
1395
|
+
// switch a readable index.
|
|
1396
|
+
function addRuntimeSpecKindColumn(db) {
|
|
1397
|
+
ensureColumn(db, "agent_runtime_specs", "runtime_kind", "ALTER TABLE agent_runtime_specs ADD COLUMN runtime_kind TEXT NOT NULL DEFAULT 'tmux' CHECK(runtime_kind IN ('tmux'))");
|
|
1398
|
+
}
|
|
1399
|
+
/** Rebuild the runtime table so its persisted shape exactly matches the runtime union. */
|
|
1400
|
+
// agent-standards-ignore prefer-inline-single-use-helper: one-function-per-
|
|
1401
|
+
// migration-case is the established schema-migration seam; keeps the version
|
|
1402
|
+
// switch a readable index.
|
|
1403
|
+
function widenRuntimeSpecPayload(db) {
|
|
1404
|
+
db.exec(`
|
|
1405
|
+
ALTER TABLE agent_runtime_specs RENAME TO agent_runtime_specs_v23;
|
|
1406
|
+
CREATE TABLE agent_runtime_specs (
|
|
1407
|
+
agent_id TEXT PRIMARY KEY NOT NULL, stable_id TEXT NOT NULL, broker_owner_id TEXT NOT NULL,
|
|
1408
|
+
cwd TEXT NOT NULL, repo_root TEXT NOT NULL, worktree_path TEXT NOT NULL,
|
|
1409
|
+
runtime_kind TEXT NOT NULL DEFAULT 'tmux' CHECK(runtime_kind IN ('tmux','herdr')),
|
|
1410
|
+
tmux_socket TEXT, tmux_session TEXT, tmux_target TEXT,
|
|
1411
|
+
herdr_session TEXT, herdr_config_dir TEXT, herdr_pane_id TEXT, herdr_shell_pid INTEGER,
|
|
1412
|
+
executable TEXT NOT NULL, argv_json TEXT NOT NULL, env_allowlist_json TEXT NOT NULL,
|
|
1413
|
+
session_resume_ref TEXT NOT NULL, config_fingerprint TEXT NOT NULL,
|
|
1414
|
+
expected_host TEXT NOT NULL, expected_user TEXT NOT NULL, launch_source TEXT NOT NULL,
|
|
1415
|
+
vcs_identity TEXT,
|
|
1416
|
+
created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
|
|
1417
|
+
CHECK(
|
|
1418
|
+
(runtime_kind = 'tmux'
|
|
1419
|
+
AND tmux_socket IS NOT NULL AND tmux_session IS NOT NULL AND tmux_target IS NOT NULL
|
|
1420
|
+
AND herdr_session IS NULL AND herdr_config_dir IS NULL
|
|
1421
|
+
AND herdr_pane_id IS NULL AND herdr_shell_pid IS NULL)
|
|
1422
|
+
OR
|
|
1423
|
+
(runtime_kind = 'herdr'
|
|
1424
|
+
AND tmux_socket IS NULL AND tmux_session IS NULL AND tmux_target IS NULL
|
|
1425
|
+
AND herdr_session IS NOT NULL AND herdr_config_dir IS NOT NULL
|
|
1426
|
+
AND herdr_pane_id IS NOT NULL AND herdr_shell_pid IS NOT NULL)
|
|
1427
|
+
)
|
|
1428
|
+
);
|
|
1429
|
+
INSERT INTO agent_runtime_specs (
|
|
1430
|
+
agent_id, stable_id, broker_owner_id, cwd, repo_root, worktree_path, runtime_kind,
|
|
1431
|
+
tmux_socket, tmux_session, tmux_target,
|
|
1432
|
+
herdr_session, herdr_config_dir, herdr_pane_id, herdr_shell_pid,
|
|
1433
|
+
executable, argv_json, env_allowlist_json, session_resume_ref, config_fingerprint,
|
|
1434
|
+
expected_host, expected_user, launch_source, vcs_identity, created_at, updated_at
|
|
1435
|
+
)
|
|
1436
|
+
SELECT
|
|
1437
|
+
agent_id, stable_id, broker_owner_id, cwd, repo_root, worktree_path, runtime_kind,
|
|
1438
|
+
tmux_socket, tmux_session, tmux_target,
|
|
1439
|
+
NULL, NULL, NULL, NULL,
|
|
1440
|
+
executable, argv_json, env_allowlist_json, session_resume_ref, config_fingerprint,
|
|
1441
|
+
expected_host, expected_user, launch_source, vcs_identity, created_at, updated_at
|
|
1442
|
+
FROM agent_runtime_specs_v23;
|
|
1443
|
+
DROP TABLE agent_runtime_specs_v23;
|
|
1444
|
+
`);
|
|
1445
|
+
}
|
|
1173
1446
|
function runSchemaMigrations(db) {
|
|
1174
1447
|
const currentVersion = getUserVersion(db);
|
|
1175
1448
|
if (currentVersion >= CURRENT_BROKER_SCHEMA_VERSION) {
|
|
@@ -1233,6 +1506,24 @@ function runSchemaMigrations(db) {
|
|
|
1233
1506
|
case 18:
|
|
1234
1507
|
addAgentHierarchyColumns(db);
|
|
1235
1508
|
break;
|
|
1509
|
+
case 19:
|
|
1510
|
+
createAgentHibernationTables(db);
|
|
1511
|
+
break;
|
|
1512
|
+
case 20:
|
|
1513
|
+
addWakeReservationNonceColumn(db);
|
|
1514
|
+
break;
|
|
1515
|
+
case 21:
|
|
1516
|
+
createWakeAcceptanceReceiptTable(db);
|
|
1517
|
+
break;
|
|
1518
|
+
case 22:
|
|
1519
|
+
addRuntimeSpecVcsIdentityColumn(db);
|
|
1520
|
+
break;
|
|
1521
|
+
case 23:
|
|
1522
|
+
addRuntimeSpecKindColumn(db);
|
|
1523
|
+
break;
|
|
1524
|
+
case 24:
|
|
1525
|
+
widenRuntimeSpecPayload(db);
|
|
1526
|
+
break;
|
|
1236
1527
|
default:
|
|
1237
1528
|
throw new Error(`Unsupported broker schema migration target: ${nextVersion}`);
|
|
1238
1529
|
}
|
|
@@ -1251,6 +1542,20 @@ function runSchemaMigrations(db) {
|
|
|
1251
1542
|
}
|
|
1252
1543
|
}
|
|
1253
1544
|
// ─── BrokerDB ────────────────────────────────────────────
|
|
1545
|
+
/**
|
|
1546
|
+
* Internal sentinel used to roll back a combined register+accept transaction:
|
|
1547
|
+
* thrown when generation acceptance is rejected so `withTransaction` rolls back
|
|
1548
|
+
* the registration mutation, then caught at the method boundary and converted
|
|
1549
|
+
* back into a normal rejection result (never propagated to callers).
|
|
1550
|
+
*/
|
|
1551
|
+
class GenerationAcceptanceRollback extends Error {
|
|
1552
|
+
rejection;
|
|
1553
|
+
constructor(rejection) {
|
|
1554
|
+
super("generation_acceptance_rollback");
|
|
1555
|
+
this.rejection = rejection;
|
|
1556
|
+
this.name = "GenerationAcceptanceRollback";
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1254
1559
|
export class BrokerDB {
|
|
1255
1560
|
db = null;
|
|
1256
1561
|
dbPath;
|
|
@@ -1428,11 +1733,895 @@ export class BrokerDB {
|
|
|
1428
1733
|
null,
|
|
1429
1734
|
};
|
|
1430
1735
|
}
|
|
1736
|
+
transitionAgentLifecycle(input) {
|
|
1737
|
+
return this.withTransaction(() => {
|
|
1738
|
+
const db = this.getDb();
|
|
1739
|
+
const current = this.getAgentById(input.agentId);
|
|
1740
|
+
if (!current?.lifecycleState || current.lifecycleVersion === undefined) {
|
|
1741
|
+
throw new Error(`Unknown lifecycle agent: ${input.agentId}`);
|
|
1742
|
+
}
|
|
1743
|
+
if (current.lifecycleVersion !== input.expectedVersion) {
|
|
1744
|
+
throw new Error(`Lifecycle CAS conflict for ${input.agentId}: expected ${input.expectedVersion}, got ${current.lifecycleVersion}`);
|
|
1745
|
+
}
|
|
1746
|
+
// Fence-identity validation: a transition that presents a fence token must
|
|
1747
|
+
// prove it holds the live, matching lease. Lease fences are monotonic per
|
|
1748
|
+
// agent (a re-acquisition after expiry bumps the fence), so matching the
|
|
1749
|
+
// fence rejects a superseded holder that would otherwise drive a fenced
|
|
1750
|
+
// transition purely on a matching version CAS. When the caller also binds
|
|
1751
|
+
// the lease identity (`leaseId`/`expectedOperation`/`now`) we additionally
|
|
1752
|
+
// reject an expired-but-unsuperseded lease and a wrong-operation lease —
|
|
1753
|
+
// the fence token alone is not sufficient authority. Unfenced
|
|
1754
|
+
// administrative/recovery transitions (no fenceToken) are unaffected.
|
|
1755
|
+
if (input.fenceToken != null) {
|
|
1756
|
+
const lease = this.getAgentLifecycleLease(input.agentId);
|
|
1757
|
+
if (!lease || lease.fenceToken !== input.fenceToken) {
|
|
1758
|
+
throw new Error(`Lifecycle fence rejected for ${input.agentId}: presented fence ${input.fenceToken} is not the currently held lease`);
|
|
1759
|
+
}
|
|
1760
|
+
if (input.leaseId != null && lease.leaseId !== input.leaseId) {
|
|
1761
|
+
throw new Error(`Lifecycle fence rejected for ${input.agentId}: presented lease is not the currently held lease`);
|
|
1762
|
+
}
|
|
1763
|
+
if (input.expectedOperation != null && lease.operation !== input.expectedOperation) {
|
|
1764
|
+
throw new Error(`Lifecycle fence rejected for ${input.agentId}: held lease operation ${lease.operation} does not authorize a ${input.expectedOperation} transition`);
|
|
1765
|
+
}
|
|
1766
|
+
if (input.now != null && Date.parse(lease.expiresAt) <= input.now) {
|
|
1767
|
+
throw new Error(`Lifecycle fence rejected for ${input.agentId}: held lease is expired`);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
assertLegalLifecycleTransition(current.lifecycleState, input.toState);
|
|
1771
|
+
const now = new Date().toISOString();
|
|
1772
|
+
const result = db
|
|
1773
|
+
.prepare(`UPDATE agents SET lifecycle_state = ?, lifecycle_version = lifecycle_version + 1,
|
|
1774
|
+
hibernated_at = CASE WHEN ? = 'hibernated' THEN ? ELSE hibernated_at END,
|
|
1775
|
+
terminated_at = CASE WHEN ? = 'terminated' THEN ? ELSE terminated_at END,
|
|
1776
|
+
hibernate_reason = CASE WHEN ? IN ('hibernating','hibernated','reap-candidate') THEN ? ELSE hibernate_reason END,
|
|
1777
|
+
last_wake_reason = CASE WHEN ? = 'waking' THEN ? ELSE last_wake_reason END
|
|
1778
|
+
WHERE id = ? AND lifecycle_version = ?`)
|
|
1779
|
+
.run(input.toState, input.toState, now, input.toState, now, input.toState, input.reason, input.toState, input.reason, input.agentId, input.expectedVersion);
|
|
1780
|
+
if (Number(result.changes) !== 1)
|
|
1781
|
+
throw new Error(`Lifecycle CAS conflict for ${input.agentId}`);
|
|
1782
|
+
const nextVersion = input.expectedVersion + 1;
|
|
1783
|
+
this.insertLifecycleEventRow({
|
|
1784
|
+
correlationId: input.correlationId,
|
|
1785
|
+
agentId: input.agentId,
|
|
1786
|
+
fromState: current.lifecycleState,
|
|
1787
|
+
toState: input.toState,
|
|
1788
|
+
lifecycleVersion: nextVersion,
|
|
1789
|
+
reason: input.reason,
|
|
1790
|
+
triggerSource: input.triggerSource,
|
|
1791
|
+
actor: input.actor,
|
|
1792
|
+
outcome: "accepted",
|
|
1793
|
+
fenceToken: input.fenceToken ?? null,
|
|
1794
|
+
queueDepth: input.queueDepth ?? null,
|
|
1795
|
+
oldestQueueAgeMs: input.oldestQueueAgeMs ?? null,
|
|
1796
|
+
durationMs: input.durationMs ?? null,
|
|
1797
|
+
rssBytesBefore: input.rssBytesBefore ?? null,
|
|
1798
|
+
rssBytesAfter: input.rssBytesAfter ?? null,
|
|
1799
|
+
createdAt: now,
|
|
1800
|
+
});
|
|
1801
|
+
const updated = this.getAgentById(input.agentId);
|
|
1802
|
+
if (!updated)
|
|
1803
|
+
throw new Error(`Lifecycle agent disappeared: ${input.agentId}`);
|
|
1804
|
+
return updated;
|
|
1805
|
+
});
|
|
1806
|
+
}
|
|
1807
|
+
/**
|
|
1808
|
+
* Append an audit-only lifecycle event (a refusal, fenced stale attempt, or
|
|
1809
|
+
* duplicate-launch prevention) without changing the agent's lifecycle state.
|
|
1810
|
+
*/
|
|
1811
|
+
recordAgentLifecycleEvent(input) {
|
|
1812
|
+
this.withTransaction(() => {
|
|
1813
|
+
this.insertLifecycleEventRow({
|
|
1814
|
+
correlationId: input.correlationId,
|
|
1815
|
+
agentId: input.agentId,
|
|
1816
|
+
fromState: input.fromState,
|
|
1817
|
+
toState: input.toState,
|
|
1818
|
+
lifecycleVersion: input.lifecycleVersion,
|
|
1819
|
+
reason: input.reason,
|
|
1820
|
+
triggerSource: input.triggerSource,
|
|
1821
|
+
actor: input.actor,
|
|
1822
|
+
outcome: input.outcome,
|
|
1823
|
+
errorCode: input.errorCode ?? null,
|
|
1824
|
+
fenceToken: input.fenceToken ?? null,
|
|
1825
|
+
queueDepth: input.queueDepth ?? null,
|
|
1826
|
+
oldestQueueAgeMs: input.oldestQueueAgeMs ?? null,
|
|
1827
|
+
durationMs: input.durationMs ?? null,
|
|
1828
|
+
rssBytesBefore: input.rssBytesBefore ?? null,
|
|
1829
|
+
rssBytesAfter: input.rssBytesAfter ?? null,
|
|
1830
|
+
createdAt: new Date().toISOString(),
|
|
1831
|
+
});
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
insertLifecycleEventRow(row) {
|
|
1835
|
+
const db = this.getDb();
|
|
1836
|
+
db.prepare(`INSERT INTO agent_lifecycle_events
|
|
1837
|
+
(correlation_id, agent_id, from_state, to_state, lifecycle_version, fence_token, reason,
|
|
1838
|
+
trigger_source, actor, outcome, error_code, queue_depth, oldest_queue_age_ms, duration_ms,
|
|
1839
|
+
rss_bytes_before, rss_bytes_after, created_at)
|
|
1840
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.correlationId, row.agentId, row.fromState, row.toState, row.lifecycleVersion, row.fenceToken, row.reason, row.triggerSource ?? null, row.actor, row.outcome, row.errorCode ?? null, row.queueDepth, row.oldestQueueAgeMs, row.durationMs, row.rssBytesBefore, row.rssBytesAfter, row.createdAt);
|
|
1841
|
+
const pruned = db
|
|
1842
|
+
.prepare(`DELETE FROM agent_lifecycle_events WHERE id IN (
|
|
1843
|
+
SELECT id FROM agent_lifecycle_events ORDER BY id DESC LIMIT -1 OFFSET 10000
|
|
1844
|
+
)`)
|
|
1845
|
+
.run();
|
|
1846
|
+
if (Number(pruned.changes) > 0) {
|
|
1847
|
+
db.prepare(`UPDATE agent_lifecycle_retention
|
|
1848
|
+
SET pruned_count = pruned_count + ?, last_pruned_at = ? WHERE singleton = 1`).run(Number(pruned.changes), row.createdAt);
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
getRecentAgentLifecycleEvents(agentId, limit = 50) {
|
|
1852
|
+
const db = this.getDb();
|
|
1853
|
+
const cappedLimit = Math.min(Math.max(Math.trunc(limit), 1), 1000);
|
|
1854
|
+
const rows = agentId
|
|
1855
|
+
? db
|
|
1856
|
+
.prepare(`SELECT * FROM agent_lifecycle_events WHERE agent_id = ? ORDER BY id DESC LIMIT ?`)
|
|
1857
|
+
.all(agentId, cappedLimit)
|
|
1858
|
+
: db
|
|
1859
|
+
.prepare(`SELECT * FROM agent_lifecycle_events ORDER BY id DESC LIMIT ?`)
|
|
1860
|
+
.all(cappedLimit);
|
|
1861
|
+
return rows.map((row) => ({
|
|
1862
|
+
id: Number(row.id),
|
|
1863
|
+
correlationId: String(row.correlation_id),
|
|
1864
|
+
agentId: String(row.agent_id),
|
|
1865
|
+
fromState: String(row.from_state),
|
|
1866
|
+
toState: String(row.to_state),
|
|
1867
|
+
lifecycleVersion: Number(row.lifecycle_version),
|
|
1868
|
+
fenceToken: row.fence_token == null ? null : Number(row.fence_token),
|
|
1869
|
+
reason: String(row.reason),
|
|
1870
|
+
triggerSource: row.trigger_source == null ? null : String(row.trigger_source),
|
|
1871
|
+
actor: String(row.actor),
|
|
1872
|
+
outcome: String(row.outcome),
|
|
1873
|
+
errorCode: row.error_code == null ? null : String(row.error_code),
|
|
1874
|
+
queueDepth: row.queue_depth == null ? null : Number(row.queue_depth),
|
|
1875
|
+
oldestQueueAgeMs: row.oldest_queue_age_ms == null ? null : Number(row.oldest_queue_age_ms),
|
|
1876
|
+
durationMs: row.duration_ms == null ? null : Number(row.duration_ms),
|
|
1877
|
+
rssBytesBefore: row.rss_bytes_before == null ? null : Number(row.rss_bytes_before),
|
|
1878
|
+
rssBytesAfter: row.rss_bytes_after == null ? null : Number(row.rss_bytes_after),
|
|
1879
|
+
createdAt: String(row.created_at),
|
|
1880
|
+
}));
|
|
1881
|
+
}
|
|
1882
|
+
getAgentLifecycleRetentionInfo() {
|
|
1883
|
+
const db = this.getDb();
|
|
1884
|
+
const retained = db.prepare("SELECT COUNT(*) AS count FROM agent_lifecycle_events").get();
|
|
1885
|
+
const retention = db
|
|
1886
|
+
.prepare("SELECT pruned_count, last_pruned_at FROM agent_lifecycle_retention WHERE singleton = 1")
|
|
1887
|
+
.get();
|
|
1888
|
+
return {
|
|
1889
|
+
retainedCount: Number(retained?.count ?? 0),
|
|
1890
|
+
prunedCount: Number(retention?.pruned_count ?? 0),
|
|
1891
|
+
lastPrunedAt: retention?.last_pruned_at ?? null,
|
|
1892
|
+
};
|
|
1893
|
+
}
|
|
1894
|
+
acquireAgentLifecycleLease(input) {
|
|
1895
|
+
return this.withTransaction(() => {
|
|
1896
|
+
const db = this.getDb();
|
|
1897
|
+
const nowMs = input.now ?? Date.now();
|
|
1898
|
+
const now = new Date(nowMs).toISOString();
|
|
1899
|
+
const expiresAt = new Date(nowMs + input.ttlMs).toISOString();
|
|
1900
|
+
const existing = db
|
|
1901
|
+
.prepare("SELECT fence_token, expires_at, attempt FROM agent_lifecycle_leases WHERE agent_id = ?")
|
|
1902
|
+
.get(input.agentId);
|
|
1903
|
+
if (existing && existing.expires_at > now)
|
|
1904
|
+
return null;
|
|
1905
|
+
const fence = (existing?.fence_token ?? 0) + 1;
|
|
1906
|
+
const attempt = (existing?.attempt ?? 0) + 1;
|
|
1907
|
+
db.prepare(`INSERT INTO agent_lifecycle_leases
|
|
1908
|
+
(agent_id, operation, fence_token, owner_broker_instance_id, lease_id, acquired_at, expires_at, attempt, trigger_message_id)
|
|
1909
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1910
|
+
ON CONFLICT(agent_id) DO UPDATE SET operation=excluded.operation, fence_token=excluded.fence_token,
|
|
1911
|
+
owner_broker_instance_id=excluded.owner_broker_instance_id, lease_id=excluded.lease_id,
|
|
1912
|
+
acquired_at=excluded.acquired_at, expires_at=excluded.expires_at, attempt=excluded.attempt,
|
|
1913
|
+
trigger_message_id=excluded.trigger_message_id`).run(input.agentId, input.operation, fence, input.ownerBrokerInstanceId, input.leaseId, now, expiresAt, attempt, input.triggerMessageId ?? null);
|
|
1914
|
+
return {
|
|
1915
|
+
agentId: input.agentId,
|
|
1916
|
+
operation: input.operation,
|
|
1917
|
+
fenceToken: fence,
|
|
1918
|
+
ownerBrokerInstanceId: input.ownerBrokerInstanceId,
|
|
1919
|
+
leaseId: input.leaseId,
|
|
1920
|
+
acquiredAt: now,
|
|
1921
|
+
expiresAt,
|
|
1922
|
+
attempt,
|
|
1923
|
+
triggerMessageId: input.triggerMessageId ?? null,
|
|
1924
|
+
};
|
|
1925
|
+
});
|
|
1926
|
+
}
|
|
1927
|
+
releaseAgentLifecycleLease(agentId, leaseId, fenceToken) {
|
|
1928
|
+
const result = this.getDb()
|
|
1929
|
+
.prepare("DELETE FROM agent_lifecycle_leases WHERE agent_id = ? AND lease_id = ? AND fence_token = ?")
|
|
1930
|
+
.run(agentId, leaseId, fenceToken);
|
|
1931
|
+
return Number(result.changes) === 1;
|
|
1932
|
+
}
|
|
1933
|
+
/**
|
|
1934
|
+
* Extend an already-held, still-valid lease's expiry WITHOUT bumping the
|
|
1935
|
+
* fence, so a legitimately long-running operation (e.g. a wake that waits on
|
|
1936
|
+
* process launch + runtime registration across several attempts) keeps a valid
|
|
1937
|
+
* lease across adapter waits and can still complete its fenced forward
|
|
1938
|
+
* transition. The fence is preserved so revival fencing is unaffected.
|
|
1939
|
+
*
|
|
1940
|
+
* Renewal only succeeds while the lease is still unexpired and held by this
|
|
1941
|
+
* exact owner (matching `leaseId` + `fenceToken`); this preserves the takeover
|
|
1942
|
+
* guarantee (a stalled owner past expiry cannot reclaim a lease another broker
|
|
1943
|
+
* may take over). Returns the refreshed lease, or null when ownership was lost
|
|
1944
|
+
* (expired, released, or the fence moved) — a null result means the caller
|
|
1945
|
+
* must fail closed rather than continue driving forward transitions.
|
|
1946
|
+
*/
|
|
1947
|
+
renewAgentLifecycleLease(input) {
|
|
1948
|
+
return this.withTransaction(() => {
|
|
1949
|
+
const db = this.getDb();
|
|
1950
|
+
const nowMs = input.now ?? Date.now();
|
|
1951
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
1952
|
+
const expiresAt = new Date(nowMs + input.ttlMs).toISOString();
|
|
1953
|
+
const result = db
|
|
1954
|
+
.prepare(`UPDATE agent_lifecycle_leases SET expires_at = ?
|
|
1955
|
+
WHERE agent_id = ? AND lease_id = ? AND fence_token = ? AND expires_at > ?`)
|
|
1956
|
+
.run(expiresAt, input.agentId, input.leaseId, input.fenceToken, nowIso);
|
|
1957
|
+
if (Number(result.changes) !== 1)
|
|
1958
|
+
return null;
|
|
1959
|
+
return this.getAgentLifecycleLease(input.agentId);
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1962
|
+
/** Set the opt-in hibernation policy for an agent (auto | manual | never). */
|
|
1963
|
+
setAgentHibernatePolicy(agentId, policy) {
|
|
1964
|
+
this.getDb()
|
|
1965
|
+
.prepare("UPDATE agents SET hibernate_policy = ? WHERE id = ?")
|
|
1966
|
+
.run(policy, agentId);
|
|
1967
|
+
}
|
|
1968
|
+
/** Record grace/idle eligibility timestamps used by the auto-hibernation scheduler. */
|
|
1969
|
+
setAgentHibernationSchedule(agentId, schedule) {
|
|
1970
|
+
const db = this.getDb();
|
|
1971
|
+
if (schedule.graceUntil !== undefined) {
|
|
1972
|
+
db.prepare("UPDATE agents SET grace_until = ? WHERE id = ?").run(schedule.graceUntil, agentId);
|
|
1973
|
+
}
|
|
1974
|
+
if (schedule.idleEligibleAt !== undefined) {
|
|
1975
|
+
db.prepare("UPDATE agents SET idle_eligible_at = ? WHERE id = ?").run(schedule.idleEligibleAt, agentId);
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
getAgentLifecycleLease(agentId) {
|
|
1979
|
+
const row = this.getDb()
|
|
1980
|
+
.prepare(`SELECT agent_id, operation, fence_token, owner_broker_instance_id, lease_id,
|
|
1981
|
+
acquired_at, expires_at, attempt, trigger_message_id
|
|
1982
|
+
FROM agent_lifecycle_leases WHERE agent_id = ?`)
|
|
1983
|
+
.get(agentId);
|
|
1984
|
+
if (!row)
|
|
1985
|
+
return null;
|
|
1986
|
+
return {
|
|
1987
|
+
agentId: row.agent_id,
|
|
1988
|
+
operation: row.operation,
|
|
1989
|
+
fenceToken: row.fence_token,
|
|
1990
|
+
ownerBrokerInstanceId: row.owner_broker_instance_id,
|
|
1991
|
+
leaseId: row.lease_id,
|
|
1992
|
+
acquiredAt: row.acquired_at,
|
|
1993
|
+
expiresAt: row.expires_at,
|
|
1994
|
+
attempt: row.attempt,
|
|
1995
|
+
triggerMessageId: row.trigger_message_id,
|
|
1996
|
+
};
|
|
1997
|
+
}
|
|
1998
|
+
// ─── Durable runtime specs (sanitized launch/resume manifest) ──────
|
|
1999
|
+
upsertAgentRuntimeSpec(input) {
|
|
2000
|
+
return this.withTransaction(() => {
|
|
2001
|
+
const db = this.getDb();
|
|
2002
|
+
const now = new Date().toISOString();
|
|
2003
|
+
const existing = db
|
|
2004
|
+
.prepare("SELECT created_at FROM agent_runtime_specs WHERE agent_id = ?")
|
|
2005
|
+
.get(input.agentId);
|
|
2006
|
+
const createdAt = existing?.created_at ?? now;
|
|
2007
|
+
let tmuxSocket = null;
|
|
2008
|
+
let tmuxSession = null;
|
|
2009
|
+
let tmuxTarget = null;
|
|
2010
|
+
let herdrSession = null;
|
|
2011
|
+
let herdrConfigDir = null;
|
|
2012
|
+
let herdrPaneId = null;
|
|
2013
|
+
let herdrShellPid = null;
|
|
2014
|
+
if (input.runtimeKind === "tmux") {
|
|
2015
|
+
if (input.tmuxSocket.trim().length === 0 ||
|
|
2016
|
+
input.tmuxSession.trim().length === 0 ||
|
|
2017
|
+
input.tmuxTarget.trim().length === 0) {
|
|
2018
|
+
throw new Error(`Invalid tmux runtime payload for agent ${input.agentId}`);
|
|
2019
|
+
}
|
|
2020
|
+
tmuxSocket = input.tmuxSocket;
|
|
2021
|
+
tmuxSession = input.tmuxSession;
|
|
2022
|
+
tmuxTarget = input.tmuxTarget;
|
|
2023
|
+
}
|
|
2024
|
+
else {
|
|
2025
|
+
if (input.herdrSession.trim().length === 0 ||
|
|
2026
|
+
input.herdrConfigDir.trim().length === 0 ||
|
|
2027
|
+
input.herdrPaneId.trim().length === 0 ||
|
|
2028
|
+
!Number.isInteger(input.herdrShellPid) ||
|
|
2029
|
+
input.herdrShellPid <= 0) {
|
|
2030
|
+
throw new Error(`Invalid Herdr runtime payload for agent ${input.agentId}`);
|
|
2031
|
+
}
|
|
2032
|
+
herdrSession = input.herdrSession;
|
|
2033
|
+
herdrConfigDir = input.herdrConfigDir;
|
|
2034
|
+
herdrPaneId = input.herdrPaneId;
|
|
2035
|
+
herdrShellPid = input.herdrShellPid;
|
|
2036
|
+
}
|
|
2037
|
+
db.prepare(`INSERT INTO agent_runtime_specs
|
|
2038
|
+
(agent_id, stable_id, broker_owner_id, cwd, repo_root, worktree_path, runtime_kind,
|
|
2039
|
+
tmux_socket, tmux_session, tmux_target,
|
|
2040
|
+
herdr_session, herdr_config_dir, herdr_pane_id, herdr_shell_pid,
|
|
2041
|
+
executable, argv_json, env_allowlist_json,
|
|
2042
|
+
session_resume_ref, config_fingerprint, expected_host, expected_user, launch_source,
|
|
2043
|
+
vcs_identity, created_at, updated_at)
|
|
2044
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2045
|
+
ON CONFLICT(agent_id) DO UPDATE SET stable_id=excluded.stable_id,
|
|
2046
|
+
broker_owner_id=excluded.broker_owner_id, cwd=excluded.cwd, repo_root=excluded.repo_root,
|
|
2047
|
+
worktree_path=excluded.worktree_path, runtime_kind=excluded.runtime_kind,
|
|
2048
|
+
tmux_socket=excluded.tmux_socket, tmux_session=excluded.tmux_session,
|
|
2049
|
+
tmux_target=excluded.tmux_target, herdr_session=excluded.herdr_session,
|
|
2050
|
+
herdr_config_dir=excluded.herdr_config_dir, herdr_pane_id=excluded.herdr_pane_id,
|
|
2051
|
+
herdr_shell_pid=excluded.herdr_shell_pid,
|
|
2052
|
+
executable=excluded.executable, argv_json=excluded.argv_json,
|
|
2053
|
+
env_allowlist_json=excluded.env_allowlist_json,
|
|
2054
|
+
session_resume_ref=excluded.session_resume_ref,
|
|
2055
|
+
config_fingerprint=excluded.config_fingerprint, expected_host=excluded.expected_host,
|
|
2056
|
+
expected_user=excluded.expected_user, launch_source=excluded.launch_source,
|
|
2057
|
+
vcs_identity=excluded.vcs_identity,
|
|
2058
|
+
updated_at=excluded.updated_at`).run(input.agentId, input.stableId, input.brokerOwnerId, input.cwd, input.repoRoot, input.worktreePath, input.runtimeKind, tmuxSocket, tmuxSession, tmuxTarget, herdrSession, herdrConfigDir, herdrPaneId, herdrShellPid, input.executable, JSON.stringify(input.argv), JSON.stringify(input.envAllowlist), input.sessionResumeRef, input.configFingerprint, input.expectedHost, input.expectedUser, input.launchSource, input.vcsIdentity ?? null, createdAt, now);
|
|
2059
|
+
const spec = this.getAgentRuntimeSpec(input.agentId);
|
|
2060
|
+
if (!spec)
|
|
2061
|
+
throw new Error(`Failed to persist runtime spec for ${input.agentId}`);
|
|
2062
|
+
return spec;
|
|
2063
|
+
});
|
|
2064
|
+
}
|
|
2065
|
+
getAgentRuntimeSpec(agentId) {
|
|
2066
|
+
const row = this.getDb()
|
|
2067
|
+
.prepare("SELECT * FROM agent_runtime_specs WHERE agent_id = ?")
|
|
2068
|
+
.get(agentId);
|
|
2069
|
+
if (!row)
|
|
2070
|
+
return null;
|
|
2071
|
+
const common = {
|
|
2072
|
+
agentId: row.agent_id,
|
|
2073
|
+
stableId: row.stable_id,
|
|
2074
|
+
brokerOwnerId: row.broker_owner_id,
|
|
2075
|
+
cwd: row.cwd,
|
|
2076
|
+
repoRoot: row.repo_root,
|
|
2077
|
+
worktreePath: row.worktree_path,
|
|
2078
|
+
executable: row.executable,
|
|
2079
|
+
argv: parseStringArray(row.argv_json),
|
|
2080
|
+
envAllowlist: parseStringArray(row.env_allowlist_json),
|
|
2081
|
+
sessionResumeRef: row.session_resume_ref,
|
|
2082
|
+
configFingerprint: row.config_fingerprint,
|
|
2083
|
+
expectedHost: row.expected_host,
|
|
2084
|
+
expectedUser: row.expected_user,
|
|
2085
|
+
launchSource: row.launch_source,
|
|
2086
|
+
vcsIdentity: row.vcs_identity ?? null,
|
|
2087
|
+
createdAt: row.created_at,
|
|
2088
|
+
updatedAt: row.updated_at,
|
|
2089
|
+
};
|
|
2090
|
+
if (row.runtime_kind === "tmux") {
|
|
2091
|
+
if (row.tmux_socket === null ||
|
|
2092
|
+
row.tmux_socket.trim().length === 0 ||
|
|
2093
|
+
row.tmux_session === null ||
|
|
2094
|
+
row.tmux_session.trim().length === 0 ||
|
|
2095
|
+
row.tmux_target === null ||
|
|
2096
|
+
row.tmux_target.trim().length === 0) {
|
|
2097
|
+
throw new Error(`Invalid tmux runtime payload for agent ${agentId}`);
|
|
2098
|
+
}
|
|
2099
|
+
return {
|
|
2100
|
+
...common,
|
|
2101
|
+
runtimeKind: "tmux",
|
|
2102
|
+
tmuxSocket: row.tmux_socket,
|
|
2103
|
+
tmuxSession: row.tmux_session,
|
|
2104
|
+
tmuxTarget: row.tmux_target,
|
|
2105
|
+
};
|
|
2106
|
+
}
|
|
2107
|
+
if (row.runtime_kind === "herdr") {
|
|
2108
|
+
if (row.herdr_session === null ||
|
|
2109
|
+
row.herdr_session.trim().length === 0 ||
|
|
2110
|
+
row.herdr_config_dir === null ||
|
|
2111
|
+
row.herdr_config_dir.trim().length === 0 ||
|
|
2112
|
+
row.herdr_pane_id === null ||
|
|
2113
|
+
row.herdr_pane_id.trim().length === 0 ||
|
|
2114
|
+
row.herdr_shell_pid === null ||
|
|
2115
|
+
!Number.isInteger(row.herdr_shell_pid) ||
|
|
2116
|
+
row.herdr_shell_pid <= 0) {
|
|
2117
|
+
throw new Error(`Invalid Herdr runtime payload for agent ${agentId}`);
|
|
2118
|
+
}
|
|
2119
|
+
return {
|
|
2120
|
+
...common,
|
|
2121
|
+
runtimeKind: "herdr",
|
|
2122
|
+
herdrSession: row.herdr_session,
|
|
2123
|
+
herdrConfigDir: row.herdr_config_dir,
|
|
2124
|
+
herdrPaneId: row.herdr_pane_id,
|
|
2125
|
+
herdrShellPid: row.herdr_shell_pid,
|
|
2126
|
+
};
|
|
2127
|
+
}
|
|
2128
|
+
throw new Error(`Invalid runtime kind for agent ${agentId}`);
|
|
2129
|
+
}
|
|
2130
|
+
deleteAgentRuntimeSpec(agentId) {
|
|
2131
|
+
this.getDb().prepare("DELETE FROM agent_runtime_specs WHERE agent_id = ?").run(agentId);
|
|
2132
|
+
}
|
|
2133
|
+
// ─── Cooperative checkpoint receipts ───────────────────────────────
|
|
2134
|
+
recordAgentCheckpointReceipt(input) {
|
|
2135
|
+
const now = new Date().toISOString();
|
|
2136
|
+
this.getDb()
|
|
2137
|
+
.prepare(`INSERT INTO agent_checkpoint_receipts
|
|
2138
|
+
(agent_id, runtime_generation, correlation_id, hibernate_safe, reason,
|
|
2139
|
+
session_resume_ref, pending_inbox_count, rss_bytes, created_at)
|
|
2140
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2141
|
+
ON CONFLICT(agent_id, runtime_generation) DO UPDATE SET correlation_id=excluded.correlation_id,
|
|
2142
|
+
hibernate_safe=excluded.hibernate_safe, reason=excluded.reason,
|
|
2143
|
+
session_resume_ref=excluded.session_resume_ref,
|
|
2144
|
+
pending_inbox_count=excluded.pending_inbox_count, rss_bytes=excluded.rss_bytes,
|
|
2145
|
+
created_at=excluded.created_at`)
|
|
2146
|
+
.run(input.agentId, input.runtimeGeneration, input.correlationId, input.hibernateSafe ? 1 : 0, input.reason ?? null, input.sessionResumeRef ?? null, input.pendingInboxCount, input.rssBytes ?? null, now);
|
|
2147
|
+
return { ...input, createdAt: now };
|
|
2148
|
+
}
|
|
2149
|
+
getLatestAgentCheckpointReceipt(agentId) {
|
|
2150
|
+
const row = this.getDb()
|
|
2151
|
+
.prepare(`SELECT agent_id, runtime_generation, correlation_id, hibernate_safe, reason,
|
|
2152
|
+
session_resume_ref, pending_inbox_count, rss_bytes, created_at
|
|
2153
|
+
FROM agent_checkpoint_receipts WHERE agent_id = ?
|
|
2154
|
+
ORDER BY runtime_generation DESC LIMIT 1`)
|
|
2155
|
+
.get(agentId);
|
|
2156
|
+
if (!row)
|
|
2157
|
+
return null;
|
|
2158
|
+
return {
|
|
2159
|
+
agentId: row.agent_id,
|
|
2160
|
+
runtimeGeneration: row.runtime_generation,
|
|
2161
|
+
correlationId: row.correlation_id,
|
|
2162
|
+
hibernateSafe: row.hibernate_safe === 1,
|
|
2163
|
+
reason: row.reason,
|
|
2164
|
+
sessionResumeRef: row.session_resume_ref,
|
|
2165
|
+
pendingInboxCount: row.pending_inbox_count,
|
|
2166
|
+
rssBytes: row.rss_bytes,
|
|
2167
|
+
createdAt: row.created_at,
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
// ─── Accepted-generation fencing (single-winner cold wake) ─────────
|
|
2171
|
+
/**
|
|
2172
|
+
* Reserve the exact runtime generation the broker will accept for a wake.
|
|
2173
|
+
* Requires an unexpired wake lease held with the given fence token. Exactly
|
|
2174
|
+
* one reservation may exist per agent (PK). The reserved generation is the
|
|
2175
|
+
* agent's current runtime_generation + 1, so any older runtime is fenced out.
|
|
2176
|
+
*/
|
|
2177
|
+
reserveWakeGeneration(input) {
|
|
2178
|
+
return this.withTransaction(() => {
|
|
2179
|
+
const db = this.getDb();
|
|
2180
|
+
const nowIso = new Date(input.now ?? Date.now()).toISOString();
|
|
2181
|
+
const lease = this.getAgentLifecycleLease(input.agentId);
|
|
2182
|
+
if (!lease ||
|
|
2183
|
+
lease.operation !== "wake" ||
|
|
2184
|
+
lease.leaseId !== input.wakeLeaseId ||
|
|
2185
|
+
lease.fenceToken !== input.fenceToken) {
|
|
2186
|
+
throw new Error(`Wake reservation requires a matching held wake lease for ${input.agentId}`);
|
|
2187
|
+
}
|
|
2188
|
+
if (lease.expiresAt <= nowIso) {
|
|
2189
|
+
throw new Error(`Wake lease for ${input.agentId} has expired`);
|
|
2190
|
+
}
|
|
2191
|
+
const agent = this.getAgentById(input.agentId);
|
|
2192
|
+
if (!agent)
|
|
2193
|
+
throw new Error(`Unknown agent for wake reservation: ${input.agentId}`);
|
|
2194
|
+
const reservedGeneration = (agent.runtimeGeneration ?? 0) + 1;
|
|
2195
|
+
// Mint a fresh per-attempt nonce. Successive wake attempts for the same
|
|
2196
|
+
// identity necessarily reuse the same lease id, fence token, and
|
|
2197
|
+
// `reserved_generation` (= runtime_generation + 1, which does not advance
|
|
2198
|
+
// until a runtime is accepted), so those three fields alone cannot tell a
|
|
2199
|
+
// slow, timed-out earlier attempt's runtime apart from the current
|
|
2200
|
+
// attempt's runtime. The nonce, minted here and threaded through the launch
|
|
2201
|
+
// context into the runtime's registration, fences the earlier runtime out.
|
|
2202
|
+
const reservationNonce = input.reservationNonce ?? crypto.randomUUID();
|
|
2203
|
+
// A NEW wake attempt supersedes any prior acceptance receipt: clear it so a
|
|
2204
|
+
// stale fence from a previously-accepted (crash-stranded) runtime can never
|
|
2205
|
+
// be replayed to rebind during this fresh wake window.
|
|
2206
|
+
db.prepare("DELETE FROM agent_wake_acceptance_receipts WHERE agent_id = ?").run(input.agentId);
|
|
2207
|
+
db.prepare(`INSERT INTO agent_wake_reservations
|
|
2208
|
+
(agent_id, wake_lease_id, fence_token, reserved_generation, reservation_nonce,
|
|
2209
|
+
correlation_id, created_at)
|
|
2210
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
2211
|
+
ON CONFLICT(agent_id) DO UPDATE SET wake_lease_id=excluded.wake_lease_id,
|
|
2212
|
+
fence_token=excluded.fence_token, reserved_generation=excluded.reserved_generation,
|
|
2213
|
+
reservation_nonce=excluded.reservation_nonce,
|
|
2214
|
+
correlation_id=excluded.correlation_id, created_at=excluded.created_at`).run(input.agentId, input.wakeLeaseId, input.fenceToken, reservedGeneration, reservationNonce, input.correlationId, nowIso);
|
|
2215
|
+
return {
|
|
2216
|
+
agentId: input.agentId,
|
|
2217
|
+
wakeLeaseId: input.wakeLeaseId,
|
|
2218
|
+
fenceToken: input.fenceToken,
|
|
2219
|
+
reservedGeneration,
|
|
2220
|
+
reservationNonce,
|
|
2221
|
+
correlationId: input.correlationId,
|
|
2222
|
+
createdAt: nowIso,
|
|
2223
|
+
};
|
|
2224
|
+
});
|
|
2225
|
+
}
|
|
2226
|
+
getAgentWakeReservation(agentId) {
|
|
2227
|
+
const row = this.getDb()
|
|
2228
|
+
.prepare(`SELECT agent_id, wake_lease_id, fence_token, reserved_generation, reservation_nonce,
|
|
2229
|
+
correlation_id, created_at
|
|
2230
|
+
FROM agent_wake_reservations WHERE agent_id = ?`)
|
|
2231
|
+
.get(agentId);
|
|
2232
|
+
if (!row)
|
|
2233
|
+
return null;
|
|
2234
|
+
return {
|
|
2235
|
+
agentId: row.agent_id,
|
|
2236
|
+
wakeLeaseId: row.wake_lease_id,
|
|
2237
|
+
fenceToken: row.fence_token,
|
|
2238
|
+
reservedGeneration: row.reserved_generation,
|
|
2239
|
+
reservationNonce: row.reservation_nonce,
|
|
2240
|
+
correlationId: row.correlation_id,
|
|
2241
|
+
createdAt: row.created_at,
|
|
2242
|
+
};
|
|
2243
|
+
}
|
|
2244
|
+
clearAgentWakeReservation(agentId) {
|
|
2245
|
+
this.getDb().prepare("DELETE FROM agent_wake_reservations WHERE agent_id = ?").run(agentId);
|
|
2246
|
+
}
|
|
2247
|
+
/**
|
|
2248
|
+
* Record the EXACT wake fence that just accepted a generation, so a runtime
|
|
2249
|
+
* whose register RPC response was lost to a crash can replay its single-use
|
|
2250
|
+
* fence and be re-bound idempotently. Called INSIDE the acceptance transaction
|
|
2251
|
+
* (via {@link acceptRuntimeGeneration} / {@link registerAgentWithGenerationAcceptance})
|
|
2252
|
+
* so the receipt is atomic with the generation advance. The stable id is read
|
|
2253
|
+
* from the just-registered row so the receipt binds to the durable identity.
|
|
2254
|
+
*/
|
|
2255
|
+
writeWakeAcceptanceReceipt(input, nowMs) {
|
|
2256
|
+
const db = this.getDb();
|
|
2257
|
+
const row = db.prepare("SELECT stable_id FROM agents WHERE id = ?").get(input.agentId);
|
|
2258
|
+
const stableId = row?.stable_id;
|
|
2259
|
+
// Only a durable stable identity can be revived by a fenced replay; a row
|
|
2260
|
+
// without a stable id cannot be a hibernation identity, so no receipt.
|
|
2261
|
+
if (!stableId)
|
|
2262
|
+
return;
|
|
2263
|
+
db.prepare(`INSERT INTO agent_wake_acceptance_receipts
|
|
2264
|
+
(agent_id, stable_id, wake_lease_id, fence_token, reserved_generation,
|
|
2265
|
+
reservation_nonce, accepted_at)
|
|
2266
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
2267
|
+
ON CONFLICT(agent_id) DO UPDATE SET stable_id=excluded.stable_id,
|
|
2268
|
+
wake_lease_id=excluded.wake_lease_id, fence_token=excluded.fence_token,
|
|
2269
|
+
reserved_generation=excluded.reserved_generation,
|
|
2270
|
+
reservation_nonce=excluded.reservation_nonce, accepted_at=excluded.accepted_at`).run(input.agentId, stableId, input.wakeLeaseId, input.fenceToken, input.reservedGeneration, input.reservationNonce, new Date(nowMs).toISOString());
|
|
2271
|
+
}
|
|
2272
|
+
getAgentWakeAcceptanceReceipt(agentId) {
|
|
2273
|
+
const row = this.getDb()
|
|
2274
|
+
.prepare(`SELECT agent_id, stable_id, wake_lease_id, fence_token, reserved_generation,
|
|
2275
|
+
reservation_nonce, accepted_at
|
|
2276
|
+
FROM agent_wake_acceptance_receipts WHERE agent_id = ?`)
|
|
2277
|
+
.get(agentId);
|
|
2278
|
+
if (!row)
|
|
2279
|
+
return null;
|
|
2280
|
+
return {
|
|
2281
|
+
agentId: row.agent_id,
|
|
2282
|
+
stableId: row.stable_id,
|
|
2283
|
+
wakeLeaseId: row.wake_lease_id,
|
|
2284
|
+
fenceToken: row.fence_token,
|
|
2285
|
+
reservedGeneration: row.reserved_generation,
|
|
2286
|
+
reservationNonce: row.reservation_nonce,
|
|
2287
|
+
acceptedAt: row.accepted_at,
|
|
2288
|
+
};
|
|
2289
|
+
}
|
|
2290
|
+
/**
|
|
2291
|
+
* Accept exactly one runtime generation for a waking agent. The registration
|
|
2292
|
+
* must present the same wake lease id, fence token, and reserved generation,
|
|
2293
|
+
* AND the bound lease must still be an unexpired `wake` lease while the agent
|
|
2294
|
+
* is still in the `waking` lifecycle state. A stale, expired, wrong-operation,
|
|
2295
|
+
* wrong-state, or duplicate registration returns `{ accepted: false }` without
|
|
2296
|
+
* mutating state. On success the agent's runtime_generation is advanced and
|
|
2297
|
+
* the reservation is consumed. `now` (epoch ms) is injectable for tests.
|
|
2298
|
+
*/
|
|
2299
|
+
/**
|
|
2300
|
+
* Non-mutating validation shared by {@link acceptRuntimeGeneration} and
|
|
2301
|
+
* {@link checkRuntimeGenerationAcceptable}. Returns a `{ accepted: false }`
|
|
2302
|
+
* rejection when the fence does not bind, or `null` when acceptance is legal.
|
|
2303
|
+
* Never advances the generation or consumes the reservation.
|
|
2304
|
+
*/
|
|
2305
|
+
validateRuntimeGenerationAcceptance(input, nowMs) {
|
|
2306
|
+
const db = this.getDb();
|
|
2307
|
+
const reservation = this.getAgentWakeReservation(input.agentId);
|
|
2308
|
+
if (!reservation)
|
|
2309
|
+
return { accepted: false, reason: "no_reservation" };
|
|
2310
|
+
if (reservation.wakeLeaseId !== input.wakeLeaseId) {
|
|
2311
|
+
return { accepted: false, reason: "lease_mismatch" };
|
|
2312
|
+
}
|
|
2313
|
+
if (reservation.fenceToken !== input.fenceToken) {
|
|
2314
|
+
return { accepted: false, reason: "fence_mismatch" };
|
|
2315
|
+
}
|
|
2316
|
+
if (reservation.reservedGeneration !== input.reservedGeneration) {
|
|
2317
|
+
return { accepted: false, reason: "generation_mismatch" };
|
|
2318
|
+
}
|
|
2319
|
+
// Per-attempt nonce: a matching lease/fence/generation is NOT sufficient,
|
|
2320
|
+
// because retries reuse all three. Only the runtime launched by THIS
|
|
2321
|
+
// attempt carries the current reservation's nonce; a slow runtime from a
|
|
2322
|
+
// superseded earlier attempt presents a stale nonce and is fenced out.
|
|
2323
|
+
if (reservation.reservationNonce !== input.reservationNonce) {
|
|
2324
|
+
return { accepted: false, reason: "nonce_mismatch" };
|
|
2325
|
+
}
|
|
2326
|
+
const lease = this.getAgentLifecycleLease(input.agentId);
|
|
2327
|
+
if (!lease || lease.leaseId !== input.wakeLeaseId || lease.fenceToken !== input.fenceToken) {
|
|
2328
|
+
return { accepted: false, reason: "lease_lost" };
|
|
2329
|
+
}
|
|
2330
|
+
// Strict lease binding: only an unexpired wake lease may accept a
|
|
2331
|
+
// generation, and only while the agent is still waking. This closes the
|
|
2332
|
+
// window where a delayed runtime presents an otherwise matching
|
|
2333
|
+
// reservation under an expired/wrong-operation lease or after the wake
|
|
2334
|
+
// was already resolved/aborted.
|
|
2335
|
+
if (lease.operation !== "wake") {
|
|
2336
|
+
return { accepted: false, reason: "lease_not_wake" };
|
|
2337
|
+
}
|
|
2338
|
+
const leaseExpiryMs = Date.parse(lease.expiresAt);
|
|
2339
|
+
if (!Number.isFinite(leaseExpiryMs) || leaseExpiryMs <= nowMs) {
|
|
2340
|
+
return { accepted: false, reason: "lease_expired" };
|
|
2341
|
+
}
|
|
2342
|
+
const stateRow = db
|
|
2343
|
+
.prepare("SELECT lifecycle_state, runtime_generation FROM agents WHERE id = ?")
|
|
2344
|
+
.get(input.agentId);
|
|
2345
|
+
if (!stateRow || stateRow.lifecycle_state !== "waking") {
|
|
2346
|
+
return { accepted: false, reason: "not_waking" };
|
|
2347
|
+
}
|
|
2348
|
+
if (Number(stateRow.runtime_generation) !== input.reservedGeneration - 1) {
|
|
2349
|
+
return { accepted: false, reason: "generation_race" };
|
|
2350
|
+
}
|
|
2351
|
+
return null;
|
|
2352
|
+
}
|
|
2353
|
+
acceptRuntimeGeneration(input) {
|
|
2354
|
+
return this.withTransaction(() => {
|
|
2355
|
+
const db = this.getDb();
|
|
2356
|
+
const nowMs = input.now ?? Date.now();
|
|
2357
|
+
const rejection = this.validateRuntimeGenerationAcceptance(input, nowMs);
|
|
2358
|
+
if (rejection)
|
|
2359
|
+
return rejection;
|
|
2360
|
+
const result = db
|
|
2361
|
+
.prepare("UPDATE agents SET runtime_generation = ? WHERE id = ? AND runtime_generation = ?")
|
|
2362
|
+
.run(input.reservedGeneration, input.agentId, input.reservedGeneration - 1);
|
|
2363
|
+
if (Number(result.changes) !== 1) {
|
|
2364
|
+
return { accepted: false, reason: "generation_race" };
|
|
2365
|
+
}
|
|
2366
|
+
db.prepare("DELETE FROM agent_wake_reservations WHERE agent_id = ?").run(input.agentId);
|
|
2367
|
+
// Persist the accepting fence so a crash between this commit and the socket
|
|
2368
|
+
// bind/response can be recovered by an idempotent fenced replay.
|
|
2369
|
+
this.writeWakeAcceptanceReceipt(input, nowMs);
|
|
2370
|
+
return { accepted: true, runtimeGeneration: input.reservedGeneration };
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
/**
|
|
2374
|
+
* Atomically settle a wake attempt against the acceptance boundary BEFORE the
|
|
2375
|
+
* orchestrator stops or quarantines a launched-but-unaccepted runtime. This
|
|
2376
|
+
* closes the timeout-boundary race: the socket layer accepts a generation
|
|
2377
|
+
* atomically, so an acceptance can land in the window between the
|
|
2378
|
+
* orchestrator's last waiter read and its decision to stop the attempt. In one
|
|
2379
|
+
* transaction:
|
|
2380
|
+
*
|
|
2381
|
+
* - If the reserved generation was ALREADY accepted (the agent's
|
|
2382
|
+
* `runtime_generation` reached `reservedGeneration` — the socket won the
|
|
2383
|
+
* race), report `{ accepted: true }` and leave the accepted runtime and its
|
|
2384
|
+
* (already consumed) reservation untouched. The caller must then treat the
|
|
2385
|
+
* attempt as the live runtime and NEVER stop it.
|
|
2386
|
+
* - Otherwise, consume ONLY this attempt's exact-nonce reservation, so any
|
|
2387
|
+
* later registration by the launched runtime can no longer be accepted
|
|
2388
|
+
* (`no_reservation`). This makes the caller's subsequent prove-stop safe:
|
|
2389
|
+
* once this returns `{ accepted: false }` the launched runtime can never
|
|
2390
|
+
* become live. A reservation minted by a superseded/newer attempt (a
|
|
2391
|
+
* different nonce) is left intact.
|
|
2392
|
+
*
|
|
2393
|
+
* `runtime_generation === reservedGeneration` uniquely identifies acceptance of
|
|
2394
|
+
* THIS attempt because reserved generations are `current_generation + 1` at
|
|
2395
|
+
* reserve time and only advance on acceptance.
|
|
2396
|
+
*/
|
|
2397
|
+
finalizeWakeAttempt(input) {
|
|
2398
|
+
return this.withTransaction(() => {
|
|
2399
|
+
const db = this.getDb();
|
|
2400
|
+
const row = db
|
|
2401
|
+
.prepare("SELECT runtime_generation FROM agents WHERE id = ?")
|
|
2402
|
+
.get(input.agentId);
|
|
2403
|
+
if (row && Number(row.runtime_generation) === input.reservedGeneration) {
|
|
2404
|
+
return { accepted: true };
|
|
2405
|
+
}
|
|
2406
|
+
db.prepare("DELETE FROM agent_wake_reservations WHERE agent_id = ? AND reservation_nonce = ?").run(input.agentId, input.reservationNonce);
|
|
2407
|
+
return { accepted: false };
|
|
2408
|
+
});
|
|
2409
|
+
}
|
|
2410
|
+
/**
|
|
2411
|
+
* Atomically revive a hibernated identity: perform the agent registration
|
|
2412
|
+
* mutation AND accept the reserved runtime generation in ONE transaction, so a
|
|
2413
|
+
* rejected acceptance rolls the registration mutation back. This closes the
|
|
2414
|
+
* window where a revival whose wake lease expires between the socket-layer
|
|
2415
|
+
* preflight and acceptance would otherwise leave the durable row with a
|
|
2416
|
+
* mutated pid/metadata/connectivity even though the socket is refused and
|
|
2417
|
+
* unbound. On rejection the transaction is rolled back and `agent` is null.
|
|
2418
|
+
*/
|
|
2419
|
+
registerAgentWithGenerationAcceptance(input) {
|
|
2420
|
+
try {
|
|
2421
|
+
return this.withTransaction(() => {
|
|
2422
|
+
const db = this.getDb();
|
|
2423
|
+
const nowMs = input.accept.now ?? Date.now();
|
|
2424
|
+
// Register first so the durable row exists/updates, then accept the
|
|
2425
|
+
// generation against that just-registered row within the same tx.
|
|
2426
|
+
const agent = this.registerAgent(input.registration.id, input.registration.name, input.registration.emoji, input.registration.pid, input.registration.metadata, input.registration.stableId);
|
|
2427
|
+
const rejection = this.validateRuntimeGenerationAcceptance(input.accept, nowMs);
|
|
2428
|
+
if (rejection)
|
|
2429
|
+
throw new GenerationAcceptanceRollback(rejection);
|
|
2430
|
+
const result = db
|
|
2431
|
+
.prepare("UPDATE agents SET runtime_generation = ? WHERE id = ? AND runtime_generation = ?")
|
|
2432
|
+
.run(input.accept.reservedGeneration, input.accept.agentId, input.accept.reservedGeneration - 1);
|
|
2433
|
+
if (Number(result.changes) !== 1) {
|
|
2434
|
+
throw new GenerationAcceptanceRollback({ accepted: false, reason: "generation_race" });
|
|
2435
|
+
}
|
|
2436
|
+
db.prepare("DELETE FROM agent_wake_reservations WHERE agent_id = ?").run(input.accept.agentId);
|
|
2437
|
+
// Persist the accepting fence (atomic with the registration + acceptance)
|
|
2438
|
+
// so a crash before this connection is bound/acknowledged can be recovered
|
|
2439
|
+
// by an idempotent fenced replay of the same single-use wake fence.
|
|
2440
|
+
this.writeWakeAcceptanceReceipt(input.accept, nowMs);
|
|
2441
|
+
return {
|
|
2442
|
+
agent,
|
|
2443
|
+
acceptance: {
|
|
2444
|
+
accepted: true,
|
|
2445
|
+
runtimeGeneration: input.accept.reservedGeneration,
|
|
2446
|
+
},
|
|
2447
|
+
};
|
|
2448
|
+
});
|
|
2449
|
+
}
|
|
2450
|
+
catch (err) {
|
|
2451
|
+
if (err instanceof GenerationAcceptanceRollback) {
|
|
2452
|
+
return { agent: null, acceptance: err.rejection };
|
|
2453
|
+
}
|
|
2454
|
+
throw err;
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
/**
|
|
2458
|
+
* Non-mutating preflight for {@link acceptRuntimeGeneration}: runs the exact
|
|
2459
|
+
* same fence validation without advancing the generation or consuming the
|
|
2460
|
+
* reservation. The socket layer uses this to validate a wake fence BEFORE
|
|
2461
|
+
* committing the agent registration, and only accepts the generation once
|
|
2462
|
+
* registration has succeeded. Because broker registration is synchronous, a
|
|
2463
|
+
* passing preflight followed immediately by `acceptRuntimeGeneration` cannot
|
|
2464
|
+
* be interleaved by another connection, so this closes the window where a
|
|
2465
|
+
* generation was advanced for a runtime whose registration then failed.
|
|
2466
|
+
*/
|
|
2467
|
+
checkRuntimeGenerationAcceptable(input) {
|
|
2468
|
+
const nowMs = input.now ?? Date.now();
|
|
2469
|
+
return (this.validateRuntimeGenerationAcceptance(input, nowMs) ?? {
|
|
2470
|
+
accepted: true,
|
|
2471
|
+
runtimeGeneration: input.reservedGeneration,
|
|
2472
|
+
});
|
|
2473
|
+
}
|
|
2474
|
+
// ─── Wake queue (ordered, capacity-bounded) ────────────────────────
|
|
2475
|
+
/**
|
|
2476
|
+
* Idempotently enqueue a wake trigger. If an active (queued/dispatching)
|
|
2477
|
+
* entry already exists for the agent it is returned unchanged except that the
|
|
2478
|
+
* effective priority is lowered to the strongest (smallest) trigger and the
|
|
2479
|
+
* trigger message id is preserved when still unset. Never fans out.
|
|
2480
|
+
*/
|
|
2481
|
+
enqueueWake(input) {
|
|
2482
|
+
return this.withTransaction(() => {
|
|
2483
|
+
const db = this.getDb();
|
|
2484
|
+
const now = new Date().toISOString();
|
|
2485
|
+
const priority = input.priority ?? 100;
|
|
2486
|
+
const existing = this.getActiveWakeQueueEntry(input.agentId);
|
|
2487
|
+
if (existing) {
|
|
2488
|
+
const nextPriority = Math.min(existing.priority, priority);
|
|
2489
|
+
const nextTriggerMessageId = existing.triggerMessageId ?? input.triggerMessageId ?? null;
|
|
2490
|
+
db.prepare(`UPDATE agent_wake_queue
|
|
2491
|
+
SET priority = ?, trigger_message_id = ?, updated_at = ?
|
|
2492
|
+
WHERE id = ?`).run(nextPriority, nextTriggerMessageId, now, existing.id);
|
|
2493
|
+
const refreshed = this.getWakeQueueEntryById(existing.id);
|
|
2494
|
+
if (!refreshed)
|
|
2495
|
+
throw new Error("Failed to refresh wake queue entry");
|
|
2496
|
+
return refreshed;
|
|
2497
|
+
}
|
|
2498
|
+
const inserted = db
|
|
2499
|
+
.prepare(`INSERT INTO agent_wake_queue
|
|
2500
|
+
(agent_id, repo_root, trigger_kind, trigger_message_id, priority, reason,
|
|
2501
|
+
correlation_id, status, attempt, enqueued_at, updated_at)
|
|
2502
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
|
|
2503
|
+
.run(input.agentId, input.repoRoot ?? null, input.triggerKind, input.triggerMessageId ?? null, priority, input.reason, input.correlationId, now, now);
|
|
2504
|
+
const entry = this.getWakeQueueEntryById(Number(inserted.lastInsertRowid));
|
|
2505
|
+
if (!entry)
|
|
2506
|
+
throw new Error("Failed to enqueue wake");
|
|
2507
|
+
return entry;
|
|
2508
|
+
});
|
|
2509
|
+
}
|
|
2510
|
+
getActiveWakeQueueEntry(agentId) {
|
|
2511
|
+
const row = this.getDb()
|
|
2512
|
+
.prepare(`SELECT * FROM agent_wake_queue
|
|
2513
|
+
WHERE agent_id = ? AND status IN ('queued','dispatching') LIMIT 1`)
|
|
2514
|
+
.get(agentId);
|
|
2515
|
+
return row ? rowToWakeQueueEntry(row) : null;
|
|
2516
|
+
}
|
|
2517
|
+
getWakeQueueEntryById(id) {
|
|
2518
|
+
const row = this.getDb().prepare("SELECT * FROM agent_wake_queue WHERE id = ?").get(id);
|
|
2519
|
+
return row ? rowToWakeQueueEntry(row) : null;
|
|
2520
|
+
}
|
|
2521
|
+
listWakeQueue(status) {
|
|
2522
|
+
const db = this.getDb();
|
|
2523
|
+
const rows = status
|
|
2524
|
+
? db
|
|
2525
|
+
.prepare("SELECT * FROM agent_wake_queue WHERE status = ? ORDER BY priority ASC, id ASC")
|
|
2526
|
+
.all(status)
|
|
2527
|
+
: db.prepare("SELECT * FROM agent_wake_queue ORDER BY priority ASC, id ASC").all();
|
|
2528
|
+
return rows.map((row) => rowToWakeQueueEntry({
|
|
2529
|
+
id: Number(row.id),
|
|
2530
|
+
agent_id: String(row.agent_id),
|
|
2531
|
+
repo_root: row.repo_root == null ? null : String(row.repo_root),
|
|
2532
|
+
trigger_kind: String(row.trigger_kind),
|
|
2533
|
+
trigger_message_id: row.trigger_message_id == null ? null : Number(row.trigger_message_id),
|
|
2534
|
+
priority: Number(row.priority),
|
|
2535
|
+
reason: String(row.reason),
|
|
2536
|
+
correlation_id: String(row.correlation_id),
|
|
2537
|
+
status: String(row.status),
|
|
2538
|
+
attempt: Number(row.attempt),
|
|
2539
|
+
enqueued_at: String(row.enqueued_at),
|
|
2540
|
+
updated_at: String(row.updated_at),
|
|
2541
|
+
}));
|
|
2542
|
+
}
|
|
2543
|
+
countInflightWakes(repoRoot) {
|
|
2544
|
+
const db = this.getDb();
|
|
2545
|
+
if (repoRoot === undefined) {
|
|
2546
|
+
const row = db
|
|
2547
|
+
.prepare("SELECT COUNT(*) AS count FROM agent_wake_queue WHERE status = 'dispatching'")
|
|
2548
|
+
.get();
|
|
2549
|
+
return Number(row?.count ?? 0);
|
|
2550
|
+
}
|
|
2551
|
+
const row = db
|
|
2552
|
+
.prepare("SELECT COUNT(*) AS count FROM agent_wake_queue WHERE status = 'dispatching' AND repo_root IS ?")
|
|
2553
|
+
.get(repoRoot);
|
|
2554
|
+
return Number(row?.count ?? 0);
|
|
2555
|
+
}
|
|
2556
|
+
markWakeDispatching(id) {
|
|
2557
|
+
return this.withTransaction(() => {
|
|
2558
|
+
const now = new Date().toISOString();
|
|
2559
|
+
const result = this.getDb()
|
|
2560
|
+
.prepare(`UPDATE agent_wake_queue SET status = 'dispatching', attempt = attempt + 1, updated_at = ?
|
|
2561
|
+
WHERE id = ? AND status = 'queued'`)
|
|
2562
|
+
.run(now, id);
|
|
2563
|
+
if (Number(result.changes) !== 1)
|
|
2564
|
+
return null;
|
|
2565
|
+
return this.getWakeQueueEntryById(id);
|
|
2566
|
+
});
|
|
2567
|
+
}
|
|
2568
|
+
requeueWake(id) {
|
|
2569
|
+
return this.withTransaction(() => {
|
|
2570
|
+
const now = new Date().toISOString();
|
|
2571
|
+
this.getDb()
|
|
2572
|
+
.prepare(`UPDATE agent_wake_queue SET status = 'queued', updated_at = ?
|
|
2573
|
+
WHERE id = ? AND status = 'dispatching'`)
|
|
2574
|
+
.run(now, id);
|
|
2575
|
+
return this.getWakeQueueEntryById(id);
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
completeWakeQueueEntry(id, status = "done") {
|
|
2579
|
+
const now = new Date().toISOString();
|
|
2580
|
+
this.getDb()
|
|
2581
|
+
.prepare("UPDATE agent_wake_queue SET status = ?, updated_at = ? WHERE id = ?")
|
|
2582
|
+
.run(status, now, id);
|
|
2583
|
+
}
|
|
2584
|
+
cancelWake(agentId) {
|
|
2585
|
+
const now = new Date().toISOString();
|
|
2586
|
+
this.getDb()
|
|
2587
|
+
.prepare(`UPDATE agent_wake_queue SET status = 'cancelled', updated_at = ?
|
|
2588
|
+
WHERE agent_id = ? AND status IN ('queued','dispatching')`)
|
|
2589
|
+
.run(now, agentId);
|
|
2590
|
+
}
|
|
2591
|
+
/** Mark any active wake-queue entry for an agent as completed (idempotent). */
|
|
2592
|
+
completeWakeForAgent(agentId) {
|
|
2593
|
+
const now = new Date().toISOString();
|
|
2594
|
+
this.getDb()
|
|
2595
|
+
.prepare(`UPDATE agent_wake_queue SET status = 'done', updated_at = ?
|
|
2596
|
+
WHERE agent_id = ? AND status IN ('queued','dispatching')`)
|
|
2597
|
+
.run(now, agentId);
|
|
2598
|
+
}
|
|
1431
2599
|
unregisterAgent(id) {
|
|
1432
2600
|
const db = this.getDb();
|
|
1433
2601
|
const now = new Date().toISOString();
|
|
1434
2602
|
this.withTransaction(() => {
|
|
1435
2603
|
const agent = this.getAgentById(id);
|
|
2604
|
+
// Durable hibernation identities MUST survive a graceful worker disconnect.
|
|
2605
|
+
// During hibernation teardown the broker stops the worker process, whose
|
|
2606
|
+
// shutdown path may send an `unregister`. A full teardown here would
|
|
2607
|
+
// delete the queued inbox and release owned threads — exactly the durable
|
|
2608
|
+
// state a later wake is supposed to drain. For hibernation lifecycle
|
|
2609
|
+
// states, treat unregister as a soft disconnect that preserves the inbox,
|
|
2610
|
+
// thread ownership, and resumability instead of tearing them down.
|
|
2611
|
+
//
|
|
2612
|
+
// `reap-candidate` is included: a hibernate/wake fault may have quarantined
|
|
2613
|
+
// the agent while its runtime was still asynchronously exiting; a late
|
|
2614
|
+
// unregister from that runtime must not destroy the inbox, ownership, and
|
|
2615
|
+
// runtime spec an operator needs to review the quarantine. This mirrors the
|
|
2616
|
+
// routine-maintenance preservation set.
|
|
2617
|
+
const state = agent?.lifecycleState;
|
|
2618
|
+
if (state === "hibernating" ||
|
|
2619
|
+
state === "hibernated" ||
|
|
2620
|
+
state === "waking" ||
|
|
2621
|
+
state === "reap-candidate") {
|
|
2622
|
+
db.prepare("UPDATE agents SET disconnected_at = ? WHERE id = ?").run(now, id);
|
|
2623
|
+
return;
|
|
2624
|
+
}
|
|
1436
2625
|
this.requeueUndeliveredMessagesInternal(id, "agent_disconnected");
|
|
1437
2626
|
db.prepare("DELETE FROM inbox WHERE agent_id = ?").run(id);
|
|
1438
2627
|
db.prepare("UPDATE agents SET disconnected_at = ?, resumable_until = NULL WHERE id = ?").run(now, id);
|
|
@@ -1599,6 +2788,7 @@ export class BrokerDB {
|
|
|
1599
2788
|
const threadId = normalizeSessionSearchNeedle(options.threadId);
|
|
1600
2789
|
const repo = normalizeSessionSearchNeedle(options.repo);
|
|
1601
2790
|
const worktreePath = normalizeSessionSearchNeedle(options.worktreePath);
|
|
2791
|
+
const runtimeLocator = normalizeSessionSearchNeedle(options.runtimeLocator);
|
|
1602
2792
|
const tmuxSession = normalizeSessionSearchNeedle(options.tmuxSession);
|
|
1603
2793
|
const sinceMs = parseSessionSearchTime(options.since);
|
|
1604
2794
|
const untilMs = parseSessionSearchTime(options.until);
|
|
@@ -1606,17 +2796,40 @@ export class BrokerDB {
|
|
|
1606
2796
|
const results = this.getAllAgents()
|
|
1607
2797
|
.map((agent) => {
|
|
1608
2798
|
const metadata = agent.metadata ?? null;
|
|
2799
|
+
const runtimeSpec = this.getAgentRuntimeSpec(agent.id);
|
|
2800
|
+
const metadataRuntimeKind = getOptionalNestedMetadataString(metadata, ["runtimeKind"]);
|
|
2801
|
+
const runtimeKind = runtimeSpec?.runtimeKind ??
|
|
2802
|
+
(metadataRuntimeKind === "tmux" || metadataRuntimeKind === "herdr"
|
|
2803
|
+
? metadataRuntimeKind
|
|
2804
|
+
: getOptionalNestedMetadataString(metadata, ["tmuxSession", "tmux"])
|
|
2805
|
+
? "tmux"
|
|
2806
|
+
: null);
|
|
2807
|
+
const runtimeLocator = runtimeSpec
|
|
2808
|
+
? runtimeSpec.runtimeKind === "tmux"
|
|
2809
|
+
? runtimeSpec.tmuxSession
|
|
2810
|
+
: runtimeSpec.herdrPaneId
|
|
2811
|
+
: (getOptionalNestedMetadataString(metadata, ["runtimeLocator"]) ??
|
|
2812
|
+
getOptionalNestedMetadataString(metadata, ["tmuxSession", "tmux"]));
|
|
1609
2813
|
const relatedThreadIds = this.getAgentRelatedThreadIds(agent.id);
|
|
1610
|
-
const matchedBy = getAgentSessionMatchedBy({
|
|
2814
|
+
const matchedBy = getAgentSessionMatchedBy({
|
|
2815
|
+
agent,
|
|
2816
|
+
metadata,
|
|
2817
|
+
runtimeKind,
|
|
2818
|
+
runtimeLocator,
|
|
2819
|
+
relatedThreadIds,
|
|
2820
|
+
options,
|
|
2821
|
+
});
|
|
1611
2822
|
return {
|
|
1612
2823
|
agent,
|
|
1613
2824
|
metadata,
|
|
2825
|
+
runtimeKind,
|
|
2826
|
+
runtimeLocator,
|
|
1614
2827
|
relatedThreadIds,
|
|
1615
2828
|
matchedBy,
|
|
1616
2829
|
lastSeenMs: Date.parse(agent.lastSeen || agent.lastHeartbeat || agent.connectedAt),
|
|
1617
2830
|
};
|
|
1618
2831
|
})
|
|
1619
|
-
.filter(({ agent, metadata, relatedThreadIds }) => {
|
|
2832
|
+
.filter(({ agent, metadata, runtimeKind, runtimeLocator: locator, relatedThreadIds }) => {
|
|
1620
2833
|
if (agentName && !matchesSessionSearchNeedle(agent.name, agentName))
|
|
1621
2834
|
return false;
|
|
1622
2835
|
if (agentId && !matchesSessionSearchPrefixOrExact(agent.id, agentId))
|
|
@@ -1644,10 +2857,11 @@ export class BrokerDB {
|
|
|
1644
2857
|
return false;
|
|
1645
2858
|
}
|
|
1646
2859
|
}
|
|
1647
|
-
if (
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
2860
|
+
if (runtimeLocator && !matchesSessionSearchNeedle(locator, runtimeLocator))
|
|
2861
|
+
return false;
|
|
2862
|
+
if (tmuxSession &&
|
|
2863
|
+
(runtimeKind !== "tmux" || !matchesSessionSearchNeedle(locator, tmuxSession))) {
|
|
2864
|
+
return false;
|
|
1651
2865
|
}
|
|
1652
2866
|
return agentSessionOverlapsRange(agent, sinceMs, untilMs);
|
|
1653
2867
|
})
|
|
@@ -1662,7 +2876,7 @@ export class BrokerDB {
|
|
|
1662
2876
|
return left.agent.name.localeCompare(right.agent.name);
|
|
1663
2877
|
})
|
|
1664
2878
|
.slice(0, limit)
|
|
1665
|
-
.map(({ agent, metadata, relatedThreadIds, matchedBy }) => ({
|
|
2879
|
+
.map(({ agent, metadata, runtimeKind, runtimeLocator, relatedThreadIds, matchedBy }) => ({
|
|
1666
2880
|
agentId: agent.id,
|
|
1667
2881
|
agentName: agent.name,
|
|
1668
2882
|
emoji: agent.emoji,
|
|
@@ -1681,7 +2895,9 @@ export class BrokerDB {
|
|
|
1681
2895
|
repoRoot: getOptionalNestedMetadataString(metadata, ["repoRoot"]),
|
|
1682
2896
|
worktreePath: getOptionalNestedMetadataString(metadata, ["worktreePath"]),
|
|
1683
2897
|
branch: getOptionalNestedMetadataString(metadata, ["branch"]),
|
|
1684
|
-
|
|
2898
|
+
runtimeKind,
|
|
2899
|
+
runtimeLocator,
|
|
2900
|
+
tmuxSession: runtimeKind === "tmux" ? runtimeLocator : null,
|
|
1685
2901
|
brokerManaged: metadata?.brokerManaged === true,
|
|
1686
2902
|
brokerManagedBy: getOptionalNestedMetadataString(metadata, ["brokerManagedBy"]),
|
|
1687
2903
|
launchSource: getOptionalNestedMetadataString(metadata, ["launchSource"]),
|
|
@@ -1925,8 +3141,9 @@ export class BrokerDB {
|
|
|
1925
3141
|
return this.withTransaction(() => {
|
|
1926
3142
|
const staleRows = db
|
|
1927
3143
|
.prepare(`SELECT * FROM agents
|
|
1928
|
-
WHERE
|
|
1929
|
-
|
|
3144
|
+
WHERE lifecycle_state NOT IN (${PRESERVED_LIFECYCLE_STATES_SQL})
|
|
3145
|
+
AND ((disconnected_at IS NULL AND last_heartbeat <= ?)
|
|
3146
|
+
OR (disconnected_at IS NOT NULL AND resumable_until IS NOT NULL AND resumable_until <= ?))`)
|
|
1930
3147
|
.all(cutoff, now);
|
|
1931
3148
|
if (staleRows.length === 0) {
|
|
1932
3149
|
return [];
|
|
@@ -1954,7 +3171,8 @@ export class BrokerDB {
|
|
|
1954
3171
|
return this.withTransaction(() => {
|
|
1955
3172
|
const rows = db
|
|
1956
3173
|
.prepare(`SELECT * FROM agents
|
|
1957
|
-
WHERE
|
|
3174
|
+
WHERE lifecycle_state NOT IN (${PRESERVED_LIFECYCLE_STATES_SQL})
|
|
3175
|
+
AND disconnected_at IS NOT NULL
|
|
1958
3176
|
AND disconnected_at <= ?
|
|
1959
3177
|
AND (resumable_until IS NULL OR resumable_until <= ?)`)
|
|
1960
3178
|
.all(cutoff, nowIso);
|
|
@@ -1977,7 +3195,8 @@ export class BrokerDB {
|
|
|
1977
3195
|
this.markDescendantsOrphaned(row.id, "parent_purged");
|
|
1978
3196
|
}
|
|
1979
3197
|
db.prepare(`DELETE FROM agents
|
|
1980
|
-
WHERE
|
|
3198
|
+
WHERE lifecycle_state NOT IN (${PRESERVED_LIFECYCLE_STATES_SQL})
|
|
3199
|
+
AND disconnected_at IS NOT NULL
|
|
1981
3200
|
AND disconnected_at <= ?
|
|
1982
3201
|
AND (resumable_until IS NULL OR resumable_until <= ?)`).run(cutoff, nowIso);
|
|
1983
3202
|
return rows.map((row) => row.id);
|
|
@@ -2796,12 +4015,17 @@ export class BrokerDB {
|
|
|
2796
4015
|
repairThreadOwnership() {
|
|
2797
4016
|
const db = this.getDb();
|
|
2798
4017
|
return this.withTransaction(() => {
|
|
4018
|
+
// Preserve ownership for live agents AND for durable hibernation /
|
|
4019
|
+
// quarantine identities: a hibernated owner is intentionally disconnected
|
|
4020
|
+
// but must keep its owned threads so a later wake resumes them.
|
|
2799
4021
|
const rows = db
|
|
2800
4022
|
.prepare(`SELECT owner_agent, COUNT(*) AS claim_count
|
|
2801
4023
|
FROM threads
|
|
2802
4024
|
WHERE owner_agent IS NOT NULL
|
|
2803
4025
|
AND owner_agent NOT IN (
|
|
2804
|
-
SELECT id FROM agents
|
|
4026
|
+
SELECT id FROM agents
|
|
4027
|
+
WHERE disconnected_at IS NULL
|
|
4028
|
+
OR lifecycle_state IN (${PRESERVED_LIFECYCLE_STATES_SQL})
|
|
2805
4029
|
)
|
|
2806
4030
|
GROUP BY owner_agent`)
|
|
2807
4031
|
.all();
|
|
@@ -2812,7 +4036,9 @@ export class BrokerDB {
|
|
|
2812
4036
|
SET owner_agent = NULL
|
|
2813
4037
|
WHERE owner_agent IS NOT NULL
|
|
2814
4038
|
AND owner_agent NOT IN (
|
|
2815
|
-
SELECT id FROM agents
|
|
4039
|
+
SELECT id FROM agents
|
|
4040
|
+
WHERE disconnected_at IS NULL
|
|
4041
|
+
OR lifecycle_state IN (${PRESERVED_LIFECYCLE_STATES_SQL})
|
|
2816
4042
|
)`).run();
|
|
2817
4043
|
return {
|
|
2818
4044
|
releasedClaimCount: rows.reduce((count, row) => count + Number(row.claim_count), 0),
|