@evident-ai/cli 3.0.1-dev.d0ec1bd → 3.0.1-dev.d3368f6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +928 -59
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve2();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -706,7 +706,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
706
706
|
if (health.healthy) {
|
|
707
707
|
return health;
|
|
708
708
|
}
|
|
709
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
710
710
|
}
|
|
711
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
712
712
|
}
|
|
@@ -1078,6 +1078,84 @@ async function getSessionMessages(port, sessionId) {
|
|
|
1078
1078
|
return null;
|
|
1079
1079
|
}
|
|
1080
1080
|
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1081
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1082
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1083
1161
|
if (directory && directory.trim()) {
|
|
@@ -1147,7 +1225,7 @@ async function sendPromptAsync(port, sessionId, content, options) {
|
|
|
1147
1225
|
if (best) return best.id;
|
|
1148
1226
|
}
|
|
1149
1227
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1150
|
-
await new Promise((
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1151
1229
|
}
|
|
1152
1230
|
}
|
|
1153
1231
|
return null;
|
|
@@ -1204,6 +1282,11 @@ function messageRunState(messages, userMessageId) {
|
|
|
1204
1282
|
if (isAssistantInFlight(reply)) return "running";
|
|
1205
1283
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1206
1284
|
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1207
1290
|
function messageError(messages, userMessageId) {
|
|
1208
1291
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1209
1292
|
const error2 = errorOf(reply);
|
|
@@ -1224,6 +1307,110 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1224
1307
|
);
|
|
1225
1308
|
}
|
|
1226
1309
|
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
return toDelete;
|
|
1356
|
+
}
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1360
|
+
}
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1369
|
+
}
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
+
}
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1227
1414
|
// src/lib/tunnel/connection.ts
|
|
1228
1415
|
import WebSocket2 from "ws";
|
|
1229
1416
|
|
|
@@ -1300,24 +1487,26 @@ var StreamForwarder = class {
|
|
|
1300
1487
|
this.send({ type: "res_end", sid });
|
|
1301
1488
|
return;
|
|
1302
1489
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1309
1498
|
const ac = new AbortController();
|
|
1310
1499
|
let bodyPromise;
|
|
1311
1500
|
let pushBody;
|
|
1312
1501
|
let endBody;
|
|
1313
1502
|
if (has_body) {
|
|
1314
1503
|
const chunks = [];
|
|
1315
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1316
1505
|
pushBody = (buf) => {
|
|
1317
1506
|
chunks.push(buf);
|
|
1318
1507
|
};
|
|
1319
1508
|
endBody = () => {
|
|
1320
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1321
1510
|
};
|
|
1322
1511
|
});
|
|
1323
1512
|
}
|
|
@@ -1352,12 +1541,14 @@ var StreamForwarder = class {
|
|
|
1352
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1353
1542
|
});
|
|
1354
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1361
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1362
1553
|
try {
|
|
1363
1554
|
if (upstream.body) {
|
|
@@ -1433,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1433
1624
|
} = options;
|
|
1434
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1435
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1436
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1437
1628
|
const ws = new WebSocket2(url, {
|
|
1438
1629
|
headers: {
|
|
1439
1630
|
Authorization: authHeader
|
|
@@ -1498,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1498
1689
|
clearTimeout(connectionTimeout);
|
|
1499
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1500
1691
|
onConnected?.(connectedAgentId);
|
|
1501
|
-
|
|
1692
|
+
resolve2({
|
|
1502
1693
|
ws,
|
|
1503
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1504
1695
|
});
|
|
@@ -1628,6 +1819,9 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1628
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1629
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1630
1821
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1631
1825
|
var ChannelAuthError = class extends Error {
|
|
1632
1826
|
constructor(message) {
|
|
1633
1827
|
super(message);
|
|
@@ -1724,6 +1918,15 @@ var ChannelDriver = class {
|
|
|
1724
1918
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1725
1919
|
*/
|
|
1726
1920
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
+
*/
|
|
1929
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1727
1930
|
/**
|
|
1728
1931
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1729
1932
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -1745,8 +1948,33 @@ var ChannelDriver = class {
|
|
|
1745
1948
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1746
1949
|
*/
|
|
1747
1950
|
opencodeDirectory = void 0;
|
|
1951
|
+
/**
|
|
1952
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1953
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1954
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1955
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1956
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1957
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1958
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1959
|
+
*/
|
|
1960
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1748
1961
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1749
1962
|
draining = false;
|
|
1963
|
+
/**
|
|
1964
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1965
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1966
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1967
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1968
|
+
*/
|
|
1969
|
+
activeDrain = null;
|
|
1970
|
+
/**
|
|
1971
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1972
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1973
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1974
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1975
|
+
* and stops opencode.
|
|
1976
|
+
*/
|
|
1977
|
+
stopped = false;
|
|
1750
1978
|
constructor(config2) {
|
|
1751
1979
|
this.agentId = config2.agentId;
|
|
1752
1980
|
this.port = config2.port;
|
|
@@ -1778,8 +2006,21 @@ var ChannelDriver = class {
|
|
|
1778
2006
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1779
2007
|
*/
|
|
1780
2008
|
async drainPending() {
|
|
2009
|
+
if (this.stopped) return 0;
|
|
1781
2010
|
if (this.draining) return 0;
|
|
1782
2011
|
this.draining = true;
|
|
2012
|
+
const run2 = this.runDrain();
|
|
2013
|
+
this.activeDrain = run2.then(
|
|
2014
|
+
() => {
|
|
2015
|
+
this.activeDrain = null;
|
|
2016
|
+
},
|
|
2017
|
+
() => {
|
|
2018
|
+
this.activeDrain = null;
|
|
2019
|
+
}
|
|
2020
|
+
);
|
|
2021
|
+
return run2;
|
|
2022
|
+
}
|
|
2023
|
+
async runDrain() {
|
|
1783
2024
|
let dispatched = 0;
|
|
1784
2025
|
try {
|
|
1785
2026
|
const conversations = await this.getPendingConversations();
|
|
@@ -1791,6 +2032,7 @@ var ChannelDriver = class {
|
|
|
1791
2032
|
});
|
|
1792
2033
|
}
|
|
1793
2034
|
for (const conv of conversations) {
|
|
2035
|
+
if (this.stopped) break;
|
|
1794
2036
|
dispatched += await this.processConversation(conv);
|
|
1795
2037
|
}
|
|
1796
2038
|
await this.readoptProcessing();
|
|
@@ -1811,6 +2053,73 @@ var ChannelDriver = class {
|
|
|
1811
2053
|
}
|
|
1812
2054
|
return false;
|
|
1813
2055
|
}
|
|
2056
|
+
/**
|
|
2057
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2058
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2059
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2060
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2061
|
+
*
|
|
2062
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2063
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2064
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2065
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2066
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2067
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2068
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2069
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2070
|
+
*/
|
|
2071
|
+
protectedSessionIds() {
|
|
2072
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2073
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2074
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2075
|
+
}
|
|
2076
|
+
return ids;
|
|
2077
|
+
}
|
|
2078
|
+
/**
|
|
2079
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2080
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2081
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2082
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2083
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2084
|
+
*/
|
|
2085
|
+
stop() {
|
|
2086
|
+
this.stopped = true;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2090
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2091
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2092
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2093
|
+
*
|
|
2094
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2095
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2096
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2097
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2098
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2099
|
+
* next runner start (ADR-0046).
|
|
2100
|
+
*
|
|
2101
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2102
|
+
* timeout elapsed with work still in flight.
|
|
2103
|
+
*/
|
|
2104
|
+
async waitForInFlight(timeoutMs) {
|
|
2105
|
+
const deadline = this.now() + timeoutMs;
|
|
2106
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2107
|
+
if (this.activeDrain) {
|
|
2108
|
+
let drainSettled = false;
|
|
2109
|
+
void this.activeDrain.then(() => {
|
|
2110
|
+
drainSettled = true;
|
|
2111
|
+
});
|
|
2112
|
+
while (!drainSettled) {
|
|
2113
|
+
if (this.now() >= deadline) return false;
|
|
2114
|
+
await this.sleep(step);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
while (this.hasInFlightWatchers()) {
|
|
2118
|
+
if (this.now() >= deadline) return false;
|
|
2119
|
+
await this.sleep(step);
|
|
2120
|
+
}
|
|
2121
|
+
return true;
|
|
2122
|
+
}
|
|
1814
2123
|
/**
|
|
1815
2124
|
* Await all outstanding per-session watchers (WI-3).
|
|
1816
2125
|
*
|
|
@@ -1847,6 +2156,7 @@ var ChannelDriver = class {
|
|
|
1847
2156
|
let dispatched = 0;
|
|
1848
2157
|
let skippedAlreadyDispatched = 0;
|
|
1849
2158
|
for (const message of messages) {
|
|
2159
|
+
if (this.stopped) break;
|
|
1850
2160
|
if (this.dispatched.has(message.id)) {
|
|
1851
2161
|
skippedAlreadyDispatched += 1;
|
|
1852
2162
|
continue;
|
|
@@ -1870,6 +2180,16 @@ var ChannelDriver = class {
|
|
|
1870
2180
|
} catch (err) {
|
|
1871
2181
|
if (err instanceof ChannelAuthError) throw err;
|
|
1872
2182
|
this.dispatched.delete(message.id);
|
|
2183
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2184
|
+
this.sessions.delete(conv.id);
|
|
2185
|
+
this.log({
|
|
2186
|
+
level: "info",
|
|
2187
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2188
|
+
conversation_id: conv.id,
|
|
2189
|
+
message_id: message.id
|
|
2190
|
+
});
|
|
2191
|
+
break;
|
|
2192
|
+
}
|
|
1873
2193
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1874
2194
|
});
|
|
1875
2195
|
this.log({
|
|
@@ -1905,16 +2225,33 @@ var ChannelDriver = class {
|
|
|
1905
2225
|
return dispatched;
|
|
1906
2226
|
}
|
|
1907
2227
|
async ensureSession(conv) {
|
|
1908
|
-
const
|
|
1909
|
-
if (
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
2228
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2229
|
+
if (bound) {
|
|
2230
|
+
const exists = await sessionExists(this.port, bound);
|
|
2231
|
+
if (exists === false) {
|
|
2232
|
+
this.log({
|
|
2233
|
+
level: "info",
|
|
2234
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2235
|
+
conversation_id: conv.id
|
|
2236
|
+
});
|
|
2237
|
+
this.sessions.delete(conv.id);
|
|
2238
|
+
return this.createAndBindSession(conv.id);
|
|
2239
|
+
}
|
|
2240
|
+
this.sessions.set(conv.id, bound);
|
|
2241
|
+
return bound;
|
|
1913
2242
|
}
|
|
2243
|
+
return this.createAndBindSession(conv.id);
|
|
2244
|
+
}
|
|
2245
|
+
/**
|
|
2246
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2247
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2248
|
+
* self-heal recreate path in `ensureSession`.
|
|
2249
|
+
*/
|
|
2250
|
+
async createAndBindSession(conversationId) {
|
|
1914
2251
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1915
2252
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1916
|
-
this.sessions.set(
|
|
1917
|
-
await this.persistSession(
|
|
2253
|
+
this.sessions.set(conversationId, sessionId);
|
|
2254
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1918
2255
|
});
|
|
1919
2256
|
return sessionId;
|
|
1920
2257
|
}
|
|
@@ -1965,7 +2302,9 @@ var ChannelDriver = class {
|
|
|
1965
2302
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1966
2303
|
loop: null,
|
|
1967
2304
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1968
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2305
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2306
|
+
lastGoodPollAt: this.now(),
|
|
2307
|
+
hadUsablePoll: false
|
|
1969
2308
|
};
|
|
1970
2309
|
this.watchers.set(sessionId, watcher);
|
|
1971
2310
|
}
|
|
@@ -1975,20 +2314,37 @@ var ChannelDriver = class {
|
|
|
1975
2314
|
opencodeMessageId,
|
|
1976
2315
|
message,
|
|
1977
2316
|
dispatchedAt: now,
|
|
2317
|
+
processingAnchorMs: now,
|
|
1978
2318
|
deadline: now + this.pausedMaxWaitMs,
|
|
1979
2319
|
started: false,
|
|
1980
2320
|
done: false,
|
|
1981
|
-
stuckReported: false
|
|
2321
|
+
stuckReported: false,
|
|
2322
|
+
lastAliveAt: 0,
|
|
2323
|
+
aliveInFlight: false,
|
|
2324
|
+
awaitingHumanLatched: false,
|
|
2325
|
+
pausedOnQuestion: false,
|
|
2326
|
+
pausedOnPermission: false,
|
|
2327
|
+
pausedClearConfirmed: false,
|
|
2328
|
+
pausedInFlight: false,
|
|
2329
|
+
deliveryDeadlineAnchored: false
|
|
1982
2330
|
});
|
|
1983
2331
|
}
|
|
1984
2332
|
/**
|
|
1985
2333
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1986
2334
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1987
2335
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1988
|
-
* `now
|
|
1989
|
-
* (10 min after `processed_at
|
|
1990
|
-
*
|
|
1991
|
-
*
|
|
2336
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2337
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2338
|
+
*
|
|
2339
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2340
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2341
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2342
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2343
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2344
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2345
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2346
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2347
|
+
* (only the appear-guard uses it).
|
|
1992
2348
|
*
|
|
1993
2349
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1994
2350
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -2006,7 +2362,9 @@ var ChannelDriver = class {
|
|
|
2006
2362
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2007
2363
|
loop: null,
|
|
2008
2364
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2009
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2365
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2366
|
+
lastGoodPollAt: this.now(),
|
|
2367
|
+
hadUsablePoll: false
|
|
2010
2368
|
};
|
|
2011
2369
|
this.watchers.set(sessionId, watcher);
|
|
2012
2370
|
}
|
|
@@ -2015,6 +2373,10 @@ var ChannelDriver = class {
|
|
|
2015
2373
|
opencodeMessageId,
|
|
2016
2374
|
message,
|
|
2017
2375
|
dispatchedAt: this.now(),
|
|
2376
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2377
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2378
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2379
|
+
processingAnchorMs: processedAtMs,
|
|
2018
2380
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2019
2381
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2020
2382
|
started: true,
|
|
@@ -2024,7 +2386,20 @@ var ChannelDriver = class {
|
|
|
2024
2386
|
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2025
2387
|
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2026
2388
|
// (#210/#220 observability).
|
|
2027
|
-
stuckReported: false
|
|
2389
|
+
stuckReported: false,
|
|
2390
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2391
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2392
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2393
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2394
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2395
|
+
lastAliveAt: 0,
|
|
2396
|
+
aliveInFlight: false,
|
|
2397
|
+
awaitingHumanLatched: false,
|
|
2398
|
+
pausedOnQuestion: false,
|
|
2399
|
+
pausedOnPermission: false,
|
|
2400
|
+
pausedClearConfirmed: false,
|
|
2401
|
+
pausedInFlight: false,
|
|
2402
|
+
deliveryDeadlineAnchored: false
|
|
2028
2403
|
});
|
|
2029
2404
|
}
|
|
2030
2405
|
/**
|
|
@@ -2075,12 +2450,30 @@ var ChannelDriver = class {
|
|
|
2075
2450
|
messages = Array.isArray(body) ? body : null;
|
|
2076
2451
|
}
|
|
2077
2452
|
} catch {
|
|
2078
|
-
continue;
|
|
2079
2453
|
}
|
|
2454
|
+
if (messages != null && messages.length > 0) {
|
|
2455
|
+
watcher.lastGoodPollAt = this.now();
|
|
2456
|
+
watcher.hadUsablePoll = true;
|
|
2457
|
+
} else {
|
|
2458
|
+
const emptyButReachable = messages != null;
|
|
2459
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2460
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2461
|
+
continue;
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2080
2465
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2081
|
-
await this.serviceInFlightMessage(
|
|
2466
|
+
await this.serviceInFlightMessage(
|
|
2467
|
+
sessionId,
|
|
2468
|
+
watcher,
|
|
2469
|
+
inFlight,
|
|
2470
|
+
messages,
|
|
2471
|
+
openQuestions,
|
|
2472
|
+
openPermissions,
|
|
2473
|
+
questionsPolledOk,
|
|
2474
|
+
permissionsPolledOk
|
|
2475
|
+
);
|
|
2082
2476
|
}
|
|
2083
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
2084
2477
|
}
|
|
2085
2478
|
} catch (err) {
|
|
2086
2479
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2102,15 +2495,40 @@ var ChannelDriver = class {
|
|
|
2102
2495
|
});
|
|
2103
2496
|
}
|
|
2104
2497
|
}
|
|
2498
|
+
/**
|
|
2499
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2500
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2501
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2502
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2503
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2504
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2505
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2506
|
+
* or past now, so a still-ample window is left untouched.
|
|
2507
|
+
*/
|
|
2508
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2509
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2510
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2511
|
+
if (this.now() >= inFlight.deadline) {
|
|
2512
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2105
2515
|
/**
|
|
2106
2516
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2107
2517
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2108
2518
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2109
2519
|
* in-flight set on completion or timeout.
|
|
2110
2520
|
*/
|
|
2111
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2521
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
2112
2522
|
const conv = watcher.conv;
|
|
2113
2523
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2524
|
+
const id = inFlight.evidentMessageId;
|
|
2525
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2526
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2527
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2528
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2529
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2530
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2531
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2114
2532
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2115
2533
|
let claimed;
|
|
2116
2534
|
try {
|
|
@@ -2141,6 +2559,7 @@ var ChannelDriver = class {
|
|
|
2141
2559
|
}
|
|
2142
2560
|
}
|
|
2143
2561
|
if (state === "done") {
|
|
2562
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2144
2563
|
if (!inFlight.done) {
|
|
2145
2564
|
this.log({
|
|
2146
2565
|
level: "info",
|
|
@@ -2191,6 +2610,7 @@ var ChannelDriver = class {
|
|
|
2191
2610
|
return;
|
|
2192
2611
|
}
|
|
2193
2612
|
if (state === "failed") {
|
|
2613
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2194
2614
|
if (!inFlight.done) {
|
|
2195
2615
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2196
2616
|
this.log({
|
|
@@ -2244,7 +2664,51 @@ var ChannelDriver = class {
|
|
|
2244
2664
|
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2245
2665
|
});
|
|
2246
2666
|
}
|
|
2247
|
-
|
|
2667
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2668
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2669
|
+
this.log({
|
|
2670
|
+
level: "error",
|
|
2671
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2672
|
+
conversation_id: conv.id,
|
|
2673
|
+
message_id: inFlight.evidentMessageId
|
|
2674
|
+
});
|
|
2675
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2676
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2677
|
+
});
|
|
2678
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2679
|
+
return;
|
|
2680
|
+
}
|
|
2681
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2682
|
+
inFlight.aliveInFlight = true;
|
|
2683
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2684
|
+
inFlight.aliveInFlight = false;
|
|
2685
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
if (awaitingHuman) {
|
|
2689
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2690
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2691
|
+
inFlight.awaitingHumanLatched = true;
|
|
2692
|
+
}
|
|
2693
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2694
|
+
inFlight.pausedInFlight = true;
|
|
2695
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2696
|
+
inFlight.pausedInFlight = false;
|
|
2697
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2700
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2701
|
+
inFlight.awaitingHumanLatched = false;
|
|
2702
|
+
inFlight.pausedOnQuestion = false;
|
|
2703
|
+
inFlight.pausedOnPermission = false;
|
|
2704
|
+
inFlight.pausedClearConfirmed = false;
|
|
2705
|
+
}
|
|
2706
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2707
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2708
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2709
|
+
);
|
|
2710
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2711
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2248
2712
|
this.log({
|
|
2249
2713
|
level: "info",
|
|
2250
2714
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
@@ -2276,12 +2740,17 @@ var ChannelDriver = class {
|
|
|
2276
2740
|
*/
|
|
2277
2741
|
async readoptProcessing() {
|
|
2278
2742
|
const rows = await this.getProcessingMessages();
|
|
2279
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2743
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2280
2744
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2281
|
-
for (const id of [
|
|
2745
|
+
for (const id of [
|
|
2746
|
+
...this.dontRedispatch,
|
|
2747
|
+
...this.doneUndeliverable,
|
|
2748
|
+
...this.readoptPollUnresolvedSignalled
|
|
2749
|
+
]) {
|
|
2282
2750
|
if (!stillProcessing.has(id)) {
|
|
2283
2751
|
const cleared = this.dontRedispatch.delete(id);
|
|
2284
2752
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2753
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2285
2754
|
if (cleared || clearedUndeliverable) {
|
|
2286
2755
|
this.log({
|
|
2287
2756
|
level: "info",
|
|
@@ -2335,8 +2804,10 @@ var ChannelDriver = class {
|
|
|
2335
2804
|
});
|
|
2336
2805
|
continue;
|
|
2337
2806
|
}
|
|
2807
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2808
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2338
2809
|
for (const row of sessionRows) {
|
|
2339
|
-
await this.readoptOne(sessionId, row, messages);
|
|
2810
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2340
2811
|
}
|
|
2341
2812
|
}
|
|
2342
2813
|
}
|
|
@@ -2358,7 +2829,7 @@ var ChannelDriver = class {
|
|
|
2358
2829
|
*
|
|
2359
2830
|
* Only `ChannelAuthError` propagates.
|
|
2360
2831
|
*/
|
|
2361
|
-
async readoptOne(sessionId, row, messages) {
|
|
2832
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2362
2833
|
if (this.isTracked(sessionId, row.id)) {
|
|
2363
2834
|
this.log({
|
|
2364
2835
|
level: "info",
|
|
@@ -2398,6 +2869,7 @@ var ChannelDriver = class {
|
|
|
2398
2869
|
conversation_id: row.conversation_id,
|
|
2399
2870
|
message_id: row.id
|
|
2400
2871
|
});
|
|
2872
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2401
2873
|
return;
|
|
2402
2874
|
}
|
|
2403
2875
|
this.log({
|
|
@@ -2409,6 +2881,7 @@ var ChannelDriver = class {
|
|
|
2409
2881
|
return;
|
|
2410
2882
|
}
|
|
2411
2883
|
this.dontRedispatch.delete(row.id);
|
|
2884
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2412
2885
|
return;
|
|
2413
2886
|
}
|
|
2414
2887
|
if (state === "failed") {
|
|
@@ -2431,6 +2904,7 @@ var ChannelDriver = class {
|
|
|
2431
2904
|
conversation_id: row.conversation_id,
|
|
2432
2905
|
message_id: row.id
|
|
2433
2906
|
});
|
|
2907
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2434
2908
|
return;
|
|
2435
2909
|
}
|
|
2436
2910
|
this.log({
|
|
@@ -2442,6 +2916,7 @@ var ChannelDriver = class {
|
|
|
2442
2916
|
return;
|
|
2443
2917
|
}
|
|
2444
2918
|
this.dontRedispatch.delete(row.id);
|
|
2919
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2445
2920
|
return;
|
|
2446
2921
|
}
|
|
2447
2922
|
if (this.dontRedispatch.has(row.id)) {
|
|
@@ -2453,6 +2928,71 @@ var ChannelDriver = class {
|
|
|
2453
2928
|
});
|
|
2454
2929
|
return;
|
|
2455
2930
|
}
|
|
2931
|
+
let statusReadableOngoing = null;
|
|
2932
|
+
if (state === "running" && ocId) {
|
|
2933
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2934
|
+
const shape = this.replyCompletionShape(reply);
|
|
2935
|
+
const ongoing = sessionOngoing;
|
|
2936
|
+
statusReadableOngoing = ongoing;
|
|
2937
|
+
if (ongoing === false) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2945
|
+
return;
|
|
2946
|
+
}
|
|
2947
|
+
if (ongoing === true) {
|
|
2948
|
+
this.log({
|
|
2949
|
+
level: "info",
|
|
2950
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2951
|
+
conversation_id: row.conversation_id,
|
|
2952
|
+
message_id: row.id
|
|
2953
|
+
});
|
|
2954
|
+
} else {
|
|
2955
|
+
if (shape === "b1") {
|
|
2956
|
+
this.log({
|
|
2957
|
+
level: "info",
|
|
2958
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2959
|
+
conversation_id: row.conversation_id,
|
|
2960
|
+
message_id: row.id
|
|
2961
|
+
});
|
|
2962
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2963
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2964
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2965
|
+
}
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
this.log({
|
|
2969
|
+
level: "info",
|
|
2970
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2971
|
+
conversation_id: row.conversation_id,
|
|
2972
|
+
message_id: row.id
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2977
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2978
|
+
if (descendantAlive === true) {
|
|
2979
|
+
this.log({
|
|
2980
|
+
level: "info",
|
|
2981
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2982
|
+
conversation_id: row.conversation_id,
|
|
2983
|
+
message_id: row.id
|
|
2984
|
+
});
|
|
2985
|
+
} else {
|
|
2986
|
+
this.log({
|
|
2987
|
+
level: "info",
|
|
2988
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2989
|
+
conversation_id: row.conversation_id,
|
|
2990
|
+
message_id: row.id
|
|
2991
|
+
});
|
|
2992
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2456
2996
|
if ((state === "running" || state === "queued") && ocId) {
|
|
2457
2997
|
const conv = this.convForRow(sessionId, row);
|
|
2458
2998
|
const message = this.queuedMessageForRow(row);
|
|
@@ -2466,6 +3006,7 @@ var ChannelDriver = class {
|
|
|
2466
3006
|
conversation_id: row.conversation_id,
|
|
2467
3007
|
message_id: row.id
|
|
2468
3008
|
});
|
|
3009
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
2469
3010
|
return;
|
|
2470
3011
|
}
|
|
2471
3012
|
await this.forceReadoptRun(sessionId, row);
|
|
@@ -2492,6 +3033,15 @@ var ChannelDriver = class {
|
|
|
2492
3033
|
* `processed_at` (Invariant 1).
|
|
2493
3034
|
*/
|
|
2494
3035
|
async forceReadoptRun(sessionId, row) {
|
|
3036
|
+
if (this.stopped) {
|
|
3037
|
+
this.log({
|
|
3038
|
+
level: "info",
|
|
3039
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3040
|
+
conversation_id: row.conversation_id,
|
|
3041
|
+
message_id: row.id
|
|
3042
|
+
});
|
|
3043
|
+
return;
|
|
3044
|
+
}
|
|
2495
3045
|
if (this.awaitingReadopt.has(row.id)) {
|
|
2496
3046
|
this.log({
|
|
2497
3047
|
level: "info",
|
|
@@ -2509,6 +3059,7 @@ var ChannelDriver = class {
|
|
|
2509
3059
|
conversation_id: row.conversation_id,
|
|
2510
3060
|
message_id: row.id
|
|
2511
3061
|
});
|
|
3062
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
2512
3063
|
return;
|
|
2513
3064
|
}
|
|
2514
3065
|
const options = {
|
|
@@ -2537,6 +3088,7 @@ var ChannelDriver = class {
|
|
|
2537
3088
|
conversation_id: row.conversation_id,
|
|
2538
3089
|
message_id: row.id
|
|
2539
3090
|
});
|
|
3091
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2540
3092
|
return;
|
|
2541
3093
|
}
|
|
2542
3094
|
if (ocId === null) {
|
|
@@ -2547,6 +3099,7 @@ var ChannelDriver = class {
|
|
|
2547
3099
|
conversation_id: row.conversation_id,
|
|
2548
3100
|
message_id: row.id
|
|
2549
3101
|
});
|
|
3102
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2550
3103
|
return;
|
|
2551
3104
|
}
|
|
2552
3105
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -2556,6 +3109,7 @@ var ChannelDriver = class {
|
|
|
2556
3109
|
this.readopted.add(row.id);
|
|
2557
3110
|
this.awaitingReadopt.delete(row.id);
|
|
2558
3111
|
this.ensureWatcherRunning(sessionId);
|
|
3112
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
2559
3113
|
}
|
|
2560
3114
|
/**
|
|
2561
3115
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -2647,21 +3201,41 @@ var ChannelDriver = class {
|
|
|
2647
3201
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2648
3202
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2649
3203
|
* oldest running message.
|
|
3204
|
+
*
|
|
3205
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3206
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3207
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3208
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3209
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3210
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3211
|
+
* even after it was already surfaced to the channel.
|
|
2650
3212
|
*/
|
|
2651
3213
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3214
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3215
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3216
|
+
let questionsPolledOk = true;
|
|
3217
|
+
let permissionsPolledOk = true;
|
|
2652
3218
|
let questions = [];
|
|
2653
3219
|
try {
|
|
2654
3220
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2655
3221
|
if (res.ok) {
|
|
2656
3222
|
const body = await res.json();
|
|
2657
|
-
|
|
3223
|
+
if (Array.isArray(body)) {
|
|
3224
|
+
questions = body;
|
|
3225
|
+
} else {
|
|
3226
|
+
questionsPolledOk = false;
|
|
3227
|
+
}
|
|
3228
|
+
} else {
|
|
3229
|
+
questionsPolledOk = false;
|
|
2658
3230
|
}
|
|
2659
3231
|
} catch {
|
|
3232
|
+
questionsPolledOk = false;
|
|
2660
3233
|
}
|
|
2661
3234
|
for (const q of questions) {
|
|
2662
|
-
if (q.sessionID
|
|
2663
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3235
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2664
3236
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3237
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3238
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2665
3239
|
const reported = await this.reportInteraction(
|
|
2666
3240
|
watcher.conv.id,
|
|
2667
3241
|
"question",
|
|
@@ -2675,14 +3249,22 @@ var ChannelDriver = class {
|
|
|
2675
3249
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2676
3250
|
if (res.ok) {
|
|
2677
3251
|
const body = await res.json();
|
|
2678
|
-
|
|
3252
|
+
if (Array.isArray(body)) {
|
|
3253
|
+
permissions = body;
|
|
3254
|
+
} else {
|
|
3255
|
+
permissionsPolledOk = false;
|
|
3256
|
+
}
|
|
3257
|
+
} else {
|
|
3258
|
+
permissionsPolledOk = false;
|
|
2679
3259
|
}
|
|
2680
3260
|
} catch {
|
|
3261
|
+
permissionsPolledOk = false;
|
|
2681
3262
|
}
|
|
2682
3263
|
for (const p of permissions) {
|
|
2683
|
-
if (p.sessionID
|
|
2684
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3264
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2685
3265
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3266
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3267
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2686
3268
|
const reported = await this.reportInteraction(
|
|
2687
3269
|
watcher.conv.id,
|
|
2688
3270
|
"permission",
|
|
@@ -2691,6 +3273,128 @@ var ChannelDriver = class {
|
|
|
2691
3273
|
);
|
|
2692
3274
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2693
3275
|
}
|
|
3276
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3277
|
+
}
|
|
3278
|
+
/**
|
|
3279
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3280
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3281
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3282
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3283
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3284
|
+
*
|
|
3285
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3286
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3287
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3288
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3289
|
+
*/
|
|
3290
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3291
|
+
let current = sessionId;
|
|
3292
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3293
|
+
if (current === rootSessionId) return true;
|
|
3294
|
+
const parent = await this.resolveSessionParent(current);
|
|
3295
|
+
if (parent === null || parent === void 0) return false;
|
|
3296
|
+
current = parent;
|
|
3297
|
+
}
|
|
3298
|
+
return false;
|
|
3299
|
+
}
|
|
3300
|
+
/**
|
|
3301
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3302
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3303
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3304
|
+
* caching a wrong answer — the next tick retries).
|
|
3305
|
+
*/
|
|
3306
|
+
async resolveSessionParent(sessionId) {
|
|
3307
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3308
|
+
if (cached !== void 0) return cached;
|
|
3309
|
+
let parent = void 0;
|
|
3310
|
+
try {
|
|
3311
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3312
|
+
if (res.ok) {
|
|
3313
|
+
const body = await res.json();
|
|
3314
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3315
|
+
}
|
|
3316
|
+
} catch {
|
|
3317
|
+
parent = void 0;
|
|
3318
|
+
}
|
|
3319
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3320
|
+
return parent;
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3324
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3325
|
+
*
|
|
3326
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3327
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3328
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3329
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3330
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3331
|
+
* provably in flight at the exact moment of recovery.
|
|
3332
|
+
*
|
|
3333
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3334
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3335
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3336
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3337
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3338
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3339
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3340
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3341
|
+
*
|
|
3342
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3343
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3344
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3345
|
+
* case), OR no descendant is found at all.
|
|
3346
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3347
|
+
*
|
|
3348
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3349
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3350
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3351
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3352
|
+
* true/false/null faithfully.
|
|
3353
|
+
*
|
|
3354
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3355
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3356
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3357
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3358
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3359
|
+
* `SessionStatus` only.
|
|
3360
|
+
*/
|
|
3361
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3362
|
+
const sessions = await listSessions(this.port);
|
|
3363
|
+
if (!sessions) {
|
|
3364
|
+
this.log({
|
|
3365
|
+
level: "error",
|
|
3366
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3367
|
+
});
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
for (const candidate of sessions) {
|
|
3371
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3372
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3373
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3374
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3375
|
+
return true;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
return false;
|
|
3379
|
+
}
|
|
3380
|
+
/**
|
|
3381
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3382
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3383
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3384
|
+
* the aborted-in-flight production bug after a restart.
|
|
3385
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3386
|
+
* (the sub-agent preamble — #253's shape).
|
|
3387
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3388
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3389
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3390
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3391
|
+
*/
|
|
3392
|
+
replyCompletionShape(reply) {
|
|
3393
|
+
if (!reply) return "other";
|
|
3394
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3395
|
+
if (completed == null) return "b1";
|
|
3396
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3397
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2694
3398
|
}
|
|
2695
3399
|
/**
|
|
2696
3400
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -2923,6 +3627,12 @@ var ChannelDriver = class {
|
|
|
2923
3627
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2924
3628
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2925
3629
|
* context (no silent catch, per development-workflow).
|
|
3630
|
+
*
|
|
3631
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3632
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3633
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3634
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3635
|
+
* leaves liveness").
|
|
2926
3636
|
*/
|
|
2927
3637
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
2928
3638
|
try {
|
|
@@ -2941,7 +3651,9 @@ var ChannelDriver = class {
|
|
|
2941
3651
|
conversation_id: conversationId,
|
|
2942
3652
|
message_id: messageId
|
|
2943
3653
|
});
|
|
3654
|
+
return false;
|
|
2944
3655
|
}
|
|
3656
|
+
return true;
|
|
2945
3657
|
} catch (err) {
|
|
2946
3658
|
this.log({
|
|
2947
3659
|
level: "error",
|
|
@@ -2949,6 +3661,7 @@ var ChannelDriver = class {
|
|
|
2949
3661
|
conversation_id: conversationId,
|
|
2950
3662
|
message_id: messageId
|
|
2951
3663
|
});
|
|
3664
|
+
return false;
|
|
2952
3665
|
}
|
|
2953
3666
|
}
|
|
2954
3667
|
async persistSession(conversationId, sessionId) {
|
|
@@ -3221,6 +3934,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3221
3934
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
3222
3935
|
}
|
|
3223
3936
|
}
|
|
3937
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3938
|
+
const apiUrl = getApiUrlConfig();
|
|
3939
|
+
try {
|
|
3940
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3941
|
+
method: "POST",
|
|
3942
|
+
headers: { Authorization: authHeader }
|
|
3943
|
+
});
|
|
3944
|
+
if (!response.ok) {
|
|
3945
|
+
const serverMessage = await readErrorMessage(response);
|
|
3946
|
+
return {
|
|
3947
|
+
ok: false,
|
|
3948
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3949
|
+
};
|
|
3950
|
+
}
|
|
3951
|
+
return { ok: true };
|
|
3952
|
+
} catch (error2) {
|
|
3953
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3224
3956
|
async function getAgentInfo(agentId, authHeader) {
|
|
3225
3957
|
const apiUrl = getApiUrlConfig();
|
|
3226
3958
|
try {
|
|
@@ -3267,6 +3999,7 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3267
3999
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
3268
4000
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
3269
4001
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4002
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3270
4003
|
function log2(state, message, isError = false) {
|
|
3271
4004
|
if (state.json) {
|
|
3272
4005
|
console.log(
|
|
@@ -3421,7 +4154,7 @@ async function driveChannels(state, driver) {
|
|
|
3421
4154
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
3422
4155
|
if (state.interactive) displayStatus(state);
|
|
3423
4156
|
}
|
|
3424
|
-
await new Promise((
|
|
4157
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
3425
4158
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
3426
4159
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
3427
4160
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -3432,8 +4165,122 @@ async function driveChannels(state, driver) {
|
|
|
3432
4165
|
}
|
|
3433
4166
|
}
|
|
3434
4167
|
}
|
|
3435
|
-
|
|
4168
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4169
|
+
async function runSweep(state, driver, config2) {
|
|
4170
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4171
|
+
try {
|
|
4172
|
+
const sessions = await listSessions(state.port);
|
|
4173
|
+
if (sessions === null) {
|
|
4174
|
+
logActivity(state, {
|
|
4175
|
+
type: "info",
|
|
4176
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4177
|
+
});
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
const toDelete = selectSessionsToDelete(
|
|
4181
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4182
|
+
{
|
|
4183
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4184
|
+
maxCount: config2.maxCount,
|
|
4185
|
+
nowMs: Date.now(),
|
|
4186
|
+
protectedIds: driver.protectedSessionIds()
|
|
4187
|
+
}
|
|
4188
|
+
);
|
|
4189
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4190
|
+
let deleted = 0;
|
|
4191
|
+
let failed = 0;
|
|
4192
|
+
let skippedNewlyActive = 0;
|
|
4193
|
+
for (const id of toDelete) {
|
|
4194
|
+
if (protectedNow.has(id)) {
|
|
4195
|
+
skippedNewlyActive++;
|
|
4196
|
+
logActivity(state, {
|
|
4197
|
+
type: "info",
|
|
4198
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4199
|
+
});
|
|
4200
|
+
continue;
|
|
4201
|
+
}
|
|
4202
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4203
|
+
else failed++;
|
|
4204
|
+
}
|
|
4205
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4206
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4207
|
+
logActivity(state, {
|
|
4208
|
+
type: "info",
|
|
4209
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4210
|
+
});
|
|
4211
|
+
} catch (error2) {
|
|
4212
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4213
|
+
logActivity(state, {
|
|
4214
|
+
type: "error",
|
|
4215
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4216
|
+
});
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4220
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4221
|
+
{
|
|
4222
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4223
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4224
|
+
interval: options.sessionCleanupInterval
|
|
4225
|
+
},
|
|
4226
|
+
process.env
|
|
4227
|
+
);
|
|
4228
|
+
for (const warning2 of config2.warnings) {
|
|
4229
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4230
|
+
}
|
|
4231
|
+
if (!config2.enabled) return;
|
|
4232
|
+
logActivity(state, {
|
|
4233
|
+
type: "info",
|
|
4234
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4235
|
+
});
|
|
4236
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4237
|
+
const firstSweep = setTimeout(
|
|
4238
|
+
() => void runSweep(state, driver, config2),
|
|
4239
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4240
|
+
);
|
|
4241
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4242
|
+
}
|
|
4243
|
+
async function notifyOffline(state) {
|
|
4244
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4245
|
+
if (!state.connected) {
|
|
4246
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4247
|
+
return;
|
|
4248
|
+
}
|
|
4249
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4250
|
+
if (result.ok) {
|
|
4251
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4252
|
+
} else {
|
|
4253
|
+
logActivity(state, {
|
|
4254
|
+
type: "error",
|
|
4255
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4256
|
+
});
|
|
4257
|
+
if (state.interactive) displayStatus(state);
|
|
4258
|
+
}
|
|
4259
|
+
}
|
|
4260
|
+
async function cleanup(state, opts = {}) {
|
|
3436
4261
|
state.running = false;
|
|
4262
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4263
|
+
clearInterval(timer);
|
|
4264
|
+
clearTimeout(timer);
|
|
4265
|
+
}
|
|
4266
|
+
state.sessionCleanupTimers = [];
|
|
4267
|
+
if (opts.graceful && state.channelDriver) {
|
|
4268
|
+
state.channelDriver.stop();
|
|
4269
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4270
|
+
if (state.interactive) {
|
|
4271
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4272
|
+
displayStatus(state);
|
|
4273
|
+
}
|
|
4274
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4275
|
+
if (!settled) {
|
|
4276
|
+
logActivity(state, {
|
|
4277
|
+
type: "info",
|
|
4278
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4279
|
+
});
|
|
4280
|
+
if (state.interactive) displayStatus(state);
|
|
4281
|
+
}
|
|
4282
|
+
}
|
|
4283
|
+
await notifyOffline(state);
|
|
3437
4284
|
if (state.connection) {
|
|
3438
4285
|
state.connection.close();
|
|
3439
4286
|
state.connection = null;
|
|
@@ -3464,10 +4311,13 @@ async function run(options) {
|
|
|
3464
4311
|
opencodeVersion: null,
|
|
3465
4312
|
opencodeProcess: null,
|
|
3466
4313
|
connection: null,
|
|
4314
|
+
channelDriver: null,
|
|
3467
4315
|
running: true,
|
|
4316
|
+
shuttingDown: false,
|
|
3468
4317
|
activityLog: [],
|
|
3469
4318
|
messageCount: 0,
|
|
3470
4319
|
lastProxiedActivityAt: null,
|
|
4320
|
+
sessionCleanupTimers: [],
|
|
3471
4321
|
authHeader: ""
|
|
3472
4322
|
};
|
|
3473
4323
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -3478,13 +4328,15 @@ async function run(options) {
|
|
|
3478
4328
|
);
|
|
3479
4329
|
}
|
|
3480
4330
|
const handleSignal = async () => {
|
|
4331
|
+
if (state.shuttingDown) return;
|
|
4332
|
+
state.shuttingDown = true;
|
|
3481
4333
|
if (state.interactive) {
|
|
3482
4334
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
3483
4335
|
displayStatus(state);
|
|
3484
4336
|
} else {
|
|
3485
4337
|
log2(state, "Shutting down...");
|
|
3486
4338
|
}
|
|
3487
|
-
await cleanup(state);
|
|
4339
|
+
await cleanup(state, { graceful: true });
|
|
3488
4340
|
await shutdownTelemetry();
|
|
3489
4341
|
process.exit(0);
|
|
3490
4342
|
};
|
|
@@ -3611,6 +4463,7 @@ async function run(options) {
|
|
|
3611
4463
|
error: entry.level === "error" ? entry.message : void 0
|
|
3612
4464
|
})
|
|
3613
4465
|
});
|
|
4466
|
+
state.channelDriver = channelDriver;
|
|
3614
4467
|
const connection = new RunnerConnection({
|
|
3615
4468
|
agentId: state.agentId,
|
|
3616
4469
|
getAuthHeader: () => state.authHeader,
|
|
@@ -3707,10 +4560,12 @@ async function run(options) {
|
|
|
3707
4560
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3708
4561
|
throw error2;
|
|
3709
4562
|
}
|
|
4563
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3710
4564
|
if (!interactive || state.json) {
|
|
3711
4565
|
log2(state, "Driving channel messages...");
|
|
3712
4566
|
}
|
|
3713
4567
|
await driveChannels(state, channelDriver);
|
|
4568
|
+
if (state.shuttingDown) return;
|
|
3714
4569
|
await cleanup(state);
|
|
3715
4570
|
if (state.json) {
|
|
3716
4571
|
console.log(
|
|
@@ -3725,6 +4580,7 @@ async function run(options) {
|
|
|
3725
4580
|
await shutdownTelemetry();
|
|
3726
4581
|
process.exit(0);
|
|
3727
4582
|
} catch (error2) {
|
|
4583
|
+
if (state.shuttingDown) return;
|
|
3728
4584
|
await cleanup(state);
|
|
3729
4585
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3730
4586
|
if (state.json) {
|
|
@@ -3759,7 +4615,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3759
4615
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3760
4616
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3761
4617
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3762
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4618
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4619
|
+
"--session-cleanup-max-age <duration>",
|
|
4620
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4621
|
+
).option(
|
|
4622
|
+
"--session-cleanup-max-count <n>",
|
|
4623
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4624
|
+
).option(
|
|
4625
|
+
"--session-cleanup-interval <duration>",
|
|
4626
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4627
|
+
).action(
|
|
3763
4628
|
(options) => {
|
|
3764
4629
|
run({
|
|
3765
4630
|
agent: options.agent,
|
|
@@ -3767,7 +4632,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3767
4632
|
verbose: options.verbose,
|
|
3768
4633
|
conversation: options.conversation,
|
|
3769
4634
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3770
|
-
json: options.json
|
|
4635
|
+
json: options.json,
|
|
4636
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4637
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4638
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4639
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3771
4640
|
});
|
|
3772
4641
|
}
|
|
3773
4642
|
);
|