@evident-ai/cli 3.0.1-dev.6a683e7 → 3.0.1-dev.6bbae3a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1255 -151
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve2();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -471,12 +471,6 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
-
// ../../packages/types/src/opencode/index.ts
|
|
475
|
-
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
-
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
-
return `msg_${sanitized}`;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
474
|
// ../../packages/types/src/telemetry/index.ts
|
|
481
475
|
var TelemetryEventTypes = {
|
|
482
476
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -515,7 +509,10 @@ function stripQuery(url) {
|
|
|
515
509
|
}
|
|
516
510
|
|
|
517
511
|
// src/lib/telemetry.ts
|
|
518
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
519
516
|
var eventBuffer = [];
|
|
520
517
|
var flushTimeout = null;
|
|
521
518
|
var isShuttingDown = false;
|
|
@@ -709,13 +706,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
709
706
|
if (health.healthy) {
|
|
710
707
|
return health;
|
|
711
708
|
}
|
|
712
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
713
710
|
}
|
|
714
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
715
712
|
}
|
|
716
713
|
|
|
717
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
718
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
719
716
|
function isQueueValidatedVersion(version2) {
|
|
720
717
|
if (!version2) return false;
|
|
721
718
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1039,7 +1036,11 @@ function roleOf(m) {
|
|
|
1039
1036
|
}
|
|
1040
1037
|
function completedOf(m) {
|
|
1041
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1042
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1043
1044
|
}
|
|
1044
1045
|
function idOf(m) {
|
|
1045
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1059,10 +1060,102 @@ function finishOf(m) {
|
|
|
1059
1060
|
const infoFinish = m.info?.finish;
|
|
1060
1061
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1061
1062
|
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1062
1067
|
function isAssistantInFlight(m) {
|
|
1063
1068
|
if (completedOf(m) == null) return true;
|
|
1064
1069
|
return finishOf(m) === "tool-calls";
|
|
1065
1070
|
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1066
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1067
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1068
1161
|
if (directory && directory.trim()) {
|
|
@@ -1080,9 +1173,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1080
1173
|
const data = await response.json();
|
|
1081
1174
|
return data.id;
|
|
1082
1175
|
}
|
|
1083
|
-
|
|
1176
|
+
function messageText(m) {
|
|
1177
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
+
}
|
|
1180
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
+
const knownUserIds = new Set(
|
|
1183
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
+
);
|
|
1084
1185
|
const body = {
|
|
1085
|
-
messageID: messageId,
|
|
1086
1186
|
parts: [{ type: "text", text: content }]
|
|
1087
1187
|
};
|
|
1088
1188
|
if (options?.agent) {
|
|
@@ -1106,6 +1206,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1106
1206
|
const text = await res.text().catch(() => "");
|
|
1107
1207
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1108
1208
|
}
|
|
1209
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
+
if (after) {
|
|
1214
|
+
let best = null;
|
|
1215
|
+
for (const m of after) {
|
|
1216
|
+
if (roleOf(m) !== "user") continue;
|
|
1217
|
+
const id = idOf(m);
|
|
1218
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
+
if (messageText(m) !== content) continue;
|
|
1220
|
+
const created = createdOf(m) ?? 0;
|
|
1221
|
+
if (best === null || created > best.created) {
|
|
1222
|
+
best = { id, created };
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (best) return best.id;
|
|
1226
|
+
}
|
|
1227
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1109
1232
|
}
|
|
1110
1233
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1111
1234
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1122,19 +1245,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1122
1245
|
}
|
|
1123
1246
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1124
1247
|
if (!messages || messages.length === 0) return null;
|
|
1248
|
+
let lastCorrelated = null;
|
|
1249
|
+
let lastNonErrored = null;
|
|
1125
1250
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1126
1251
|
const m = messages[i];
|
|
1127
|
-
if (roleOf(m)
|
|
1252
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1253
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1254
|
+
if (errorOf(m) == null) {
|
|
1255
|
+
lastNonErrored = m;
|
|
1256
|
+
break;
|
|
1257
|
+
}
|
|
1128
1258
|
}
|
|
1259
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1129
1260
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1130
1261
|
if (userIndex === -1) return null;
|
|
1131
1262
|
let last = null;
|
|
1263
|
+
let lastOk = null;
|
|
1132
1264
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1133
1265
|
const role = roleOf(messages[i]);
|
|
1134
1266
|
if (role === "user") break;
|
|
1135
|
-
if (role === "assistant")
|
|
1267
|
+
if (role === "assistant") {
|
|
1268
|
+
last = messages[i];
|
|
1269
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1270
|
+
}
|
|
1136
1271
|
}
|
|
1137
|
-
return last;
|
|
1272
|
+
return lastOk ?? last;
|
|
1138
1273
|
}
|
|
1139
1274
|
function messageRunState(messages, userMessageId) {
|
|
1140
1275
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1144,7 +1279,26 @@ function messageRunState(messages, userMessageId) {
|
|
|
1144
1279
|
if (!reply) return "unknown";
|
|
1145
1280
|
}
|
|
1146
1281
|
if (!reply) return "queued";
|
|
1147
|
-
|
|
1282
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1283
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1284
|
+
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1290
|
+
function messageError(messages, userMessageId) {
|
|
1291
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1292
|
+
const error2 = errorOf(reply);
|
|
1293
|
+
if (error2 == null) return null;
|
|
1294
|
+
if (typeof error2 === "string") return error2;
|
|
1295
|
+
if (typeof error2 === "object") {
|
|
1296
|
+
const e = error2;
|
|
1297
|
+
const dataMessage = e.data?.message;
|
|
1298
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1299
|
+
if (typeof e.message === "string") return e.message;
|
|
1300
|
+
}
|
|
1301
|
+
return "The agent run failed.";
|
|
1148
1302
|
}
|
|
1149
1303
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1150
1304
|
if (!messages || messages.length === 0) return false;
|
|
@@ -1152,8 +1306,109 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1152
1306
|
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1153
1307
|
);
|
|
1154
1308
|
}
|
|
1155
|
-
|
|
1156
|
-
|
|
1309
|
+
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
return toDelete;
|
|
1356
|
+
}
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1360
|
+
}
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1369
|
+
}
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
+
}
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1157
1412
|
}
|
|
1158
1413
|
|
|
1159
1414
|
// src/lib/tunnel/connection.ts
|
|
@@ -1232,24 +1487,26 @@ var StreamForwarder = class {
|
|
|
1232
1487
|
this.send({ type: "res_end", sid });
|
|
1233
1488
|
return;
|
|
1234
1489
|
}
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1241
1498
|
const ac = new AbortController();
|
|
1242
1499
|
let bodyPromise;
|
|
1243
1500
|
let pushBody;
|
|
1244
1501
|
let endBody;
|
|
1245
1502
|
if (has_body) {
|
|
1246
1503
|
const chunks = [];
|
|
1247
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1248
1505
|
pushBody = (buf) => {
|
|
1249
1506
|
chunks.push(buf);
|
|
1250
1507
|
};
|
|
1251
1508
|
endBody = () => {
|
|
1252
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1253
1510
|
};
|
|
1254
1511
|
});
|
|
1255
1512
|
}
|
|
@@ -1284,12 +1541,14 @@ var StreamForwarder = class {
|
|
|
1284
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1285
1542
|
});
|
|
1286
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1293
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1294
1553
|
try {
|
|
1295
1554
|
if (upstream.body) {
|
|
@@ -1365,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1365
1624
|
} = options;
|
|
1366
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1367
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1368
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1369
1628
|
const ws = new WebSocket2(url, {
|
|
1370
1629
|
headers: {
|
|
1371
1630
|
Authorization: authHeader
|
|
@@ -1430,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1430
1689
|
clearTimeout(connectionTimeout);
|
|
1431
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1432
1691
|
onConnected?.(connectedAgentId);
|
|
1433
|
-
|
|
1692
|
+
resolve2({
|
|
1434
1693
|
ws,
|
|
1435
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1436
1695
|
});
|
|
@@ -1559,8 +1818,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1559
1818
|
};
|
|
1560
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1561
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1562
|
-
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1563
1821
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1564
1825
|
var ChannelAuthError = class extends Error {
|
|
1565
1826
|
constructor(message) {
|
|
1566
1827
|
super(message);
|
|
@@ -1595,11 +1856,18 @@ var ChannelDriver = class {
|
|
|
1595
1856
|
sleep;
|
|
1596
1857
|
pausedPollIntervalMs;
|
|
1597
1858
|
pausedMaxWaitMs;
|
|
1598
|
-
dispatchConfirmMs;
|
|
1599
1859
|
stuckQueuedMs;
|
|
1600
1860
|
now;
|
|
1601
1861
|
/** Cache of conversationId → opencode sessionId. */
|
|
1602
1862
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
+
/**
|
|
1864
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
+
*/
|
|
1870
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1603
1871
|
/**
|
|
1604
1872
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1605
1873
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1650,6 +1918,29 @@ var ChannelDriver = class {
|
|
|
1650
1918
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1651
1919
|
*/
|
|
1652
1920
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
+
*/
|
|
1929
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1930
|
+
/**
|
|
1931
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1932
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1933
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1934
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1935
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1936
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1937
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1938
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1939
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1940
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1941
|
+
* so the NEXT tick may retry exactly once more).
|
|
1942
|
+
*/
|
|
1943
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1653
1944
|
/**
|
|
1654
1945
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1655
1946
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1657,8 +1948,33 @@ var ChannelDriver = class {
|
|
|
1657
1948
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1658
1949
|
*/
|
|
1659
1950
|
opencodeDirectory = void 0;
|
|
1951
|
+
/**
|
|
1952
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1953
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1954
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1955
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1956
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1957
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1958
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1959
|
+
*/
|
|
1960
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1660
1961
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1661
1962
|
draining = false;
|
|
1963
|
+
/**
|
|
1964
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1965
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1966
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1967
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1968
|
+
*/
|
|
1969
|
+
activeDrain = null;
|
|
1970
|
+
/**
|
|
1971
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1972
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1973
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1974
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1975
|
+
* and stops opencode.
|
|
1976
|
+
*/
|
|
1977
|
+
stopped = false;
|
|
1662
1978
|
constructor(config2) {
|
|
1663
1979
|
this.agentId = config2.agentId;
|
|
1664
1980
|
this.port = config2.port;
|
|
@@ -1672,7 +1988,6 @@ var ChannelDriver = class {
|
|
|
1672
1988
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1673
1989
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1674
1990
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1675
|
-
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1676
1991
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1677
1992
|
this.now = config2.now ?? (() => Date.now());
|
|
1678
1993
|
}
|
|
@@ -1691,8 +2006,21 @@ var ChannelDriver = class {
|
|
|
1691
2006
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1692
2007
|
*/
|
|
1693
2008
|
async drainPending() {
|
|
2009
|
+
if (this.stopped) return 0;
|
|
1694
2010
|
if (this.draining) return 0;
|
|
1695
2011
|
this.draining = true;
|
|
2012
|
+
const run2 = this.runDrain();
|
|
2013
|
+
this.activeDrain = run2.then(
|
|
2014
|
+
() => {
|
|
2015
|
+
this.activeDrain = null;
|
|
2016
|
+
},
|
|
2017
|
+
() => {
|
|
2018
|
+
this.activeDrain = null;
|
|
2019
|
+
}
|
|
2020
|
+
);
|
|
2021
|
+
return run2;
|
|
2022
|
+
}
|
|
2023
|
+
async runDrain() {
|
|
1696
2024
|
let dispatched = 0;
|
|
1697
2025
|
try {
|
|
1698
2026
|
const conversations = await this.getPendingConversations();
|
|
@@ -1704,6 +2032,7 @@ var ChannelDriver = class {
|
|
|
1704
2032
|
});
|
|
1705
2033
|
}
|
|
1706
2034
|
for (const conv of conversations) {
|
|
2035
|
+
if (this.stopped) break;
|
|
1707
2036
|
dispatched += await this.processConversation(conv);
|
|
1708
2037
|
}
|
|
1709
2038
|
await this.readoptProcessing();
|
|
@@ -1724,6 +2053,73 @@ var ChannelDriver = class {
|
|
|
1724
2053
|
}
|
|
1725
2054
|
return false;
|
|
1726
2055
|
}
|
|
2056
|
+
/**
|
|
2057
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2058
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2059
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2060
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2061
|
+
*
|
|
2062
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2063
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2064
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2065
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2066
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2067
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2068
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2069
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2070
|
+
*/
|
|
2071
|
+
protectedSessionIds() {
|
|
2072
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2073
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2074
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2075
|
+
}
|
|
2076
|
+
return ids;
|
|
2077
|
+
}
|
|
2078
|
+
/**
|
|
2079
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2080
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2081
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2082
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2083
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2084
|
+
*/
|
|
2085
|
+
stop() {
|
|
2086
|
+
this.stopped = true;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2090
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2091
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2092
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2093
|
+
*
|
|
2094
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2095
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2096
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2097
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2098
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2099
|
+
* next runner start (ADR-0046).
|
|
2100
|
+
*
|
|
2101
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2102
|
+
* timeout elapsed with work still in flight.
|
|
2103
|
+
*/
|
|
2104
|
+
async waitForInFlight(timeoutMs) {
|
|
2105
|
+
const deadline = this.now() + timeoutMs;
|
|
2106
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2107
|
+
if (this.activeDrain) {
|
|
2108
|
+
let drainSettled = false;
|
|
2109
|
+
void this.activeDrain.then(() => {
|
|
2110
|
+
drainSettled = true;
|
|
2111
|
+
});
|
|
2112
|
+
while (!drainSettled) {
|
|
2113
|
+
if (this.now() >= deadline) return false;
|
|
2114
|
+
await this.sleep(step);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
while (this.hasInFlightWatchers()) {
|
|
2118
|
+
if (this.now() >= deadline) return false;
|
|
2119
|
+
await this.sleep(step);
|
|
2120
|
+
}
|
|
2121
|
+
return true;
|
|
2122
|
+
}
|
|
1727
2123
|
/**
|
|
1728
2124
|
* Await all outstanding per-session watchers (WI-3).
|
|
1729
2125
|
*
|
|
@@ -1760,15 +2156,16 @@ var ChannelDriver = class {
|
|
|
1760
2156
|
let dispatched = 0;
|
|
1761
2157
|
let skippedAlreadyDispatched = 0;
|
|
1762
2158
|
for (const message of messages) {
|
|
2159
|
+
if (this.stopped) break;
|
|
1763
2160
|
if (this.dispatched.has(message.id)) {
|
|
1764
2161
|
skippedAlreadyDispatched += 1;
|
|
1765
2162
|
continue;
|
|
1766
2163
|
}
|
|
1767
|
-
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1768
2164
|
const options = {
|
|
1769
2165
|
agent: message.opencode_agent ?? void 0,
|
|
1770
2166
|
model: message.opencode_model ?? void 0
|
|
1771
2167
|
};
|
|
2168
|
+
let opencodeMessageId;
|
|
1772
2169
|
try {
|
|
1773
2170
|
this.log({
|
|
1774
2171
|
level: "info",
|
|
@@ -1776,10 +2173,23 @@ var ChannelDriver = class {
|
|
|
1776
2173
|
conversation_id: conv.id,
|
|
1777
2174
|
message_id: message.id
|
|
1778
2175
|
});
|
|
1779
|
-
await
|
|
2176
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2177
|
+
sessionId,
|
|
2178
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2179
|
+
);
|
|
1780
2180
|
} catch (err) {
|
|
1781
2181
|
if (err instanceof ChannelAuthError) throw err;
|
|
1782
2182
|
this.dispatched.delete(message.id);
|
|
2183
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2184
|
+
this.sessions.delete(conv.id);
|
|
2185
|
+
this.log({
|
|
2186
|
+
level: "info",
|
|
2187
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2188
|
+
conversation_id: conv.id,
|
|
2189
|
+
message_id: message.id
|
|
2190
|
+
});
|
|
2191
|
+
break;
|
|
2192
|
+
}
|
|
1783
2193
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1784
2194
|
});
|
|
1785
2195
|
this.log({
|
|
@@ -1790,6 +2200,15 @@ var ChannelDriver = class {
|
|
|
1790
2200
|
});
|
|
1791
2201
|
continue;
|
|
1792
2202
|
}
|
|
2203
|
+
if (opencodeMessageId === null) {
|
|
2204
|
+
this.log({
|
|
2205
|
+
level: "error",
|
|
2206
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2207
|
+
conversation_id: conv.id,
|
|
2208
|
+
message_id: message.id
|
|
2209
|
+
});
|
|
2210
|
+
continue;
|
|
2211
|
+
}
|
|
1793
2212
|
this.dispatched.add(message.id);
|
|
1794
2213
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1795
2214
|
dispatched += 1;
|
|
@@ -1806,16 +2225,33 @@ var ChannelDriver = class {
|
|
|
1806
2225
|
return dispatched;
|
|
1807
2226
|
}
|
|
1808
2227
|
async ensureSession(conv) {
|
|
1809
|
-
const
|
|
1810
|
-
if (
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
2228
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2229
|
+
if (bound) {
|
|
2230
|
+
const exists = await sessionExists(this.port, bound);
|
|
2231
|
+
if (exists === false) {
|
|
2232
|
+
this.log({
|
|
2233
|
+
level: "info",
|
|
2234
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2235
|
+
conversation_id: conv.id
|
|
2236
|
+
});
|
|
2237
|
+
this.sessions.delete(conv.id);
|
|
2238
|
+
return this.createAndBindSession(conv.id);
|
|
2239
|
+
}
|
|
2240
|
+
this.sessions.set(conv.id, bound);
|
|
2241
|
+
return bound;
|
|
1814
2242
|
}
|
|
2243
|
+
return this.createAndBindSession(conv.id);
|
|
2244
|
+
}
|
|
2245
|
+
/**
|
|
2246
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2247
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2248
|
+
* self-heal recreate path in `ensureSession`.
|
|
2249
|
+
*/
|
|
2250
|
+
async createAndBindSession(conversationId) {
|
|
1815
2251
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1816
2252
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1817
|
-
this.sessions.set(
|
|
1818
|
-
await this.persistSession(
|
|
2253
|
+
this.sessions.set(conversationId, sessionId);
|
|
2254
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1819
2255
|
});
|
|
1820
2256
|
return sessionId;
|
|
1821
2257
|
}
|
|
@@ -1838,6 +2274,25 @@ var ChannelDriver = class {
|
|
|
1838
2274
|
// -------------------------------------------------------------------------
|
|
1839
2275
|
// Per-session watcher (WI-3)
|
|
1840
2276
|
// -------------------------------------------------------------------------
|
|
2277
|
+
/**
|
|
2278
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2279
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2280
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2281
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2282
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2283
|
+
*/
|
|
2284
|
+
dispatchLocked(sessionId, fn) {
|
|
2285
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2286
|
+
const run2 = prior.then(fn, fn);
|
|
2287
|
+
this.sessionDispatchLocks.set(
|
|
2288
|
+
sessionId,
|
|
2289
|
+
run2.then(
|
|
2290
|
+
() => void 0,
|
|
2291
|
+
() => void 0
|
|
2292
|
+
)
|
|
2293
|
+
);
|
|
2294
|
+
return run2;
|
|
2295
|
+
}
|
|
1841
2296
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1842
2297
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1843
2298
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1847,7 +2302,9 @@ var ChannelDriver = class {
|
|
|
1847
2302
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1848
2303
|
loop: null,
|
|
1849
2304
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1850
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2305
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2306
|
+
lastGoodPollAt: this.now(),
|
|
2307
|
+
hadUsablePoll: false
|
|
1851
2308
|
};
|
|
1852
2309
|
this.watchers.set(sessionId, watcher);
|
|
1853
2310
|
}
|
|
@@ -1857,20 +2314,37 @@ var ChannelDriver = class {
|
|
|
1857
2314
|
opencodeMessageId,
|
|
1858
2315
|
message,
|
|
1859
2316
|
dispatchedAt: now,
|
|
2317
|
+
processingAnchorMs: now,
|
|
1860
2318
|
deadline: now + this.pausedMaxWaitMs,
|
|
1861
2319
|
started: false,
|
|
1862
2320
|
done: false,
|
|
1863
|
-
stuckReported: false
|
|
2321
|
+
stuckReported: false,
|
|
2322
|
+
lastAliveAt: 0,
|
|
2323
|
+
aliveInFlight: false,
|
|
2324
|
+
awaitingHumanLatched: false,
|
|
2325
|
+
pausedOnQuestion: false,
|
|
2326
|
+
pausedOnPermission: false,
|
|
2327
|
+
pausedClearConfirmed: false,
|
|
2328
|
+
pausedInFlight: false,
|
|
2329
|
+
deliveryDeadlineAnchored: false
|
|
1864
2330
|
});
|
|
1865
2331
|
}
|
|
1866
2332
|
/**
|
|
1867
2333
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1868
2334
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1869
2335
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1870
|
-
* `now
|
|
1871
|
-
* (10 min after `processed_at
|
|
1872
|
-
*
|
|
1873
|
-
*
|
|
2336
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2337
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2338
|
+
*
|
|
2339
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2340
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2341
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2342
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2343
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2344
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2345
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2346
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2347
|
+
* (only the appear-guard uses it).
|
|
1874
2348
|
*
|
|
1875
2349
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1876
2350
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -1888,7 +2362,9 @@ var ChannelDriver = class {
|
|
|
1888
2362
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1889
2363
|
loop: null,
|
|
1890
2364
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1891
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2365
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2366
|
+
lastGoodPollAt: this.now(),
|
|
2367
|
+
hadUsablePoll: false
|
|
1892
2368
|
};
|
|
1893
2369
|
this.watchers.set(sessionId, watcher);
|
|
1894
2370
|
}
|
|
@@ -1897,16 +2373,33 @@ var ChannelDriver = class {
|
|
|
1897
2373
|
opencodeMessageId,
|
|
1898
2374
|
message,
|
|
1899
2375
|
dispatchedAt: this.now(),
|
|
2376
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2377
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2378
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2379
|
+
processingAnchorMs: processedAtMs,
|
|
1900
2380
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1901
2381
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1902
2382
|
started: true,
|
|
1903
2383
|
done: false,
|
|
1904
2384
|
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1905
|
-
// AND the stuck-queued observer
|
|
1906
|
-
//
|
|
1907
|
-
//
|
|
1908
|
-
// (
|
|
1909
|
-
stuckReported: false
|
|
2385
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2386
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2387
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2388
|
+
// (#210/#220 observability).
|
|
2389
|
+
stuckReported: false,
|
|
2390
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2391
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2392
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2393
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2394
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2395
|
+
lastAliveAt: 0,
|
|
2396
|
+
aliveInFlight: false,
|
|
2397
|
+
awaitingHumanLatched: false,
|
|
2398
|
+
pausedOnQuestion: false,
|
|
2399
|
+
pausedOnPermission: false,
|
|
2400
|
+
pausedClearConfirmed: false,
|
|
2401
|
+
pausedInFlight: false,
|
|
2402
|
+
deliveryDeadlineAnchored: false
|
|
1910
2403
|
});
|
|
1911
2404
|
}
|
|
1912
2405
|
/**
|
|
@@ -1957,12 +2450,30 @@ var ChannelDriver = class {
|
|
|
1957
2450
|
messages = Array.isArray(body) ? body : null;
|
|
1958
2451
|
}
|
|
1959
2452
|
} catch {
|
|
1960
|
-
continue;
|
|
1961
2453
|
}
|
|
2454
|
+
if (messages != null && messages.length > 0) {
|
|
2455
|
+
watcher.lastGoodPollAt = this.now();
|
|
2456
|
+
watcher.hadUsablePoll = true;
|
|
2457
|
+
} else {
|
|
2458
|
+
const emptyButReachable = messages != null;
|
|
2459
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2460
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2461
|
+
continue;
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
1962
2465
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
1963
|
-
await this.serviceInFlightMessage(
|
|
2466
|
+
await this.serviceInFlightMessage(
|
|
2467
|
+
sessionId,
|
|
2468
|
+
watcher,
|
|
2469
|
+
inFlight,
|
|
2470
|
+
messages,
|
|
2471
|
+
openQuestions,
|
|
2472
|
+
openPermissions,
|
|
2473
|
+
questionsPolledOk,
|
|
2474
|
+
permissionsPolledOk
|
|
2475
|
+
);
|
|
1964
2476
|
}
|
|
1965
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
1966
2477
|
}
|
|
1967
2478
|
} catch (err) {
|
|
1968
2479
|
if (err instanceof ChannelAuthError) {
|
|
@@ -1984,19 +2495,49 @@ var ChannelDriver = class {
|
|
|
1984
2495
|
});
|
|
1985
2496
|
}
|
|
1986
2497
|
}
|
|
2498
|
+
/**
|
|
2499
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2500
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2501
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2502
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2503
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2504
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2505
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2506
|
+
* or past now, so a still-ample window is left untouched.
|
|
2507
|
+
*/
|
|
2508
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2509
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2510
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2511
|
+
if (this.now() >= inFlight.deadline) {
|
|
2512
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
1987
2515
|
/**
|
|
1988
2516
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
1989
2517
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
1990
2518
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
1991
2519
|
* in-flight set on completion or timeout.
|
|
1992
2520
|
*/
|
|
1993
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2521
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
1994
2522
|
const conv = watcher.conv;
|
|
1995
2523
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1996
|
-
|
|
2524
|
+
const id = inFlight.evidentMessageId;
|
|
2525
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2526
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2527
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2528
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2529
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2530
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2531
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2532
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1997
2533
|
let claimed;
|
|
1998
2534
|
try {
|
|
1999
|
-
claimed = await this.markProcessing(
|
|
2535
|
+
claimed = await this.markProcessing(
|
|
2536
|
+
conv.id,
|
|
2537
|
+
inFlight.evidentMessageId,
|
|
2538
|
+
sessionId,
|
|
2539
|
+
inFlight.opencodeMessageId
|
|
2540
|
+
);
|
|
2000
2541
|
} catch (err) {
|
|
2001
2542
|
if (err instanceof ChannelAuthError) throw err;
|
|
2002
2543
|
this.log({
|
|
@@ -2018,6 +2559,7 @@ var ChannelDriver = class {
|
|
|
2018
2559
|
}
|
|
2019
2560
|
}
|
|
2020
2561
|
if (state === "done") {
|
|
2562
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2021
2563
|
if (!inFlight.done) {
|
|
2022
2564
|
this.log({
|
|
2023
2565
|
level: "info",
|
|
@@ -2026,7 +2568,12 @@ var ChannelDriver = class {
|
|
|
2026
2568
|
message_id: inFlight.evidentMessageId
|
|
2027
2569
|
});
|
|
2028
2570
|
try {
|
|
2029
|
-
await this.markDone(
|
|
2571
|
+
await this.markDone(
|
|
2572
|
+
conv.id,
|
|
2573
|
+
inFlight.evidentMessageId,
|
|
2574
|
+
sessionId,
|
|
2575
|
+
inFlight.opencodeMessageId
|
|
2576
|
+
);
|
|
2030
2577
|
} catch (err) {
|
|
2031
2578
|
if (err instanceof ChannelAuthError) throw err;
|
|
2032
2579
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -2062,59 +2609,117 @@ var ChannelDriver = class {
|
|
|
2062
2609
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2063
2610
|
return;
|
|
2064
2611
|
}
|
|
2065
|
-
if (state === "
|
|
2066
|
-
|
|
2067
|
-
|
|
2612
|
+
if (state === "failed") {
|
|
2613
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2614
|
+
if (!inFlight.done) {
|
|
2615
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2616
|
+
this.log({
|
|
2617
|
+
level: "error",
|
|
2618
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2619
|
+
conversation_id: conv.id,
|
|
2620
|
+
message_id: inFlight.evidentMessageId
|
|
2621
|
+
});
|
|
2622
|
+
try {
|
|
2623
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2624
|
+
} catch (err) {
|
|
2625
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2626
|
+
if (err instanceof ChannelTerminalError) {
|
|
2627
|
+
this.log({
|
|
2628
|
+
level: "error",
|
|
2629
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2630
|
+
conversation_id: conv.id,
|
|
2631
|
+
message_id: inFlight.evidentMessageId
|
|
2632
|
+
});
|
|
2633
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
if (this.now() >= inFlight.deadline) {
|
|
2637
|
+
this.log({
|
|
2638
|
+
level: "error",
|
|
2639
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2640
|
+
conversation_id: conv.id,
|
|
2641
|
+
message_id: inFlight.evidentMessageId
|
|
2642
|
+
});
|
|
2643
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2644
|
+
return;
|
|
2645
|
+
}
|
|
2646
|
+
this.log({
|
|
2647
|
+
level: "error",
|
|
2648
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2649
|
+
conversation_id: conv.id,
|
|
2650
|
+
message_id: inFlight.evidentMessageId
|
|
2651
|
+
});
|
|
2652
|
+
return;
|
|
2653
|
+
}
|
|
2654
|
+
inFlight.done = true;
|
|
2068
2655
|
}
|
|
2656
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2657
|
+
return;
|
|
2069
2658
|
}
|
|
2070
|
-
|
|
2659
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2660
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2661
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2071
2662
|
inFlight.stuckReported = true;
|
|
2072
2663
|
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2073
2664
|
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2074
2665
|
});
|
|
2075
2666
|
}
|
|
2076
|
-
|
|
2667
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2668
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2077
2669
|
this.log({
|
|
2078
|
-
level: "
|
|
2079
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2670
|
+
level: "error",
|
|
2671
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2080
2672
|
conversation_id: conv.id,
|
|
2081
2673
|
message_id: inFlight.evidentMessageId
|
|
2082
2674
|
});
|
|
2675
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2676
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2677
|
+
});
|
|
2083
2678
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2679
|
+
return;
|
|
2084
2680
|
}
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2681
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2682
|
+
inFlight.aliveInFlight = true;
|
|
2683
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2684
|
+
inFlight.aliveInFlight = false;
|
|
2685
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
if (awaitingHuman) {
|
|
2689
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2690
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2691
|
+
inFlight.awaitingHumanLatched = true;
|
|
2692
|
+
}
|
|
2693
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2694
|
+
inFlight.pausedInFlight = true;
|
|
2695
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2696
|
+
inFlight.pausedInFlight = false;
|
|
2697
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2700
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2701
|
+
inFlight.awaitingHumanLatched = false;
|
|
2702
|
+
inFlight.pausedOnQuestion = false;
|
|
2703
|
+
inFlight.pausedOnPermission = false;
|
|
2704
|
+
inFlight.pausedClearConfirmed = false;
|
|
2705
|
+
}
|
|
2706
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2707
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2708
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2709
|
+
);
|
|
2710
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2711
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2111
2712
|
this.log({
|
|
2112
|
-
level: "
|
|
2113
|
-
message: `
|
|
2713
|
+
level: "info",
|
|
2714
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2715
|
+
conversation_id: conv.id,
|
|
2114
2716
|
message_id: inFlight.evidentMessageId
|
|
2115
2717
|
});
|
|
2718
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2719
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2720
|
+
});
|
|
2721
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2116
2722
|
}
|
|
2117
|
-
inFlight.dispatchedAt = this.now();
|
|
2118
2723
|
}
|
|
2119
2724
|
// -------------------------------------------------------------------------
|
|
2120
2725
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -2135,12 +2740,17 @@ var ChannelDriver = class {
|
|
|
2135
2740
|
*/
|
|
2136
2741
|
async readoptProcessing() {
|
|
2137
2742
|
const rows = await this.getProcessingMessages();
|
|
2138
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2743
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2139
2744
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2140
|
-
for (const id of [
|
|
2745
|
+
for (const id of [
|
|
2746
|
+
...this.dontRedispatch,
|
|
2747
|
+
...this.doneUndeliverable,
|
|
2748
|
+
...this.readoptPollUnresolvedSignalled
|
|
2749
|
+
]) {
|
|
2141
2750
|
if (!stillProcessing.has(id)) {
|
|
2142
2751
|
const cleared = this.dontRedispatch.delete(id);
|
|
2143
2752
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2753
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2144
2754
|
if (cleared || clearedUndeliverable) {
|
|
2145
2755
|
this.log({
|
|
2146
2756
|
level: "info",
|
|
@@ -2194,8 +2804,10 @@ var ChannelDriver = class {
|
|
|
2194
2804
|
});
|
|
2195
2805
|
continue;
|
|
2196
2806
|
}
|
|
2807
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2808
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2197
2809
|
for (const row of sessionRows) {
|
|
2198
|
-
await this.readoptOne(sessionId, row, messages);
|
|
2810
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2199
2811
|
}
|
|
2200
2812
|
}
|
|
2201
2813
|
}
|
|
@@ -2203,14 +2815,21 @@ var ChannelDriver = class {
|
|
|
2203
2815
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2204
2816
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2205
2817
|
*
|
|
2206
|
-
* Branches on `messageRunState(messages,
|
|
2818
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2819
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2820
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2821
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2822
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2207
2823
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2208
|
-
* - `
|
|
2209
|
-
*
|
|
2824
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2825
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2826
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2827
|
+
* tracking the stored id so the reply correlates by it;
|
|
2828
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2210
2829
|
*
|
|
2211
2830
|
* Only `ChannelAuthError` propagates.
|
|
2212
2831
|
*/
|
|
2213
|
-
async readoptOne(sessionId, row, messages) {
|
|
2832
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2214
2833
|
if (this.isTracked(sessionId, row.id)) {
|
|
2215
2834
|
this.log({
|
|
2216
2835
|
level: "info",
|
|
@@ -2220,8 +2839,8 @@ var ChannelDriver = class {
|
|
|
2220
2839
|
});
|
|
2221
2840
|
return;
|
|
2222
2841
|
}
|
|
2223
|
-
const ocId =
|
|
2224
|
-
const state = messageRunState(messages, ocId);
|
|
2842
|
+
const ocId = row.opencode_message_id;
|
|
2843
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2225
2844
|
if (state === "done") {
|
|
2226
2845
|
if (this.doneUndeliverable.has(row.id)) {
|
|
2227
2846
|
this.log({
|
|
@@ -2239,7 +2858,7 @@ var ChannelDriver = class {
|
|
|
2239
2858
|
message_id: row.id
|
|
2240
2859
|
});
|
|
2241
2860
|
try {
|
|
2242
|
-
await this.markDone(row.conversation_id, row.id, sessionId);
|
|
2861
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2243
2862
|
} catch (err) {
|
|
2244
2863
|
if (err instanceof ChannelAuthError) throw err;
|
|
2245
2864
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -2250,6 +2869,7 @@ var ChannelDriver = class {
|
|
|
2250
2869
|
conversation_id: row.conversation_id,
|
|
2251
2870
|
message_id: row.id
|
|
2252
2871
|
});
|
|
2872
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2253
2873
|
return;
|
|
2254
2874
|
}
|
|
2255
2875
|
this.log({
|
|
@@ -2261,6 +2881,42 @@ var ChannelDriver = class {
|
|
|
2261
2881
|
return;
|
|
2262
2882
|
}
|
|
2263
2883
|
this.dontRedispatch.delete(row.id);
|
|
2884
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2885
|
+
return;
|
|
2886
|
+
}
|
|
2887
|
+
if (state === "failed") {
|
|
2888
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2889
|
+
this.log({
|
|
2890
|
+
level: "error",
|
|
2891
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2892
|
+
conversation_id: row.conversation_id,
|
|
2893
|
+
message_id: row.id
|
|
2894
|
+
});
|
|
2895
|
+
try {
|
|
2896
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2897
|
+
} catch (err) {
|
|
2898
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2899
|
+
if (err instanceof ChannelTerminalError) {
|
|
2900
|
+
this.doneUndeliverable.add(row.id);
|
|
2901
|
+
this.log({
|
|
2902
|
+
level: "error",
|
|
2903
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2904
|
+
conversation_id: row.conversation_id,
|
|
2905
|
+
message_id: row.id
|
|
2906
|
+
});
|
|
2907
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2908
|
+
return;
|
|
2909
|
+
}
|
|
2910
|
+
this.log({
|
|
2911
|
+
level: "error",
|
|
2912
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2913
|
+
conversation_id: row.conversation_id,
|
|
2914
|
+
message_id: row.id
|
|
2915
|
+
});
|
|
2916
|
+
return;
|
|
2917
|
+
}
|
|
2918
|
+
this.dontRedispatch.delete(row.id);
|
|
2919
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2264
2920
|
return;
|
|
2265
2921
|
}
|
|
2266
2922
|
if (this.dontRedispatch.has(row.id)) {
|
|
@@ -2272,7 +2928,72 @@ var ChannelDriver = class {
|
|
|
2272
2928
|
});
|
|
2273
2929
|
return;
|
|
2274
2930
|
}
|
|
2275
|
-
|
|
2931
|
+
let statusReadableOngoing = null;
|
|
2932
|
+
if (state === "running" && ocId) {
|
|
2933
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2934
|
+
const shape = this.replyCompletionShape(reply);
|
|
2935
|
+
const ongoing = sessionOngoing;
|
|
2936
|
+
statusReadableOngoing = ongoing;
|
|
2937
|
+
if (ongoing === false) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2945
|
+
return;
|
|
2946
|
+
}
|
|
2947
|
+
if (ongoing === true) {
|
|
2948
|
+
this.log({
|
|
2949
|
+
level: "info",
|
|
2950
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2951
|
+
conversation_id: row.conversation_id,
|
|
2952
|
+
message_id: row.id
|
|
2953
|
+
});
|
|
2954
|
+
} else {
|
|
2955
|
+
if (shape === "b1") {
|
|
2956
|
+
this.log({
|
|
2957
|
+
level: "info",
|
|
2958
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2959
|
+
conversation_id: row.conversation_id,
|
|
2960
|
+
message_id: row.id
|
|
2961
|
+
});
|
|
2962
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2963
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2964
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2965
|
+
}
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
this.log({
|
|
2969
|
+
level: "info",
|
|
2970
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2971
|
+
conversation_id: row.conversation_id,
|
|
2972
|
+
message_id: row.id
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2977
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2978
|
+
if (descendantAlive === true) {
|
|
2979
|
+
this.log({
|
|
2980
|
+
level: "info",
|
|
2981
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2982
|
+
conversation_id: row.conversation_id,
|
|
2983
|
+
message_id: row.id
|
|
2984
|
+
});
|
|
2985
|
+
} else {
|
|
2986
|
+
this.log({
|
|
2987
|
+
level: "info",
|
|
2988
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2989
|
+
conversation_id: row.conversation_id,
|
|
2990
|
+
message_id: row.id
|
|
2991
|
+
});
|
|
2992
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2276
2997
|
const conv = this.convForRow(sessionId, row);
|
|
2277
2998
|
const message = this.queuedMessageForRow(row);
|
|
2278
2999
|
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
@@ -2281,32 +3002,55 @@ var ChannelDriver = class {
|
|
|
2281
3002
|
this.ensureWatcherRunning(sessionId);
|
|
2282
3003
|
this.log({
|
|
2283
3004
|
level: "info",
|
|
2284
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (
|
|
3005
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2285
3006
|
conversation_id: row.conversation_id,
|
|
2286
3007
|
message_id: row.id
|
|
2287
3008
|
});
|
|
3009
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
2288
3010
|
return;
|
|
2289
3011
|
}
|
|
2290
3012
|
await this.forceReadoptRun(sessionId, row);
|
|
2291
3013
|
}
|
|
2292
3014
|
/**
|
|
2293
|
-
* Re-dispatch an orphaned (`unknown
|
|
3015
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2294
3016
|
*
|
|
2295
|
-
*
|
|
2296
|
-
*
|
|
2297
|
-
*
|
|
2298
|
-
*
|
|
2299
|
-
* (which hangs off the stable id) is found and delivered. The residual
|
|
2300
|
-
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2301
|
-
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2302
|
-
* to swallow the duplicate.
|
|
3017
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
3018
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
3019
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
3020
|
+
* correlates server-side.
|
|
2303
3021
|
*
|
|
2304
|
-
*
|
|
2305
|
-
*
|
|
3022
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3023
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3024
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3025
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3026
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3027
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3028
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3029
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3030
|
+
* may retry exactly once more).
|
|
3031
|
+
*
|
|
3032
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2306
3033
|
* `processed_at` (Invariant 1).
|
|
2307
3034
|
*/
|
|
2308
3035
|
async forceReadoptRun(sessionId, row) {
|
|
2309
|
-
|
|
3036
|
+
if (this.stopped) {
|
|
3037
|
+
this.log({
|
|
3038
|
+
level: "info",
|
|
3039
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3040
|
+
conversation_id: row.conversation_id,
|
|
3041
|
+
message_id: row.id
|
|
3042
|
+
});
|
|
3043
|
+
return;
|
|
3044
|
+
}
|
|
3045
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3046
|
+
this.log({
|
|
3047
|
+
level: "info",
|
|
3048
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3049
|
+
conversation_id: row.conversation_id,
|
|
3050
|
+
message_id: row.id
|
|
3051
|
+
});
|
|
3052
|
+
return;
|
|
3053
|
+
}
|
|
2310
3054
|
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2311
3055
|
this.dontRedispatch.add(row.id);
|
|
2312
3056
|
this.log({
|
|
@@ -2315,6 +3059,7 @@ var ChannelDriver = class {
|
|
|
2315
3059
|
conversation_id: row.conversation_id,
|
|
2316
3060
|
message_id: row.id
|
|
2317
3061
|
});
|
|
3062
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
2318
3063
|
return;
|
|
2319
3064
|
}
|
|
2320
3065
|
const options = {
|
|
@@ -2323,13 +3068,19 @@ var ChannelDriver = class {
|
|
|
2323
3068
|
};
|
|
2324
3069
|
this.log({
|
|
2325
3070
|
level: "info",
|
|
2326
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching
|
|
3071
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2327
3072
|
conversation_id: row.conversation_id,
|
|
2328
3073
|
message_id: row.id
|
|
2329
3074
|
});
|
|
3075
|
+
this.awaitingReadopt.add(row.id);
|
|
3076
|
+
let ocId;
|
|
2330
3077
|
try {
|
|
2331
|
-
await
|
|
3078
|
+
ocId = await this.dispatchLocked(
|
|
3079
|
+
sessionId,
|
|
3080
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
3081
|
+
);
|
|
2332
3082
|
} catch (err) {
|
|
3083
|
+
this.awaitingReadopt.delete(row.id);
|
|
2333
3084
|
if (err instanceof ChannelAuthError) throw err;
|
|
2334
3085
|
this.log({
|
|
2335
3086
|
level: "error",
|
|
@@ -2337,6 +3088,18 @@ var ChannelDriver = class {
|
|
|
2337
3088
|
conversation_id: row.conversation_id,
|
|
2338
3089
|
message_id: row.id
|
|
2339
3090
|
});
|
|
3091
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3092
|
+
return;
|
|
3093
|
+
}
|
|
3094
|
+
if (ocId === null) {
|
|
3095
|
+
this.awaitingReadopt.delete(row.id);
|
|
3096
|
+
this.log({
|
|
3097
|
+
level: "error",
|
|
3098
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3099
|
+
conversation_id: row.conversation_id,
|
|
3100
|
+
message_id: row.id
|
|
3101
|
+
});
|
|
3102
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2340
3103
|
return;
|
|
2341
3104
|
}
|
|
2342
3105
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -2344,7 +3107,9 @@ var ChannelDriver = class {
|
|
|
2344
3107
|
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2345
3108
|
this.dispatched.add(row.id);
|
|
2346
3109
|
this.readopted.add(row.id);
|
|
3110
|
+
this.awaitingReadopt.delete(row.id);
|
|
2347
3111
|
this.ensureWatcherRunning(sessionId);
|
|
3112
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
2348
3113
|
}
|
|
2349
3114
|
/**
|
|
2350
3115
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -2436,21 +3201,41 @@ var ChannelDriver = class {
|
|
|
2436
3201
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2437
3202
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2438
3203
|
* oldest running message.
|
|
3204
|
+
*
|
|
3205
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3206
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3207
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3208
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3209
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3210
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3211
|
+
* even after it was already surfaced to the channel.
|
|
2439
3212
|
*/
|
|
2440
3213
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3214
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3215
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3216
|
+
let questionsPolledOk = true;
|
|
3217
|
+
let permissionsPolledOk = true;
|
|
2441
3218
|
let questions = [];
|
|
2442
3219
|
try {
|
|
2443
3220
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2444
3221
|
if (res.ok) {
|
|
2445
3222
|
const body = await res.json();
|
|
2446
|
-
|
|
3223
|
+
if (Array.isArray(body)) {
|
|
3224
|
+
questions = body;
|
|
3225
|
+
} else {
|
|
3226
|
+
questionsPolledOk = false;
|
|
3227
|
+
}
|
|
3228
|
+
} else {
|
|
3229
|
+
questionsPolledOk = false;
|
|
2447
3230
|
}
|
|
2448
3231
|
} catch {
|
|
3232
|
+
questionsPolledOk = false;
|
|
2449
3233
|
}
|
|
2450
3234
|
for (const q of questions) {
|
|
2451
|
-
if (q.sessionID
|
|
2452
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3235
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2453
3236
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3237
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3238
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2454
3239
|
const reported = await this.reportInteraction(
|
|
2455
3240
|
watcher.conv.id,
|
|
2456
3241
|
"question",
|
|
@@ -2464,14 +3249,22 @@ var ChannelDriver = class {
|
|
|
2464
3249
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2465
3250
|
if (res.ok) {
|
|
2466
3251
|
const body = await res.json();
|
|
2467
|
-
|
|
3252
|
+
if (Array.isArray(body)) {
|
|
3253
|
+
permissions = body;
|
|
3254
|
+
} else {
|
|
3255
|
+
permissionsPolledOk = false;
|
|
3256
|
+
}
|
|
3257
|
+
} else {
|
|
3258
|
+
permissionsPolledOk = false;
|
|
2468
3259
|
}
|
|
2469
3260
|
} catch {
|
|
3261
|
+
permissionsPolledOk = false;
|
|
2470
3262
|
}
|
|
2471
3263
|
for (const p of permissions) {
|
|
2472
|
-
if (p.sessionID
|
|
2473
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3264
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2474
3265
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3266
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3267
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2475
3268
|
const reported = await this.reportInteraction(
|
|
2476
3269
|
watcher.conv.id,
|
|
2477
3270
|
"permission",
|
|
@@ -2480,6 +3273,128 @@ var ChannelDriver = class {
|
|
|
2480
3273
|
);
|
|
2481
3274
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2482
3275
|
}
|
|
3276
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3277
|
+
}
|
|
3278
|
+
/**
|
|
3279
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3280
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3281
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3282
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3283
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3284
|
+
*
|
|
3285
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3286
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3287
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3288
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3289
|
+
*/
|
|
3290
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3291
|
+
let current = sessionId;
|
|
3292
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3293
|
+
if (current === rootSessionId) return true;
|
|
3294
|
+
const parent = await this.resolveSessionParent(current);
|
|
3295
|
+
if (parent === null || parent === void 0) return false;
|
|
3296
|
+
current = parent;
|
|
3297
|
+
}
|
|
3298
|
+
return false;
|
|
3299
|
+
}
|
|
3300
|
+
/**
|
|
3301
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3302
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3303
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3304
|
+
* caching a wrong answer — the next tick retries).
|
|
3305
|
+
*/
|
|
3306
|
+
async resolveSessionParent(sessionId) {
|
|
3307
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3308
|
+
if (cached !== void 0) return cached;
|
|
3309
|
+
let parent = void 0;
|
|
3310
|
+
try {
|
|
3311
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3312
|
+
if (res.ok) {
|
|
3313
|
+
const body = await res.json();
|
|
3314
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3315
|
+
}
|
|
3316
|
+
} catch {
|
|
3317
|
+
parent = void 0;
|
|
3318
|
+
}
|
|
3319
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3320
|
+
return parent;
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3324
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3325
|
+
*
|
|
3326
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3327
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3328
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3329
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3330
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3331
|
+
* provably in flight at the exact moment of recovery.
|
|
3332
|
+
*
|
|
3333
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3334
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3335
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3336
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3337
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3338
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3339
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3340
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3341
|
+
*
|
|
3342
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3343
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3344
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3345
|
+
* case), OR no descendant is found at all.
|
|
3346
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3347
|
+
*
|
|
3348
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3349
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3350
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3351
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3352
|
+
* true/false/null faithfully.
|
|
3353
|
+
*
|
|
3354
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3355
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3356
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3357
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3358
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3359
|
+
* `SessionStatus` only.
|
|
3360
|
+
*/
|
|
3361
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3362
|
+
const sessions = await listSessions(this.port);
|
|
3363
|
+
if (!sessions) {
|
|
3364
|
+
this.log({
|
|
3365
|
+
level: "error",
|
|
3366
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3367
|
+
});
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
for (const candidate of sessions) {
|
|
3371
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3372
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3373
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3374
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3375
|
+
return true;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
return false;
|
|
3379
|
+
}
|
|
3380
|
+
/**
|
|
3381
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3382
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3383
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3384
|
+
* the aborted-in-flight production bug after a restart.
|
|
3385
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3386
|
+
* (the sub-agent preamble — #253's shape).
|
|
3387
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3388
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3389
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3390
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3391
|
+
*/
|
|
3392
|
+
replyCompletionShape(reply) {
|
|
3393
|
+
if (!reply) return "other";
|
|
3394
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3395
|
+
if (completed == null) return "b1";
|
|
3396
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3397
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2483
3398
|
}
|
|
2484
3399
|
/**
|
|
2485
3400
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -2613,13 +3528,17 @@ var ChannelDriver = class {
|
|
|
2613
3528
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2614
3529
|
* retry vehicle for the swap-to-running.
|
|
2615
3530
|
*/
|
|
2616
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
3531
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2617
3532
|
const res = await this.fetchImpl(
|
|
2618
3533
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2619
3534
|
{
|
|
2620
3535
|
method: "PATCH",
|
|
2621
3536
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2622
|
-
body: JSON.stringify({
|
|
3537
|
+
body: JSON.stringify({
|
|
3538
|
+
status: "processing",
|
|
3539
|
+
opencode_session_id: sessionId,
|
|
3540
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3541
|
+
})
|
|
2623
3542
|
}
|
|
2624
3543
|
);
|
|
2625
3544
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2657,13 +3576,17 @@ var ChannelDriver = class {
|
|
|
2657
3576
|
* watcher retries next tick within the
|
|
2658
3577
|
* deadline, Finding 4).
|
|
2659
3578
|
*/
|
|
2660
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3579
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2661
3580
|
const res = await this.fetchImpl(
|
|
2662
3581
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2663
3582
|
{
|
|
2664
3583
|
method: "PATCH",
|
|
2665
3584
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2666
|
-
body: JSON.stringify({
|
|
3585
|
+
body: JSON.stringify({
|
|
3586
|
+
status: "done",
|
|
3587
|
+
opencode_session_id: sessionId,
|
|
3588
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3589
|
+
})
|
|
2667
3590
|
}
|
|
2668
3591
|
);
|
|
2669
3592
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2673,7 +3596,17 @@ var ChannelDriver = class {
|
|
|
2673
3596
|
}
|
|
2674
3597
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2675
3598
|
}
|
|
2676
|
-
|
|
3599
|
+
/**
|
|
3600
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3601
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3602
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3603
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3604
|
+
* failure reason reaches the channel.
|
|
3605
|
+
*/
|
|
3606
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3607
|
+
const body = { status: "failed" };
|
|
3608
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3609
|
+
if (error2 !== void 0) body.error = error2;
|
|
2677
3610
|
await this.callWithRetry(
|
|
2678
3611
|
"marking message as failed",
|
|
2679
3612
|
() => this.fetchImpl(
|
|
@@ -2681,7 +3614,7 @@ var ChannelDriver = class {
|
|
|
2681
3614
|
{
|
|
2682
3615
|
method: "PATCH",
|
|
2683
3616
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2684
|
-
body: JSON.stringify(
|
|
3617
|
+
body: JSON.stringify(body)
|
|
2685
3618
|
}
|
|
2686
3619
|
)
|
|
2687
3620
|
);
|
|
@@ -2694,6 +3627,12 @@ var ChannelDriver = class {
|
|
|
2694
3627
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2695
3628
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2696
3629
|
* context (no silent catch, per development-workflow).
|
|
3630
|
+
*
|
|
3631
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3632
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3633
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3634
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3635
|
+
* leaves liveness").
|
|
2697
3636
|
*/
|
|
2698
3637
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
2699
3638
|
try {
|
|
@@ -2712,7 +3651,9 @@ var ChannelDriver = class {
|
|
|
2712
3651
|
conversation_id: conversationId,
|
|
2713
3652
|
message_id: messageId
|
|
2714
3653
|
});
|
|
3654
|
+
return false;
|
|
2715
3655
|
}
|
|
3656
|
+
return true;
|
|
2716
3657
|
} catch (err) {
|
|
2717
3658
|
this.log({
|
|
2718
3659
|
level: "error",
|
|
@@ -2720,6 +3661,7 @@ var ChannelDriver = class {
|
|
|
2720
3661
|
conversation_id: conversationId,
|
|
2721
3662
|
message_id: messageId
|
|
2722
3663
|
});
|
|
3664
|
+
return false;
|
|
2723
3665
|
}
|
|
2724
3666
|
}
|
|
2725
3667
|
async persistSession(conversationId, sessionId) {
|
|
@@ -2992,6 +3934,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
2992
3934
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
2993
3935
|
}
|
|
2994
3936
|
}
|
|
3937
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3938
|
+
const apiUrl = getApiUrlConfig();
|
|
3939
|
+
try {
|
|
3940
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3941
|
+
method: "POST",
|
|
3942
|
+
headers: { Authorization: authHeader }
|
|
3943
|
+
});
|
|
3944
|
+
if (!response.ok) {
|
|
3945
|
+
const serverMessage = await readErrorMessage(response);
|
|
3946
|
+
return {
|
|
3947
|
+
ok: false,
|
|
3948
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3949
|
+
};
|
|
3950
|
+
}
|
|
3951
|
+
return { ok: true };
|
|
3952
|
+
} catch (error2) {
|
|
3953
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
2995
3956
|
async function getAgentInfo(agentId, authHeader) {
|
|
2996
3957
|
const apiUrl = getApiUrlConfig();
|
|
2997
3958
|
try {
|
|
@@ -3037,6 +3998,8 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3037
3998
|
// src/commands/run.ts
|
|
3038
3999
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
3039
4000
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
4001
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4002
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3040
4003
|
function log2(state, message, isError = false) {
|
|
3041
4004
|
if (state.json) {
|
|
3042
4005
|
console.log(
|
|
@@ -3191,7 +4154,7 @@ async function driveChannels(state, driver) {
|
|
|
3191
4154
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
3192
4155
|
if (state.interactive) displayStatus(state);
|
|
3193
4156
|
}
|
|
3194
|
-
await new Promise((
|
|
4157
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
3195
4158
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
3196
4159
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
3197
4160
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -3202,8 +4165,122 @@ async function driveChannels(state, driver) {
|
|
|
3202
4165
|
}
|
|
3203
4166
|
}
|
|
3204
4167
|
}
|
|
3205
|
-
|
|
4168
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4169
|
+
async function runSweep(state, driver, config2) {
|
|
4170
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4171
|
+
try {
|
|
4172
|
+
const sessions = await listSessions(state.port);
|
|
4173
|
+
if (sessions === null) {
|
|
4174
|
+
logActivity(state, {
|
|
4175
|
+
type: "info",
|
|
4176
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4177
|
+
});
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
const toDelete = selectSessionsToDelete(
|
|
4181
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4182
|
+
{
|
|
4183
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4184
|
+
maxCount: config2.maxCount,
|
|
4185
|
+
nowMs: Date.now(),
|
|
4186
|
+
protectedIds: driver.protectedSessionIds()
|
|
4187
|
+
}
|
|
4188
|
+
);
|
|
4189
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4190
|
+
let deleted = 0;
|
|
4191
|
+
let failed = 0;
|
|
4192
|
+
let skippedNewlyActive = 0;
|
|
4193
|
+
for (const id of toDelete) {
|
|
4194
|
+
if (protectedNow.has(id)) {
|
|
4195
|
+
skippedNewlyActive++;
|
|
4196
|
+
logActivity(state, {
|
|
4197
|
+
type: "info",
|
|
4198
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4199
|
+
});
|
|
4200
|
+
continue;
|
|
4201
|
+
}
|
|
4202
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4203
|
+
else failed++;
|
|
4204
|
+
}
|
|
4205
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4206
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4207
|
+
logActivity(state, {
|
|
4208
|
+
type: "info",
|
|
4209
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4210
|
+
});
|
|
4211
|
+
} catch (error2) {
|
|
4212
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4213
|
+
logActivity(state, {
|
|
4214
|
+
type: "error",
|
|
4215
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4216
|
+
});
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4220
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4221
|
+
{
|
|
4222
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4223
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4224
|
+
interval: options.sessionCleanupInterval
|
|
4225
|
+
},
|
|
4226
|
+
process.env
|
|
4227
|
+
);
|
|
4228
|
+
for (const warning2 of config2.warnings) {
|
|
4229
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4230
|
+
}
|
|
4231
|
+
if (!config2.enabled) return;
|
|
4232
|
+
logActivity(state, {
|
|
4233
|
+
type: "info",
|
|
4234
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4235
|
+
});
|
|
4236
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4237
|
+
const firstSweep = setTimeout(
|
|
4238
|
+
() => void runSweep(state, driver, config2),
|
|
4239
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4240
|
+
);
|
|
4241
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4242
|
+
}
|
|
4243
|
+
async function notifyOffline(state) {
|
|
4244
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4245
|
+
if (!state.connected) {
|
|
4246
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4247
|
+
return;
|
|
4248
|
+
}
|
|
4249
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4250
|
+
if (result.ok) {
|
|
4251
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4252
|
+
} else {
|
|
4253
|
+
logActivity(state, {
|
|
4254
|
+
type: "error",
|
|
4255
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4256
|
+
});
|
|
4257
|
+
if (state.interactive) displayStatus(state);
|
|
4258
|
+
}
|
|
4259
|
+
}
|
|
4260
|
+
async function cleanup(state, opts = {}) {
|
|
3206
4261
|
state.running = false;
|
|
4262
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4263
|
+
clearInterval(timer);
|
|
4264
|
+
clearTimeout(timer);
|
|
4265
|
+
}
|
|
4266
|
+
state.sessionCleanupTimers = [];
|
|
4267
|
+
if (opts.graceful && state.channelDriver) {
|
|
4268
|
+
state.channelDriver.stop();
|
|
4269
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4270
|
+
if (state.interactive) {
|
|
4271
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4272
|
+
displayStatus(state);
|
|
4273
|
+
}
|
|
4274
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4275
|
+
if (!settled) {
|
|
4276
|
+
logActivity(state, {
|
|
4277
|
+
type: "info",
|
|
4278
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4279
|
+
});
|
|
4280
|
+
if (state.interactive) displayStatus(state);
|
|
4281
|
+
}
|
|
4282
|
+
}
|
|
4283
|
+
await notifyOffline(state);
|
|
3207
4284
|
if (state.connection) {
|
|
3208
4285
|
state.connection.close();
|
|
3209
4286
|
state.connection = null;
|
|
@@ -3234,10 +4311,13 @@ async function run(options) {
|
|
|
3234
4311
|
opencodeVersion: null,
|
|
3235
4312
|
opencodeProcess: null,
|
|
3236
4313
|
connection: null,
|
|
4314
|
+
channelDriver: null,
|
|
3237
4315
|
running: true,
|
|
4316
|
+
shuttingDown: false,
|
|
3238
4317
|
activityLog: [],
|
|
3239
4318
|
messageCount: 0,
|
|
3240
4319
|
lastProxiedActivityAt: null,
|
|
4320
|
+
sessionCleanupTimers: [],
|
|
3241
4321
|
authHeader: ""
|
|
3242
4322
|
};
|
|
3243
4323
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -3248,13 +4328,15 @@ async function run(options) {
|
|
|
3248
4328
|
);
|
|
3249
4329
|
}
|
|
3250
4330
|
const handleSignal = async () => {
|
|
4331
|
+
if (state.shuttingDown) return;
|
|
4332
|
+
state.shuttingDown = true;
|
|
3251
4333
|
if (state.interactive) {
|
|
3252
4334
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
3253
4335
|
displayStatus(state);
|
|
3254
4336
|
} else {
|
|
3255
4337
|
log2(state, "Shutting down...");
|
|
3256
4338
|
}
|
|
3257
|
-
await cleanup(state);
|
|
4339
|
+
await cleanup(state, { graceful: true });
|
|
3258
4340
|
await shutdownTelemetry();
|
|
3259
4341
|
process.exit(0);
|
|
3260
4342
|
};
|
|
@@ -3374,12 +4456,14 @@ async function run(options) {
|
|
|
3374
4456
|
apiUrl: getApiUrlConfig(),
|
|
3375
4457
|
getAuthHeader: () => state.authHeader,
|
|
3376
4458
|
conversationFilter: state.conversationFilter,
|
|
4459
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3377
4460
|
log: (entry) => logActivity(state, {
|
|
3378
4461
|
type: entry.level === "error" ? "error" : "info",
|
|
3379
4462
|
message: entry.message,
|
|
3380
4463
|
error: entry.level === "error" ? entry.message : void 0
|
|
3381
4464
|
})
|
|
3382
4465
|
});
|
|
4466
|
+
state.channelDriver = channelDriver;
|
|
3383
4467
|
const connection = new RunnerConnection({
|
|
3384
4468
|
agentId: state.agentId,
|
|
3385
4469
|
getAuthHeader: () => state.authHeader,
|
|
@@ -3393,7 +4477,11 @@ async function run(options) {
|
|
|
3393
4477
|
type: "info",
|
|
3394
4478
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
3395
4479
|
});
|
|
3396
|
-
emitAgentConnected(state.agentId, {
|
|
4480
|
+
emitAgentConnected(state.agentId, {
|
|
4481
|
+
port: state.port,
|
|
4482
|
+
cli_version: getCliVersion(),
|
|
4483
|
+
opencode_version: state.opencodeVersion
|
|
4484
|
+
});
|
|
3397
4485
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
3398
4486
|
if (state.interactive) displayStatus(state);
|
|
3399
4487
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -3472,10 +4560,12 @@ async function run(options) {
|
|
|
3472
4560
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3473
4561
|
throw error2;
|
|
3474
4562
|
}
|
|
4563
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3475
4564
|
if (!interactive || state.json) {
|
|
3476
4565
|
log2(state, "Driving channel messages...");
|
|
3477
4566
|
}
|
|
3478
4567
|
await driveChannels(state, channelDriver);
|
|
4568
|
+
if (state.shuttingDown) return;
|
|
3479
4569
|
await cleanup(state);
|
|
3480
4570
|
if (state.json) {
|
|
3481
4571
|
console.log(
|
|
@@ -3490,6 +4580,7 @@ async function run(options) {
|
|
|
3490
4580
|
await shutdownTelemetry();
|
|
3491
4581
|
process.exit(0);
|
|
3492
4582
|
} catch (error2) {
|
|
4583
|
+
if (state.shuttingDown) return;
|
|
3493
4584
|
await cleanup(state);
|
|
3494
4585
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3495
4586
|
if (state.json) {
|
|
@@ -3524,7 +4615,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3524
4615
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3525
4616
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3526
4617
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3527
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4618
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4619
|
+
"--session-cleanup-max-age <duration>",
|
|
4620
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4621
|
+
).option(
|
|
4622
|
+
"--session-cleanup-max-count <n>",
|
|
4623
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4624
|
+
).option(
|
|
4625
|
+
"--session-cleanup-interval <duration>",
|
|
4626
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4627
|
+
).action(
|
|
3528
4628
|
(options) => {
|
|
3529
4629
|
run({
|
|
3530
4630
|
agent: options.agent,
|
|
@@ -3532,7 +4632,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3532
4632
|
verbose: options.verbose,
|
|
3533
4633
|
conversation: options.conversation,
|
|
3534
4634
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3535
|
-
json: options.json
|
|
4635
|
+
json: options.json,
|
|
4636
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4637
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4638
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4639
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3536
4640
|
});
|
|
3537
4641
|
}
|
|
3538
4642
|
);
|