@evident-ai/cli 3.0.1-dev.892519e → 3.0.1-dev.8e92c69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1734 -104
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve2();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -471,12 +471,6 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
-
// ../../packages/types/src/opencode/index.ts
|
|
475
|
-
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
-
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
-
return `msg_${sanitized}`;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
474
|
// ../../packages/types/src/telemetry/index.ts
|
|
481
475
|
var TelemetryEventTypes = {
|
|
482
476
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -515,7 +509,10 @@ function stripQuery(url) {
|
|
|
515
509
|
}
|
|
516
510
|
|
|
517
511
|
// src/lib/telemetry.ts
|
|
518
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
519
516
|
var eventBuffer = [];
|
|
520
517
|
var flushTimeout = null;
|
|
521
518
|
var isShuttingDown = false;
|
|
@@ -709,13 +706,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
709
706
|
if (health.healthy) {
|
|
710
707
|
return health;
|
|
711
708
|
}
|
|
712
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
713
710
|
}
|
|
714
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
715
712
|
}
|
|
716
713
|
|
|
717
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
718
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
719
716
|
function isQueueValidatedVersion(version2) {
|
|
720
717
|
if (!version2) return false;
|
|
721
718
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1039,7 +1036,11 @@ function roleOf(m) {
|
|
|
1039
1036
|
}
|
|
1040
1037
|
function completedOf(m) {
|
|
1041
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1042
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1043
1044
|
}
|
|
1044
1045
|
function idOf(m) {
|
|
1045
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1059,6 +1060,102 @@ function finishOf(m) {
|
|
|
1059
1060
|
const infoFinish = m.info?.finish;
|
|
1060
1061
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1061
1062
|
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1070
|
+
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1062
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1063
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1064
1161
|
if (directory && directory.trim()) {
|
|
@@ -1076,9 +1173,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1076
1173
|
const data = await response.json();
|
|
1077
1174
|
return data.id;
|
|
1078
1175
|
}
|
|
1079
|
-
|
|
1176
|
+
function messageText(m) {
|
|
1177
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
+
}
|
|
1180
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
+
const knownUserIds = new Set(
|
|
1183
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
+
);
|
|
1080
1185
|
const body = {
|
|
1081
|
-
messageID: messageId,
|
|
1082
1186
|
parts: [{ type: "text", text: content }]
|
|
1083
1187
|
};
|
|
1084
1188
|
if (options?.agent) {
|
|
@@ -1102,6 +1206,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1102
1206
|
const text = await res.text().catch(() => "");
|
|
1103
1207
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1104
1208
|
}
|
|
1209
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
+
if (after) {
|
|
1214
|
+
let best = null;
|
|
1215
|
+
for (const m of after) {
|
|
1216
|
+
if (roleOf(m) !== "user") continue;
|
|
1217
|
+
const id = idOf(m);
|
|
1218
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
+
if (messageText(m) !== content) continue;
|
|
1220
|
+
const created = createdOf(m) ?? 0;
|
|
1221
|
+
if (best === null || created > best.created) {
|
|
1222
|
+
best = { id, created };
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (best) return best.id;
|
|
1226
|
+
}
|
|
1227
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1105
1232
|
}
|
|
1106
1233
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1107
1234
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1118,19 +1245,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1118
1245
|
}
|
|
1119
1246
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1120
1247
|
if (!messages || messages.length === 0) return null;
|
|
1248
|
+
let lastCorrelated = null;
|
|
1249
|
+
let lastNonErrored = null;
|
|
1121
1250
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1122
1251
|
const m = messages[i];
|
|
1123
|
-
if (roleOf(m)
|
|
1252
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1253
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1254
|
+
if (errorOf(m) == null) {
|
|
1255
|
+
lastNonErrored = m;
|
|
1256
|
+
break;
|
|
1257
|
+
}
|
|
1124
1258
|
}
|
|
1259
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1125
1260
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1126
1261
|
if (userIndex === -1) return null;
|
|
1127
1262
|
let last = null;
|
|
1263
|
+
let lastOk = null;
|
|
1128
1264
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1129
1265
|
const role = roleOf(messages[i]);
|
|
1130
1266
|
if (role === "user") break;
|
|
1131
|
-
if (role === "assistant")
|
|
1267
|
+
if (role === "assistant") {
|
|
1268
|
+
last = messages[i];
|
|
1269
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1270
|
+
}
|
|
1132
1271
|
}
|
|
1133
|
-
return last;
|
|
1272
|
+
return lastOk ?? last;
|
|
1134
1273
|
}
|
|
1135
1274
|
function messageRunState(messages, userMessageId) {
|
|
1136
1275
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1140,12 +1279,136 @@ function messageRunState(messages, userMessageId) {
|
|
|
1140
1279
|
if (!reply) return "unknown";
|
|
1141
1280
|
}
|
|
1142
1281
|
if (!reply) return "queued";
|
|
1143
|
-
if (
|
|
1144
|
-
|
|
1145
|
-
|
|
1282
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1283
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1284
|
+
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1290
|
+
function messageError(messages, userMessageId) {
|
|
1291
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1292
|
+
const error2 = errorOf(reply);
|
|
1293
|
+
if (error2 == null) return null;
|
|
1294
|
+
if (typeof error2 === "string") return error2;
|
|
1295
|
+
if (typeof error2 === "object") {
|
|
1296
|
+
const e = error2;
|
|
1297
|
+
const dataMessage = e.data?.message;
|
|
1298
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1299
|
+
if (typeof e.message === "string") return e.message;
|
|
1300
|
+
}
|
|
1301
|
+
return "The agent run failed.";
|
|
1302
|
+
}
|
|
1303
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1304
|
+
if (!messages || messages.length === 0) return false;
|
|
1305
|
+
return messages.some(
|
|
1306
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1307
|
+
);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
return toDelete;
|
|
1356
|
+
}
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1146
1360
|
}
|
|
1147
|
-
function
|
|
1148
|
-
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1369
|
+
}
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
+
}
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1149
1412
|
}
|
|
1150
1413
|
|
|
1151
1414
|
// src/lib/tunnel/connection.ts
|
|
@@ -1224,24 +1487,26 @@ var StreamForwarder = class {
|
|
|
1224
1487
|
this.send({ type: "res_end", sid });
|
|
1225
1488
|
return;
|
|
1226
1489
|
}
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1233
1498
|
const ac = new AbortController();
|
|
1234
1499
|
let bodyPromise;
|
|
1235
1500
|
let pushBody;
|
|
1236
1501
|
let endBody;
|
|
1237
1502
|
if (has_body) {
|
|
1238
1503
|
const chunks = [];
|
|
1239
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1240
1505
|
pushBody = (buf) => {
|
|
1241
1506
|
chunks.push(buf);
|
|
1242
1507
|
};
|
|
1243
1508
|
endBody = () => {
|
|
1244
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1245
1510
|
};
|
|
1246
1511
|
});
|
|
1247
1512
|
}
|
|
@@ -1276,12 +1541,14 @@ var StreamForwarder = class {
|
|
|
1276
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1277
1542
|
});
|
|
1278
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1285
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1286
1553
|
try {
|
|
1287
1554
|
if (upstream.body) {
|
|
@@ -1357,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1357
1624
|
} = options;
|
|
1358
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1359
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1360
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1361
1628
|
const ws = new WebSocket2(url, {
|
|
1362
1629
|
headers: {
|
|
1363
1630
|
Authorization: authHeader
|
|
@@ -1422,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1422
1689
|
clearTimeout(connectionTimeout);
|
|
1423
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1424
1691
|
onConnected?.(connectedAgentId);
|
|
1425
|
-
|
|
1692
|
+
resolve2({
|
|
1426
1693
|
ws,
|
|
1427
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1428
1695
|
});
|
|
@@ -1551,7 +1818,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1551
1818
|
};
|
|
1552
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1553
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1554
|
-
var
|
|
1821
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1555
1825
|
var ChannelAuthError = class extends Error {
|
|
1556
1826
|
constructor(message) {
|
|
1557
1827
|
super(message);
|
|
@@ -1586,10 +1856,18 @@ var ChannelDriver = class {
|
|
|
1586
1856
|
sleep;
|
|
1587
1857
|
pausedPollIntervalMs;
|
|
1588
1858
|
pausedMaxWaitMs;
|
|
1589
|
-
|
|
1859
|
+
stuckQueuedMs;
|
|
1590
1860
|
now;
|
|
1591
1861
|
/** Cache of conversationId → opencode sessionId. */
|
|
1592
1862
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
+
/**
|
|
1864
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
+
*/
|
|
1870
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1593
1871
|
/**
|
|
1594
1872
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1595
1873
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1606,6 +1884,63 @@ var ChannelDriver = class {
|
|
|
1606
1884
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1607
1885
|
*/
|
|
1608
1886
|
dispatched = /* @__PURE__ */ new Set();
|
|
1887
|
+
/**
|
|
1888
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1889
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1890
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1891
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1892
|
+
* the processing list.
|
|
1893
|
+
*/
|
|
1894
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1895
|
+
/**
|
|
1896
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1897
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1898
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1899
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1900
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1901
|
+
*
|
|
1902
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1903
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1904
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1905
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1906
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1907
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1908
|
+
*/
|
|
1909
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1910
|
+
/**
|
|
1911
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1912
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1913
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1914
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1915
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1916
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1917
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1918
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1919
|
+
*/
|
|
1920
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
+
*/
|
|
1929
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1930
|
+
/**
|
|
1931
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1932
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1933
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1934
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1935
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1936
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1937
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1938
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1939
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1940
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1941
|
+
* so the NEXT tick may retry exactly once more).
|
|
1942
|
+
*/
|
|
1943
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1609
1944
|
/**
|
|
1610
1945
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1611
1946
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1613,8 +1948,43 @@ var ChannelDriver = class {
|
|
|
1613
1948
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1614
1949
|
*/
|
|
1615
1950
|
opencodeDirectory = void 0;
|
|
1951
|
+
/**
|
|
1952
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1953
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1954
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1955
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1956
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1957
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1958
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1959
|
+
*/
|
|
1960
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1961
|
+
/**
|
|
1962
|
+
* Per-session OpenCode title cache (#310), keyed by sessionId. Only a resolved
|
|
1963
|
+
* NON-EMPTY name is stored (terminal — a real session name won't later un-name),
|
|
1964
|
+
* so we do NOT re-GET `/session/:id` every tick. A missing entry = not yet
|
|
1965
|
+
* resolved OR resolved-but-still-empty → re-fetch on next need, since OpenCode
|
|
1966
|
+
* names sessions asynchronously mid-turn. Driver-level (not per-watcher) so both
|
|
1967
|
+
* the watcher completion path AND the restart-recovery re-adopt path (which has
|
|
1968
|
+
* no watcher) can resolve the title.
|
|
1969
|
+
*/
|
|
1970
|
+
sessionTitles = /* @__PURE__ */ new Map();
|
|
1616
1971
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1617
1972
|
draining = false;
|
|
1973
|
+
/**
|
|
1974
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1975
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1976
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1977
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1978
|
+
*/
|
|
1979
|
+
activeDrain = null;
|
|
1980
|
+
/**
|
|
1981
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1982
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1983
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1984
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1985
|
+
* and stops opencode.
|
|
1986
|
+
*/
|
|
1987
|
+
stopped = false;
|
|
1618
1988
|
constructor(config2) {
|
|
1619
1989
|
this.agentId = config2.agentId;
|
|
1620
1990
|
this.port = config2.port;
|
|
@@ -1628,7 +1998,7 @@ var ChannelDriver = class {
|
|
|
1628
1998
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1629
1999
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1630
2000
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1631
|
-
this.
|
|
2001
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1632
2002
|
this.now = config2.now ?? (() => Date.now());
|
|
1633
2003
|
}
|
|
1634
2004
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1646,8 +2016,21 @@ var ChannelDriver = class {
|
|
|
1646
2016
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1647
2017
|
*/
|
|
1648
2018
|
async drainPending() {
|
|
2019
|
+
if (this.stopped) return 0;
|
|
1649
2020
|
if (this.draining) return 0;
|
|
1650
2021
|
this.draining = true;
|
|
2022
|
+
const run2 = this.runDrain();
|
|
2023
|
+
this.activeDrain = run2.then(
|
|
2024
|
+
() => {
|
|
2025
|
+
this.activeDrain = null;
|
|
2026
|
+
},
|
|
2027
|
+
() => {
|
|
2028
|
+
this.activeDrain = null;
|
|
2029
|
+
}
|
|
2030
|
+
);
|
|
2031
|
+
return run2;
|
|
2032
|
+
}
|
|
2033
|
+
async runDrain() {
|
|
1651
2034
|
let dispatched = 0;
|
|
1652
2035
|
try {
|
|
1653
2036
|
const conversations = await this.getPendingConversations();
|
|
@@ -1659,8 +2042,10 @@ var ChannelDriver = class {
|
|
|
1659
2042
|
});
|
|
1660
2043
|
}
|
|
1661
2044
|
for (const conv of conversations) {
|
|
2045
|
+
if (this.stopped) break;
|
|
1662
2046
|
dispatched += await this.processConversation(conv);
|
|
1663
2047
|
}
|
|
2048
|
+
await this.readoptProcessing();
|
|
1664
2049
|
} finally {
|
|
1665
2050
|
this.draining = false;
|
|
1666
2051
|
}
|
|
@@ -1678,6 +2063,73 @@ var ChannelDriver = class {
|
|
|
1678
2063
|
}
|
|
1679
2064
|
return false;
|
|
1680
2065
|
}
|
|
2066
|
+
/**
|
|
2067
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2068
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2069
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2070
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2071
|
+
*
|
|
2072
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2073
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2074
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2075
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2076
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2077
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2078
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2079
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2080
|
+
*/
|
|
2081
|
+
protectedSessionIds() {
|
|
2082
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2083
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2084
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2085
|
+
}
|
|
2086
|
+
return ids;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2090
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2091
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2092
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2093
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2094
|
+
*/
|
|
2095
|
+
stop() {
|
|
2096
|
+
this.stopped = true;
|
|
2097
|
+
}
|
|
2098
|
+
/**
|
|
2099
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2100
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2101
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2102
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2103
|
+
*
|
|
2104
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2105
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2106
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2107
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2108
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2109
|
+
* next runner start (ADR-0046).
|
|
2110
|
+
*
|
|
2111
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2112
|
+
* timeout elapsed with work still in flight.
|
|
2113
|
+
*/
|
|
2114
|
+
async waitForInFlight(timeoutMs) {
|
|
2115
|
+
const deadline = this.now() + timeoutMs;
|
|
2116
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2117
|
+
if (this.activeDrain) {
|
|
2118
|
+
let drainSettled = false;
|
|
2119
|
+
void this.activeDrain.then(() => {
|
|
2120
|
+
drainSettled = true;
|
|
2121
|
+
});
|
|
2122
|
+
while (!drainSettled) {
|
|
2123
|
+
if (this.now() >= deadline) return false;
|
|
2124
|
+
await this.sleep(step);
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
while (this.hasInFlightWatchers()) {
|
|
2128
|
+
if (this.now() >= deadline) return false;
|
|
2129
|
+
await this.sleep(step);
|
|
2130
|
+
}
|
|
2131
|
+
return true;
|
|
2132
|
+
}
|
|
1681
2133
|
/**
|
|
1682
2134
|
* Await all outstanding per-session watchers (WI-3).
|
|
1683
2135
|
*
|
|
@@ -1714,15 +2166,16 @@ var ChannelDriver = class {
|
|
|
1714
2166
|
let dispatched = 0;
|
|
1715
2167
|
let skippedAlreadyDispatched = 0;
|
|
1716
2168
|
for (const message of messages) {
|
|
2169
|
+
if (this.stopped) break;
|
|
1717
2170
|
if (this.dispatched.has(message.id)) {
|
|
1718
2171
|
skippedAlreadyDispatched += 1;
|
|
1719
2172
|
continue;
|
|
1720
2173
|
}
|
|
1721
|
-
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1722
2174
|
const options = {
|
|
1723
2175
|
agent: message.opencode_agent ?? void 0,
|
|
1724
2176
|
model: message.opencode_model ?? void 0
|
|
1725
2177
|
};
|
|
2178
|
+
let opencodeMessageId;
|
|
1726
2179
|
try {
|
|
1727
2180
|
this.log({
|
|
1728
2181
|
level: "info",
|
|
@@ -1730,10 +2183,23 @@ var ChannelDriver = class {
|
|
|
1730
2183
|
conversation_id: conv.id,
|
|
1731
2184
|
message_id: message.id
|
|
1732
2185
|
});
|
|
1733
|
-
await
|
|
2186
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2187
|
+
sessionId,
|
|
2188
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2189
|
+
);
|
|
1734
2190
|
} catch (err) {
|
|
1735
2191
|
if (err instanceof ChannelAuthError) throw err;
|
|
1736
2192
|
this.dispatched.delete(message.id);
|
|
2193
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2194
|
+
this.sessions.delete(conv.id);
|
|
2195
|
+
this.log({
|
|
2196
|
+
level: "info",
|
|
2197
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2198
|
+
conversation_id: conv.id,
|
|
2199
|
+
message_id: message.id
|
|
2200
|
+
});
|
|
2201
|
+
break;
|
|
2202
|
+
}
|
|
1737
2203
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1738
2204
|
});
|
|
1739
2205
|
this.log({
|
|
@@ -1744,9 +2210,19 @@ var ChannelDriver = class {
|
|
|
1744
2210
|
});
|
|
1745
2211
|
continue;
|
|
1746
2212
|
}
|
|
2213
|
+
if (opencodeMessageId === null) {
|
|
2214
|
+
this.log({
|
|
2215
|
+
level: "error",
|
|
2216
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2217
|
+
conversation_id: conv.id,
|
|
2218
|
+
message_id: message.id
|
|
2219
|
+
});
|
|
2220
|
+
continue;
|
|
2221
|
+
}
|
|
1747
2222
|
this.dispatched.add(message.id);
|
|
1748
2223
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1749
2224
|
dispatched += 1;
|
|
2225
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1750
2226
|
}
|
|
1751
2227
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1752
2228
|
this.log({
|
|
@@ -1759,16 +2235,33 @@ var ChannelDriver = class {
|
|
|
1759
2235
|
return dispatched;
|
|
1760
2236
|
}
|
|
1761
2237
|
async ensureSession(conv) {
|
|
1762
|
-
const
|
|
1763
|
-
if (
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
2238
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2239
|
+
if (bound) {
|
|
2240
|
+
const exists = await sessionExists(this.port, bound);
|
|
2241
|
+
if (exists === false) {
|
|
2242
|
+
this.log({
|
|
2243
|
+
level: "info",
|
|
2244
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2245
|
+
conversation_id: conv.id
|
|
2246
|
+
});
|
|
2247
|
+
this.sessions.delete(conv.id);
|
|
2248
|
+
return this.createAndBindSession(conv.id);
|
|
2249
|
+
}
|
|
2250
|
+
this.sessions.set(conv.id, bound);
|
|
2251
|
+
return bound;
|
|
1767
2252
|
}
|
|
2253
|
+
return this.createAndBindSession(conv.id);
|
|
2254
|
+
}
|
|
2255
|
+
/**
|
|
2256
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2257
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2258
|
+
* self-heal recreate path in `ensureSession`.
|
|
2259
|
+
*/
|
|
2260
|
+
async createAndBindSession(conversationId) {
|
|
1768
2261
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1769
2262
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1770
|
-
this.sessions.set(
|
|
1771
|
-
await this.persistSession(
|
|
2263
|
+
this.sessions.set(conversationId, sessionId);
|
|
2264
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1772
2265
|
});
|
|
1773
2266
|
return sessionId;
|
|
1774
2267
|
}
|
|
@@ -1791,6 +2284,25 @@ var ChannelDriver = class {
|
|
|
1791
2284
|
// -------------------------------------------------------------------------
|
|
1792
2285
|
// Per-session watcher (WI-3)
|
|
1793
2286
|
// -------------------------------------------------------------------------
|
|
2287
|
+
/**
|
|
2288
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2289
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2290
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2291
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2292
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2293
|
+
*/
|
|
2294
|
+
dispatchLocked(sessionId, fn) {
|
|
2295
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2296
|
+
const run2 = prior.then(fn, fn);
|
|
2297
|
+
this.sessionDispatchLocks.set(
|
|
2298
|
+
sessionId,
|
|
2299
|
+
run2.then(
|
|
2300
|
+
() => void 0,
|
|
2301
|
+
() => void 0
|
|
2302
|
+
)
|
|
2303
|
+
);
|
|
2304
|
+
return run2;
|
|
2305
|
+
}
|
|
1794
2306
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1795
2307
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1796
2308
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1800,7 +2312,9 @@ var ChannelDriver = class {
|
|
|
1800
2312
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1801
2313
|
loop: null,
|
|
1802
2314
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1803
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2315
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2316
|
+
lastGoodPollAt: this.now(),
|
|
2317
|
+
hadUsablePoll: false
|
|
1804
2318
|
};
|
|
1805
2319
|
this.watchers.set(sessionId, watcher);
|
|
1806
2320
|
}
|
|
@@ -1810,9 +2324,92 @@ var ChannelDriver = class {
|
|
|
1810
2324
|
opencodeMessageId,
|
|
1811
2325
|
message,
|
|
1812
2326
|
dispatchedAt: now,
|
|
2327
|
+
processingAnchorMs: now,
|
|
1813
2328
|
deadline: now + this.pausedMaxWaitMs,
|
|
1814
2329
|
started: false,
|
|
1815
|
-
done: false
|
|
2330
|
+
done: false,
|
|
2331
|
+
stuckReported: false,
|
|
2332
|
+
lastAliveAt: 0,
|
|
2333
|
+
aliveInFlight: false,
|
|
2334
|
+
awaitingHumanLatched: false,
|
|
2335
|
+
pausedOnQuestion: false,
|
|
2336
|
+
pausedOnPermission: false,
|
|
2337
|
+
pausedClearConfirmed: false,
|
|
2338
|
+
pausedInFlight: false,
|
|
2339
|
+
deliveryDeadlineAnchored: false
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
/**
|
|
2343
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2344
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2345
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2346
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2347
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2348
|
+
*
|
|
2349
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2350
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2351
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2352
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2353
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2354
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2355
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2356
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2357
|
+
* (only the appear-guard uses it).
|
|
2358
|
+
*
|
|
2359
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2360
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
2361
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
2362
|
+
*
|
|
2363
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
2364
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
2365
|
+
* fire from the watcher's normal branches.
|
|
2366
|
+
*/
|
|
2367
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2368
|
+
let watcher = this.watchers.get(sessionId);
|
|
2369
|
+
if (!watcher) {
|
|
2370
|
+
watcher = {
|
|
2371
|
+
conv,
|
|
2372
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2373
|
+
loop: null,
|
|
2374
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2375
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2376
|
+
lastGoodPollAt: this.now(),
|
|
2377
|
+
hadUsablePoll: false
|
|
2378
|
+
};
|
|
2379
|
+
this.watchers.set(sessionId, watcher);
|
|
2380
|
+
}
|
|
2381
|
+
watcher.inFlight.set(message.id, {
|
|
2382
|
+
evidentMessageId: message.id,
|
|
2383
|
+
opencodeMessageId,
|
|
2384
|
+
message,
|
|
2385
|
+
dispatchedAt: this.now(),
|
|
2386
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2387
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2388
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2389
|
+
processingAnchorMs: processedAtMs,
|
|
2390
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2391
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2392
|
+
started: true,
|
|
2393
|
+
done: false,
|
|
2394
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2395
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2396
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2397
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2398
|
+
// (#210/#220 observability).
|
|
2399
|
+
stuckReported: false,
|
|
2400
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2401
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2402
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2403
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2404
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2405
|
+
lastAliveAt: 0,
|
|
2406
|
+
aliveInFlight: false,
|
|
2407
|
+
awaitingHumanLatched: false,
|
|
2408
|
+
pausedOnQuestion: false,
|
|
2409
|
+
pausedOnPermission: false,
|
|
2410
|
+
pausedClearConfirmed: false,
|
|
2411
|
+
pausedInFlight: false,
|
|
2412
|
+
deliveryDeadlineAnchored: false
|
|
1816
2413
|
});
|
|
1817
2414
|
}
|
|
1818
2415
|
/**
|
|
@@ -1863,12 +2460,30 @@ var ChannelDriver = class {
|
|
|
1863
2460
|
messages = Array.isArray(body) ? body : null;
|
|
1864
2461
|
}
|
|
1865
2462
|
} catch {
|
|
1866
|
-
continue;
|
|
1867
2463
|
}
|
|
2464
|
+
if (messages != null && messages.length > 0) {
|
|
2465
|
+
watcher.lastGoodPollAt = this.now();
|
|
2466
|
+
watcher.hadUsablePoll = true;
|
|
2467
|
+
} else {
|
|
2468
|
+
const emptyButReachable = messages != null;
|
|
2469
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2470
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2471
|
+
continue;
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
1868
2475
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
1869
|
-
await this.serviceInFlightMessage(
|
|
2476
|
+
await this.serviceInFlightMessage(
|
|
2477
|
+
sessionId,
|
|
2478
|
+
watcher,
|
|
2479
|
+
inFlight,
|
|
2480
|
+
messages,
|
|
2481
|
+
openQuestions,
|
|
2482
|
+
openPermissions,
|
|
2483
|
+
questionsPolledOk,
|
|
2484
|
+
permissionsPolledOk
|
|
2485
|
+
);
|
|
1870
2486
|
}
|
|
1871
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
1872
2487
|
}
|
|
1873
2488
|
} catch (err) {
|
|
1874
2489
|
if (err instanceof ChannelAuthError) {
|
|
@@ -1878,6 +2493,7 @@ var ChannelDriver = class {
|
|
|
1878
2493
|
conversation_id: watcher.conv.id
|
|
1879
2494
|
});
|
|
1880
2495
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2496
|
+
this.readopted.delete(evidentMessageId);
|
|
1881
2497
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1882
2498
|
}
|
|
1883
2499
|
return;
|
|
@@ -1889,19 +2505,51 @@ var ChannelDriver = class {
|
|
|
1889
2505
|
});
|
|
1890
2506
|
}
|
|
1891
2507
|
}
|
|
2508
|
+
/**
|
|
2509
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2510
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2511
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2512
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2513
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2514
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2515
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2516
|
+
* or past now, so a still-ample window is left untouched.
|
|
2517
|
+
*/
|
|
2518
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2519
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2520
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2521
|
+
if (this.now() >= inFlight.deadline) {
|
|
2522
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
1892
2525
|
/**
|
|
1893
2526
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
1894
2527
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
1895
2528
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
1896
2529
|
* in-flight set on completion or timeout.
|
|
1897
2530
|
*/
|
|
1898
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2531
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
1899
2532
|
const conv = watcher.conv;
|
|
1900
2533
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1901
|
-
|
|
2534
|
+
const id = inFlight.evidentMessageId;
|
|
2535
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2536
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2537
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2538
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2539
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2540
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2541
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2542
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2543
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
1902
2544
|
let claimed;
|
|
1903
2545
|
try {
|
|
1904
|
-
claimed = await this.markProcessing(
|
|
2546
|
+
claimed = await this.markProcessing(
|
|
2547
|
+
conv.id,
|
|
2548
|
+
inFlight.evidentMessageId,
|
|
2549
|
+
sessionId,
|
|
2550
|
+
inFlight.opencodeMessageId,
|
|
2551
|
+
title
|
|
2552
|
+
);
|
|
1905
2553
|
} catch (err) {
|
|
1906
2554
|
if (err instanceof ChannelAuthError) throw err;
|
|
1907
2555
|
this.log({
|
|
@@ -1923,6 +2571,7 @@ var ChannelDriver = class {
|
|
|
1923
2571
|
}
|
|
1924
2572
|
}
|
|
1925
2573
|
if (state === "done") {
|
|
2574
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
1926
2575
|
if (!inFlight.done) {
|
|
1927
2576
|
this.log({
|
|
1928
2577
|
level: "info",
|
|
@@ -1930,8 +2579,15 @@ var ChannelDriver = class {
|
|
|
1930
2579
|
conversation_id: conv.id,
|
|
1931
2580
|
message_id: inFlight.evidentMessageId
|
|
1932
2581
|
});
|
|
2582
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
1933
2583
|
try {
|
|
1934
|
-
await this.markDone(
|
|
2584
|
+
await this.markDone(
|
|
2585
|
+
conv.id,
|
|
2586
|
+
inFlight.evidentMessageId,
|
|
2587
|
+
sessionId,
|
|
2588
|
+
inFlight.opencodeMessageId,
|
|
2589
|
+
title
|
|
2590
|
+
);
|
|
1935
2591
|
} catch (err) {
|
|
1936
2592
|
if (err instanceof ChannelAuthError) throw err;
|
|
1937
2593
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -1967,60 +2623,583 @@ var ChannelDriver = class {
|
|
|
1967
2623
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1968
2624
|
return;
|
|
1969
2625
|
}
|
|
1970
|
-
if (state === "
|
|
1971
|
-
|
|
1972
|
-
|
|
2626
|
+
if (state === "failed") {
|
|
2627
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2628
|
+
if (!inFlight.done) {
|
|
2629
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2630
|
+
this.log({
|
|
2631
|
+
level: "error",
|
|
2632
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2633
|
+
conversation_id: conv.id,
|
|
2634
|
+
message_id: inFlight.evidentMessageId
|
|
2635
|
+
});
|
|
2636
|
+
try {
|
|
2637
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2638
|
+
} catch (err) {
|
|
2639
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2640
|
+
if (err instanceof ChannelTerminalError) {
|
|
2641
|
+
this.log({
|
|
2642
|
+
level: "error",
|
|
2643
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2644
|
+
conversation_id: conv.id,
|
|
2645
|
+
message_id: inFlight.evidentMessageId
|
|
2646
|
+
});
|
|
2647
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2648
|
+
return;
|
|
2649
|
+
}
|
|
2650
|
+
if (this.now() >= inFlight.deadline) {
|
|
2651
|
+
this.log({
|
|
2652
|
+
level: "error",
|
|
2653
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2654
|
+
conversation_id: conv.id,
|
|
2655
|
+
message_id: inFlight.evidentMessageId
|
|
2656
|
+
});
|
|
2657
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
this.log({
|
|
2661
|
+
level: "error",
|
|
2662
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2663
|
+
conversation_id: conv.id,
|
|
2664
|
+
message_id: inFlight.evidentMessageId
|
|
2665
|
+
});
|
|
2666
|
+
return;
|
|
2667
|
+
}
|
|
2668
|
+
inFlight.done = true;
|
|
1973
2669
|
}
|
|
2670
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2671
|
+
return;
|
|
1974
2672
|
}
|
|
1975
|
-
|
|
2673
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2674
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2675
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2676
|
+
inFlight.stuckReported = true;
|
|
2677
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2678
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2679
|
+
});
|
|
2680
|
+
}
|
|
2681
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2682
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
1976
2683
|
this.log({
|
|
1977
|
-
level: "
|
|
1978
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2684
|
+
level: "error",
|
|
2685
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
1979
2686
|
conversation_id: conv.id,
|
|
1980
2687
|
message_id: inFlight.evidentMessageId
|
|
1981
2688
|
});
|
|
2689
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2690
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2691
|
+
});
|
|
2692
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2693
|
+
return;
|
|
2694
|
+
}
|
|
2695
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2696
|
+
inFlight.aliveInFlight = true;
|
|
2697
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2698
|
+
inFlight.aliveInFlight = false;
|
|
2699
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2700
|
+
});
|
|
2701
|
+
}
|
|
2702
|
+
if (awaitingHuman) {
|
|
2703
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2704
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2705
|
+
inFlight.awaitingHumanLatched = true;
|
|
2706
|
+
}
|
|
2707
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2708
|
+
inFlight.pausedInFlight = true;
|
|
2709
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2710
|
+
inFlight.pausedInFlight = false;
|
|
2711
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2712
|
+
});
|
|
2713
|
+
}
|
|
2714
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2715
|
+
inFlight.awaitingHumanLatched = false;
|
|
2716
|
+
inFlight.pausedOnQuestion = false;
|
|
2717
|
+
inFlight.pausedOnPermission = false;
|
|
2718
|
+
inFlight.pausedClearConfirmed = false;
|
|
2719
|
+
}
|
|
2720
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2721
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2722
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2723
|
+
);
|
|
2724
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2725
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2726
|
+
this.log({
|
|
2727
|
+
level: "info",
|
|
2728
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2729
|
+
conversation_id: conv.id,
|
|
2730
|
+
message_id: inFlight.evidentMessageId
|
|
2731
|
+
});
|
|
2732
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2733
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2734
|
+
});
|
|
1982
2735
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1983
2736
|
}
|
|
1984
2737
|
}
|
|
2738
|
+
// -------------------------------------------------------------------------
|
|
2739
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2740
|
+
// -------------------------------------------------------------------------
|
|
1985
2741
|
/**
|
|
1986
|
-
* Re-
|
|
1987
|
-
*
|
|
1988
|
-
*
|
|
1989
|
-
* the
|
|
2742
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2743
|
+
*
|
|
2744
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2745
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2746
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2747
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2748
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2749
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2750
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2751
|
+
*
|
|
2752
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2753
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
1990
2754
|
*/
|
|
1991
|
-
async
|
|
2755
|
+
async readoptProcessing() {
|
|
2756
|
+
const rows = await this.getProcessingMessages();
|
|
2757
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2758
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2759
|
+
for (const id of [
|
|
2760
|
+
...this.dontRedispatch,
|
|
2761
|
+
...this.doneUndeliverable,
|
|
2762
|
+
...this.readoptPollUnresolvedSignalled
|
|
2763
|
+
]) {
|
|
2764
|
+
if (!stillProcessing.has(id)) {
|
|
2765
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2766
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2767
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2768
|
+
if (cleared || clearedUndeliverable) {
|
|
2769
|
+
this.log({
|
|
2770
|
+
level: "info",
|
|
2771
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2772
|
+
message_id: id
|
|
2773
|
+
});
|
|
2774
|
+
}
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
if (rows.length === 0) return;
|
|
2779
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2780
|
+
for (const row of rows) {
|
|
2781
|
+
if (!row.opencode_session_id) {
|
|
2782
|
+
this.log({
|
|
2783
|
+
level: "error",
|
|
2784
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2785
|
+
conversation_id: row.conversation_id,
|
|
2786
|
+
message_id: row.id
|
|
2787
|
+
});
|
|
2788
|
+
continue;
|
|
2789
|
+
}
|
|
2790
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2791
|
+
list.push(row);
|
|
2792
|
+
bySession.set(row.opencode_session_id, list);
|
|
2793
|
+
}
|
|
2794
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2795
|
+
let messages;
|
|
2796
|
+
try {
|
|
2797
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2798
|
+
if (!res.ok) {
|
|
2799
|
+
this.log({
|
|
2800
|
+
level: "error",
|
|
2801
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2802
|
+
});
|
|
2803
|
+
continue;
|
|
2804
|
+
}
|
|
2805
|
+
const body = await res.json();
|
|
2806
|
+
if (!Array.isArray(body)) {
|
|
2807
|
+
this.log({
|
|
2808
|
+
level: "error",
|
|
2809
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2810
|
+
});
|
|
2811
|
+
continue;
|
|
2812
|
+
}
|
|
2813
|
+
messages = body;
|
|
2814
|
+
} catch (err) {
|
|
2815
|
+
this.log({
|
|
2816
|
+
level: "error",
|
|
2817
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2818
|
+
});
|
|
2819
|
+
continue;
|
|
2820
|
+
}
|
|
2821
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2822
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2823
|
+
for (const row of sessionRows) {
|
|
2824
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
}
|
|
2828
|
+
/**
|
|
2829
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2830
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2831
|
+
*
|
|
2832
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2833
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2834
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2835
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2836
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2837
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2838
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2839
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2840
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2841
|
+
* tracking the stored id so the reply correlates by it;
|
|
2842
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2843
|
+
*
|
|
2844
|
+
* Only `ChannelAuthError` propagates.
|
|
2845
|
+
*/
|
|
2846
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2847
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2848
|
+
this.log({
|
|
2849
|
+
level: "info",
|
|
2850
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2851
|
+
conversation_id: row.conversation_id,
|
|
2852
|
+
message_id: row.id
|
|
2853
|
+
});
|
|
2854
|
+
return;
|
|
2855
|
+
}
|
|
2856
|
+
const ocId = row.opencode_message_id;
|
|
2857
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2858
|
+
if (state === "done") {
|
|
2859
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2860
|
+
this.log({
|
|
2861
|
+
level: "info",
|
|
2862
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2863
|
+
conversation_id: row.conversation_id,
|
|
2864
|
+
message_id: row.id
|
|
2865
|
+
});
|
|
2866
|
+
return;
|
|
2867
|
+
}
|
|
2868
|
+
this.log({
|
|
2869
|
+
level: "info",
|
|
2870
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2871
|
+
conversation_id: row.conversation_id,
|
|
2872
|
+
message_id: row.id
|
|
2873
|
+
});
|
|
2874
|
+
try {
|
|
2875
|
+
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
2876
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId, title);
|
|
2877
|
+
} catch (err) {
|
|
2878
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2879
|
+
if (err instanceof ChannelTerminalError) {
|
|
2880
|
+
this.doneUndeliverable.add(row.id);
|
|
2881
|
+
this.log({
|
|
2882
|
+
level: "error",
|
|
2883
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2884
|
+
conversation_id: row.conversation_id,
|
|
2885
|
+
message_id: row.id
|
|
2886
|
+
});
|
|
2887
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2888
|
+
return;
|
|
2889
|
+
}
|
|
2890
|
+
this.log({
|
|
2891
|
+
level: "error",
|
|
2892
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2893
|
+
conversation_id: row.conversation_id,
|
|
2894
|
+
message_id: row.id
|
|
2895
|
+
});
|
|
2896
|
+
return;
|
|
2897
|
+
}
|
|
2898
|
+
this.dontRedispatch.delete(row.id);
|
|
2899
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2900
|
+
return;
|
|
2901
|
+
}
|
|
2902
|
+
if (state === "failed") {
|
|
2903
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2904
|
+
this.log({
|
|
2905
|
+
level: "error",
|
|
2906
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2907
|
+
conversation_id: row.conversation_id,
|
|
2908
|
+
message_id: row.id
|
|
2909
|
+
});
|
|
2910
|
+
try {
|
|
2911
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2912
|
+
} catch (err) {
|
|
2913
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2914
|
+
if (err instanceof ChannelTerminalError) {
|
|
2915
|
+
this.doneUndeliverable.add(row.id);
|
|
2916
|
+
this.log({
|
|
2917
|
+
level: "error",
|
|
2918
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2919
|
+
conversation_id: row.conversation_id,
|
|
2920
|
+
message_id: row.id
|
|
2921
|
+
});
|
|
2922
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2923
|
+
return;
|
|
2924
|
+
}
|
|
2925
|
+
this.log({
|
|
2926
|
+
level: "error",
|
|
2927
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2928
|
+
conversation_id: row.conversation_id,
|
|
2929
|
+
message_id: row.id
|
|
2930
|
+
});
|
|
2931
|
+
return;
|
|
2932
|
+
}
|
|
2933
|
+
this.dontRedispatch.delete(row.id);
|
|
2934
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2935
|
+
return;
|
|
2936
|
+
}
|
|
2937
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
return;
|
|
2945
|
+
}
|
|
2946
|
+
let statusReadableOngoing = null;
|
|
2947
|
+
if (state === "running" && ocId) {
|
|
2948
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2949
|
+
const shape = this.replyCompletionShape(reply);
|
|
2950
|
+
const ongoing = sessionOngoing;
|
|
2951
|
+
statusReadableOngoing = ongoing;
|
|
2952
|
+
if (ongoing === false) {
|
|
2953
|
+
this.log({
|
|
2954
|
+
level: "info",
|
|
2955
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2956
|
+
conversation_id: row.conversation_id,
|
|
2957
|
+
message_id: row.id
|
|
2958
|
+
});
|
|
2959
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2960
|
+
return;
|
|
2961
|
+
}
|
|
2962
|
+
if (ongoing === true) {
|
|
2963
|
+
this.log({
|
|
2964
|
+
level: "info",
|
|
2965
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2966
|
+
conversation_id: row.conversation_id,
|
|
2967
|
+
message_id: row.id
|
|
2968
|
+
});
|
|
2969
|
+
} else {
|
|
2970
|
+
if (shape === "b1") {
|
|
2971
|
+
this.log({
|
|
2972
|
+
level: "info",
|
|
2973
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2974
|
+
conversation_id: row.conversation_id,
|
|
2975
|
+
message_id: row.id
|
|
2976
|
+
});
|
|
2977
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2978
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2979
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2980
|
+
}
|
|
2981
|
+
return;
|
|
2982
|
+
}
|
|
2983
|
+
this.log({
|
|
2984
|
+
level: "info",
|
|
2985
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2986
|
+
conversation_id: row.conversation_id,
|
|
2987
|
+
message_id: row.id
|
|
2988
|
+
});
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2992
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2993
|
+
if (descendantAlive === true) {
|
|
2994
|
+
this.log({
|
|
2995
|
+
level: "info",
|
|
2996
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2997
|
+
conversation_id: row.conversation_id,
|
|
2998
|
+
message_id: row.id
|
|
2999
|
+
});
|
|
3000
|
+
} else {
|
|
3001
|
+
this.log({
|
|
3002
|
+
level: "info",
|
|
3003
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
3004
|
+
conversation_id: row.conversation_id,
|
|
3005
|
+
message_id: row.id
|
|
3006
|
+
});
|
|
3007
|
+
await this.forceReadoptRun(sessionId, row);
|
|
3008
|
+
return;
|
|
3009
|
+
}
|
|
3010
|
+
}
|
|
3011
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
3012
|
+
const conv = this.convForRow(sessionId, row);
|
|
3013
|
+
const message = this.queuedMessageForRow(row);
|
|
3014
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
3015
|
+
this.dispatched.add(row.id);
|
|
3016
|
+
this.readopted.add(row.id);
|
|
3017
|
+
this.ensureWatcherRunning(sessionId);
|
|
3018
|
+
this.log({
|
|
3019
|
+
level: "info",
|
|
3020
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
3021
|
+
conversation_id: row.conversation_id,
|
|
3022
|
+
message_id: row.id
|
|
3023
|
+
});
|
|
3024
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
3025
|
+
return;
|
|
3026
|
+
}
|
|
3027
|
+
await this.forceReadoptRun(sessionId, row);
|
|
3028
|
+
}
|
|
3029
|
+
/**
|
|
3030
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
3031
|
+
*
|
|
3032
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
3033
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
3034
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
3035
|
+
* correlates server-side.
|
|
3036
|
+
*
|
|
3037
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3038
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3039
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3040
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3041
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3042
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3043
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3044
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3045
|
+
* may retry exactly once more).
|
|
3046
|
+
*
|
|
3047
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
3048
|
+
* `processed_at` (Invariant 1).
|
|
3049
|
+
*/
|
|
3050
|
+
async forceReadoptRun(sessionId, row) {
|
|
3051
|
+
if (this.stopped) {
|
|
3052
|
+
this.log({
|
|
3053
|
+
level: "info",
|
|
3054
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3055
|
+
conversation_id: row.conversation_id,
|
|
3056
|
+
message_id: row.id
|
|
3057
|
+
});
|
|
3058
|
+
return;
|
|
3059
|
+
}
|
|
3060
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3061
|
+
this.log({
|
|
3062
|
+
level: "info",
|
|
3063
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3064
|
+
conversation_id: row.conversation_id,
|
|
3065
|
+
message_id: row.id
|
|
3066
|
+
});
|
|
3067
|
+
return;
|
|
3068
|
+
}
|
|
3069
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
3070
|
+
this.dontRedispatch.add(row.id);
|
|
3071
|
+
this.log({
|
|
3072
|
+
level: "info",
|
|
3073
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
3074
|
+
conversation_id: row.conversation_id,
|
|
3075
|
+
message_id: row.id
|
|
3076
|
+
});
|
|
3077
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
3078
|
+
return;
|
|
3079
|
+
}
|
|
1992
3080
|
const options = {
|
|
1993
|
-
agent:
|
|
1994
|
-
model:
|
|
3081
|
+
agent: row.opencode_agent ?? void 0,
|
|
3082
|
+
model: row.opencode_model ?? void 0
|
|
1995
3083
|
};
|
|
1996
3084
|
this.log({
|
|
1997
3085
|
level: "info",
|
|
1998
|
-
message: `
|
|
1999
|
-
|
|
3086
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
3087
|
+
conversation_id: row.conversation_id,
|
|
3088
|
+
message_id: row.id
|
|
2000
3089
|
});
|
|
3090
|
+
this.awaitingReadopt.add(row.id);
|
|
3091
|
+
let ocId;
|
|
2001
3092
|
try {
|
|
2002
|
-
await
|
|
2003
|
-
this.port,
|
|
3093
|
+
ocId = await this.dispatchLocked(
|
|
2004
3094
|
sessionId,
|
|
2005
|
-
|
|
2006
|
-
options,
|
|
2007
|
-
inFlight.opencodeMessageId
|
|
3095
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
2008
3096
|
);
|
|
2009
3097
|
} catch (err) {
|
|
3098
|
+
this.awaitingReadopt.delete(row.id);
|
|
3099
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2010
3100
|
this.log({
|
|
2011
3101
|
level: "error",
|
|
2012
|
-
message: `Re-dispatch failed for message ${
|
|
2013
|
-
|
|
3102
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3103
|
+
conversation_id: row.conversation_id,
|
|
3104
|
+
message_id: row.id
|
|
3105
|
+
});
|
|
3106
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3107
|
+
return;
|
|
3108
|
+
}
|
|
3109
|
+
if (ocId === null) {
|
|
3110
|
+
this.awaitingReadopt.delete(row.id);
|
|
3111
|
+
this.log({
|
|
3112
|
+
level: "error",
|
|
3113
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3114
|
+
conversation_id: row.conversation_id,
|
|
3115
|
+
message_id: row.id
|
|
2014
3116
|
});
|
|
3117
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3118
|
+
return;
|
|
2015
3119
|
}
|
|
2016
|
-
|
|
3120
|
+
const conv = this.convForRow(sessionId, row);
|
|
3121
|
+
const message = this.queuedMessageForRow(row);
|
|
3122
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
3123
|
+
this.dispatched.add(row.id);
|
|
3124
|
+
this.readopted.add(row.id);
|
|
3125
|
+
this.awaitingReadopt.delete(row.id);
|
|
3126
|
+
this.ensureWatcherRunning(sessionId);
|
|
3127
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
3128
|
+
}
|
|
3129
|
+
/**
|
|
3130
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
3131
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
3132
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
3133
|
+
*/
|
|
3134
|
+
isTracked(sessionId, evidentMessageId) {
|
|
3135
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
3136
|
+
const watcher = this.watchers.get(sessionId);
|
|
3137
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
3138
|
+
}
|
|
3139
|
+
/**
|
|
3140
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
3141
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
3142
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
3143
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
3144
|
+
* intended, which is worth surfacing.
|
|
3145
|
+
*/
|
|
3146
|
+
processedAtMs(row) {
|
|
3147
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
3148
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
3149
|
+
this.log({
|
|
3150
|
+
level: "error",
|
|
3151
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
3152
|
+
conversation_id: row.conversation_id,
|
|
3153
|
+
message_id: row.id
|
|
3154
|
+
});
|
|
3155
|
+
return this.now();
|
|
3156
|
+
}
|
|
3157
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
3158
|
+
convForRow(sessionId, row) {
|
|
3159
|
+
return {
|
|
3160
|
+
id: row.conversation_id,
|
|
3161
|
+
agent_id: this.agentId,
|
|
3162
|
+
opencode_session_id: sessionId,
|
|
3163
|
+
pending_message_count: 0,
|
|
3164
|
+
oldest_pending_at: row.processed_at
|
|
3165
|
+
};
|
|
3166
|
+
}
|
|
3167
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
3168
|
+
queuedMessageForRow(row) {
|
|
3169
|
+
return {
|
|
3170
|
+
id: row.id,
|
|
3171
|
+
content: row.content,
|
|
3172
|
+
status: "processing",
|
|
3173
|
+
opencode_agent: row.opencode_agent,
|
|
3174
|
+
opencode_model: row.opencode_model,
|
|
3175
|
+
source_message_id: row.source_message_id,
|
|
3176
|
+
slack_user_id: row.slack_user_id
|
|
3177
|
+
};
|
|
2017
3178
|
}
|
|
2018
3179
|
/**
|
|
2019
3180
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2020
3181
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2021
3182
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
3183
|
+
*
|
|
3184
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
3185
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
3186
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
3187
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
3188
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
3189
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
3190
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2022
3191
|
*/
|
|
2023
3192
|
removeInFlight(watcher, evidentMessageId) {
|
|
3193
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
3194
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
3195
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
3196
|
+
this.log({
|
|
3197
|
+
level: "info",
|
|
3198
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
3199
|
+
conversation_id: watcher.conv.id,
|
|
3200
|
+
message_id: evidentMessageId
|
|
3201
|
+
});
|
|
3202
|
+
}
|
|
2024
3203
|
watcher.inFlight.delete(evidentMessageId);
|
|
2025
3204
|
this.dispatched.delete(evidentMessageId);
|
|
2026
3205
|
}
|
|
@@ -2037,21 +3216,41 @@ var ChannelDriver = class {
|
|
|
2037
3216
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2038
3217
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2039
3218
|
* oldest running message.
|
|
3219
|
+
*
|
|
3220
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3221
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3222
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3223
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3224
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3225
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3226
|
+
* even after it was already surfaced to the channel.
|
|
2040
3227
|
*/
|
|
2041
3228
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3229
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3230
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3231
|
+
let questionsPolledOk = true;
|
|
3232
|
+
let permissionsPolledOk = true;
|
|
2042
3233
|
let questions = [];
|
|
2043
3234
|
try {
|
|
2044
3235
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2045
3236
|
if (res.ok) {
|
|
2046
3237
|
const body = await res.json();
|
|
2047
|
-
|
|
3238
|
+
if (Array.isArray(body)) {
|
|
3239
|
+
questions = body;
|
|
3240
|
+
} else {
|
|
3241
|
+
questionsPolledOk = false;
|
|
3242
|
+
}
|
|
3243
|
+
} else {
|
|
3244
|
+
questionsPolledOk = false;
|
|
2048
3245
|
}
|
|
2049
3246
|
} catch {
|
|
3247
|
+
questionsPolledOk = false;
|
|
2050
3248
|
}
|
|
2051
3249
|
for (const q of questions) {
|
|
2052
|
-
if (q.sessionID
|
|
2053
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3250
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2054
3251
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3252
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3253
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2055
3254
|
const reported = await this.reportInteraction(
|
|
2056
3255
|
watcher.conv.id,
|
|
2057
3256
|
"question",
|
|
@@ -2065,14 +3264,22 @@ var ChannelDriver = class {
|
|
|
2065
3264
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2066
3265
|
if (res.ok) {
|
|
2067
3266
|
const body = await res.json();
|
|
2068
|
-
|
|
3267
|
+
if (Array.isArray(body)) {
|
|
3268
|
+
permissions = body;
|
|
3269
|
+
} else {
|
|
3270
|
+
permissionsPolledOk = false;
|
|
3271
|
+
}
|
|
3272
|
+
} else {
|
|
3273
|
+
permissionsPolledOk = false;
|
|
2069
3274
|
}
|
|
2070
3275
|
} catch {
|
|
3276
|
+
permissionsPolledOk = false;
|
|
2071
3277
|
}
|
|
2072
3278
|
for (const p of permissions) {
|
|
2073
|
-
if (p.sessionID
|
|
2074
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3279
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2075
3280
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3281
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3282
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2076
3283
|
const reported = await this.reportInteraction(
|
|
2077
3284
|
watcher.conv.id,
|
|
2078
3285
|
"permission",
|
|
@@ -2081,6 +3288,173 @@ var ChannelDriver = class {
|
|
|
2081
3288
|
);
|
|
2082
3289
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2083
3290
|
}
|
|
3291
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3292
|
+
}
|
|
3293
|
+
/**
|
|
3294
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3295
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3296
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3297
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3298
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3299
|
+
*
|
|
3300
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3301
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3302
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3303
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3304
|
+
*/
|
|
3305
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3306
|
+
let current = sessionId;
|
|
3307
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3308
|
+
if (current === rootSessionId) return true;
|
|
3309
|
+
const parent = await this.resolveSessionParent(current);
|
|
3310
|
+
if (parent === null || parent === void 0) return false;
|
|
3311
|
+
current = parent;
|
|
3312
|
+
}
|
|
3313
|
+
return false;
|
|
3314
|
+
}
|
|
3315
|
+
/**
|
|
3316
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3317
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3318
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3319
|
+
* caching a wrong answer — the next tick retries).
|
|
3320
|
+
*/
|
|
3321
|
+
async resolveSessionParent(sessionId) {
|
|
3322
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3323
|
+
if (cached !== void 0) return cached;
|
|
3324
|
+
let parent = void 0;
|
|
3325
|
+
try {
|
|
3326
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3327
|
+
if (res.ok) {
|
|
3328
|
+
const body = await res.json();
|
|
3329
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3330
|
+
}
|
|
3331
|
+
} catch {
|
|
3332
|
+
parent = void 0;
|
|
3333
|
+
}
|
|
3334
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3335
|
+
return parent;
|
|
3336
|
+
}
|
|
3337
|
+
/**
|
|
3338
|
+
* Resolve (and cache in `sessionTitles`) the OpenCode session TITLE (#310) so the
|
|
3339
|
+
* status PATCH can carry it into the "Live sessions" list. Driver-level cache so
|
|
3340
|
+
* BOTH the watcher completion path and the restart-recovery re-adopt path (which
|
|
3341
|
+
* has no watcher) can use it. `conversationId` is passed only for log context.
|
|
3342
|
+
* Best-effort:
|
|
3343
|
+
* - a resolved NON-EMPTY title is cached and terminal (a real session name
|
|
3344
|
+
* won't later un-name), so we do NOT re-GET `/session/:id` every tick;
|
|
3345
|
+
* - while the title is still absent/empty we do NOT latch it — OpenCode names
|
|
3346
|
+
* sessions asynchronously mid-turn, so an early call (e.g. at `processing`)
|
|
3347
|
+
* must leave the cache unresolved and re-fetch on the next need so a later
|
|
3348
|
+
* call (e.g. at `done`) picks up the name assigned in the meantime. Such a
|
|
3349
|
+
* call returns `null` (omit the title on THIS PATCH) without caching;
|
|
3350
|
+
* - a failed request likewise leaves the cache unresolved (retry next need)
|
|
3351
|
+
* and returns `null` — it must NEVER throw or block completion.
|
|
3352
|
+
* A failure is logged with agent/session context (no silent catch).
|
|
3353
|
+
*/
|
|
3354
|
+
async resolveSessionTitle(sessionId, conversationId) {
|
|
3355
|
+
const cached = this.sessionTitles.get(sessionId);
|
|
3356
|
+
if (cached != null) return cached;
|
|
3357
|
+
try {
|
|
3358
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3359
|
+
if (res.ok) {
|
|
3360
|
+
const body = await res.json();
|
|
3361
|
+
const title = body && typeof body.title === "string" ? body.title.trim() : "";
|
|
3362
|
+
if (title.length > 0) {
|
|
3363
|
+
this.sessionTitles.set(sessionId, title);
|
|
3364
|
+
return title;
|
|
3365
|
+
}
|
|
3366
|
+
return null;
|
|
3367
|
+
}
|
|
3368
|
+
this.log({
|
|
3369
|
+
level: "info",
|
|
3370
|
+
message: `Session title fetch for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) returned HTTP ${res.status} \u2014 omitting title`,
|
|
3371
|
+
conversation_id: conversationId
|
|
3372
|
+
});
|
|
3373
|
+
} catch (err) {
|
|
3374
|
+
this.log({
|
|
3375
|
+
level: "info",
|
|
3376
|
+
message: `Best-effort session title fetch failed for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) \u2014 omitting title: ${err instanceof Error ? err.message : String(err)}`,
|
|
3377
|
+
conversation_id: conversationId
|
|
3378
|
+
});
|
|
3379
|
+
}
|
|
3380
|
+
return null;
|
|
3381
|
+
}
|
|
3382
|
+
/**
|
|
3383
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3384
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3385
|
+
*
|
|
3386
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3387
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3388
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3389
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3390
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3391
|
+
* provably in flight at the exact moment of recovery.
|
|
3392
|
+
*
|
|
3393
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3394
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3395
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3396
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3397
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3398
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3399
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3400
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3401
|
+
*
|
|
3402
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3403
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3404
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3405
|
+
* case), OR no descendant is found at all.
|
|
3406
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3407
|
+
*
|
|
3408
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3409
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3410
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3411
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3412
|
+
* true/false/null faithfully.
|
|
3413
|
+
*
|
|
3414
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3415
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3416
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3417
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3418
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3419
|
+
* `SessionStatus` only.
|
|
3420
|
+
*/
|
|
3421
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3422
|
+
const sessions = await listSessions(this.port);
|
|
3423
|
+
if (!sessions) {
|
|
3424
|
+
this.log({
|
|
3425
|
+
level: "error",
|
|
3426
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3427
|
+
});
|
|
3428
|
+
return null;
|
|
3429
|
+
}
|
|
3430
|
+
for (const candidate of sessions) {
|
|
3431
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3432
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3433
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3434
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3435
|
+
return true;
|
|
3436
|
+
}
|
|
3437
|
+
}
|
|
3438
|
+
return false;
|
|
3439
|
+
}
|
|
3440
|
+
/**
|
|
3441
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3442
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3443
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3444
|
+
* the aborted-in-flight production bug after a restart.
|
|
3445
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3446
|
+
* (the sub-agent preamble — #253's shape).
|
|
3447
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3448
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3449
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3450
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3451
|
+
*/
|
|
3452
|
+
replyCompletionShape(reply) {
|
|
3453
|
+
if (!reply) return "other";
|
|
3454
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3455
|
+
if (completed == null) return "b1";
|
|
3456
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3457
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2084
3458
|
}
|
|
2085
3459
|
/**
|
|
2086
3460
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -2164,6 +3538,35 @@ var ChannelDriver = class {
|
|
|
2164
3538
|
}
|
|
2165
3539
|
return await res.json();
|
|
2166
3540
|
}
|
|
3541
|
+
/**
|
|
3542
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
3543
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
3544
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
3545
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
3546
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
3547
|
+
* `opencode_session_id`, routing).
|
|
3548
|
+
*
|
|
3549
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
3550
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
3551
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
3552
|
+
* next tick retries.
|
|
3553
|
+
*/
|
|
3554
|
+
async getProcessingMessages() {
|
|
3555
|
+
const res = await this.fetchImpl(
|
|
3556
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
3557
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
3558
|
+
);
|
|
3559
|
+
this.assertAuth(res, "fetching processing messages");
|
|
3560
|
+
if (!res.ok) {
|
|
3561
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
3562
|
+
}
|
|
3563
|
+
const data = await res.json();
|
|
3564
|
+
let messages = data.messages ?? [];
|
|
3565
|
+
if (this.conversationFilter) {
|
|
3566
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
3567
|
+
}
|
|
3568
|
+
return messages;
|
|
3569
|
+
}
|
|
2167
3570
|
/**
|
|
2168
3571
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2169
3572
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2185,13 +3588,18 @@ var ChannelDriver = class {
|
|
|
2185
3588
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2186
3589
|
* retry vehicle for the swap-to-running.
|
|
2187
3590
|
*/
|
|
2188
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
3591
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId, title) {
|
|
2189
3592
|
const res = await this.fetchImpl(
|
|
2190
3593
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2191
3594
|
{
|
|
2192
3595
|
method: "PATCH",
|
|
2193
3596
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2194
|
-
body: JSON.stringify({
|
|
3597
|
+
body: JSON.stringify({
|
|
3598
|
+
status: "processing",
|
|
3599
|
+
opencode_session_id: sessionId,
|
|
3600
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
3601
|
+
...title ? { title } : {}
|
|
3602
|
+
})
|
|
2195
3603
|
}
|
|
2196
3604
|
);
|
|
2197
3605
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2229,13 +3637,18 @@ var ChannelDriver = class {
|
|
|
2229
3637
|
* watcher retries next tick within the
|
|
2230
3638
|
* deadline, Finding 4).
|
|
2231
3639
|
*/
|
|
2232
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3640
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title) {
|
|
2233
3641
|
const res = await this.fetchImpl(
|
|
2234
3642
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2235
3643
|
{
|
|
2236
3644
|
method: "PATCH",
|
|
2237
3645
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2238
|
-
body: JSON.stringify({
|
|
3646
|
+
body: JSON.stringify({
|
|
3647
|
+
status: "done",
|
|
3648
|
+
opencode_session_id: sessionId,
|
|
3649
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
3650
|
+
...title ? { title } : {}
|
|
3651
|
+
})
|
|
2239
3652
|
}
|
|
2240
3653
|
);
|
|
2241
3654
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2245,7 +3658,17 @@ var ChannelDriver = class {
|
|
|
2245
3658
|
}
|
|
2246
3659
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2247
3660
|
}
|
|
2248
|
-
|
|
3661
|
+
/**
|
|
3662
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3663
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3664
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3665
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3666
|
+
* failure reason reaches the channel.
|
|
3667
|
+
*/
|
|
3668
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3669
|
+
const body = { status: "failed" };
|
|
3670
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3671
|
+
if (error2 !== void 0) body.error = error2;
|
|
2249
3672
|
await this.callWithRetry(
|
|
2250
3673
|
"marking message as failed",
|
|
2251
3674
|
() => this.fetchImpl(
|
|
@@ -2253,11 +3676,56 @@ var ChannelDriver = class {
|
|
|
2253
3676
|
{
|
|
2254
3677
|
method: "PATCH",
|
|
2255
3678
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2256
|
-
body: JSON.stringify(
|
|
3679
|
+
body: JSON.stringify(body)
|
|
2257
3680
|
}
|
|
2258
3681
|
)
|
|
2259
3682
|
);
|
|
2260
3683
|
}
|
|
3684
|
+
/**
|
|
3685
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3686
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3687
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3688
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3689
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3690
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3691
|
+
* context (no silent catch, per development-workflow).
|
|
3692
|
+
*
|
|
3693
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3694
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3695
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3696
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3697
|
+
* leaves liveness").
|
|
3698
|
+
*/
|
|
3699
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3700
|
+
try {
|
|
3701
|
+
const res = await this.fetchImpl(
|
|
3702
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
3703
|
+
{
|
|
3704
|
+
method: "POST",
|
|
3705
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3706
|
+
body: JSON.stringify({ signal, ...extra })
|
|
3707
|
+
}
|
|
3708
|
+
);
|
|
3709
|
+
if (!res.ok) {
|
|
3710
|
+
this.log({
|
|
3711
|
+
level: "error",
|
|
3712
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3713
|
+
conversation_id: conversationId,
|
|
3714
|
+
message_id: messageId
|
|
3715
|
+
});
|
|
3716
|
+
return false;
|
|
3717
|
+
}
|
|
3718
|
+
return true;
|
|
3719
|
+
} catch (err) {
|
|
3720
|
+
this.log({
|
|
3721
|
+
level: "error",
|
|
3722
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3723
|
+
conversation_id: conversationId,
|
|
3724
|
+
message_id: messageId
|
|
3725
|
+
});
|
|
3726
|
+
return false;
|
|
3727
|
+
}
|
|
3728
|
+
}
|
|
2261
3729
|
async persistSession(conversationId, sessionId) {
|
|
2262
3730
|
const res = await this.fetchImpl(
|
|
2263
3731
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2528,6 +3996,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
2528
3996
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
2529
3997
|
}
|
|
2530
3998
|
}
|
|
3999
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
4000
|
+
const apiUrl = getApiUrlConfig();
|
|
4001
|
+
try {
|
|
4002
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
4003
|
+
method: "POST",
|
|
4004
|
+
headers: { Authorization: authHeader }
|
|
4005
|
+
});
|
|
4006
|
+
if (!response.ok) {
|
|
4007
|
+
const serverMessage = await readErrorMessage(response);
|
|
4008
|
+
return {
|
|
4009
|
+
ok: false,
|
|
4010
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4011
|
+
};
|
|
4012
|
+
}
|
|
4013
|
+
return { ok: true };
|
|
4014
|
+
} catch (error2) {
|
|
4015
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
4016
|
+
}
|
|
4017
|
+
}
|
|
2531
4018
|
async function getAgentInfo(agentId, authHeader) {
|
|
2532
4019
|
const apiUrl = getApiUrlConfig();
|
|
2533
4020
|
try {
|
|
@@ -2573,6 +4060,8 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2573
4060
|
// src/commands/run.ts
|
|
2574
4061
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2575
4062
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
4063
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4064
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
2576
4065
|
function log2(state, message, isError = false) {
|
|
2577
4066
|
if (state.json) {
|
|
2578
4067
|
console.log(
|
|
@@ -2727,7 +4216,7 @@ async function driveChannels(state, driver) {
|
|
|
2727
4216
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
2728
4217
|
if (state.interactive) displayStatus(state);
|
|
2729
4218
|
}
|
|
2730
|
-
await new Promise((
|
|
4219
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
2731
4220
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
2732
4221
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
2733
4222
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -2738,8 +4227,122 @@ async function driveChannels(state, driver) {
|
|
|
2738
4227
|
}
|
|
2739
4228
|
}
|
|
2740
4229
|
}
|
|
2741
|
-
|
|
4230
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4231
|
+
async function runSweep(state, driver, config2) {
|
|
4232
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4233
|
+
try {
|
|
4234
|
+
const sessions = await listSessions(state.port);
|
|
4235
|
+
if (sessions === null) {
|
|
4236
|
+
logActivity(state, {
|
|
4237
|
+
type: "info",
|
|
4238
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4239
|
+
});
|
|
4240
|
+
return;
|
|
4241
|
+
}
|
|
4242
|
+
const toDelete = selectSessionsToDelete(
|
|
4243
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4244
|
+
{
|
|
4245
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4246
|
+
maxCount: config2.maxCount,
|
|
4247
|
+
nowMs: Date.now(),
|
|
4248
|
+
protectedIds: driver.protectedSessionIds()
|
|
4249
|
+
}
|
|
4250
|
+
);
|
|
4251
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4252
|
+
let deleted = 0;
|
|
4253
|
+
let failed = 0;
|
|
4254
|
+
let skippedNewlyActive = 0;
|
|
4255
|
+
for (const id of toDelete) {
|
|
4256
|
+
if (protectedNow.has(id)) {
|
|
4257
|
+
skippedNewlyActive++;
|
|
4258
|
+
logActivity(state, {
|
|
4259
|
+
type: "info",
|
|
4260
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4261
|
+
});
|
|
4262
|
+
continue;
|
|
4263
|
+
}
|
|
4264
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4265
|
+
else failed++;
|
|
4266
|
+
}
|
|
4267
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4268
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4269
|
+
logActivity(state, {
|
|
4270
|
+
type: "info",
|
|
4271
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4272
|
+
});
|
|
4273
|
+
} catch (error2) {
|
|
4274
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4275
|
+
logActivity(state, {
|
|
4276
|
+
type: "error",
|
|
4277
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4278
|
+
});
|
|
4279
|
+
}
|
|
4280
|
+
}
|
|
4281
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4282
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4283
|
+
{
|
|
4284
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4285
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4286
|
+
interval: options.sessionCleanupInterval
|
|
4287
|
+
},
|
|
4288
|
+
process.env
|
|
4289
|
+
);
|
|
4290
|
+
for (const warning2 of config2.warnings) {
|
|
4291
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4292
|
+
}
|
|
4293
|
+
if (!config2.enabled) return;
|
|
4294
|
+
logActivity(state, {
|
|
4295
|
+
type: "info",
|
|
4296
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4297
|
+
});
|
|
4298
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4299
|
+
const firstSweep = setTimeout(
|
|
4300
|
+
() => void runSweep(state, driver, config2),
|
|
4301
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4302
|
+
);
|
|
4303
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4304
|
+
}
|
|
4305
|
+
async function notifyOffline(state) {
|
|
4306
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4307
|
+
if (!state.connected) {
|
|
4308
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4309
|
+
return;
|
|
4310
|
+
}
|
|
4311
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4312
|
+
if (result.ok) {
|
|
4313
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4314
|
+
} else {
|
|
4315
|
+
logActivity(state, {
|
|
4316
|
+
type: "error",
|
|
4317
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4318
|
+
});
|
|
4319
|
+
if (state.interactive) displayStatus(state);
|
|
4320
|
+
}
|
|
4321
|
+
}
|
|
4322
|
+
async function cleanup(state, opts = {}) {
|
|
2742
4323
|
state.running = false;
|
|
4324
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4325
|
+
clearInterval(timer);
|
|
4326
|
+
clearTimeout(timer);
|
|
4327
|
+
}
|
|
4328
|
+
state.sessionCleanupTimers = [];
|
|
4329
|
+
if (opts.graceful && state.channelDriver) {
|
|
4330
|
+
state.channelDriver.stop();
|
|
4331
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4332
|
+
if (state.interactive) {
|
|
4333
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4334
|
+
displayStatus(state);
|
|
4335
|
+
}
|
|
4336
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4337
|
+
if (!settled) {
|
|
4338
|
+
logActivity(state, {
|
|
4339
|
+
type: "info",
|
|
4340
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4341
|
+
});
|
|
4342
|
+
if (state.interactive) displayStatus(state);
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4345
|
+
await notifyOffline(state);
|
|
2743
4346
|
if (state.connection) {
|
|
2744
4347
|
state.connection.close();
|
|
2745
4348
|
state.connection = null;
|
|
@@ -2770,10 +4373,13 @@ async function run(options) {
|
|
|
2770
4373
|
opencodeVersion: null,
|
|
2771
4374
|
opencodeProcess: null,
|
|
2772
4375
|
connection: null,
|
|
4376
|
+
channelDriver: null,
|
|
2773
4377
|
running: true,
|
|
4378
|
+
shuttingDown: false,
|
|
2774
4379
|
activityLog: [],
|
|
2775
4380
|
messageCount: 0,
|
|
2776
4381
|
lastProxiedActivityAt: null,
|
|
4382
|
+
sessionCleanupTimers: [],
|
|
2777
4383
|
authHeader: ""
|
|
2778
4384
|
};
|
|
2779
4385
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -2784,13 +4390,15 @@ async function run(options) {
|
|
|
2784
4390
|
);
|
|
2785
4391
|
}
|
|
2786
4392
|
const handleSignal = async () => {
|
|
4393
|
+
if (state.shuttingDown) return;
|
|
4394
|
+
state.shuttingDown = true;
|
|
2787
4395
|
if (state.interactive) {
|
|
2788
4396
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2789
4397
|
displayStatus(state);
|
|
2790
4398
|
} else {
|
|
2791
4399
|
log2(state, "Shutting down...");
|
|
2792
4400
|
}
|
|
2793
|
-
await cleanup(state);
|
|
4401
|
+
await cleanup(state, { graceful: true });
|
|
2794
4402
|
await shutdownTelemetry();
|
|
2795
4403
|
process.exit(0);
|
|
2796
4404
|
};
|
|
@@ -2910,12 +4518,14 @@ async function run(options) {
|
|
|
2910
4518
|
apiUrl: getApiUrlConfig(),
|
|
2911
4519
|
getAuthHeader: () => state.authHeader,
|
|
2912
4520
|
conversationFilter: state.conversationFilter,
|
|
4521
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
2913
4522
|
log: (entry) => logActivity(state, {
|
|
2914
4523
|
type: entry.level === "error" ? "error" : "info",
|
|
2915
4524
|
message: entry.message,
|
|
2916
4525
|
error: entry.level === "error" ? entry.message : void 0
|
|
2917
4526
|
})
|
|
2918
4527
|
});
|
|
4528
|
+
state.channelDriver = channelDriver;
|
|
2919
4529
|
const connection = new RunnerConnection({
|
|
2920
4530
|
agentId: state.agentId,
|
|
2921
4531
|
getAuthHeader: () => state.authHeader,
|
|
@@ -2929,7 +4539,11 @@ async function run(options) {
|
|
|
2929
4539
|
type: "info",
|
|
2930
4540
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2931
4541
|
});
|
|
2932
|
-
emitAgentConnected(state.agentId, {
|
|
4542
|
+
emitAgentConnected(state.agentId, {
|
|
4543
|
+
port: state.port,
|
|
4544
|
+
cli_version: getCliVersion(),
|
|
4545
|
+
opencode_version: state.opencodeVersion
|
|
4546
|
+
});
|
|
2933
4547
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2934
4548
|
if (state.interactive) displayStatus(state);
|
|
2935
4549
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -3008,10 +4622,12 @@ async function run(options) {
|
|
|
3008
4622
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3009
4623
|
throw error2;
|
|
3010
4624
|
}
|
|
4625
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3011
4626
|
if (!interactive || state.json) {
|
|
3012
4627
|
log2(state, "Driving channel messages...");
|
|
3013
4628
|
}
|
|
3014
4629
|
await driveChannels(state, channelDriver);
|
|
4630
|
+
if (state.shuttingDown) return;
|
|
3015
4631
|
await cleanup(state);
|
|
3016
4632
|
if (state.json) {
|
|
3017
4633
|
console.log(
|
|
@@ -3026,6 +4642,7 @@ async function run(options) {
|
|
|
3026
4642
|
await shutdownTelemetry();
|
|
3027
4643
|
process.exit(0);
|
|
3028
4644
|
} catch (error2) {
|
|
4645
|
+
if (state.shuttingDown) return;
|
|
3029
4646
|
await cleanup(state);
|
|
3030
4647
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3031
4648
|
if (state.json) {
|
|
@@ -3060,7 +4677,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3060
4677
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3061
4678
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3062
4679
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3063
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4680
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4681
|
+
"--session-cleanup-max-age <duration>",
|
|
4682
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4683
|
+
).option(
|
|
4684
|
+
"--session-cleanup-max-count <n>",
|
|
4685
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4686
|
+
).option(
|
|
4687
|
+
"--session-cleanup-interval <duration>",
|
|
4688
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4689
|
+
).action(
|
|
3064
4690
|
(options) => {
|
|
3065
4691
|
run({
|
|
3066
4692
|
agent: options.agent,
|
|
@@ -3068,7 +4694,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3068
4694
|
verbose: options.verbose,
|
|
3069
4695
|
conversation: options.conversation,
|
|
3070
4696
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3071
|
-
json: options.json
|
|
4697
|
+
json: options.json,
|
|
4698
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4699
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4700
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4701
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3072
4702
|
});
|
|
3073
4703
|
}
|
|
3074
4704
|
);
|