@evident-ai/cli 3.0.1-dev.4b2470a → 3.0.1-dev.4be1aa5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1713 -110
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -284,14 +285,14 @@ function blank() {
|
|
|
284
285
|
console.log();
|
|
285
286
|
}
|
|
286
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
287
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
288
289
|
process.stdout.write(chalk.dim(prompt));
|
|
289
290
|
const handler = () => {
|
|
290
291
|
process.stdin.removeListener("data", handler);
|
|
291
292
|
process.stdin.setRawMode?.(false);
|
|
292
293
|
process.stdin.pause();
|
|
293
294
|
console.log();
|
|
294
|
-
|
|
295
|
+
resolve2();
|
|
295
296
|
};
|
|
296
297
|
if (process.stdin.isTTY) {
|
|
297
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -301,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
301
302
|
});
|
|
302
303
|
}
|
|
303
304
|
function sleep(ms) {
|
|
304
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
305
306
|
}
|
|
306
307
|
|
|
307
308
|
// src/commands/login.ts
|
|
@@ -375,19 +376,19 @@ async function tokenLogin() {
|
|
|
375
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
376
377
|
blank();
|
|
377
378
|
process.stdout.write("Paste token: ");
|
|
378
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
379
380
|
let data = "";
|
|
380
381
|
process.stdin.setEncoding("utf8");
|
|
381
382
|
process.stdin.on("data", (chunk) => {
|
|
382
383
|
data += chunk;
|
|
383
384
|
});
|
|
384
385
|
process.stdin.on("end", () => {
|
|
385
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
386
387
|
});
|
|
387
388
|
if (process.stdin.isTTY) {
|
|
388
389
|
process.stdin.once("data", (chunk) => {
|
|
389
390
|
process.stdin.pause();
|
|
390
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
391
392
|
});
|
|
392
393
|
process.stdin.resume();
|
|
393
394
|
}
|
|
@@ -484,8 +485,34 @@ var TelemetryEventTypes = {
|
|
|
484
485
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
486
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
487
|
|
|
488
|
+
// ../../packages/types/src/logging/index.ts
|
|
489
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
|
+
function log(level, event, fields) {
|
|
491
|
+
const method = level === "debug" ? "log" : level;
|
|
492
|
+
try {
|
|
493
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
494
|
+
} catch (err) {
|
|
495
|
+
console.error(
|
|
496
|
+
"[evident] log_serialize_failed",
|
|
497
|
+
event,
|
|
498
|
+
err instanceof Error ? err.message : String(err)
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
function stripQuery(url) {
|
|
503
|
+
try {
|
|
504
|
+
return new URL(url).pathname;
|
|
505
|
+
} catch {
|
|
506
|
+
const q = url.indexOf("?");
|
|
507
|
+
return q === -1 ? url : url.slice(0, q);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
487
511
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
489
516
|
var eventBuffer = [];
|
|
490
517
|
var flushTimeout = null;
|
|
491
518
|
var isShuttingDown = false;
|
|
@@ -679,20 +706,20 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
679
706
|
if (health.healthy) {
|
|
680
707
|
return health;
|
|
681
708
|
}
|
|
682
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
683
710
|
}
|
|
684
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
685
712
|
}
|
|
686
713
|
|
|
687
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
688
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
689
|
-
function isQueueValidatedVersion(
|
|
690
|
-
if (!
|
|
691
|
-
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(
|
|
692
|
-
}
|
|
693
|
-
function buildOpenCodeVersionWarning(
|
|
694
|
-
if (isQueueValidatedVersion(
|
|
695
|
-
const detected =
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
716
|
+
function isQueueValidatedVersion(version2) {
|
|
717
|
+
if (!version2) return false;
|
|
718
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
719
|
+
}
|
|
720
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
721
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
722
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
696
723
|
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
697
724
|
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
698
725
|
}
|
|
@@ -1009,7 +1036,11 @@ function roleOf(m) {
|
|
|
1009
1036
|
}
|
|
1010
1037
|
function completedOf(m) {
|
|
1011
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1012
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1013
1044
|
}
|
|
1014
1045
|
function idOf(m) {
|
|
1015
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1029,6 +1060,102 @@ function finishOf(m) {
|
|
|
1029
1060
|
const infoFinish = m.info?.finish;
|
|
1030
1061
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1031
1062
|
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1070
|
+
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1032
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1033
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1034
1161
|
if (directory && directory.trim()) {
|
|
@@ -1046,9 +1173,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1046
1173
|
const data = await response.json();
|
|
1047
1174
|
return data.id;
|
|
1048
1175
|
}
|
|
1049
|
-
|
|
1176
|
+
function messageText(m) {
|
|
1177
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
+
}
|
|
1180
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
+
const knownUserIds = new Set(
|
|
1183
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
+
);
|
|
1050
1185
|
const body = {
|
|
1051
|
-
messageID: messageId,
|
|
1052
1186
|
parts: [{ type: "text", text: content }]
|
|
1053
1187
|
};
|
|
1054
1188
|
if (options?.agent) {
|
|
@@ -1072,6 +1206,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1072
1206
|
const text = await res.text().catch(() => "");
|
|
1073
1207
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1074
1208
|
}
|
|
1209
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
+
if (after) {
|
|
1214
|
+
let best = null;
|
|
1215
|
+
for (const m of after) {
|
|
1216
|
+
if (roleOf(m) !== "user") continue;
|
|
1217
|
+
const id = idOf(m);
|
|
1218
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
+
if (messageText(m) !== content) continue;
|
|
1220
|
+
const created = createdOf(m) ?? 0;
|
|
1221
|
+
if (best === null || created > best.created) {
|
|
1222
|
+
best = { id, created };
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (best) return best.id;
|
|
1226
|
+
}
|
|
1227
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1075
1232
|
}
|
|
1076
1233
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1077
1234
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1088,19 +1245,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1088
1245
|
}
|
|
1089
1246
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1090
1247
|
if (!messages || messages.length === 0) return null;
|
|
1248
|
+
let lastCorrelated = null;
|
|
1249
|
+
let lastNonErrored = null;
|
|
1091
1250
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1092
1251
|
const m = messages[i];
|
|
1093
|
-
if (roleOf(m)
|
|
1252
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1253
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1254
|
+
if (errorOf(m) == null) {
|
|
1255
|
+
lastNonErrored = m;
|
|
1256
|
+
break;
|
|
1257
|
+
}
|
|
1094
1258
|
}
|
|
1259
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1095
1260
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1096
1261
|
if (userIndex === -1) return null;
|
|
1097
1262
|
let last = null;
|
|
1263
|
+
let lastOk = null;
|
|
1098
1264
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1099
1265
|
const role = roleOf(messages[i]);
|
|
1100
1266
|
if (role === "user") break;
|
|
1101
|
-
if (role === "assistant")
|
|
1267
|
+
if (role === "assistant") {
|
|
1268
|
+
last = messages[i];
|
|
1269
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1270
|
+
}
|
|
1102
1271
|
}
|
|
1103
|
-
return last;
|
|
1272
|
+
return lastOk ?? last;
|
|
1104
1273
|
}
|
|
1105
1274
|
function messageRunState(messages, userMessageId) {
|
|
1106
1275
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1110,13 +1279,136 @@ function messageRunState(messages, userMessageId) {
|
|
|
1110
1279
|
if (!reply) return "unknown";
|
|
1111
1280
|
}
|
|
1112
1281
|
if (!reply) return "queued";
|
|
1113
|
-
if (
|
|
1114
|
-
|
|
1115
|
-
|
|
1282
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1283
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1284
|
+
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1290
|
+
function messageError(messages, userMessageId) {
|
|
1291
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1292
|
+
const error2 = errorOf(reply);
|
|
1293
|
+
if (error2 == null) return null;
|
|
1294
|
+
if (typeof error2 === "string") return error2;
|
|
1295
|
+
if (typeof error2 === "object") {
|
|
1296
|
+
const e = error2;
|
|
1297
|
+
const dataMessage = e.data?.message;
|
|
1298
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1299
|
+
if (typeof e.message === "string") return e.message;
|
|
1300
|
+
}
|
|
1301
|
+
return "The agent run failed.";
|
|
1302
|
+
}
|
|
1303
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1304
|
+
if (!messages || messages.length === 0) return false;
|
|
1305
|
+
return messages.some(
|
|
1306
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1307
|
+
);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
return toDelete;
|
|
1356
|
+
}
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1360
|
+
}
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1369
|
+
}
|
|
1370
|
+
return value;
|
|
1116
1371
|
}
|
|
1117
|
-
function
|
|
1118
|
-
const
|
|
1119
|
-
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
+
}
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1120
1412
|
}
|
|
1121
1413
|
|
|
1122
1414
|
// src/lib/tunnel/connection.ts
|
|
@@ -1187,24 +1479,34 @@ var StreamForwarder = class {
|
|
|
1187
1479
|
}
|
|
1188
1480
|
async handleOpen(frame) {
|
|
1189
1481
|
const { sid, method, path, headers, has_body } = frame;
|
|
1482
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1483
|
+
const startedAt = Date.now();
|
|
1190
1484
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1191
1485
|
this.callbacks.onDrainPing?.();
|
|
1192
1486
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1193
1487
|
this.send({ type: "res_end", sid });
|
|
1194
1488
|
return;
|
|
1195
1489
|
}
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1196
1498
|
const ac = new AbortController();
|
|
1197
1499
|
let bodyPromise;
|
|
1198
1500
|
let pushBody;
|
|
1199
1501
|
let endBody;
|
|
1200
1502
|
if (has_body) {
|
|
1201
1503
|
const chunks = [];
|
|
1202
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1203
1505
|
pushBody = (buf) => {
|
|
1204
1506
|
chunks.push(buf);
|
|
1205
1507
|
};
|
|
1206
1508
|
endBody = () => {
|
|
1207
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1208
1510
|
};
|
|
1209
1511
|
});
|
|
1210
1512
|
}
|
|
@@ -1239,6 +1541,14 @@ var StreamForwarder = class {
|
|
|
1239
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1240
1542
|
});
|
|
1241
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1242
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1243
1553
|
try {
|
|
1244
1554
|
if (upstream.body) {
|
|
@@ -1314,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1314
1624
|
} = options;
|
|
1315
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1316
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1317
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1318
1628
|
const ws = new WebSocket2(url, {
|
|
1319
1629
|
headers: {
|
|
1320
1630
|
Authorization: authHeader
|
|
@@ -1379,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1379
1689
|
clearTimeout(connectionTimeout);
|
|
1380
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1381
1691
|
onConnected?.(connectedAgentId);
|
|
1382
|
-
|
|
1692
|
+
resolve2({
|
|
1383
1693
|
ws,
|
|
1384
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1385
1695
|
});
|
|
@@ -1508,7 +1818,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1508
1818
|
};
|
|
1509
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1510
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1511
|
-
var
|
|
1821
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1512
1825
|
var ChannelAuthError = class extends Error {
|
|
1513
1826
|
constructor(message) {
|
|
1514
1827
|
super(message);
|
|
@@ -1543,10 +1856,18 @@ var ChannelDriver = class {
|
|
|
1543
1856
|
sleep;
|
|
1544
1857
|
pausedPollIntervalMs;
|
|
1545
1858
|
pausedMaxWaitMs;
|
|
1546
|
-
|
|
1859
|
+
stuckQueuedMs;
|
|
1547
1860
|
now;
|
|
1548
1861
|
/** Cache of conversationId → opencode sessionId. */
|
|
1549
1862
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
+
/**
|
|
1864
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
+
*/
|
|
1870
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1550
1871
|
/**
|
|
1551
1872
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1552
1873
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1563,6 +1884,54 @@ var ChannelDriver = class {
|
|
|
1563
1884
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1564
1885
|
*/
|
|
1565
1886
|
dispatched = /* @__PURE__ */ new Set();
|
|
1887
|
+
/**
|
|
1888
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1889
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1890
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1891
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1892
|
+
* the processing list.
|
|
1893
|
+
*/
|
|
1894
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1895
|
+
/**
|
|
1896
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1897
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1898
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1899
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1900
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1901
|
+
*
|
|
1902
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1903
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1904
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1905
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1906
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1907
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1908
|
+
*/
|
|
1909
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1910
|
+
/**
|
|
1911
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1912
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1913
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1914
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1915
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1916
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1917
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1918
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1919
|
+
*/
|
|
1920
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1923
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1924
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1925
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1926
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1927
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1928
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1929
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1930
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1931
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1932
|
+
* so the NEXT tick may retry exactly once more).
|
|
1933
|
+
*/
|
|
1934
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1566
1935
|
/**
|
|
1567
1936
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1568
1937
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1570,8 +1939,33 @@ var ChannelDriver = class {
|
|
|
1570
1939
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1571
1940
|
*/
|
|
1572
1941
|
opencodeDirectory = void 0;
|
|
1942
|
+
/**
|
|
1943
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1944
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1945
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1946
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1947
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1948
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1949
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1950
|
+
*/
|
|
1951
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1573
1952
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1574
1953
|
draining = false;
|
|
1954
|
+
/**
|
|
1955
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1956
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1957
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1958
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1959
|
+
*/
|
|
1960
|
+
activeDrain = null;
|
|
1961
|
+
/**
|
|
1962
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1963
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1964
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1965
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1966
|
+
* and stops opencode.
|
|
1967
|
+
*/
|
|
1968
|
+
stopped = false;
|
|
1575
1969
|
constructor(config2) {
|
|
1576
1970
|
this.agentId = config2.agentId;
|
|
1577
1971
|
this.port = config2.port;
|
|
@@ -1585,7 +1979,7 @@ var ChannelDriver = class {
|
|
|
1585
1979
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1586
1980
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1587
1981
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1588
|
-
this.
|
|
1982
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1589
1983
|
this.now = config2.now ?? (() => Date.now());
|
|
1590
1984
|
}
|
|
1591
1985
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1603,8 +1997,21 @@ var ChannelDriver = class {
|
|
|
1603
1997
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1604
1998
|
*/
|
|
1605
1999
|
async drainPending() {
|
|
2000
|
+
if (this.stopped) return 0;
|
|
1606
2001
|
if (this.draining) return 0;
|
|
1607
2002
|
this.draining = true;
|
|
2003
|
+
const run2 = this.runDrain();
|
|
2004
|
+
this.activeDrain = run2.then(
|
|
2005
|
+
() => {
|
|
2006
|
+
this.activeDrain = null;
|
|
2007
|
+
},
|
|
2008
|
+
() => {
|
|
2009
|
+
this.activeDrain = null;
|
|
2010
|
+
}
|
|
2011
|
+
);
|
|
2012
|
+
return run2;
|
|
2013
|
+
}
|
|
2014
|
+
async runDrain() {
|
|
1608
2015
|
let dispatched = 0;
|
|
1609
2016
|
try {
|
|
1610
2017
|
const conversations = await this.getPendingConversations();
|
|
@@ -1616,8 +2023,10 @@ var ChannelDriver = class {
|
|
|
1616
2023
|
});
|
|
1617
2024
|
}
|
|
1618
2025
|
for (const conv of conversations) {
|
|
2026
|
+
if (this.stopped) break;
|
|
1619
2027
|
dispatched += await this.processConversation(conv);
|
|
1620
2028
|
}
|
|
2029
|
+
await this.readoptProcessing();
|
|
1621
2030
|
} finally {
|
|
1622
2031
|
this.draining = false;
|
|
1623
2032
|
}
|
|
@@ -1635,6 +2044,73 @@ var ChannelDriver = class {
|
|
|
1635
2044
|
}
|
|
1636
2045
|
return false;
|
|
1637
2046
|
}
|
|
2047
|
+
/**
|
|
2048
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2049
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2050
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2051
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2052
|
+
*
|
|
2053
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2054
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2055
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2056
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2057
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2058
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2059
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2060
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2061
|
+
*/
|
|
2062
|
+
protectedSessionIds() {
|
|
2063
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2064
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2065
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2066
|
+
}
|
|
2067
|
+
return ids;
|
|
2068
|
+
}
|
|
2069
|
+
/**
|
|
2070
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2071
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2072
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2073
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2074
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2075
|
+
*/
|
|
2076
|
+
stop() {
|
|
2077
|
+
this.stopped = true;
|
|
2078
|
+
}
|
|
2079
|
+
/**
|
|
2080
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2081
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2082
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2083
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2084
|
+
*
|
|
2085
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2086
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2087
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2088
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2089
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2090
|
+
* next runner start (ADR-0046).
|
|
2091
|
+
*
|
|
2092
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2093
|
+
* timeout elapsed with work still in flight.
|
|
2094
|
+
*/
|
|
2095
|
+
async waitForInFlight(timeoutMs) {
|
|
2096
|
+
const deadline = this.now() + timeoutMs;
|
|
2097
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2098
|
+
if (this.activeDrain) {
|
|
2099
|
+
let drainSettled = false;
|
|
2100
|
+
void this.activeDrain.then(() => {
|
|
2101
|
+
drainSettled = true;
|
|
2102
|
+
});
|
|
2103
|
+
while (!drainSettled) {
|
|
2104
|
+
if (this.now() >= deadline) return false;
|
|
2105
|
+
await this.sleep(step);
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
while (this.hasInFlightWatchers()) {
|
|
2109
|
+
if (this.now() >= deadline) return false;
|
|
2110
|
+
await this.sleep(step);
|
|
2111
|
+
}
|
|
2112
|
+
return true;
|
|
2113
|
+
}
|
|
1638
2114
|
/**
|
|
1639
2115
|
* Await all outstanding per-session watchers (WI-3).
|
|
1640
2116
|
*
|
|
@@ -1669,15 +2145,18 @@ var ChannelDriver = class {
|
|
|
1669
2145
|
const sessionId = await this.ensureSession(conv);
|
|
1670
2146
|
const messages = await this.getPendingMessages(conv.id);
|
|
1671
2147
|
let dispatched = 0;
|
|
2148
|
+
let skippedAlreadyDispatched = 0;
|
|
1672
2149
|
for (const message of messages) {
|
|
2150
|
+
if (this.stopped) break;
|
|
1673
2151
|
if (this.dispatched.has(message.id)) {
|
|
2152
|
+
skippedAlreadyDispatched += 1;
|
|
1674
2153
|
continue;
|
|
1675
2154
|
}
|
|
1676
|
-
const opencodeMessageId = opencodeMessageIdFor(message.id);
|
|
1677
2155
|
const options = {
|
|
1678
2156
|
agent: message.opencode_agent ?? void 0,
|
|
1679
2157
|
model: message.opencode_model ?? void 0
|
|
1680
2158
|
};
|
|
2159
|
+
let opencodeMessageId;
|
|
1681
2160
|
try {
|
|
1682
2161
|
this.log({
|
|
1683
2162
|
level: "info",
|
|
@@ -1685,10 +2164,23 @@ var ChannelDriver = class {
|
|
|
1685
2164
|
conversation_id: conv.id,
|
|
1686
2165
|
message_id: message.id
|
|
1687
2166
|
});
|
|
1688
|
-
await
|
|
2167
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2168
|
+
sessionId,
|
|
2169
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2170
|
+
);
|
|
1689
2171
|
} catch (err) {
|
|
1690
2172
|
if (err instanceof ChannelAuthError) throw err;
|
|
1691
2173
|
this.dispatched.delete(message.id);
|
|
2174
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2175
|
+
this.sessions.delete(conv.id);
|
|
2176
|
+
this.log({
|
|
2177
|
+
level: "info",
|
|
2178
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2179
|
+
conversation_id: conv.id,
|
|
2180
|
+
message_id: message.id
|
|
2181
|
+
});
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
1692
2184
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1693
2185
|
});
|
|
1694
2186
|
this.log({
|
|
@@ -1699,24 +2191,58 @@ var ChannelDriver = class {
|
|
|
1699
2191
|
});
|
|
1700
2192
|
continue;
|
|
1701
2193
|
}
|
|
2194
|
+
if (opencodeMessageId === null) {
|
|
2195
|
+
this.log({
|
|
2196
|
+
level: "error",
|
|
2197
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2198
|
+
conversation_id: conv.id,
|
|
2199
|
+
message_id: message.id
|
|
2200
|
+
});
|
|
2201
|
+
continue;
|
|
2202
|
+
}
|
|
1702
2203
|
this.dispatched.add(message.id);
|
|
1703
2204
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1704
2205
|
dispatched += 1;
|
|
2206
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
2207
|
+
}
|
|
2208
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
2209
|
+
this.log({
|
|
2210
|
+
level: "error",
|
|
2211
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
2212
|
+
conversation_id: conv.id
|
|
2213
|
+
});
|
|
1705
2214
|
}
|
|
1706
2215
|
this.ensureWatcherRunning(sessionId);
|
|
1707
2216
|
return dispatched;
|
|
1708
2217
|
}
|
|
1709
2218
|
async ensureSession(conv) {
|
|
1710
|
-
const
|
|
1711
|
-
if (
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
2219
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2220
|
+
if (bound) {
|
|
2221
|
+
const exists = await sessionExists(this.port, bound);
|
|
2222
|
+
if (exists === false) {
|
|
2223
|
+
this.log({
|
|
2224
|
+
level: "info",
|
|
2225
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2226
|
+
conversation_id: conv.id
|
|
2227
|
+
});
|
|
2228
|
+
this.sessions.delete(conv.id);
|
|
2229
|
+
return this.createAndBindSession(conv.id);
|
|
2230
|
+
}
|
|
2231
|
+
this.sessions.set(conv.id, bound);
|
|
2232
|
+
return bound;
|
|
1715
2233
|
}
|
|
2234
|
+
return this.createAndBindSession(conv.id);
|
|
2235
|
+
}
|
|
2236
|
+
/**
|
|
2237
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2238
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2239
|
+
* self-heal recreate path in `ensureSession`.
|
|
2240
|
+
*/
|
|
2241
|
+
async createAndBindSession(conversationId) {
|
|
1716
2242
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1717
2243
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1718
|
-
this.sessions.set(
|
|
1719
|
-
await this.persistSession(
|
|
2244
|
+
this.sessions.set(conversationId, sessionId);
|
|
2245
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1720
2246
|
});
|
|
1721
2247
|
return sessionId;
|
|
1722
2248
|
}
|
|
@@ -1739,6 +2265,25 @@ var ChannelDriver = class {
|
|
|
1739
2265
|
// -------------------------------------------------------------------------
|
|
1740
2266
|
// Per-session watcher (WI-3)
|
|
1741
2267
|
// -------------------------------------------------------------------------
|
|
2268
|
+
/**
|
|
2269
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2270
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2271
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2272
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2273
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2274
|
+
*/
|
|
2275
|
+
dispatchLocked(sessionId, fn) {
|
|
2276
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2277
|
+
const run2 = prior.then(fn, fn);
|
|
2278
|
+
this.sessionDispatchLocks.set(
|
|
2279
|
+
sessionId,
|
|
2280
|
+
run2.then(
|
|
2281
|
+
() => void 0,
|
|
2282
|
+
() => void 0
|
|
2283
|
+
)
|
|
2284
|
+
);
|
|
2285
|
+
return run2;
|
|
2286
|
+
}
|
|
1742
2287
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1743
2288
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1744
2289
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1748,7 +2293,9 @@ var ChannelDriver = class {
|
|
|
1748
2293
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1749
2294
|
loop: null,
|
|
1750
2295
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1751
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2296
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2297
|
+
lastGoodPollAt: this.now(),
|
|
2298
|
+
hadUsablePoll: false
|
|
1752
2299
|
};
|
|
1753
2300
|
this.watchers.set(sessionId, watcher);
|
|
1754
2301
|
}
|
|
@@ -1758,9 +2305,92 @@ var ChannelDriver = class {
|
|
|
1758
2305
|
opencodeMessageId,
|
|
1759
2306
|
message,
|
|
1760
2307
|
dispatchedAt: now,
|
|
2308
|
+
processingAnchorMs: now,
|
|
1761
2309
|
deadline: now + this.pausedMaxWaitMs,
|
|
1762
2310
|
started: false,
|
|
1763
|
-
done: false
|
|
2311
|
+
done: false,
|
|
2312
|
+
stuckReported: false,
|
|
2313
|
+
lastAliveAt: 0,
|
|
2314
|
+
aliveInFlight: false,
|
|
2315
|
+
awaitingHumanLatched: false,
|
|
2316
|
+
pausedOnQuestion: false,
|
|
2317
|
+
pausedOnPermission: false,
|
|
2318
|
+
pausedClearConfirmed: false,
|
|
2319
|
+
pausedInFlight: false,
|
|
2320
|
+
deliveryDeadlineAnchored: false
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
/**
|
|
2324
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2325
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2326
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2327
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2328
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2329
|
+
*
|
|
2330
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2331
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2332
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2333
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2334
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2335
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2336
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2337
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2338
|
+
* (only the appear-guard uses it).
|
|
2339
|
+
*
|
|
2340
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2341
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
2342
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
2343
|
+
*
|
|
2344
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
2345
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
2346
|
+
* fire from the watcher's normal branches.
|
|
2347
|
+
*/
|
|
2348
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2349
|
+
let watcher = this.watchers.get(sessionId);
|
|
2350
|
+
if (!watcher) {
|
|
2351
|
+
watcher = {
|
|
2352
|
+
conv,
|
|
2353
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2354
|
+
loop: null,
|
|
2355
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2356
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2357
|
+
lastGoodPollAt: this.now(),
|
|
2358
|
+
hadUsablePoll: false
|
|
2359
|
+
};
|
|
2360
|
+
this.watchers.set(sessionId, watcher);
|
|
2361
|
+
}
|
|
2362
|
+
watcher.inFlight.set(message.id, {
|
|
2363
|
+
evidentMessageId: message.id,
|
|
2364
|
+
opencodeMessageId,
|
|
2365
|
+
message,
|
|
2366
|
+
dispatchedAt: this.now(),
|
|
2367
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2368
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2369
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2370
|
+
processingAnchorMs: processedAtMs,
|
|
2371
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2372
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2373
|
+
started: true,
|
|
2374
|
+
done: false,
|
|
2375
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2376
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2377
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2378
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2379
|
+
// (#210/#220 observability).
|
|
2380
|
+
stuckReported: false,
|
|
2381
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2382
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2383
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2384
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2385
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2386
|
+
lastAliveAt: 0,
|
|
2387
|
+
aliveInFlight: false,
|
|
2388
|
+
awaitingHumanLatched: false,
|
|
2389
|
+
pausedOnQuestion: false,
|
|
2390
|
+
pausedOnPermission: false,
|
|
2391
|
+
pausedClearConfirmed: false,
|
|
2392
|
+
pausedInFlight: false,
|
|
2393
|
+
deliveryDeadlineAnchored: false
|
|
1764
2394
|
});
|
|
1765
2395
|
}
|
|
1766
2396
|
/**
|
|
@@ -1811,12 +2441,30 @@ var ChannelDriver = class {
|
|
|
1811
2441
|
messages = Array.isArray(body) ? body : null;
|
|
1812
2442
|
}
|
|
1813
2443
|
} catch {
|
|
1814
|
-
continue;
|
|
1815
2444
|
}
|
|
2445
|
+
if (messages != null && messages.length > 0) {
|
|
2446
|
+
watcher.lastGoodPollAt = this.now();
|
|
2447
|
+
watcher.hadUsablePoll = true;
|
|
2448
|
+
} else {
|
|
2449
|
+
const emptyButReachable = messages != null;
|
|
2450
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2451
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2452
|
+
continue;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
1816
2456
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
1817
|
-
await this.serviceInFlightMessage(
|
|
2457
|
+
await this.serviceInFlightMessage(
|
|
2458
|
+
sessionId,
|
|
2459
|
+
watcher,
|
|
2460
|
+
inFlight,
|
|
2461
|
+
messages,
|
|
2462
|
+
openQuestions,
|
|
2463
|
+
openPermissions,
|
|
2464
|
+
questionsPolledOk,
|
|
2465
|
+
permissionsPolledOk
|
|
2466
|
+
);
|
|
1818
2467
|
}
|
|
1819
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
1820
2468
|
}
|
|
1821
2469
|
} catch (err) {
|
|
1822
2470
|
if (err instanceof ChannelAuthError) {
|
|
@@ -1826,6 +2474,7 @@ var ChannelDriver = class {
|
|
|
1826
2474
|
conversation_id: watcher.conv.id
|
|
1827
2475
|
});
|
|
1828
2476
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2477
|
+
this.readopted.delete(evidentMessageId);
|
|
1829
2478
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1830
2479
|
}
|
|
1831
2480
|
return;
|
|
@@ -1837,19 +2486,49 @@ var ChannelDriver = class {
|
|
|
1837
2486
|
});
|
|
1838
2487
|
}
|
|
1839
2488
|
}
|
|
2489
|
+
/**
|
|
2490
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2491
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2492
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2493
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2494
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2495
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2496
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2497
|
+
* or past now, so a still-ample window is left untouched.
|
|
2498
|
+
*/
|
|
2499
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2500
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2501
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2502
|
+
if (this.now() >= inFlight.deadline) {
|
|
2503
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
1840
2506
|
/**
|
|
1841
2507
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
1842
2508
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
1843
2509
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
1844
2510
|
* in-flight set on completion or timeout.
|
|
1845
2511
|
*/
|
|
1846
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2512
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
1847
2513
|
const conv = watcher.conv;
|
|
1848
2514
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1849
|
-
|
|
2515
|
+
const id = inFlight.evidentMessageId;
|
|
2516
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2517
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2518
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2519
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2520
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2521
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2522
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2523
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1850
2524
|
let claimed;
|
|
1851
2525
|
try {
|
|
1852
|
-
claimed = await this.markProcessing(
|
|
2526
|
+
claimed = await this.markProcessing(
|
|
2527
|
+
conv.id,
|
|
2528
|
+
inFlight.evidentMessageId,
|
|
2529
|
+
sessionId,
|
|
2530
|
+
inFlight.opencodeMessageId
|
|
2531
|
+
);
|
|
1853
2532
|
} catch (err) {
|
|
1854
2533
|
if (err instanceof ChannelAuthError) throw err;
|
|
1855
2534
|
this.log({
|
|
@@ -1871,6 +2550,7 @@ var ChannelDriver = class {
|
|
|
1871
2550
|
}
|
|
1872
2551
|
}
|
|
1873
2552
|
if (state === "done") {
|
|
2553
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
1874
2554
|
if (!inFlight.done) {
|
|
1875
2555
|
this.log({
|
|
1876
2556
|
level: "info",
|
|
@@ -1879,7 +2559,12 @@ var ChannelDriver = class {
|
|
|
1879
2559
|
message_id: inFlight.evidentMessageId
|
|
1880
2560
|
});
|
|
1881
2561
|
try {
|
|
1882
|
-
await this.markDone(
|
|
2562
|
+
await this.markDone(
|
|
2563
|
+
conv.id,
|
|
2564
|
+
inFlight.evidentMessageId,
|
|
2565
|
+
sessionId,
|
|
2566
|
+
inFlight.opencodeMessageId
|
|
2567
|
+
);
|
|
1883
2568
|
} catch (err) {
|
|
1884
2569
|
if (err instanceof ChannelAuthError) throw err;
|
|
1885
2570
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -1915,60 +2600,564 @@ var ChannelDriver = class {
|
|
|
1915
2600
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1916
2601
|
return;
|
|
1917
2602
|
}
|
|
1918
|
-
if (state === "
|
|
1919
|
-
|
|
1920
|
-
|
|
2603
|
+
if (state === "failed") {
|
|
2604
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2605
|
+
if (!inFlight.done) {
|
|
2606
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2607
|
+
this.log({
|
|
2608
|
+
level: "error",
|
|
2609
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2610
|
+
conversation_id: conv.id,
|
|
2611
|
+
message_id: inFlight.evidentMessageId
|
|
2612
|
+
});
|
|
2613
|
+
try {
|
|
2614
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2615
|
+
} catch (err) {
|
|
2616
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2617
|
+
if (err instanceof ChannelTerminalError) {
|
|
2618
|
+
this.log({
|
|
2619
|
+
level: "error",
|
|
2620
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2621
|
+
conversation_id: conv.id,
|
|
2622
|
+
message_id: inFlight.evidentMessageId
|
|
2623
|
+
});
|
|
2624
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2625
|
+
return;
|
|
2626
|
+
}
|
|
2627
|
+
if (this.now() >= inFlight.deadline) {
|
|
2628
|
+
this.log({
|
|
2629
|
+
level: "error",
|
|
2630
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2631
|
+
conversation_id: conv.id,
|
|
2632
|
+
message_id: inFlight.evidentMessageId
|
|
2633
|
+
});
|
|
2634
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2635
|
+
return;
|
|
2636
|
+
}
|
|
2637
|
+
this.log({
|
|
2638
|
+
level: "error",
|
|
2639
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2640
|
+
conversation_id: conv.id,
|
|
2641
|
+
message_id: inFlight.evidentMessageId
|
|
2642
|
+
});
|
|
2643
|
+
return;
|
|
2644
|
+
}
|
|
2645
|
+
inFlight.done = true;
|
|
1921
2646
|
}
|
|
2647
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2648
|
+
return;
|
|
1922
2649
|
}
|
|
1923
|
-
|
|
2650
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2651
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2652
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2653
|
+
inFlight.stuckReported = true;
|
|
2654
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2655
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2656
|
+
});
|
|
2657
|
+
}
|
|
2658
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2659
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
1924
2660
|
this.log({
|
|
1925
|
-
level: "
|
|
1926
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2661
|
+
level: "error",
|
|
2662
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
1927
2663
|
conversation_id: conv.id,
|
|
1928
2664
|
message_id: inFlight.evidentMessageId
|
|
1929
2665
|
});
|
|
2666
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2667
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2668
|
+
});
|
|
1930
2669
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2670
|
+
return;
|
|
2671
|
+
}
|
|
2672
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2673
|
+
inFlight.aliveInFlight = true;
|
|
2674
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2675
|
+
inFlight.aliveInFlight = false;
|
|
2676
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2677
|
+
});
|
|
2678
|
+
}
|
|
2679
|
+
if (awaitingHuman) {
|
|
2680
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2681
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2682
|
+
inFlight.awaitingHumanLatched = true;
|
|
2683
|
+
}
|
|
2684
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2685
|
+
inFlight.pausedInFlight = true;
|
|
2686
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2687
|
+
inFlight.pausedInFlight = false;
|
|
2688
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2689
|
+
});
|
|
2690
|
+
}
|
|
2691
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2692
|
+
inFlight.awaitingHumanLatched = false;
|
|
2693
|
+
inFlight.pausedOnQuestion = false;
|
|
2694
|
+
inFlight.pausedOnPermission = false;
|
|
2695
|
+
inFlight.pausedClearConfirmed = false;
|
|
2696
|
+
}
|
|
2697
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2698
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2699
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2700
|
+
);
|
|
2701
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2702
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2703
|
+
this.log({
|
|
2704
|
+
level: "info",
|
|
2705
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2706
|
+
conversation_id: conv.id,
|
|
2707
|
+
message_id: inFlight.evidentMessageId
|
|
2708
|
+
});
|
|
2709
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2710
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2711
|
+
});
|
|
2712
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
// -------------------------------------------------------------------------
|
|
2716
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2717
|
+
// -------------------------------------------------------------------------
|
|
2718
|
+
/**
|
|
2719
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2720
|
+
*
|
|
2721
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2722
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2723
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2724
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2725
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2726
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2727
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2728
|
+
*
|
|
2729
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2730
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2731
|
+
*/
|
|
2732
|
+
async readoptProcessing() {
|
|
2733
|
+
const rows = await this.getProcessingMessages();
|
|
2734
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2735
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2736
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2737
|
+
if (!stillProcessing.has(id)) {
|
|
2738
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2739
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2740
|
+
if (cleared || clearedUndeliverable) {
|
|
2741
|
+
this.log({
|
|
2742
|
+
level: "info",
|
|
2743
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2744
|
+
message_id: id
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
}
|
|
2750
|
+
if (rows.length === 0) return;
|
|
2751
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2752
|
+
for (const row of rows) {
|
|
2753
|
+
if (!row.opencode_session_id) {
|
|
2754
|
+
this.log({
|
|
2755
|
+
level: "error",
|
|
2756
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2757
|
+
conversation_id: row.conversation_id,
|
|
2758
|
+
message_id: row.id
|
|
2759
|
+
});
|
|
2760
|
+
continue;
|
|
2761
|
+
}
|
|
2762
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2763
|
+
list.push(row);
|
|
2764
|
+
bySession.set(row.opencode_session_id, list);
|
|
2765
|
+
}
|
|
2766
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2767
|
+
let messages;
|
|
2768
|
+
try {
|
|
2769
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2770
|
+
if (!res.ok) {
|
|
2771
|
+
this.log({
|
|
2772
|
+
level: "error",
|
|
2773
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2774
|
+
});
|
|
2775
|
+
continue;
|
|
2776
|
+
}
|
|
2777
|
+
const body = await res.json();
|
|
2778
|
+
if (!Array.isArray(body)) {
|
|
2779
|
+
this.log({
|
|
2780
|
+
level: "error",
|
|
2781
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2782
|
+
});
|
|
2783
|
+
continue;
|
|
2784
|
+
}
|
|
2785
|
+
messages = body;
|
|
2786
|
+
} catch (err) {
|
|
2787
|
+
this.log({
|
|
2788
|
+
level: "error",
|
|
2789
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2790
|
+
});
|
|
2791
|
+
continue;
|
|
2792
|
+
}
|
|
2793
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2794
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2795
|
+
for (const row of sessionRows) {
|
|
2796
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2797
|
+
}
|
|
1931
2798
|
}
|
|
1932
2799
|
}
|
|
1933
2800
|
/**
|
|
1934
|
-
* Re-
|
|
1935
|
-
*
|
|
1936
|
-
*
|
|
1937
|
-
*
|
|
2801
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2802
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2803
|
+
*
|
|
2804
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2805
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2806
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2807
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2808
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2809
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2810
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2811
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2812
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2813
|
+
* tracking the stored id so the reply correlates by it;
|
|
2814
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2815
|
+
*
|
|
2816
|
+
* Only `ChannelAuthError` propagates.
|
|
1938
2817
|
*/
|
|
1939
|
-
async
|
|
2818
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2819
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2820
|
+
this.log({
|
|
2821
|
+
level: "info",
|
|
2822
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2823
|
+
conversation_id: row.conversation_id,
|
|
2824
|
+
message_id: row.id
|
|
2825
|
+
});
|
|
2826
|
+
return;
|
|
2827
|
+
}
|
|
2828
|
+
const ocId = row.opencode_message_id;
|
|
2829
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2830
|
+
if (state === "done") {
|
|
2831
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2832
|
+
this.log({
|
|
2833
|
+
level: "info",
|
|
2834
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2835
|
+
conversation_id: row.conversation_id,
|
|
2836
|
+
message_id: row.id
|
|
2837
|
+
});
|
|
2838
|
+
return;
|
|
2839
|
+
}
|
|
2840
|
+
this.log({
|
|
2841
|
+
level: "info",
|
|
2842
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2843
|
+
conversation_id: row.conversation_id,
|
|
2844
|
+
message_id: row.id
|
|
2845
|
+
});
|
|
2846
|
+
try {
|
|
2847
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2848
|
+
} catch (err) {
|
|
2849
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2850
|
+
if (err instanceof ChannelTerminalError) {
|
|
2851
|
+
this.doneUndeliverable.add(row.id);
|
|
2852
|
+
this.log({
|
|
2853
|
+
level: "error",
|
|
2854
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2855
|
+
conversation_id: row.conversation_id,
|
|
2856
|
+
message_id: row.id
|
|
2857
|
+
});
|
|
2858
|
+
return;
|
|
2859
|
+
}
|
|
2860
|
+
this.log({
|
|
2861
|
+
level: "error",
|
|
2862
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2863
|
+
conversation_id: row.conversation_id,
|
|
2864
|
+
message_id: row.id
|
|
2865
|
+
});
|
|
2866
|
+
return;
|
|
2867
|
+
}
|
|
2868
|
+
this.dontRedispatch.delete(row.id);
|
|
2869
|
+
return;
|
|
2870
|
+
}
|
|
2871
|
+
if (state === "failed") {
|
|
2872
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2873
|
+
this.log({
|
|
2874
|
+
level: "error",
|
|
2875
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2876
|
+
conversation_id: row.conversation_id,
|
|
2877
|
+
message_id: row.id
|
|
2878
|
+
});
|
|
2879
|
+
try {
|
|
2880
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2881
|
+
} catch (err) {
|
|
2882
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2883
|
+
if (err instanceof ChannelTerminalError) {
|
|
2884
|
+
this.doneUndeliverable.add(row.id);
|
|
2885
|
+
this.log({
|
|
2886
|
+
level: "error",
|
|
2887
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2888
|
+
conversation_id: row.conversation_id,
|
|
2889
|
+
message_id: row.id
|
|
2890
|
+
});
|
|
2891
|
+
return;
|
|
2892
|
+
}
|
|
2893
|
+
this.log({
|
|
2894
|
+
level: "error",
|
|
2895
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2896
|
+
conversation_id: row.conversation_id,
|
|
2897
|
+
message_id: row.id
|
|
2898
|
+
});
|
|
2899
|
+
return;
|
|
2900
|
+
}
|
|
2901
|
+
this.dontRedispatch.delete(row.id);
|
|
2902
|
+
return;
|
|
2903
|
+
}
|
|
2904
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2905
|
+
this.log({
|
|
2906
|
+
level: "info",
|
|
2907
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2908
|
+
conversation_id: row.conversation_id,
|
|
2909
|
+
message_id: row.id
|
|
2910
|
+
});
|
|
2911
|
+
return;
|
|
2912
|
+
}
|
|
2913
|
+
let statusReadableOngoing = null;
|
|
2914
|
+
if (state === "running" && ocId) {
|
|
2915
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2916
|
+
const shape = this.replyCompletionShape(reply);
|
|
2917
|
+
const ongoing = sessionOngoing;
|
|
2918
|
+
statusReadableOngoing = ongoing;
|
|
2919
|
+
if (ongoing === false) {
|
|
2920
|
+
this.log({
|
|
2921
|
+
level: "info",
|
|
2922
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2923
|
+
conversation_id: row.conversation_id,
|
|
2924
|
+
message_id: row.id
|
|
2925
|
+
});
|
|
2926
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2927
|
+
return;
|
|
2928
|
+
}
|
|
2929
|
+
if (ongoing === true) {
|
|
2930
|
+
this.log({
|
|
2931
|
+
level: "info",
|
|
2932
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2933
|
+
conversation_id: row.conversation_id,
|
|
2934
|
+
message_id: row.id
|
|
2935
|
+
});
|
|
2936
|
+
} else {
|
|
2937
|
+
if (shape === "b1") {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
return;
|
|
2945
|
+
}
|
|
2946
|
+
this.log({
|
|
2947
|
+
level: "info",
|
|
2948
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2949
|
+
conversation_id: row.conversation_id,
|
|
2950
|
+
message_id: row.id
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2955
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2956
|
+
if (descendantAlive === true) {
|
|
2957
|
+
this.log({
|
|
2958
|
+
level: "info",
|
|
2959
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2960
|
+
conversation_id: row.conversation_id,
|
|
2961
|
+
message_id: row.id
|
|
2962
|
+
});
|
|
2963
|
+
} else {
|
|
2964
|
+
this.log({
|
|
2965
|
+
level: "info",
|
|
2966
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2967
|
+
conversation_id: row.conversation_id,
|
|
2968
|
+
message_id: row.id
|
|
2969
|
+
});
|
|
2970
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2971
|
+
return;
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2975
|
+
const conv = this.convForRow(sessionId, row);
|
|
2976
|
+
const message = this.queuedMessageForRow(row);
|
|
2977
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2978
|
+
this.dispatched.add(row.id);
|
|
2979
|
+
this.readopted.add(row.id);
|
|
2980
|
+
this.ensureWatcherRunning(sessionId);
|
|
2981
|
+
this.log({
|
|
2982
|
+
level: "info",
|
|
2983
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2984
|
+
conversation_id: row.conversation_id,
|
|
2985
|
+
message_id: row.id
|
|
2986
|
+
});
|
|
2987
|
+
return;
|
|
2988
|
+
}
|
|
2989
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2990
|
+
}
|
|
2991
|
+
/**
|
|
2992
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2993
|
+
*
|
|
2994
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
2995
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
2996
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
2997
|
+
* correlates server-side.
|
|
2998
|
+
*
|
|
2999
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3000
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3001
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3002
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3003
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3004
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3005
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3006
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3007
|
+
* may retry exactly once more).
|
|
3008
|
+
*
|
|
3009
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
3010
|
+
* `processed_at` (Invariant 1).
|
|
3011
|
+
*/
|
|
3012
|
+
async forceReadoptRun(sessionId, row) {
|
|
3013
|
+
if (this.stopped) {
|
|
3014
|
+
this.log({
|
|
3015
|
+
level: "info",
|
|
3016
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3017
|
+
conversation_id: row.conversation_id,
|
|
3018
|
+
message_id: row.id
|
|
3019
|
+
});
|
|
3020
|
+
return;
|
|
3021
|
+
}
|
|
3022
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3023
|
+
this.log({
|
|
3024
|
+
level: "info",
|
|
3025
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3026
|
+
conversation_id: row.conversation_id,
|
|
3027
|
+
message_id: row.id
|
|
3028
|
+
});
|
|
3029
|
+
return;
|
|
3030
|
+
}
|
|
3031
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
3032
|
+
this.dontRedispatch.add(row.id);
|
|
3033
|
+
this.log({
|
|
3034
|
+
level: "info",
|
|
3035
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
3036
|
+
conversation_id: row.conversation_id,
|
|
3037
|
+
message_id: row.id
|
|
3038
|
+
});
|
|
3039
|
+
return;
|
|
3040
|
+
}
|
|
1940
3041
|
const options = {
|
|
1941
|
-
agent:
|
|
1942
|
-
model:
|
|
3042
|
+
agent: row.opencode_agent ?? void 0,
|
|
3043
|
+
model: row.opencode_model ?? void 0
|
|
1943
3044
|
};
|
|
1944
3045
|
this.log({
|
|
1945
3046
|
level: "info",
|
|
1946
|
-
message: `
|
|
1947
|
-
|
|
3047
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
3048
|
+
conversation_id: row.conversation_id,
|
|
3049
|
+
message_id: row.id
|
|
1948
3050
|
});
|
|
3051
|
+
this.awaitingReadopt.add(row.id);
|
|
3052
|
+
let ocId;
|
|
1949
3053
|
try {
|
|
1950
|
-
await
|
|
1951
|
-
this.port,
|
|
3054
|
+
ocId = await this.dispatchLocked(
|
|
1952
3055
|
sessionId,
|
|
1953
|
-
|
|
1954
|
-
options,
|
|
1955
|
-
inFlight.opencodeMessageId
|
|
3056
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
1956
3057
|
);
|
|
1957
3058
|
} catch (err) {
|
|
3059
|
+
this.awaitingReadopt.delete(row.id);
|
|
3060
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
1958
3061
|
this.log({
|
|
1959
3062
|
level: "error",
|
|
1960
|
-
message: `Re-dispatch failed for message ${
|
|
1961
|
-
|
|
3063
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3064
|
+
conversation_id: row.conversation_id,
|
|
3065
|
+
message_id: row.id
|
|
1962
3066
|
});
|
|
3067
|
+
return;
|
|
1963
3068
|
}
|
|
1964
|
-
|
|
3069
|
+
if (ocId === null) {
|
|
3070
|
+
this.awaitingReadopt.delete(row.id);
|
|
3071
|
+
this.log({
|
|
3072
|
+
level: "error",
|
|
3073
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3074
|
+
conversation_id: row.conversation_id,
|
|
3075
|
+
message_id: row.id
|
|
3076
|
+
});
|
|
3077
|
+
return;
|
|
3078
|
+
}
|
|
3079
|
+
const conv = this.convForRow(sessionId, row);
|
|
3080
|
+
const message = this.queuedMessageForRow(row);
|
|
3081
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
3082
|
+
this.dispatched.add(row.id);
|
|
3083
|
+
this.readopted.add(row.id);
|
|
3084
|
+
this.awaitingReadopt.delete(row.id);
|
|
3085
|
+
this.ensureWatcherRunning(sessionId);
|
|
3086
|
+
}
|
|
3087
|
+
/**
|
|
3088
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
3089
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
3090
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
3091
|
+
*/
|
|
3092
|
+
isTracked(sessionId, evidentMessageId) {
|
|
3093
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
3094
|
+
const watcher = this.watchers.get(sessionId);
|
|
3095
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
3096
|
+
}
|
|
3097
|
+
/**
|
|
3098
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
3099
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
3100
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
3101
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
3102
|
+
* intended, which is worth surfacing.
|
|
3103
|
+
*/
|
|
3104
|
+
processedAtMs(row) {
|
|
3105
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
3106
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
3107
|
+
this.log({
|
|
3108
|
+
level: "error",
|
|
3109
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
3110
|
+
conversation_id: row.conversation_id,
|
|
3111
|
+
message_id: row.id
|
|
3112
|
+
});
|
|
3113
|
+
return this.now();
|
|
3114
|
+
}
|
|
3115
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
3116
|
+
convForRow(sessionId, row) {
|
|
3117
|
+
return {
|
|
3118
|
+
id: row.conversation_id,
|
|
3119
|
+
agent_id: this.agentId,
|
|
3120
|
+
opencode_session_id: sessionId,
|
|
3121
|
+
pending_message_count: 0,
|
|
3122
|
+
oldest_pending_at: row.processed_at
|
|
3123
|
+
};
|
|
3124
|
+
}
|
|
3125
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
3126
|
+
queuedMessageForRow(row) {
|
|
3127
|
+
return {
|
|
3128
|
+
id: row.id,
|
|
3129
|
+
content: row.content,
|
|
3130
|
+
status: "processing",
|
|
3131
|
+
opencode_agent: row.opencode_agent,
|
|
3132
|
+
opencode_model: row.opencode_model,
|
|
3133
|
+
source_message_id: row.source_message_id,
|
|
3134
|
+
slack_user_id: row.slack_user_id
|
|
3135
|
+
};
|
|
1965
3136
|
}
|
|
1966
3137
|
/**
|
|
1967
3138
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
1968
3139
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
1969
3140
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
3141
|
+
*
|
|
3142
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
3143
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
3144
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
3145
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
3146
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
3147
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
3148
|
+
* the done branch still delivers it (Bugbot #202).
|
|
1970
3149
|
*/
|
|
1971
3150
|
removeInFlight(watcher, evidentMessageId) {
|
|
3151
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
3152
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
3153
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
3154
|
+
this.log({
|
|
3155
|
+
level: "info",
|
|
3156
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
3157
|
+
conversation_id: watcher.conv.id,
|
|
3158
|
+
message_id: evidentMessageId
|
|
3159
|
+
});
|
|
3160
|
+
}
|
|
1972
3161
|
watcher.inFlight.delete(evidentMessageId);
|
|
1973
3162
|
this.dispatched.delete(evidentMessageId);
|
|
1974
3163
|
}
|
|
@@ -1985,21 +3174,41 @@ var ChannelDriver = class {
|
|
|
1985
3174
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
1986
3175
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
1987
3176
|
* oldest running message.
|
|
3177
|
+
*
|
|
3178
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3179
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3180
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3181
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3182
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3183
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3184
|
+
* even after it was already surfaced to the channel.
|
|
1988
3185
|
*/
|
|
1989
3186
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3187
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3188
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3189
|
+
let questionsPolledOk = true;
|
|
3190
|
+
let permissionsPolledOk = true;
|
|
1990
3191
|
let questions = [];
|
|
1991
3192
|
try {
|
|
1992
3193
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
1993
3194
|
if (res.ok) {
|
|
1994
3195
|
const body = await res.json();
|
|
1995
|
-
|
|
3196
|
+
if (Array.isArray(body)) {
|
|
3197
|
+
questions = body;
|
|
3198
|
+
} else {
|
|
3199
|
+
questionsPolledOk = false;
|
|
3200
|
+
}
|
|
3201
|
+
} else {
|
|
3202
|
+
questionsPolledOk = false;
|
|
1996
3203
|
}
|
|
1997
3204
|
} catch {
|
|
3205
|
+
questionsPolledOk = false;
|
|
1998
3206
|
}
|
|
1999
3207
|
for (const q of questions) {
|
|
2000
|
-
if (q.sessionID
|
|
2001
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3208
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2002
3209
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3210
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3211
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2003
3212
|
const reported = await this.reportInteraction(
|
|
2004
3213
|
watcher.conv.id,
|
|
2005
3214
|
"question",
|
|
@@ -2013,14 +3222,22 @@ var ChannelDriver = class {
|
|
|
2013
3222
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2014
3223
|
if (res.ok) {
|
|
2015
3224
|
const body = await res.json();
|
|
2016
|
-
|
|
3225
|
+
if (Array.isArray(body)) {
|
|
3226
|
+
permissions = body;
|
|
3227
|
+
} else {
|
|
3228
|
+
permissionsPolledOk = false;
|
|
3229
|
+
}
|
|
3230
|
+
} else {
|
|
3231
|
+
permissionsPolledOk = false;
|
|
2017
3232
|
}
|
|
2018
3233
|
} catch {
|
|
3234
|
+
permissionsPolledOk = false;
|
|
2019
3235
|
}
|
|
2020
3236
|
for (const p of permissions) {
|
|
2021
|
-
if (p.sessionID
|
|
2022
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3237
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2023
3238
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3239
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3240
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2024
3241
|
const reported = await this.reportInteraction(
|
|
2025
3242
|
watcher.conv.id,
|
|
2026
3243
|
"permission",
|
|
@@ -2029,6 +3246,128 @@ var ChannelDriver = class {
|
|
|
2029
3246
|
);
|
|
2030
3247
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2031
3248
|
}
|
|
3249
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3250
|
+
}
|
|
3251
|
+
/**
|
|
3252
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3253
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3254
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3255
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3256
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3257
|
+
*
|
|
3258
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3259
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3260
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3261
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3262
|
+
*/
|
|
3263
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3264
|
+
let current = sessionId;
|
|
3265
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3266
|
+
if (current === rootSessionId) return true;
|
|
3267
|
+
const parent = await this.resolveSessionParent(current);
|
|
3268
|
+
if (parent === null || parent === void 0) return false;
|
|
3269
|
+
current = parent;
|
|
3270
|
+
}
|
|
3271
|
+
return false;
|
|
3272
|
+
}
|
|
3273
|
+
/**
|
|
3274
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3275
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3276
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3277
|
+
* caching a wrong answer — the next tick retries).
|
|
3278
|
+
*/
|
|
3279
|
+
async resolveSessionParent(sessionId) {
|
|
3280
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3281
|
+
if (cached !== void 0) return cached;
|
|
3282
|
+
let parent = void 0;
|
|
3283
|
+
try {
|
|
3284
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3285
|
+
if (res.ok) {
|
|
3286
|
+
const body = await res.json();
|
|
3287
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3288
|
+
}
|
|
3289
|
+
} catch {
|
|
3290
|
+
parent = void 0;
|
|
3291
|
+
}
|
|
3292
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3293
|
+
return parent;
|
|
3294
|
+
}
|
|
3295
|
+
/**
|
|
3296
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3297
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3298
|
+
*
|
|
3299
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3300
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3301
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3302
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3303
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3304
|
+
* provably in flight at the exact moment of recovery.
|
|
3305
|
+
*
|
|
3306
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3307
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3308
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3309
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3310
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3311
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3312
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3313
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3314
|
+
*
|
|
3315
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3316
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3317
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3318
|
+
* case), OR no descendant is found at all.
|
|
3319
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3320
|
+
*
|
|
3321
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3322
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3323
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3324
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3325
|
+
* true/false/null faithfully.
|
|
3326
|
+
*
|
|
3327
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3328
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3329
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3330
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3331
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3332
|
+
* `SessionStatus` only.
|
|
3333
|
+
*/
|
|
3334
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3335
|
+
const sessions = await listSessions(this.port);
|
|
3336
|
+
if (!sessions) {
|
|
3337
|
+
this.log({
|
|
3338
|
+
level: "error",
|
|
3339
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3340
|
+
});
|
|
3341
|
+
return null;
|
|
3342
|
+
}
|
|
3343
|
+
for (const candidate of sessions) {
|
|
3344
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3345
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3346
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3347
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3348
|
+
return true;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
return false;
|
|
3352
|
+
}
|
|
3353
|
+
/**
|
|
3354
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3355
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3356
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3357
|
+
* the aborted-in-flight production bug after a restart.
|
|
3358
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3359
|
+
* (the sub-agent preamble — #253's shape).
|
|
3360
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3361
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3362
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3363
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3364
|
+
*/
|
|
3365
|
+
replyCompletionShape(reply) {
|
|
3366
|
+
if (!reply) return "other";
|
|
3367
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3368
|
+
if (completed == null) return "b1";
|
|
3369
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3370
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2032
3371
|
}
|
|
2033
3372
|
/**
|
|
2034
3373
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -2112,6 +3451,35 @@ var ChannelDriver = class {
|
|
|
2112
3451
|
}
|
|
2113
3452
|
return await res.json();
|
|
2114
3453
|
}
|
|
3454
|
+
/**
|
|
3455
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
3456
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
3457
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
3458
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
3459
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
3460
|
+
* `opencode_session_id`, routing).
|
|
3461
|
+
*
|
|
3462
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
3463
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
3464
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
3465
|
+
* next tick retries.
|
|
3466
|
+
*/
|
|
3467
|
+
async getProcessingMessages() {
|
|
3468
|
+
const res = await this.fetchImpl(
|
|
3469
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
3470
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
3471
|
+
);
|
|
3472
|
+
this.assertAuth(res, "fetching processing messages");
|
|
3473
|
+
if (!res.ok) {
|
|
3474
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
3475
|
+
}
|
|
3476
|
+
const data = await res.json();
|
|
3477
|
+
let messages = data.messages ?? [];
|
|
3478
|
+
if (this.conversationFilter) {
|
|
3479
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
3480
|
+
}
|
|
3481
|
+
return messages;
|
|
3482
|
+
}
|
|
2115
3483
|
/**
|
|
2116
3484
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2117
3485
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2133,13 +3501,17 @@ var ChannelDriver = class {
|
|
|
2133
3501
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2134
3502
|
* retry vehicle for the swap-to-running.
|
|
2135
3503
|
*/
|
|
2136
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
3504
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2137
3505
|
const res = await this.fetchImpl(
|
|
2138
3506
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2139
3507
|
{
|
|
2140
3508
|
method: "PATCH",
|
|
2141
3509
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2142
|
-
body: JSON.stringify({
|
|
3510
|
+
body: JSON.stringify({
|
|
3511
|
+
status: "processing",
|
|
3512
|
+
opencode_session_id: sessionId,
|
|
3513
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3514
|
+
})
|
|
2143
3515
|
}
|
|
2144
3516
|
);
|
|
2145
3517
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2177,13 +3549,17 @@ var ChannelDriver = class {
|
|
|
2177
3549
|
* watcher retries next tick within the
|
|
2178
3550
|
* deadline, Finding 4).
|
|
2179
3551
|
*/
|
|
2180
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3552
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2181
3553
|
const res = await this.fetchImpl(
|
|
2182
3554
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2183
3555
|
{
|
|
2184
3556
|
method: "PATCH",
|
|
2185
3557
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2186
|
-
body: JSON.stringify({
|
|
3558
|
+
body: JSON.stringify({
|
|
3559
|
+
status: "done",
|
|
3560
|
+
opencode_session_id: sessionId,
|
|
3561
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3562
|
+
})
|
|
2187
3563
|
}
|
|
2188
3564
|
);
|
|
2189
3565
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2193,7 +3569,17 @@ var ChannelDriver = class {
|
|
|
2193
3569
|
}
|
|
2194
3570
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2195
3571
|
}
|
|
2196
|
-
|
|
3572
|
+
/**
|
|
3573
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3574
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3575
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3576
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3577
|
+
* failure reason reaches the channel.
|
|
3578
|
+
*/
|
|
3579
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3580
|
+
const body = { status: "failed" };
|
|
3581
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3582
|
+
if (error2 !== void 0) body.error = error2;
|
|
2197
3583
|
await this.callWithRetry(
|
|
2198
3584
|
"marking message as failed",
|
|
2199
3585
|
() => this.fetchImpl(
|
|
@@ -2201,11 +3587,56 @@ var ChannelDriver = class {
|
|
|
2201
3587
|
{
|
|
2202
3588
|
method: "PATCH",
|
|
2203
3589
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2204
|
-
body: JSON.stringify(
|
|
3590
|
+
body: JSON.stringify(body)
|
|
2205
3591
|
}
|
|
2206
3592
|
)
|
|
2207
3593
|
);
|
|
2208
3594
|
}
|
|
3595
|
+
/**
|
|
3596
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3597
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3598
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3599
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3600
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3601
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3602
|
+
* context (no silent catch, per development-workflow).
|
|
3603
|
+
*
|
|
3604
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3605
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3606
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3607
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3608
|
+
* leaves liveness").
|
|
3609
|
+
*/
|
|
3610
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3611
|
+
try {
|
|
3612
|
+
const res = await this.fetchImpl(
|
|
3613
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
3614
|
+
{
|
|
3615
|
+
method: "POST",
|
|
3616
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3617
|
+
body: JSON.stringify({ signal, ...extra })
|
|
3618
|
+
}
|
|
3619
|
+
);
|
|
3620
|
+
if (!res.ok) {
|
|
3621
|
+
this.log({
|
|
3622
|
+
level: "error",
|
|
3623
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3624
|
+
conversation_id: conversationId,
|
|
3625
|
+
message_id: messageId
|
|
3626
|
+
});
|
|
3627
|
+
return false;
|
|
3628
|
+
}
|
|
3629
|
+
return true;
|
|
3630
|
+
} catch (err) {
|
|
3631
|
+
this.log({
|
|
3632
|
+
level: "error",
|
|
3633
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3634
|
+
conversation_id: conversationId,
|
|
3635
|
+
message_id: messageId
|
|
3636
|
+
});
|
|
3637
|
+
return false;
|
|
3638
|
+
}
|
|
3639
|
+
}
|
|
2209
3640
|
async persistSession(conversationId, sessionId) {
|
|
2210
3641
|
const res = await this.fetchImpl(
|
|
2211
3642
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2476,6 +3907,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
2476
3907
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
2477
3908
|
}
|
|
2478
3909
|
}
|
|
3910
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3911
|
+
const apiUrl = getApiUrlConfig();
|
|
3912
|
+
try {
|
|
3913
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3914
|
+
method: "POST",
|
|
3915
|
+
headers: { Authorization: authHeader }
|
|
3916
|
+
});
|
|
3917
|
+
if (!response.ok) {
|
|
3918
|
+
const serverMessage = await readErrorMessage(response);
|
|
3919
|
+
return {
|
|
3920
|
+
ok: false,
|
|
3921
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3922
|
+
};
|
|
3923
|
+
}
|
|
3924
|
+
return { ok: true };
|
|
3925
|
+
} catch (error2) {
|
|
3926
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3927
|
+
}
|
|
3928
|
+
}
|
|
2479
3929
|
async function getAgentInfo(agentId, authHeader) {
|
|
2480
3930
|
const apiUrl = getApiUrlConfig();
|
|
2481
3931
|
try {
|
|
@@ -2521,7 +3971,9 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2521
3971
|
// src/commands/run.ts
|
|
2522
3972
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2523
3973
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2524
|
-
|
|
3974
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3975
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3976
|
+
function log2(state, message, isError = false) {
|
|
2525
3977
|
if (state.json) {
|
|
2526
3978
|
console.log(
|
|
2527
3979
|
JSON.stringify({
|
|
@@ -2546,9 +3998,9 @@ function logActivity(state, entry) {
|
|
|
2546
3998
|
}
|
|
2547
3999
|
if (!state.interactive) {
|
|
2548
4000
|
if (entry.type === "error") {
|
|
2549
|
-
|
|
4001
|
+
log2(state, entry.error ?? "Unknown error", true);
|
|
2550
4002
|
} else if (entry.type === "info" && entry.message) {
|
|
2551
|
-
|
|
4003
|
+
log2(state, entry.message);
|
|
2552
4004
|
}
|
|
2553
4005
|
}
|
|
2554
4006
|
}
|
|
@@ -2634,6 +4086,7 @@ async function handleAuthError(state, error2) {
|
|
|
2634
4086
|
}
|
|
2635
4087
|
async function driveChannels(state, driver) {
|
|
2636
4088
|
let idlePolls = 0;
|
|
4089
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2637
4090
|
while (state.running) {
|
|
2638
4091
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2639
4092
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2643,7 +4096,9 @@ async function driveChannels(state, driver) {
|
|
|
2643
4096
|
try {
|
|
2644
4097
|
const processed = await driver.drainPending();
|
|
2645
4098
|
state.messageCount += processed;
|
|
2646
|
-
|
|
4099
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
4100
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
4101
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2647
4102
|
idlePolls = 0;
|
|
2648
4103
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2649
4104
|
} else if (state.idleTimeout !== null) {
|
|
@@ -2672,7 +4127,7 @@ async function driveChannels(state, driver) {
|
|
|
2672
4127
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
2673
4128
|
if (state.interactive) displayStatus(state);
|
|
2674
4129
|
}
|
|
2675
|
-
await new Promise((
|
|
4130
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
2676
4131
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
2677
4132
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
2678
4133
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -2683,8 +4138,122 @@ async function driveChannels(state, driver) {
|
|
|
2683
4138
|
}
|
|
2684
4139
|
}
|
|
2685
4140
|
}
|
|
2686
|
-
|
|
4141
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4142
|
+
async function runSweep(state, driver, config2) {
|
|
4143
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4144
|
+
try {
|
|
4145
|
+
const sessions = await listSessions(state.port);
|
|
4146
|
+
if (sessions === null) {
|
|
4147
|
+
logActivity(state, {
|
|
4148
|
+
type: "info",
|
|
4149
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4150
|
+
});
|
|
4151
|
+
return;
|
|
4152
|
+
}
|
|
4153
|
+
const toDelete = selectSessionsToDelete(
|
|
4154
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4155
|
+
{
|
|
4156
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4157
|
+
maxCount: config2.maxCount,
|
|
4158
|
+
nowMs: Date.now(),
|
|
4159
|
+
protectedIds: driver.protectedSessionIds()
|
|
4160
|
+
}
|
|
4161
|
+
);
|
|
4162
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4163
|
+
let deleted = 0;
|
|
4164
|
+
let failed = 0;
|
|
4165
|
+
let skippedNewlyActive = 0;
|
|
4166
|
+
for (const id of toDelete) {
|
|
4167
|
+
if (protectedNow.has(id)) {
|
|
4168
|
+
skippedNewlyActive++;
|
|
4169
|
+
logActivity(state, {
|
|
4170
|
+
type: "info",
|
|
4171
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4172
|
+
});
|
|
4173
|
+
continue;
|
|
4174
|
+
}
|
|
4175
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4176
|
+
else failed++;
|
|
4177
|
+
}
|
|
4178
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4179
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4180
|
+
logActivity(state, {
|
|
4181
|
+
type: "info",
|
|
4182
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4183
|
+
});
|
|
4184
|
+
} catch (error2) {
|
|
4185
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4186
|
+
logActivity(state, {
|
|
4187
|
+
type: "error",
|
|
4188
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4189
|
+
});
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
4192
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4193
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4194
|
+
{
|
|
4195
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4196
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4197
|
+
interval: options.sessionCleanupInterval
|
|
4198
|
+
},
|
|
4199
|
+
process.env
|
|
4200
|
+
);
|
|
4201
|
+
for (const warning2 of config2.warnings) {
|
|
4202
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4203
|
+
}
|
|
4204
|
+
if (!config2.enabled) return;
|
|
4205
|
+
logActivity(state, {
|
|
4206
|
+
type: "info",
|
|
4207
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4208
|
+
});
|
|
4209
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4210
|
+
const firstSweep = setTimeout(
|
|
4211
|
+
() => void runSweep(state, driver, config2),
|
|
4212
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4213
|
+
);
|
|
4214
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4215
|
+
}
|
|
4216
|
+
async function notifyOffline(state) {
|
|
4217
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4218
|
+
if (!state.connected) {
|
|
4219
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4220
|
+
return;
|
|
4221
|
+
}
|
|
4222
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4223
|
+
if (result.ok) {
|
|
4224
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4225
|
+
} else {
|
|
4226
|
+
logActivity(state, {
|
|
4227
|
+
type: "error",
|
|
4228
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4229
|
+
});
|
|
4230
|
+
if (state.interactive) displayStatus(state);
|
|
4231
|
+
}
|
|
4232
|
+
}
|
|
4233
|
+
async function cleanup(state, opts = {}) {
|
|
2687
4234
|
state.running = false;
|
|
4235
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4236
|
+
clearInterval(timer);
|
|
4237
|
+
clearTimeout(timer);
|
|
4238
|
+
}
|
|
4239
|
+
state.sessionCleanupTimers = [];
|
|
4240
|
+
if (opts.graceful && state.channelDriver) {
|
|
4241
|
+
state.channelDriver.stop();
|
|
4242
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4243
|
+
if (state.interactive) {
|
|
4244
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4245
|
+
displayStatus(state);
|
|
4246
|
+
}
|
|
4247
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4248
|
+
if (!settled) {
|
|
4249
|
+
logActivity(state, {
|
|
4250
|
+
type: "info",
|
|
4251
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4252
|
+
});
|
|
4253
|
+
if (state.interactive) displayStatus(state);
|
|
4254
|
+
}
|
|
4255
|
+
}
|
|
4256
|
+
await notifyOffline(state);
|
|
2688
4257
|
if (state.connection) {
|
|
2689
4258
|
state.connection.close();
|
|
2690
4259
|
state.connection = null;
|
|
@@ -2695,7 +4264,7 @@ async function cleanup(state) {
|
|
|
2695
4264
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2696
4265
|
displayStatus(state);
|
|
2697
4266
|
} else {
|
|
2698
|
-
|
|
4267
|
+
log2(state, "Stopped OpenCode process");
|
|
2699
4268
|
}
|
|
2700
4269
|
state.opencodeProcess = null;
|
|
2701
4270
|
}
|
|
@@ -2715,26 +4284,32 @@ async function run(options) {
|
|
|
2715
4284
|
opencodeVersion: null,
|
|
2716
4285
|
opencodeProcess: null,
|
|
2717
4286
|
connection: null,
|
|
4287
|
+
channelDriver: null,
|
|
2718
4288
|
running: true,
|
|
4289
|
+
shuttingDown: false,
|
|
2719
4290
|
activityLog: [],
|
|
2720
4291
|
messageCount: 0,
|
|
4292
|
+
lastProxiedActivityAt: null,
|
|
4293
|
+
sessionCleanupTimers: [],
|
|
2721
4294
|
authHeader: ""
|
|
2722
4295
|
};
|
|
2723
4296
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2724
|
-
|
|
4297
|
+
log2(
|
|
2725
4298
|
state,
|
|
2726
4299
|
"Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
2727
4300
|
false
|
|
2728
4301
|
);
|
|
2729
4302
|
}
|
|
2730
4303
|
const handleSignal = async () => {
|
|
4304
|
+
if (state.shuttingDown) return;
|
|
4305
|
+
state.shuttingDown = true;
|
|
2731
4306
|
if (state.interactive) {
|
|
2732
4307
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2733
4308
|
displayStatus(state);
|
|
2734
4309
|
} else {
|
|
2735
|
-
|
|
4310
|
+
log2(state, "Shutting down...");
|
|
2736
4311
|
}
|
|
2737
|
-
await cleanup(state);
|
|
4312
|
+
await cleanup(state, { graceful: true });
|
|
2738
4313
|
await shutdownTelemetry();
|
|
2739
4314
|
process.exit(0);
|
|
2740
4315
|
};
|
|
@@ -2765,7 +4340,7 @@ async function run(options) {
|
|
|
2765
4340
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2766
4341
|
if (resolved.agent_id) {
|
|
2767
4342
|
state.agentId = resolved.agent_id;
|
|
2768
|
-
|
|
4343
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2769
4344
|
if (state.interactive && !state.json) {
|
|
2770
4345
|
logActivity(state, {
|
|
2771
4346
|
type: "info",
|
|
@@ -2828,17 +4403,17 @@ async function run(options) {
|
|
|
2828
4403
|
port: state.port,
|
|
2829
4404
|
interactive: state.interactive,
|
|
2830
4405
|
agentId: state.agentId,
|
|
2831
|
-
log: (message) =>
|
|
4406
|
+
log: (message) => log2(state, message)
|
|
2832
4407
|
});
|
|
2833
4408
|
state.port = oc.port;
|
|
2834
4409
|
state.opencodeProcess = oc.process;
|
|
2835
4410
|
state.opencodeVersion = oc.version;
|
|
2836
4411
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2837
|
-
const
|
|
2838
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
4412
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
4413
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
2839
4414
|
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
2840
4415
|
if (versionWarning) {
|
|
2841
|
-
|
|
4416
|
+
log2(state, versionWarning, false);
|
|
2842
4417
|
if (state.interactive && !state.json) {
|
|
2843
4418
|
logActivity(state, { type: "info", message: versionWarning });
|
|
2844
4419
|
}
|
|
@@ -2854,12 +4429,14 @@ async function run(options) {
|
|
|
2854
4429
|
apiUrl: getApiUrlConfig(),
|
|
2855
4430
|
getAuthHeader: () => state.authHeader,
|
|
2856
4431
|
conversationFilter: state.conversationFilter,
|
|
4432
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
2857
4433
|
log: (entry) => logActivity(state, {
|
|
2858
4434
|
type: entry.level === "error" ? "error" : "info",
|
|
2859
4435
|
message: entry.message,
|
|
2860
4436
|
error: entry.level === "error" ? entry.message : void 0
|
|
2861
4437
|
})
|
|
2862
4438
|
});
|
|
4439
|
+
state.channelDriver = channelDriver;
|
|
2863
4440
|
const connection = new RunnerConnection({
|
|
2864
4441
|
agentId: state.agentId,
|
|
2865
4442
|
getAuthHeader: () => state.authHeader,
|
|
@@ -2873,7 +4450,11 @@ async function run(options) {
|
|
|
2873
4450
|
type: "info",
|
|
2874
4451
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2875
4452
|
});
|
|
2876
|
-
emitAgentConnected(state.agentId, {
|
|
4453
|
+
emitAgentConnected(state.agentId, {
|
|
4454
|
+
port: state.port,
|
|
4455
|
+
cli_version: getCliVersion(),
|
|
4456
|
+
opencode_version: state.opencodeVersion
|
|
4457
|
+
});
|
|
2877
4458
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2878
4459
|
if (state.interactive) displayStatus(state);
|
|
2879
4460
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2907,9 +4488,14 @@ async function run(options) {
|
|
|
2907
4488
|
logActivity(state, { type: "error", error: error2 });
|
|
2908
4489
|
if (state.interactive) displayStatus(state);
|
|
2909
4490
|
},
|
|
2910
|
-
// Web traffic is proxied transparently;
|
|
4491
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
4492
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
4493
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
4494
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
4495
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2911
4496
|
onResponse: () => {
|
|
2912
4497
|
state.opencodeConnected = true;
|
|
4498
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2913
4499
|
},
|
|
2914
4500
|
// A channel message was queued and the api-worker pinged us over the
|
|
2915
4501
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2947,10 +4533,12 @@ async function run(options) {
|
|
|
2947
4533
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
2948
4534
|
throw error2;
|
|
2949
4535
|
}
|
|
4536
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
2950
4537
|
if (!interactive || state.json) {
|
|
2951
|
-
|
|
4538
|
+
log2(state, "Driving channel messages...");
|
|
2952
4539
|
}
|
|
2953
4540
|
await driveChannels(state, channelDriver);
|
|
4541
|
+
if (state.shuttingDown) return;
|
|
2954
4542
|
await cleanup(state);
|
|
2955
4543
|
if (state.json) {
|
|
2956
4544
|
console.log(
|
|
@@ -2960,11 +4548,12 @@ async function run(options) {
|
|
|
2960
4548
|
})
|
|
2961
4549
|
);
|
|
2962
4550
|
} else if (!interactive) {
|
|
2963
|
-
|
|
4551
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2964
4552
|
}
|
|
2965
4553
|
await shutdownTelemetry();
|
|
2966
4554
|
process.exit(0);
|
|
2967
4555
|
} catch (error2) {
|
|
4556
|
+
if (state.shuttingDown) return;
|
|
2968
4557
|
await cleanup(state);
|
|
2969
4558
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
2970
4559
|
if (state.json) {
|
|
@@ -2982,8 +4571,9 @@ async function run(options) {
|
|
|
2982
4571
|
}
|
|
2983
4572
|
|
|
2984
4573
|
// src/index.ts
|
|
4574
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2985
4575
|
var program = new Command();
|
|
2986
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
4576
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2987
4577
|
"--endpoint <url>",
|
|
2988
4578
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2989
4579
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|
|
@@ -2998,7 +4588,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
2998
4588
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
2999
4589
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3000
4590
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3001
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4591
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4592
|
+
"--session-cleanup-max-age <duration>",
|
|
4593
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4594
|
+
).option(
|
|
4595
|
+
"--session-cleanup-max-count <n>",
|
|
4596
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4597
|
+
).option(
|
|
4598
|
+
"--session-cleanup-interval <duration>",
|
|
4599
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4600
|
+
).action(
|
|
3002
4601
|
(options) => {
|
|
3003
4602
|
run({
|
|
3004
4603
|
agent: options.agent,
|
|
@@ -3006,7 +4605,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3006
4605
|
verbose: options.verbose,
|
|
3007
4606
|
conversation: options.conversation,
|
|
3008
4607
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3009
|
-
json: options.json
|
|
4608
|
+
json: options.json,
|
|
4609
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4610
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4611
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4612
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3010
4613
|
});
|
|
3011
4614
|
}
|
|
3012
4615
|
);
|