@evident-ai/cli 3.0.1-dev.476bdd6 → 3.0.1-dev.4b2c312
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -87
- package/dist/index.js +1846 -260
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -284,14 +285,14 @@ function blank() {
|
|
|
284
285
|
console.log();
|
|
285
286
|
}
|
|
286
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
287
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
288
289
|
process.stdout.write(chalk.dim(prompt));
|
|
289
290
|
const handler = () => {
|
|
290
291
|
process.stdin.removeListener("data", handler);
|
|
291
292
|
process.stdin.setRawMode?.(false);
|
|
292
293
|
process.stdin.pause();
|
|
293
294
|
console.log();
|
|
294
|
-
|
|
295
|
+
resolve2();
|
|
295
296
|
};
|
|
296
297
|
if (process.stdin.isTTY) {
|
|
297
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -301,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
301
302
|
});
|
|
302
303
|
}
|
|
303
304
|
function sleep(ms) {
|
|
304
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
305
306
|
}
|
|
306
307
|
|
|
307
308
|
// src/commands/login.ts
|
|
@@ -375,19 +376,19 @@ async function tokenLogin() {
|
|
|
375
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
376
377
|
blank();
|
|
377
378
|
process.stdout.write("Paste token: ");
|
|
378
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
379
380
|
let data = "";
|
|
380
381
|
process.stdin.setEncoding("utf8");
|
|
381
382
|
process.stdin.on("data", (chunk) => {
|
|
382
383
|
data += chunk;
|
|
383
384
|
});
|
|
384
385
|
process.stdin.on("end", () => {
|
|
385
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
386
387
|
});
|
|
387
388
|
if (process.stdin.isTTY) {
|
|
388
389
|
process.stdin.once("data", (chunk) => {
|
|
389
390
|
process.stdin.pause();
|
|
390
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
391
392
|
});
|
|
392
393
|
process.stdin.resume();
|
|
393
394
|
}
|
|
@@ -484,8 +485,34 @@ var TelemetryEventTypes = {
|
|
|
484
485
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
486
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
487
|
|
|
488
|
+
// ../../packages/types/src/logging/index.ts
|
|
489
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
|
+
function log(level, event, fields) {
|
|
491
|
+
const method = level === "debug" ? "log" : level;
|
|
492
|
+
try {
|
|
493
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
494
|
+
} catch (err) {
|
|
495
|
+
console.error(
|
|
496
|
+
"[evident] log_serialize_failed",
|
|
497
|
+
event,
|
|
498
|
+
err instanceof Error ? err.message : String(err)
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
function stripQuery(url) {
|
|
503
|
+
try {
|
|
504
|
+
return new URL(url).pathname;
|
|
505
|
+
} catch {
|
|
506
|
+
const q = url.indexOf("?");
|
|
507
|
+
return q === -1 ? url : url.slice(0, q);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
487
511
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
489
516
|
var eventBuffer = [];
|
|
490
517
|
var flushTimeout = null;
|
|
491
518
|
var isShuttingDown = false;
|
|
@@ -679,11 +706,24 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
679
706
|
if (health.healthy) {
|
|
680
707
|
return health;
|
|
681
708
|
}
|
|
682
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
683
710
|
}
|
|
684
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
685
712
|
}
|
|
686
713
|
|
|
714
|
+
// src/lib/opencode/opencode-version-gate.ts
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
716
|
+
function isQueueValidatedVersion(version2) {
|
|
717
|
+
if (!version2) return false;
|
|
718
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
719
|
+
}
|
|
720
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
721
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
722
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
723
|
+
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
724
|
+
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
725
|
+
}
|
|
726
|
+
|
|
687
727
|
// src/lib/opencode/process.ts
|
|
688
728
|
import { execSync, spawn } from "child_process";
|
|
689
729
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
@@ -996,7 +1036,37 @@ function roleOf(m) {
|
|
|
996
1036
|
}
|
|
997
1037
|
function completedOf(m) {
|
|
998
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
999
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1044
|
+
}
|
|
1045
|
+
function idOf(m) {
|
|
1046
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1047
|
+
if (typeof m.id === "string") return m.id;
|
|
1048
|
+
const infoId = m.info?.id;
|
|
1049
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1050
|
+
}
|
|
1051
|
+
function parentIdOf(m) {
|
|
1052
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1053
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1054
|
+
const infoParent = m.info?.parentID;
|
|
1055
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1056
|
+
}
|
|
1057
|
+
function finishOf(m) {
|
|
1058
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1059
|
+
if (typeof m.finish === "string") return m.finish;
|
|
1060
|
+
const infoFinish = m.info?.finish;
|
|
1061
|
+
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1062
|
+
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1000
1070
|
}
|
|
1001
1071
|
async function getSessionMessages(port, sessionId) {
|
|
1002
1072
|
try {
|
|
@@ -1008,11 +1078,47 @@ async function getSessionMessages(port, sessionId) {
|
|
|
1008
1078
|
return null;
|
|
1009
1079
|
}
|
|
1010
1080
|
}
|
|
1011
|
-
function
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1081
|
+
function sessionLastActivityMs(session) {
|
|
1082
|
+
const candidates = [
|
|
1083
|
+
session.time?.updated,
|
|
1084
|
+
session.time?.created,
|
|
1085
|
+
session.time_updated,
|
|
1086
|
+
session.time_created,
|
|
1087
|
+
session.updated,
|
|
1088
|
+
session.created
|
|
1089
|
+
];
|
|
1090
|
+
for (const c of candidates) {
|
|
1091
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1092
|
+
}
|
|
1093
|
+
return null;
|
|
1094
|
+
}
|
|
1095
|
+
async function listSessions(port) {
|
|
1096
|
+
try {
|
|
1097
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1098
|
+
if (!res.ok) return null;
|
|
1099
|
+
const body = await res.json();
|
|
1100
|
+
return Array.isArray(body) ? body : null;
|
|
1101
|
+
} catch {
|
|
1102
|
+
return null;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
async function deleteSession(port, id) {
|
|
1106
|
+
try {
|
|
1107
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1108
|
+
return res.status >= 200 && res.status < 300;
|
|
1109
|
+
} catch {
|
|
1110
|
+
return false;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
async function sessionExists(port, id) {
|
|
1114
|
+
try {
|
|
1115
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1116
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1117
|
+
if (res.status === 404) return false;
|
|
1118
|
+
return null;
|
|
1119
|
+
} catch {
|
|
1120
|
+
return null;
|
|
1121
|
+
}
|
|
1016
1122
|
}
|
|
1017
1123
|
async function createOpenCodeSession(port, directory) {
|
|
1018
1124
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
@@ -1031,7 +1137,15 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1031
1137
|
const data = await response.json();
|
|
1032
1138
|
return data.id;
|
|
1033
1139
|
}
|
|
1034
|
-
|
|
1140
|
+
function messageText(m) {
|
|
1141
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1142
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1143
|
+
}
|
|
1144
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1145
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1146
|
+
const knownUserIds = new Set(
|
|
1147
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1148
|
+
);
|
|
1035
1149
|
const body = {
|
|
1036
1150
|
parts: [{ type: "text", text: content }]
|
|
1037
1151
|
};
|
|
@@ -1047,79 +1161,213 @@ async function sendMessageToOpenCode(port, sessionId, content, options, hooks, m
|
|
|
1047
1161
|
};
|
|
1048
1162
|
}
|
|
1049
1163
|
}
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
try {
|
|
1074
|
-
const res = await fetch(`${opencodeBase(port)}/permission`);
|
|
1075
|
-
if (res.ok) {
|
|
1076
|
-
const permissions = await res.json();
|
|
1077
|
-
for (const p of permissions) {
|
|
1078
|
-
if (p.sessionID === sessionId && !reportedPermissions.has(p.id)) {
|
|
1079
|
-
reportedPermissions.add(p.id);
|
|
1080
|
-
await hooks.onPermission(p);
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
}
|
|
1084
|
-
} catch {
|
|
1164
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1165
|
+
method: "POST",
|
|
1166
|
+
headers: { "Content-Type": "application/json" },
|
|
1167
|
+
body: JSON.stringify(body)
|
|
1168
|
+
});
|
|
1169
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1170
|
+
const text = await res.text().catch(() => "");
|
|
1171
|
+
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1172
|
+
}
|
|
1173
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1174
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1175
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1176
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1177
|
+
if (after) {
|
|
1178
|
+
let best = null;
|
|
1179
|
+
for (const m of after) {
|
|
1180
|
+
if (roleOf(m) !== "user") continue;
|
|
1181
|
+
const id = idOf(m);
|
|
1182
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1183
|
+
if (messageText(m) !== content) continue;
|
|
1184
|
+
const created = createdOf(m) ?? 0;
|
|
1185
|
+
if (best === null || created > best.created) {
|
|
1186
|
+
best = { id, created };
|
|
1085
1187
|
}
|
|
1086
1188
|
}
|
|
1189
|
+
if (best) return best.id;
|
|
1190
|
+
}
|
|
1191
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1192
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
return null;
|
|
1196
|
+
}
|
|
1197
|
+
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1198
|
+
if (!messages || messages.length === 0) return null;
|
|
1199
|
+
const byParent = messages.find(
|
|
1200
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1201
|
+
);
|
|
1202
|
+
if (byParent) return byParent;
|
|
1203
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1204
|
+
if (userIndex === -1) return null;
|
|
1205
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1206
|
+
if (roleOf(messages[i]) === "assistant") return messages[i];
|
|
1207
|
+
}
|
|
1208
|
+
return null;
|
|
1209
|
+
}
|
|
1210
|
+
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1211
|
+
if (!messages || messages.length === 0) return null;
|
|
1212
|
+
let lastCorrelated = null;
|
|
1213
|
+
let lastNonErrored = null;
|
|
1214
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1215
|
+
const m = messages[i];
|
|
1216
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1217
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1218
|
+
if (errorOf(m) == null) {
|
|
1219
|
+
lastNonErrored = m;
|
|
1220
|
+
break;
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1224
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1225
|
+
if (userIndex === -1) return null;
|
|
1226
|
+
let last = null;
|
|
1227
|
+
let lastOk = null;
|
|
1228
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1229
|
+
const role = roleOf(messages[i]);
|
|
1230
|
+
if (role === "user") break;
|
|
1231
|
+
if (role === "assistant") {
|
|
1232
|
+
last = messages[i];
|
|
1233
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1087
1234
|
}
|
|
1235
|
+
}
|
|
1236
|
+
return lastOk ?? last;
|
|
1237
|
+
}
|
|
1238
|
+
function messageRunState(messages, userMessageId) {
|
|
1239
|
+
if (!messages || messages.length === 0) return "unknown";
|
|
1240
|
+
const hasUser = messages.some((m) => idOf(m) === userMessageId);
|
|
1241
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1242
|
+
if (!hasUser) {
|
|
1243
|
+
if (!reply) return "unknown";
|
|
1244
|
+
}
|
|
1245
|
+
if (!reply) return "queued";
|
|
1246
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1247
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1248
|
+
}
|
|
1249
|
+
function messageError(messages, userMessageId) {
|
|
1250
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1251
|
+
const error2 = errorOf(reply);
|
|
1252
|
+
if (error2 == null) return null;
|
|
1253
|
+
if (typeof error2 === "string") return error2;
|
|
1254
|
+
if (typeof error2 === "object") {
|
|
1255
|
+
const e = error2;
|
|
1256
|
+
const dataMessage = e.data?.message;
|
|
1257
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1258
|
+
if (typeof e.message === "string") return e.message;
|
|
1259
|
+
}
|
|
1260
|
+
return "The agent run failed.";
|
|
1261
|
+
}
|
|
1262
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1263
|
+
if (!messages || messages.length === 0) return false;
|
|
1264
|
+
return messages.some(
|
|
1265
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1266
|
+
);
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1270
|
+
var DURATION_UNIT_MS = {
|
|
1271
|
+
s: 1e3,
|
|
1272
|
+
m: 60 * 1e3,
|
|
1273
|
+
h: 60 * 60 * 1e3,
|
|
1274
|
+
d: 24 * 60 * 60 * 1e3
|
|
1275
|
+
};
|
|
1276
|
+
function parseDurationMs(input) {
|
|
1277
|
+
const trimmed = input.trim();
|
|
1278
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1279
|
+
if (!match) {
|
|
1280
|
+
throw new Error(
|
|
1281
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1282
|
+
);
|
|
1283
|
+
}
|
|
1284
|
+
const value = Number(match[1]);
|
|
1285
|
+
if (value <= 0) {
|
|
1286
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1287
|
+
}
|
|
1288
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1289
|
+
}
|
|
1290
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1291
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1292
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1293
|
+
const ageEligible = (s) => {
|
|
1294
|
+
if (maxAgeMs === void 0) return false;
|
|
1295
|
+
if (s.lastActivityMs === null) return true;
|
|
1296
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1088
1297
|
};
|
|
1089
|
-
const
|
|
1090
|
-
|
|
1091
|
-
const
|
|
1298
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1299
|
+
if (maxCount !== void 0) {
|
|
1300
|
+
const byActivityDesc = [...sessions].sort(
|
|
1301
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1302
|
+
);
|
|
1303
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1304
|
+
countEligibleIds.add(s.id);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
const toDelete = [];
|
|
1308
|
+
for (const s of sessions) {
|
|
1309
|
+
if (protectedIds.has(s.id)) continue;
|
|
1310
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1311
|
+
toDelete.push(s.id);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
return toDelete;
|
|
1315
|
+
}
|
|
1316
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1317
|
+
function resolve(flag, envValue, fallback) {
|
|
1318
|
+
return flag ?? envValue ?? fallback;
|
|
1319
|
+
}
|
|
1320
|
+
function parseMaxCount(input) {
|
|
1321
|
+
const trimmed = input.trim();
|
|
1322
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1323
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(trimmed);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value;
|
|
1330
|
+
}
|
|
1331
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1332
|
+
const warnings = [];
|
|
1333
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1334
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1335
|
+
const intervalRaw = resolve(
|
|
1336
|
+
flags.interval,
|
|
1337
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1338
|
+
DEFAULT_INTERVAL
|
|
1339
|
+
);
|
|
1340
|
+
let maxAgeMs;
|
|
1341
|
+
if (maxAgeRaw !== void 0) {
|
|
1092
1342
|
try {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
signal: controller.signal
|
|
1098
|
-
});
|
|
1099
|
-
if (!res.ok) {
|
|
1100
|
-
const text = await res.text().catch(() => "");
|
|
1101
|
-
throw new Error(`OpenCode message failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1102
|
-
}
|
|
1103
|
-
const sessionRes = await fetch(`${opencodeBase(port)}/session/${sessionId}`).catch(
|
|
1104
|
-
() => null
|
|
1343
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1344
|
+
} catch (err) {
|
|
1345
|
+
warnings.push(
|
|
1346
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1105
1347
|
);
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
let maxCount;
|
|
1351
|
+
if (maxCountRaw !== void 0) {
|
|
1352
|
+
try {
|
|
1353
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1111
1354
|
} catch (err) {
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
throw err;
|
|
1116
|
-
} finally {
|
|
1117
|
-
clearTimeout(timer);
|
|
1118
|
-
pollDone = true;
|
|
1355
|
+
warnings.push(
|
|
1356
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1357
|
+
);
|
|
1119
1358
|
}
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1359
|
+
}
|
|
1360
|
+
let intervalMs;
|
|
1361
|
+
try {
|
|
1362
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1363
|
+
} catch (err) {
|
|
1364
|
+
warnings.push(
|
|
1365
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1366
|
+
);
|
|
1367
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1368
|
+
}
|
|
1369
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1370
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1123
1371
|
}
|
|
1124
1372
|
|
|
1125
1373
|
// src/lib/tunnel/connection.ts
|
|
@@ -1190,24 +1438,34 @@ var StreamForwarder = class {
|
|
|
1190
1438
|
}
|
|
1191
1439
|
async handleOpen(frame) {
|
|
1192
1440
|
const { sid, method, path, headers, has_body } = frame;
|
|
1441
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1442
|
+
const startedAt = Date.now();
|
|
1193
1443
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1194
1444
|
this.callbacks.onDrainPing?.();
|
|
1195
1445
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1196
1446
|
this.send({ type: "res_end", sid });
|
|
1197
1447
|
return;
|
|
1198
1448
|
}
|
|
1449
|
+
if (process.env.DEBUG) {
|
|
1450
|
+
log("debug", "agent_request", {
|
|
1451
|
+
correlation_id: correlationId,
|
|
1452
|
+
sid,
|
|
1453
|
+
method,
|
|
1454
|
+
path: stripQuery(path)
|
|
1455
|
+
});
|
|
1456
|
+
}
|
|
1199
1457
|
const ac = new AbortController();
|
|
1200
1458
|
let bodyPromise;
|
|
1201
1459
|
let pushBody;
|
|
1202
1460
|
let endBody;
|
|
1203
1461
|
if (has_body) {
|
|
1204
1462
|
const chunks = [];
|
|
1205
|
-
bodyPromise = new Promise((
|
|
1463
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1206
1464
|
pushBody = (buf) => {
|
|
1207
1465
|
chunks.push(buf);
|
|
1208
1466
|
};
|
|
1209
1467
|
endBody = () => {
|
|
1210
|
-
|
|
1468
|
+
resolve2(Buffer.concat(chunks));
|
|
1211
1469
|
};
|
|
1212
1470
|
});
|
|
1213
1471
|
}
|
|
@@ -1242,6 +1500,14 @@ var StreamForwarder = class {
|
|
|
1242
1500
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1243
1501
|
});
|
|
1244
1502
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1503
|
+
if (process.env.DEBUG) {
|
|
1504
|
+
log("debug", "agent_response", {
|
|
1505
|
+
correlation_id: correlationId,
|
|
1506
|
+
sid,
|
|
1507
|
+
status: upstream.status,
|
|
1508
|
+
duration_ms: Date.now() - startedAt
|
|
1509
|
+
});
|
|
1510
|
+
}
|
|
1245
1511
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1246
1512
|
try {
|
|
1247
1513
|
if (upstream.body) {
|
|
@@ -1317,7 +1583,7 @@ function connectTunnel(options) {
|
|
|
1317
1583
|
} = options;
|
|
1318
1584
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1319
1585
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1320
|
-
return new Promise((
|
|
1586
|
+
return new Promise((resolve2, reject) => {
|
|
1321
1587
|
const ws = new WebSocket2(url, {
|
|
1322
1588
|
headers: {
|
|
1323
1589
|
Authorization: authHeader
|
|
@@ -1382,7 +1648,7 @@ function connectTunnel(options) {
|
|
|
1382
1648
|
clearTimeout(connectionTimeout);
|
|
1383
1649
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1384
1650
|
onConnected?.(connectedAgentId);
|
|
1385
|
-
|
|
1651
|
+
resolve2({
|
|
1386
1652
|
ws,
|
|
1387
1653
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1388
1654
|
});
|
|
@@ -1498,6 +1764,12 @@ var RunnerConnection = class {
|
|
|
1498
1764
|
};
|
|
1499
1765
|
|
|
1500
1766
|
// src/lib/channels/driver.ts
|
|
1767
|
+
function messageIdOf(m) {
|
|
1768
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1769
|
+
if (typeof m.id === "string") return m.id;
|
|
1770
|
+
const infoId = m.info?.id;
|
|
1771
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1772
|
+
}
|
|
1501
1773
|
var DEFAULT_RETRY_POLICY = {
|
|
1502
1774
|
maxAttempts: 6,
|
|
1503
1775
|
baseDelayMs: 500,
|
|
@@ -1505,12 +1777,21 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1505
1777
|
};
|
|
1506
1778
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1507
1779
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1780
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1508
1781
|
var ChannelAuthError = class extends Error {
|
|
1509
1782
|
constructor(message) {
|
|
1510
1783
|
super(message);
|
|
1511
1784
|
this.name = "ChannelAuthError";
|
|
1512
1785
|
}
|
|
1513
1786
|
};
|
|
1787
|
+
var ChannelTerminalError = class extends Error {
|
|
1788
|
+
status;
|
|
1789
|
+
constructor(message, status) {
|
|
1790
|
+
super(message);
|
|
1791
|
+
this.name = "ChannelTerminalError";
|
|
1792
|
+
this.status = status;
|
|
1793
|
+
}
|
|
1794
|
+
};
|
|
1514
1795
|
function backoffDelay(attempt, policy) {
|
|
1515
1796
|
const exp = policy.baseDelayMs * Math.pow(2, attempt);
|
|
1516
1797
|
const capped = Math.min(policy.maxDelayMs, exp);
|
|
@@ -1531,15 +1812,82 @@ var ChannelDriver = class {
|
|
|
1531
1812
|
sleep;
|
|
1532
1813
|
pausedPollIntervalMs;
|
|
1533
1814
|
pausedMaxWaitMs;
|
|
1815
|
+
stuckQueuedMs;
|
|
1816
|
+
now;
|
|
1534
1817
|
/** Cache of conversationId → opencode sessionId. */
|
|
1535
1818
|
sessions = /* @__PURE__ */ new Map();
|
|
1536
1819
|
/**
|
|
1537
|
-
*
|
|
1538
|
-
*
|
|
1539
|
-
*
|
|
1540
|
-
*
|
|
1820
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1821
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1822
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1823
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1824
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1825
|
+
*/
|
|
1826
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1827
|
+
/**
|
|
1828
|
+
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1829
|
+
* session: one polling loop services all of that session's in-flight messages.
|
|
1830
|
+
* A session entry exists while it has any in-flight (dispatched-but-not-done)
|
|
1831
|
+
* message; it is removed once its in-flight set empties.
|
|
1541
1832
|
*/
|
|
1542
1833
|
watchers = /* @__PURE__ */ new Map();
|
|
1834
|
+
/**
|
|
1835
|
+
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
1836
|
+
* dispatched and are still in-flight. A message in this set is never
|
|
1837
|
+
* re-`prompt_async`-ed by a subsequent poll tick while it is queued/running.
|
|
1838
|
+
* Backed by a stable minted opencode `messageID` whose duplicate re-enqueue is
|
|
1839
|
+
* idempotent on opencode (PoC fact 9) — so even if this set is lost on restart,
|
|
1840
|
+
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1841
|
+
*/
|
|
1842
|
+
dispatched = /* @__PURE__ */ new Set();
|
|
1843
|
+
/**
|
|
1844
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1845
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1846
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1847
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1848
|
+
* the processing list.
|
|
1849
|
+
*/
|
|
1850
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1851
|
+
/**
|
|
1852
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1853
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1854
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1855
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1856
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1857
|
+
*
|
|
1858
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1859
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1860
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1861
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1862
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1863
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1864
|
+
*/
|
|
1865
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1866
|
+
/**
|
|
1867
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1868
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1869
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1870
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1871
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1872
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1873
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1874
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1875
|
+
*/
|
|
1876
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1877
|
+
/**
|
|
1878
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1879
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1880
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1881
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1882
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1883
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1884
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1885
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1886
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1887
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1888
|
+
* so the NEXT tick may retry exactly once more).
|
|
1889
|
+
*/
|
|
1890
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1543
1891
|
/**
|
|
1544
1892
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1545
1893
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1547,8 +1895,33 @@ var ChannelDriver = class {
|
|
|
1547
1895
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1548
1896
|
*/
|
|
1549
1897
|
opencodeDirectory = void 0;
|
|
1898
|
+
/**
|
|
1899
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1900
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1901
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1902
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1903
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1904
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1905
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1906
|
+
*/
|
|
1907
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1550
1908
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1551
1909
|
draining = false;
|
|
1910
|
+
/**
|
|
1911
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1912
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1913
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1914
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1915
|
+
*/
|
|
1916
|
+
activeDrain = null;
|
|
1917
|
+
/**
|
|
1918
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1919
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1920
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1921
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1922
|
+
* and stops opencode.
|
|
1923
|
+
*/
|
|
1924
|
+
stopped = false;
|
|
1552
1925
|
constructor(config2) {
|
|
1553
1926
|
this.agentId = config2.agentId;
|
|
1554
1927
|
this.port = config2.port;
|
|
@@ -1562,6 +1935,8 @@ var ChannelDriver = class {
|
|
|
1562
1935
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1563
1936
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1564
1937
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1938
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1939
|
+
this.now = config2.now ?? (() => Date.now());
|
|
1565
1940
|
}
|
|
1566
1941
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
1567
1942
|
get opencodeBase() {
|
|
@@ -1571,16 +1946,29 @@ var ChannelDriver = class {
|
|
|
1571
1946
|
// Public API
|
|
1572
1947
|
// -------------------------------------------------------------------------
|
|
1573
1948
|
/**
|
|
1574
|
-
* Drain all pending channel conversations once: poll →
|
|
1949
|
+
* Drain all pending channel conversations once: poll → dispatch → register.
|
|
1575
1950
|
* Called on tunnel `connected` (WI-CHAN-4) and on each poll tick by `run.ts`.
|
|
1576
1951
|
* Re-entrant calls while a drain is in flight are skipped (return 0).
|
|
1577
1952
|
*
|
|
1578
|
-
* @returns the number of messages
|
|
1953
|
+
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1579
1954
|
*/
|
|
1580
1955
|
async drainPending() {
|
|
1956
|
+
if (this.stopped) return 0;
|
|
1581
1957
|
if (this.draining) return 0;
|
|
1582
1958
|
this.draining = true;
|
|
1583
|
-
|
|
1959
|
+
const run2 = this.runDrain();
|
|
1960
|
+
this.activeDrain = run2.then(
|
|
1961
|
+
() => {
|
|
1962
|
+
this.activeDrain = null;
|
|
1963
|
+
},
|
|
1964
|
+
() => {
|
|
1965
|
+
this.activeDrain = null;
|
|
1966
|
+
}
|
|
1967
|
+
);
|
|
1968
|
+
return run2;
|
|
1969
|
+
}
|
|
1970
|
+
async runDrain() {
|
|
1971
|
+
let dispatched = 0;
|
|
1584
1972
|
try {
|
|
1585
1973
|
const conversations = await this.getPendingConversations();
|
|
1586
1974
|
if (conversations.length > 0) {
|
|
@@ -1591,107 +1979,226 @@ var ChannelDriver = class {
|
|
|
1591
1979
|
});
|
|
1592
1980
|
}
|
|
1593
1981
|
for (const conv of conversations) {
|
|
1594
|
-
|
|
1982
|
+
if (this.stopped) break;
|
|
1983
|
+
dispatched += await this.processConversation(conv);
|
|
1595
1984
|
}
|
|
1985
|
+
await this.readoptProcessing();
|
|
1596
1986
|
} finally {
|
|
1597
1987
|
this.draining = false;
|
|
1598
1988
|
}
|
|
1599
|
-
return
|
|
1989
|
+
return dispatched;
|
|
1990
|
+
}
|
|
1991
|
+
/**
|
|
1992
|
+
* True while any per-session watcher has a non-empty in-flight dispatched set
|
|
1993
|
+
* (Task 3.7). `run.ts` treats this as NON-idle so `--idle-timeout` cannot exit
|
|
1994
|
+
* the process while a dispatched message is still queued/running — which would
|
|
1995
|
+
* kill the turn and orphan its reply.
|
|
1996
|
+
*/
|
|
1997
|
+
hasInFlightWatchers() {
|
|
1998
|
+
for (const watcher of this.watchers.values()) {
|
|
1999
|
+
if (watcher.inFlight.size > 0) return true;
|
|
2000
|
+
}
|
|
2001
|
+
return false;
|
|
2002
|
+
}
|
|
2003
|
+
/**
|
|
2004
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2005
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2006
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2007
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2008
|
+
*
|
|
2009
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2010
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2011
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2012
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2013
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2014
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2015
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2016
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2017
|
+
*/
|
|
2018
|
+
protectedSessionIds() {
|
|
2019
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2020
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2021
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2022
|
+
}
|
|
2023
|
+
return ids;
|
|
2024
|
+
}
|
|
2025
|
+
/**
|
|
2026
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2027
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2028
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2029
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2030
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2031
|
+
*/
|
|
2032
|
+
stop() {
|
|
2033
|
+
this.stopped = true;
|
|
2034
|
+
}
|
|
2035
|
+
/**
|
|
2036
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2037
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2038
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2039
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2040
|
+
*
|
|
2041
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2042
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2043
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2044
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2045
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2046
|
+
* next runner start (ADR-0046).
|
|
2047
|
+
*
|
|
2048
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2049
|
+
* timeout elapsed with work still in flight.
|
|
2050
|
+
*/
|
|
2051
|
+
async waitForInFlight(timeoutMs) {
|
|
2052
|
+
const deadline = this.now() + timeoutMs;
|
|
2053
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2054
|
+
if (this.activeDrain) {
|
|
2055
|
+
let drainSettled = false;
|
|
2056
|
+
void this.activeDrain.then(() => {
|
|
2057
|
+
drainSettled = true;
|
|
2058
|
+
});
|
|
2059
|
+
while (!drainSettled) {
|
|
2060
|
+
if (this.now() >= deadline) return false;
|
|
2061
|
+
await this.sleep(step);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
while (this.hasInFlightWatchers()) {
|
|
2065
|
+
if (this.now() >= deadline) return false;
|
|
2066
|
+
await this.sleep(step);
|
|
2067
|
+
}
|
|
2068
|
+
return true;
|
|
1600
2069
|
}
|
|
1601
2070
|
/**
|
|
1602
|
-
* Await all outstanding
|
|
2071
|
+
* Await all outstanding per-session watchers (WI-3).
|
|
1603
2072
|
*
|
|
1604
|
-
* In production the
|
|
1605
|
-
* loop never blocks on them and process exit is not held up (the cron
|
|
1606
|
-
* any abandoned ones). This helper exists primarily for deterministic
|
|
1607
|
-
* that need to observe
|
|
1608
|
-
* after a non-blocking `drainPending`.
|
|
2073
|
+
* In production the watcher loops are deliberately started-not-awaited so the
|
|
2074
|
+
* drain loop never blocks on them and process exit is not held up (the cron
|
|
2075
|
+
* recovers any abandoned ones). This helper exists primarily for deterministic
|
|
2076
|
+
* tests that need to observe a watcher's effect (the `processing`/`done` PATCH
|
|
2077
|
+
* or its giving up) after a non-blocking `drainPending`. Watcher loops never
|
|
2078
|
+
* reject, so this resolves.
|
|
1609
2079
|
*/
|
|
1610
2080
|
async flushPausedWatchers() {
|
|
1611
|
-
|
|
2081
|
+
while (true) {
|
|
2082
|
+
const loops = [...this.watchers.values()].map((w) => w.loop).filter((l) => l != null);
|
|
2083
|
+
if (loops.length === 0) return;
|
|
2084
|
+
await Promise.all(loops);
|
|
2085
|
+
const stillLive = [...this.watchers.values()].some((w) => w.loop != null);
|
|
2086
|
+
if (!stillLive) return;
|
|
2087
|
+
}
|
|
1612
2088
|
}
|
|
1613
2089
|
// -------------------------------------------------------------------------
|
|
1614
|
-
// Conversation processing
|
|
2090
|
+
// Conversation processing (WI-3 — async dispatch)
|
|
1615
2091
|
// -------------------------------------------------------------------------
|
|
2092
|
+
/**
|
|
2093
|
+
* Dispatch each pending message for a conversation to opencode's native queue
|
|
2094
|
+
* via `prompt_async` (Task 3.2) and register it with the conversation's
|
|
2095
|
+
* per-session watcher. Does NOT block on the turn and does NOT call
|
|
2096
|
+
* `markProcessing` here — that fires from the watcher on running-start.
|
|
2097
|
+
*
|
|
2098
|
+
* @returns the count of messages NEWLY dispatched (not already in-flight).
|
|
2099
|
+
*/
|
|
1616
2100
|
async processConversation(conv) {
|
|
1617
2101
|
const sessionId = await this.ensureSession(conv);
|
|
1618
2102
|
const messages = await this.getPendingMessages(conv.id);
|
|
1619
|
-
let
|
|
2103
|
+
let dispatched = 0;
|
|
2104
|
+
let skippedAlreadyDispatched = 0;
|
|
1620
2105
|
for (const message of messages) {
|
|
1621
|
-
|
|
1622
|
-
if (
|
|
1623
|
-
|
|
1624
|
-
level: "info",
|
|
1625
|
-
message: `Message ${message.id.slice(0, 8)} already claimed \u2014 skipping`,
|
|
1626
|
-
conversation_id: conv.id,
|
|
1627
|
-
message_id: message.id
|
|
1628
|
-
});
|
|
2106
|
+
if (this.stopped) break;
|
|
2107
|
+
if (this.dispatched.has(message.id)) {
|
|
2108
|
+
skippedAlreadyDispatched += 1;
|
|
1629
2109
|
continue;
|
|
1630
2110
|
}
|
|
2111
|
+
const options = {
|
|
2112
|
+
agent: message.opencode_agent ?? void 0,
|
|
2113
|
+
model: message.opencode_model ?? void 0
|
|
2114
|
+
};
|
|
2115
|
+
let opencodeMessageId;
|
|
1631
2116
|
try {
|
|
1632
2117
|
this.log({
|
|
1633
2118
|
level: "info",
|
|
1634
|
-
message: `
|
|
2119
|
+
message: `Dispatching message ${message.id.slice(0, 8)} to OpenCode native queue (session ${sessionId.slice(0, 8)})`,
|
|
1635
2120
|
conversation_id: conv.id,
|
|
1636
2121
|
message_id: message.id
|
|
1637
2122
|
});
|
|
1638
|
-
|
|
1639
|
-
this.port,
|
|
2123
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
1640
2124
|
sessionId,
|
|
1641
|
-
message.content,
|
|
1642
|
-
{
|
|
1643
|
-
agent: message.opencode_agent ?? void 0,
|
|
1644
|
-
model: message.opencode_model ?? void 0
|
|
1645
|
-
},
|
|
1646
|
-
{
|
|
1647
|
-
onQuestion: (question) => this.reportInteraction(conv.id, "question", question),
|
|
1648
|
-
onPermission: (permission) => this.reportInteraction(conv.id, "permission", permission)
|
|
1649
|
-
}
|
|
2125
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
1650
2126
|
);
|
|
1651
|
-
|
|
2127
|
+
} catch (err) {
|
|
2128
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2129
|
+
this.dispatched.delete(message.id);
|
|
2130
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2131
|
+
this.sessions.delete(conv.id);
|
|
1652
2132
|
this.log({
|
|
1653
2133
|
level: "info",
|
|
1654
|
-
message: `Message ${message.id.slice(0, 8)}
|
|
2134
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
1655
2135
|
conversation_id: conv.id,
|
|
1656
2136
|
message_id: message.id
|
|
1657
2137
|
});
|
|
1658
|
-
|
|
1659
|
-
continue;
|
|
2138
|
+
break;
|
|
1660
2139
|
}
|
|
1661
|
-
await this.
|
|
1662
|
-
|
|
1663
|
-
processed += 1;
|
|
2140
|
+
await this.markFailed(conv.id, message.id).catch(() => {
|
|
2141
|
+
});
|
|
1664
2142
|
this.log({
|
|
1665
|
-
level: "
|
|
1666
|
-
message: `Message ${message.id.slice(0, 8)}
|
|
2143
|
+
level: "error",
|
|
2144
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1667
2145
|
conversation_id: conv.id,
|
|
1668
2146
|
message_id: message.id
|
|
1669
2147
|
});
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
});
|
|
2148
|
+
continue;
|
|
2149
|
+
}
|
|
2150
|
+
if (opencodeMessageId === null) {
|
|
1674
2151
|
this.log({
|
|
1675
2152
|
level: "error",
|
|
1676
|
-
message: `Message ${message.id.slice(0, 8)}
|
|
2153
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
1677
2154
|
conversation_id: conv.id,
|
|
1678
2155
|
message_id: message.id
|
|
1679
2156
|
});
|
|
2157
|
+
continue;
|
|
1680
2158
|
}
|
|
2159
|
+
this.dispatched.add(message.id);
|
|
2160
|
+
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
2161
|
+
dispatched += 1;
|
|
2162
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
2163
|
+
}
|
|
2164
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
2165
|
+
this.log({
|
|
2166
|
+
level: "error",
|
|
2167
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
2168
|
+
conversation_id: conv.id
|
|
2169
|
+
});
|
|
1681
2170
|
}
|
|
1682
|
-
|
|
2171
|
+
this.ensureWatcherRunning(sessionId);
|
|
2172
|
+
return dispatched;
|
|
1683
2173
|
}
|
|
1684
2174
|
async ensureSession(conv) {
|
|
1685
|
-
const
|
|
1686
|
-
if (
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
2175
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2176
|
+
if (bound) {
|
|
2177
|
+
const exists = await sessionExists(this.port, bound);
|
|
2178
|
+
if (exists === false) {
|
|
2179
|
+
this.log({
|
|
2180
|
+
level: "info",
|
|
2181
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2182
|
+
conversation_id: conv.id
|
|
2183
|
+
});
|
|
2184
|
+
this.sessions.delete(conv.id);
|
|
2185
|
+
return this.createAndBindSession(conv.id);
|
|
2186
|
+
}
|
|
2187
|
+
this.sessions.set(conv.id, bound);
|
|
2188
|
+
return bound;
|
|
1690
2189
|
}
|
|
2190
|
+
return this.createAndBindSession(conv.id);
|
|
2191
|
+
}
|
|
2192
|
+
/**
|
|
2193
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2194
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2195
|
+
* self-heal recreate path in `ensureSession`.
|
|
2196
|
+
*/
|
|
2197
|
+
async createAndBindSession(conversationId) {
|
|
1691
2198
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1692
2199
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1693
|
-
this.sessions.set(
|
|
1694
|
-
await this.persistSession(
|
|
2200
|
+
this.sessions.set(conversationId, sessionId);
|
|
2201
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1695
2202
|
});
|
|
1696
2203
|
return sessionId;
|
|
1697
2204
|
}
|
|
@@ -1711,111 +2218,866 @@ var ChannelDriver = class {
|
|
|
1711
2218
|
}
|
|
1712
2219
|
return this.opencodeDirectory;
|
|
1713
2220
|
}
|
|
2221
|
+
// -------------------------------------------------------------------------
|
|
2222
|
+
// Per-session watcher (WI-3)
|
|
2223
|
+
// -------------------------------------------------------------------------
|
|
1714
2224
|
/**
|
|
1715
|
-
*
|
|
1716
|
-
*
|
|
1717
|
-
*
|
|
1718
|
-
*
|
|
1719
|
-
*
|
|
1720
|
-
* the messages itself (via `extractTextFromMessages`) when delivering the
|
|
1721
|
-
* reply — so this round-trip never gates delivery. We keep it only to surface a
|
|
1722
|
-
* truthful diagnostic when opencode hasn't yet recorded a completed assistant
|
|
1723
|
-
* turn at reconcile time, then proceed to `markDone` regardless. Uses the
|
|
1724
|
-
* injected `fetchImpl` and reuses only the pure `isTurnComplete` predicate.
|
|
2225
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2226
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2227
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2228
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2229
|
+
* (each dispatch reports its own outcome to its caller).
|
|
1725
2230
|
*/
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
2231
|
+
dispatchLocked(sessionId, fn) {
|
|
2232
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2233
|
+
const run2 = prior.then(fn, fn);
|
|
2234
|
+
this.sessionDispatchLocks.set(
|
|
2235
|
+
sessionId,
|
|
2236
|
+
run2.then(
|
|
2237
|
+
() => void 0,
|
|
2238
|
+
() => void 0
|
|
2239
|
+
)
|
|
2240
|
+
);
|
|
2241
|
+
return run2;
|
|
2242
|
+
}
|
|
2243
|
+
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
2244
|
+
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
2245
|
+
let watcher = this.watchers.get(sessionId);
|
|
2246
|
+
if (!watcher) {
|
|
2247
|
+
watcher = {
|
|
2248
|
+
conv,
|
|
2249
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2250
|
+
loop: null,
|
|
2251
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2252
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2253
|
+
};
|
|
2254
|
+
this.watchers.set(sessionId, watcher);
|
|
1739
2255
|
}
|
|
2256
|
+
const now = this.now();
|
|
2257
|
+
watcher.inFlight.set(message.id, {
|
|
2258
|
+
evidentMessageId: message.id,
|
|
2259
|
+
opencodeMessageId,
|
|
2260
|
+
message,
|
|
2261
|
+
dispatchedAt: now,
|
|
2262
|
+
deadline: now + this.pausedMaxWaitMs,
|
|
2263
|
+
started: false,
|
|
2264
|
+
done: false,
|
|
2265
|
+
stuckReported: false
|
|
2266
|
+
});
|
|
1740
2267
|
}
|
|
1741
|
-
// -------------------------------------------------------------------------
|
|
1742
|
-
// Paused-session watcher (WI-2-CLI)
|
|
1743
|
-
// -------------------------------------------------------------------------
|
|
1744
2268
|
/**
|
|
1745
|
-
*
|
|
2269
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2270
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2271
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2272
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
2273
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
2274
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
2275
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1746
2276
|
*
|
|
1747
|
-
*
|
|
1748
|
-
*
|
|
1749
|
-
*
|
|
1750
|
-
*
|
|
2277
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2278
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
2279
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
2280
|
+
*
|
|
2281
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
2282
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
2283
|
+
* fire from the watcher's normal branches.
|
|
1751
2284
|
*/
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
2285
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2286
|
+
let watcher = this.watchers.get(sessionId);
|
|
2287
|
+
if (!watcher) {
|
|
2288
|
+
watcher = {
|
|
2289
|
+
conv,
|
|
2290
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2291
|
+
loop: null,
|
|
2292
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2293
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2294
|
+
};
|
|
2295
|
+
this.watchers.set(sessionId, watcher);
|
|
2296
|
+
}
|
|
2297
|
+
watcher.inFlight.set(message.id, {
|
|
2298
|
+
evidentMessageId: message.id,
|
|
2299
|
+
opencodeMessageId,
|
|
2300
|
+
message,
|
|
2301
|
+
dispatchedAt: this.now(),
|
|
2302
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2303
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2304
|
+
started: true,
|
|
2305
|
+
done: false,
|
|
2306
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2307
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2308
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2309
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2310
|
+
// (#210/#220 observability).
|
|
2311
|
+
stuckReported: false
|
|
1756
2312
|
});
|
|
1757
|
-
this.watchers.set(message.id, watcher);
|
|
1758
2313
|
}
|
|
1759
2314
|
/**
|
|
1760
|
-
*
|
|
1761
|
-
*
|
|
1762
|
-
*
|
|
1763
|
-
*
|
|
1764
|
-
*
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2315
|
+
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
2316
|
+
* work and is not already running. Single-flight per session. The loop is
|
|
2317
|
+
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
2318
|
+
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
2319
|
+
* stays as the safety net.
|
|
2320
|
+
*/
|
|
2321
|
+
ensureWatcherRunning(sessionId) {
|
|
2322
|
+
const watcher = this.watchers.get(sessionId);
|
|
2323
|
+
if (!watcher) return;
|
|
2324
|
+
if (watcher.loop) return;
|
|
2325
|
+
if (watcher.inFlight.size === 0) {
|
|
2326
|
+
this.watchers.delete(sessionId);
|
|
2327
|
+
return;
|
|
2328
|
+
}
|
|
2329
|
+
const loop = this.runWatcherLoop(sessionId, watcher).finally(() => {
|
|
2330
|
+
watcher.loop = null;
|
|
2331
|
+
if (watcher.inFlight.size === 0) {
|
|
2332
|
+
this.watchers.delete(sessionId);
|
|
2333
|
+
}
|
|
2334
|
+
});
|
|
2335
|
+
watcher.loop = loop;
|
|
2336
|
+
}
|
|
2337
|
+
/**
|
|
2338
|
+
* The per-session polling loop (WI-3). Once per tick it:
|
|
2339
|
+
* 1. polls `GET /session/:id/message` once and, per in-flight message,
|
|
2340
|
+
* computes `messageRunState` and fires markProcessing (queued→running) /
|
|
2341
|
+
* markDone (done) exactly once per transition;
|
|
2342
|
+
* 2. applies the idle-path re-dispatch guard (a dispatched message that never
|
|
2343
|
+
* APPEARS → re-dispatch — D1 obligation 2);
|
|
2344
|
+
* 3. polls `/question` + `/permission` (scoped to the session) and surfaces
|
|
2345
|
+
* NEW ones via `reportInteraction`, carrying the PAUSED message's own
|
|
2346
|
+
* `source_message_id`;
|
|
2347
|
+
* 4. drops messages that completed or timed out from the in-flight set.
|
|
2348
|
+
* Exits when the in-flight set empties. Never throws.
|
|
1778
2349
|
*/
|
|
1779
|
-
async
|
|
1780
|
-
const deadline = Date.now() + this.pausedMaxWaitMs;
|
|
2350
|
+
async runWatcherLoop(sessionId, watcher) {
|
|
1781
2351
|
try {
|
|
1782
|
-
while (
|
|
2352
|
+
while (watcher.inFlight.size > 0) {
|
|
1783
2353
|
await this.sleep(this.pausedPollIntervalMs);
|
|
1784
|
-
let
|
|
2354
|
+
let messages = null;
|
|
1785
2355
|
try {
|
|
1786
2356
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
1787
2357
|
if (res.ok) {
|
|
1788
2358
|
const body = await res.json();
|
|
1789
|
-
|
|
1790
|
-
completed = isTurnComplete(messages);
|
|
2359
|
+
messages = Array.isArray(body) ? body : null;
|
|
1791
2360
|
}
|
|
1792
2361
|
} catch {
|
|
1793
2362
|
continue;
|
|
1794
2363
|
}
|
|
1795
|
-
|
|
2364
|
+
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2365
|
+
await this.serviceInFlightMessage(sessionId, watcher, inFlight, messages);
|
|
2366
|
+
}
|
|
2367
|
+
await this.pollInteractions(sessionId, watcher, messages);
|
|
2368
|
+
}
|
|
2369
|
+
} catch (err) {
|
|
2370
|
+
if (err instanceof ChannelAuthError) {
|
|
1796
2371
|
this.log({
|
|
1797
|
-
level: "
|
|
1798
|
-
message: `
|
|
1799
|
-
conversation_id: conv.id
|
|
1800
|
-
message_id: message.id
|
|
2372
|
+
level: "error",
|
|
2373
|
+
message: `Session watcher aborted on auth failure for session ${sessionId.slice(0, 8)} \u2014 clearing in-flight state for re-drive after re-auth: ${err.message}`,
|
|
2374
|
+
conversation_id: watcher.conv.id
|
|
1801
2375
|
});
|
|
1802
|
-
|
|
2376
|
+
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2377
|
+
this.readopted.delete(evidentMessageId);
|
|
2378
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
2379
|
+
}
|
|
1803
2380
|
return;
|
|
1804
2381
|
}
|
|
2382
|
+
this.log({
|
|
2383
|
+
level: "error",
|
|
2384
|
+
message: `Session watcher failed for session ${sessionId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2385
|
+
conversation_id: watcher.conv.id
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
/**
|
|
2390
|
+
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2391
|
+
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2392
|
+
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2393
|
+
* in-flight set on completion or timeout.
|
|
2394
|
+
*/
|
|
2395
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2396
|
+
const conv = watcher.conv;
|
|
2397
|
+
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2398
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2399
|
+
let claimed;
|
|
2400
|
+
try {
|
|
2401
|
+
claimed = await this.markProcessing(
|
|
2402
|
+
conv.id,
|
|
2403
|
+
inFlight.evidentMessageId,
|
|
2404
|
+
sessionId,
|
|
2405
|
+
inFlight.opencodeMessageId
|
|
2406
|
+
);
|
|
2407
|
+
} catch (err) {
|
|
2408
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2409
|
+
this.log({
|
|
2410
|
+
level: "error",
|
|
2411
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2412
|
+
conversation_id: conv.id,
|
|
2413
|
+
message_id: inFlight.evidentMessageId
|
|
2414
|
+
});
|
|
2415
|
+
return;
|
|
2416
|
+
}
|
|
2417
|
+
inFlight.started = true;
|
|
2418
|
+
if (!claimed) {
|
|
2419
|
+
this.log({
|
|
2420
|
+
level: "info",
|
|
2421
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
2422
|
+
conversation_id: conv.id,
|
|
2423
|
+
message_id: inFlight.evidentMessageId
|
|
2424
|
+
});
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
if (state === "done") {
|
|
2428
|
+
if (!inFlight.done) {
|
|
2429
|
+
this.log({
|
|
2430
|
+
level: "info",
|
|
2431
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
2432
|
+
conversation_id: conv.id,
|
|
2433
|
+
message_id: inFlight.evidentMessageId
|
|
2434
|
+
});
|
|
2435
|
+
try {
|
|
2436
|
+
await this.markDone(
|
|
2437
|
+
conv.id,
|
|
2438
|
+
inFlight.evidentMessageId,
|
|
2439
|
+
sessionId,
|
|
2440
|
+
inFlight.opencodeMessageId
|
|
2441
|
+
);
|
|
2442
|
+
} catch (err) {
|
|
2443
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2444
|
+
if (err instanceof ChannelTerminalError) {
|
|
2445
|
+
this.log({
|
|
2446
|
+
level: "error",
|
|
2447
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2448
|
+
conversation_id: conv.id,
|
|
2449
|
+
message_id: inFlight.evidentMessageId
|
|
2450
|
+
});
|
|
2451
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
if (this.now() >= inFlight.deadline) {
|
|
2455
|
+
this.log({
|
|
2456
|
+
level: "error",
|
|
2457
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2458
|
+
conversation_id: conv.id,
|
|
2459
|
+
message_id: inFlight.evidentMessageId
|
|
2460
|
+
});
|
|
2461
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2462
|
+
return;
|
|
2463
|
+
}
|
|
2464
|
+
this.log({
|
|
2465
|
+
level: "error",
|
|
2466
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2467
|
+
conversation_id: conv.id,
|
|
2468
|
+
message_id: inFlight.evidentMessageId
|
|
2469
|
+
});
|
|
2470
|
+
return;
|
|
2471
|
+
}
|
|
2472
|
+
inFlight.done = true;
|
|
2473
|
+
}
|
|
2474
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2475
|
+
return;
|
|
2476
|
+
}
|
|
2477
|
+
if (state === "failed") {
|
|
2478
|
+
if (!inFlight.done) {
|
|
2479
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2480
|
+
this.log({
|
|
2481
|
+
level: "error",
|
|
2482
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2483
|
+
conversation_id: conv.id,
|
|
2484
|
+
message_id: inFlight.evidentMessageId
|
|
2485
|
+
});
|
|
2486
|
+
try {
|
|
2487
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2488
|
+
} catch (err) {
|
|
2489
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2490
|
+
if (err instanceof ChannelTerminalError) {
|
|
2491
|
+
this.log({
|
|
2492
|
+
level: "error",
|
|
2493
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2494
|
+
conversation_id: conv.id,
|
|
2495
|
+
message_id: inFlight.evidentMessageId
|
|
2496
|
+
});
|
|
2497
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
if (this.now() >= inFlight.deadline) {
|
|
2501
|
+
this.log({
|
|
2502
|
+
level: "error",
|
|
2503
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2504
|
+
conversation_id: conv.id,
|
|
2505
|
+
message_id: inFlight.evidentMessageId
|
|
2506
|
+
});
|
|
2507
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2508
|
+
return;
|
|
2509
|
+
}
|
|
2510
|
+
this.log({
|
|
2511
|
+
level: "error",
|
|
2512
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2513
|
+
conversation_id: conv.id,
|
|
2514
|
+
message_id: inFlight.evidentMessageId
|
|
2515
|
+
});
|
|
2516
|
+
return;
|
|
2517
|
+
}
|
|
2518
|
+
inFlight.done = true;
|
|
2519
|
+
}
|
|
2520
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2521
|
+
return;
|
|
2522
|
+
}
|
|
2523
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2524
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2525
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2526
|
+
inFlight.stuckReported = true;
|
|
2527
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2528
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2529
|
+
});
|
|
2530
|
+
}
|
|
2531
|
+
if (this.now() >= inFlight.deadline) {
|
|
1805
2532
|
this.log({
|
|
1806
2533
|
level: "info",
|
|
1807
|
-
message: `
|
|
2534
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
1808
2535
|
conversation_id: conv.id,
|
|
1809
|
-
message_id:
|
|
2536
|
+
message_id: inFlight.evidentMessageId
|
|
2537
|
+
});
|
|
2538
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2539
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
1810
2540
|
});
|
|
2541
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2542
|
+
}
|
|
2543
|
+
}
|
|
2544
|
+
// -------------------------------------------------------------------------
|
|
2545
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2546
|
+
// -------------------------------------------------------------------------
|
|
2547
|
+
/**
|
|
2548
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2549
|
+
*
|
|
2550
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2551
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2552
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2553
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2554
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2555
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2556
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2557
|
+
*
|
|
2558
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2559
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2560
|
+
*/
|
|
2561
|
+
async readoptProcessing() {
|
|
2562
|
+
const rows = await this.getProcessingMessages();
|
|
2563
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2564
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2565
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2566
|
+
if (!stillProcessing.has(id)) {
|
|
2567
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2568
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2569
|
+
if (cleared || clearedUndeliverable) {
|
|
2570
|
+
this.log({
|
|
2571
|
+
level: "info",
|
|
2572
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2573
|
+
message_id: id
|
|
2574
|
+
});
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
if (rows.length === 0) return;
|
|
2580
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2581
|
+
for (const row of rows) {
|
|
2582
|
+
if (!row.opencode_session_id) {
|
|
2583
|
+
this.log({
|
|
2584
|
+
level: "error",
|
|
2585
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2586
|
+
conversation_id: row.conversation_id,
|
|
2587
|
+
message_id: row.id
|
|
2588
|
+
});
|
|
2589
|
+
continue;
|
|
2590
|
+
}
|
|
2591
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2592
|
+
list.push(row);
|
|
2593
|
+
bySession.set(row.opencode_session_id, list);
|
|
2594
|
+
}
|
|
2595
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2596
|
+
let messages;
|
|
2597
|
+
try {
|
|
2598
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2599
|
+
if (!res.ok) {
|
|
2600
|
+
this.log({
|
|
2601
|
+
level: "error",
|
|
2602
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2603
|
+
});
|
|
2604
|
+
continue;
|
|
2605
|
+
}
|
|
2606
|
+
const body = await res.json();
|
|
2607
|
+
if (!Array.isArray(body)) {
|
|
2608
|
+
this.log({
|
|
2609
|
+
level: "error",
|
|
2610
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2611
|
+
});
|
|
2612
|
+
continue;
|
|
2613
|
+
}
|
|
2614
|
+
messages = body;
|
|
2615
|
+
} catch (err) {
|
|
2616
|
+
this.log({
|
|
2617
|
+
level: "error",
|
|
2618
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2619
|
+
});
|
|
2620
|
+
continue;
|
|
2621
|
+
}
|
|
2622
|
+
for (const row of sessionRows) {
|
|
2623
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
/**
|
|
2628
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2629
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2630
|
+
*
|
|
2631
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2632
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2633
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2634
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2635
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2636
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2637
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2638
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2639
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2640
|
+
* tracking the stored id so the reply correlates by it;
|
|
2641
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2642
|
+
*
|
|
2643
|
+
* Only `ChannelAuthError` propagates.
|
|
2644
|
+
*/
|
|
2645
|
+
async readoptOne(sessionId, row, messages) {
|
|
2646
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2647
|
+
this.log({
|
|
2648
|
+
level: "info",
|
|
2649
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2650
|
+
conversation_id: row.conversation_id,
|
|
2651
|
+
message_id: row.id
|
|
2652
|
+
});
|
|
2653
|
+
return;
|
|
2654
|
+
}
|
|
2655
|
+
const ocId = row.opencode_message_id;
|
|
2656
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2657
|
+
if (state === "done") {
|
|
2658
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2659
|
+
this.log({
|
|
2660
|
+
level: "info",
|
|
2661
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2662
|
+
conversation_id: row.conversation_id,
|
|
2663
|
+
message_id: row.id
|
|
2664
|
+
});
|
|
2665
|
+
return;
|
|
2666
|
+
}
|
|
2667
|
+
this.log({
|
|
2668
|
+
level: "info",
|
|
2669
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2670
|
+
conversation_id: row.conversation_id,
|
|
2671
|
+
message_id: row.id
|
|
2672
|
+
});
|
|
2673
|
+
try {
|
|
2674
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2675
|
+
} catch (err) {
|
|
2676
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2677
|
+
if (err instanceof ChannelTerminalError) {
|
|
2678
|
+
this.doneUndeliverable.add(row.id);
|
|
2679
|
+
this.log({
|
|
2680
|
+
level: "error",
|
|
2681
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2682
|
+
conversation_id: row.conversation_id,
|
|
2683
|
+
message_id: row.id
|
|
2684
|
+
});
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
this.log({
|
|
2688
|
+
level: "error",
|
|
2689
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2690
|
+
conversation_id: row.conversation_id,
|
|
2691
|
+
message_id: row.id
|
|
2692
|
+
});
|
|
2693
|
+
return;
|
|
2694
|
+
}
|
|
2695
|
+
this.dontRedispatch.delete(row.id);
|
|
2696
|
+
return;
|
|
2697
|
+
}
|
|
2698
|
+
if (state === "failed") {
|
|
2699
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2700
|
+
this.log({
|
|
2701
|
+
level: "error",
|
|
2702
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2703
|
+
conversation_id: row.conversation_id,
|
|
2704
|
+
message_id: row.id
|
|
2705
|
+
});
|
|
2706
|
+
try {
|
|
2707
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2708
|
+
} catch (err) {
|
|
2709
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2710
|
+
if (err instanceof ChannelTerminalError) {
|
|
2711
|
+
this.doneUndeliverable.add(row.id);
|
|
2712
|
+
this.log({
|
|
2713
|
+
level: "error",
|
|
2714
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2715
|
+
conversation_id: row.conversation_id,
|
|
2716
|
+
message_id: row.id
|
|
2717
|
+
});
|
|
2718
|
+
return;
|
|
2719
|
+
}
|
|
2720
|
+
this.log({
|
|
2721
|
+
level: "error",
|
|
2722
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2723
|
+
conversation_id: row.conversation_id,
|
|
2724
|
+
message_id: row.id
|
|
2725
|
+
});
|
|
2726
|
+
return;
|
|
2727
|
+
}
|
|
2728
|
+
this.dontRedispatch.delete(row.id);
|
|
2729
|
+
return;
|
|
2730
|
+
}
|
|
2731
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2732
|
+
this.log({
|
|
2733
|
+
level: "info",
|
|
2734
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2735
|
+
conversation_id: row.conversation_id,
|
|
2736
|
+
message_id: row.id
|
|
2737
|
+
});
|
|
2738
|
+
return;
|
|
2739
|
+
}
|
|
2740
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2741
|
+
const conv = this.convForRow(sessionId, row);
|
|
2742
|
+
const message = this.queuedMessageForRow(row);
|
|
2743
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2744
|
+
this.dispatched.add(row.id);
|
|
2745
|
+
this.readopted.add(row.id);
|
|
2746
|
+
this.ensureWatcherRunning(sessionId);
|
|
2747
|
+
this.log({
|
|
2748
|
+
level: "info",
|
|
2749
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2750
|
+
conversation_id: row.conversation_id,
|
|
2751
|
+
message_id: row.id
|
|
2752
|
+
});
|
|
2753
|
+
return;
|
|
2754
|
+
}
|
|
2755
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2756
|
+
}
|
|
2757
|
+
/**
|
|
2758
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2759
|
+
*
|
|
2760
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
2761
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
2762
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
2763
|
+
* correlates server-side.
|
|
2764
|
+
*
|
|
2765
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
2766
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
2767
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
2768
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
2769
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
2770
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
2771
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
2772
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
2773
|
+
* may retry exactly once more).
|
|
2774
|
+
*
|
|
2775
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2776
|
+
* `processed_at` (Invariant 1).
|
|
2777
|
+
*/
|
|
2778
|
+
async forceReadoptRun(sessionId, row) {
|
|
2779
|
+
if (this.stopped) {
|
|
2780
|
+
this.log({
|
|
2781
|
+
level: "info",
|
|
2782
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
2783
|
+
conversation_id: row.conversation_id,
|
|
2784
|
+
message_id: row.id
|
|
2785
|
+
});
|
|
2786
|
+
return;
|
|
2787
|
+
}
|
|
2788
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
2789
|
+
this.log({
|
|
2790
|
+
level: "info",
|
|
2791
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
2792
|
+
conversation_id: row.conversation_id,
|
|
2793
|
+
message_id: row.id
|
|
2794
|
+
});
|
|
2795
|
+
return;
|
|
2796
|
+
}
|
|
2797
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2798
|
+
this.dontRedispatch.add(row.id);
|
|
2799
|
+
this.log({
|
|
2800
|
+
level: "info",
|
|
2801
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2802
|
+
conversation_id: row.conversation_id,
|
|
2803
|
+
message_id: row.id
|
|
2804
|
+
});
|
|
2805
|
+
return;
|
|
2806
|
+
}
|
|
2807
|
+
const options = {
|
|
2808
|
+
agent: row.opencode_agent ?? void 0,
|
|
2809
|
+
model: row.opencode_model ?? void 0
|
|
2810
|
+
};
|
|
2811
|
+
this.log({
|
|
2812
|
+
level: "info",
|
|
2813
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2814
|
+
conversation_id: row.conversation_id,
|
|
2815
|
+
message_id: row.id
|
|
2816
|
+
});
|
|
2817
|
+
this.awaitingReadopt.add(row.id);
|
|
2818
|
+
let ocId;
|
|
2819
|
+
try {
|
|
2820
|
+
ocId = await this.dispatchLocked(
|
|
2821
|
+
sessionId,
|
|
2822
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
2823
|
+
);
|
|
1811
2824
|
} catch (err) {
|
|
2825
|
+
this.awaitingReadopt.delete(row.id);
|
|
2826
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
1812
2827
|
this.log({
|
|
1813
2828
|
level: "error",
|
|
1814
|
-
message: `
|
|
1815
|
-
conversation_id:
|
|
1816
|
-
message_id:
|
|
2829
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2830
|
+
conversation_id: row.conversation_id,
|
|
2831
|
+
message_id: row.id
|
|
2832
|
+
});
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2835
|
+
if (ocId === null) {
|
|
2836
|
+
this.awaitingReadopt.delete(row.id);
|
|
2837
|
+
this.log({
|
|
2838
|
+
level: "error",
|
|
2839
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
2840
|
+
conversation_id: row.conversation_id,
|
|
2841
|
+
message_id: row.id
|
|
2842
|
+
});
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
const conv = this.convForRow(sessionId, row);
|
|
2846
|
+
const message = this.queuedMessageForRow(row);
|
|
2847
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2848
|
+
this.dispatched.add(row.id);
|
|
2849
|
+
this.readopted.add(row.id);
|
|
2850
|
+
this.awaitingReadopt.delete(row.id);
|
|
2851
|
+
this.ensureWatcherRunning(sessionId);
|
|
2852
|
+
}
|
|
2853
|
+
/**
|
|
2854
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2855
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2856
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2857
|
+
*/
|
|
2858
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2859
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2860
|
+
const watcher = this.watchers.get(sessionId);
|
|
2861
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2862
|
+
}
|
|
2863
|
+
/**
|
|
2864
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2865
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2866
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2867
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2868
|
+
* intended, which is worth surfacing.
|
|
2869
|
+
*/
|
|
2870
|
+
processedAtMs(row) {
|
|
2871
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2872
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2873
|
+
this.log({
|
|
2874
|
+
level: "error",
|
|
2875
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2876
|
+
conversation_id: row.conversation_id,
|
|
2877
|
+
message_id: row.id
|
|
2878
|
+
});
|
|
2879
|
+
return this.now();
|
|
2880
|
+
}
|
|
2881
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2882
|
+
convForRow(sessionId, row) {
|
|
2883
|
+
return {
|
|
2884
|
+
id: row.conversation_id,
|
|
2885
|
+
agent_id: this.agentId,
|
|
2886
|
+
opencode_session_id: sessionId,
|
|
2887
|
+
pending_message_count: 0,
|
|
2888
|
+
oldest_pending_at: row.processed_at
|
|
2889
|
+
};
|
|
2890
|
+
}
|
|
2891
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2892
|
+
queuedMessageForRow(row) {
|
|
2893
|
+
return {
|
|
2894
|
+
id: row.id,
|
|
2895
|
+
content: row.content,
|
|
2896
|
+
status: "processing",
|
|
2897
|
+
opencode_agent: row.opencode_agent,
|
|
2898
|
+
opencode_model: row.opencode_model,
|
|
2899
|
+
source_message_id: row.source_message_id,
|
|
2900
|
+
slack_user_id: row.slack_user_id
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
/**
|
|
2904
|
+
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2905
|
+
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2906
|
+
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2907
|
+
*
|
|
2908
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2909
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2910
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2911
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2912
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2913
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2914
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2915
|
+
*/
|
|
2916
|
+
removeInFlight(watcher, evidentMessageId) {
|
|
2917
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2918
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2919
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2920
|
+
this.log({
|
|
2921
|
+
level: "info",
|
|
2922
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2923
|
+
conversation_id: watcher.conv.id,
|
|
2924
|
+
message_id: evidentMessageId
|
|
2925
|
+
});
|
|
2926
|
+
}
|
|
2927
|
+
watcher.inFlight.delete(evidentMessageId);
|
|
2928
|
+
this.dispatched.delete(evidentMessageId);
|
|
2929
|
+
}
|
|
2930
|
+
/**
|
|
2931
|
+
* Poll `/question` + `/permission` (scoped to the session) and surface NEW ones
|
|
2932
|
+
* via `reportInteraction` (Task 3.5), carrying the PAUSED message's own
|
|
2933
|
+
* `source_message_id` so the server @mentions the correct person under
|
|
2934
|
+
* concurrency. Dedups by interaction id across ticks (reused per-session sets).
|
|
2935
|
+
*
|
|
2936
|
+
* The interaction is attributed to the in-flight message it paused on. opencode
|
|
2937
|
+
* stamps a `messageID` on a permission (and `tool.messageID` on a question) =
|
|
2938
|
+
* the assistant message id, whose `parentID` is the user message id — but the
|
|
2939
|
+
* simplest robust attribution here is: the single in-flight message that is
|
|
2940
|
+
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2941
|
+
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2942
|
+
* oldest running message.
|
|
2943
|
+
*/
|
|
2944
|
+
async pollInteractions(sessionId, watcher, messages) {
|
|
2945
|
+
let questions = [];
|
|
2946
|
+
try {
|
|
2947
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2948
|
+
if (res.ok) {
|
|
2949
|
+
const body = await res.json();
|
|
2950
|
+
questions = Array.isArray(body) ? body : [];
|
|
2951
|
+
}
|
|
2952
|
+
} catch {
|
|
2953
|
+
}
|
|
2954
|
+
for (const q of questions) {
|
|
2955
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2956
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2957
|
+
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
2958
|
+
const reported = await this.reportInteraction(
|
|
2959
|
+
watcher.conv.id,
|
|
2960
|
+
"question",
|
|
2961
|
+
q,
|
|
2962
|
+
paused?.message.source_message_id ?? void 0
|
|
2963
|
+
);
|
|
2964
|
+
if (reported) watcher.reportedQuestions.add(q.id);
|
|
2965
|
+
}
|
|
2966
|
+
let permissions = [];
|
|
2967
|
+
try {
|
|
2968
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2969
|
+
if (res.ok) {
|
|
2970
|
+
const body = await res.json();
|
|
2971
|
+
permissions = Array.isArray(body) ? body : [];
|
|
2972
|
+
}
|
|
2973
|
+
} catch {
|
|
2974
|
+
}
|
|
2975
|
+
for (const p of permissions) {
|
|
2976
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2977
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2978
|
+
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
2979
|
+
const reported = await this.reportInteraction(
|
|
2980
|
+
watcher.conv.id,
|
|
2981
|
+
"permission",
|
|
2982
|
+
p,
|
|
2983
|
+
paused?.message.source_message_id ?? void 0
|
|
2984
|
+
);
|
|
2985
|
+
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2986
|
+
}
|
|
2987
|
+
}
|
|
2988
|
+
/**
|
|
2989
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
2990
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
2991
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
2992
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
2993
|
+
* still be attributed to the root conversation the watcher owns.
|
|
2994
|
+
*
|
|
2995
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
2996
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
2997
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
2998
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
2999
|
+
*/
|
|
3000
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3001
|
+
let current = sessionId;
|
|
3002
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3003
|
+
if (current === rootSessionId) return true;
|
|
3004
|
+
const parent = await this.resolveSessionParent(current);
|
|
3005
|
+
if (parent === null || parent === void 0) return false;
|
|
3006
|
+
current = parent;
|
|
3007
|
+
}
|
|
3008
|
+
return false;
|
|
3009
|
+
}
|
|
3010
|
+
/**
|
|
3011
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3012
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3013
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3014
|
+
* caching a wrong answer — the next tick retries).
|
|
3015
|
+
*/
|
|
3016
|
+
async resolveSessionParent(sessionId) {
|
|
3017
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3018
|
+
if (cached !== void 0) return cached;
|
|
3019
|
+
let parent = void 0;
|
|
3020
|
+
try {
|
|
3021
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3022
|
+
if (res.ok) {
|
|
3023
|
+
const body = await res.json();
|
|
3024
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3025
|
+
}
|
|
3026
|
+
} catch {
|
|
3027
|
+
parent = void 0;
|
|
3028
|
+
}
|
|
3029
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3030
|
+
return parent;
|
|
3031
|
+
}
|
|
3032
|
+
/**
|
|
3033
|
+
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
3034
|
+
*
|
|
3035
|
+
* The interaction carries `interactionMessageId` — the ASSISTANT message id
|
|
3036
|
+
* that raised it (a question's `tool.messageID` / a permission's `messageID`).
|
|
3037
|
+
* That assistant message is the reply to ONE of our minted user messages
|
|
3038
|
+
* (correlated by `parentID`, GATE-B). So when we have the tick's message
|
|
3039
|
+
* snapshot, we resolve each running in-flight message's correlated assistant
|
|
3040
|
+
* reply (`findAssistantReplyAfter`) and match its id against
|
|
3041
|
+
* `interactionMessageId` — giving an EXACT attribution even with several
|
|
3042
|
+
* messages in flight concurrently in one session.
|
|
3043
|
+
*
|
|
3044
|
+
* We fall back to the oldest running message ONLY when no exact match is
|
|
3045
|
+
* possible (the id is absent, the snapshot is missing, or the reply has not yet
|
|
3046
|
+
* been correlated). With a single running message either path is exact. Never
|
|
3047
|
+
* throws.
|
|
3048
|
+
*
|
|
3049
|
+
* Attribution must NOT depend on our own `started` PATCH flag: opencode can
|
|
3050
|
+
* START a turn AND raise a question/permission BEFORE our next tick fires
|
|
3051
|
+
* `markProcessing` (which sets `started`). Relying on `started` would leave the
|
|
3052
|
+
* running set empty in that window and let the server fall back to "newest
|
|
3053
|
+
* processing/pending" — possibly @mentioning a FOLLOW-UP author rather than the
|
|
3054
|
+
* person whose active turn actually paused. So we derive "running" from the
|
|
3055
|
+
* tick's `messages` snapshot via `messageRunState` instead.
|
|
3056
|
+
*/
|
|
3057
|
+
attributeInteraction(watcher, interactionMessageId, messages) {
|
|
3058
|
+
const inFlight = [...watcher.inFlight.values()].filter((m) => !m.done);
|
|
3059
|
+
if (inFlight.length === 0) return void 0;
|
|
3060
|
+
if (interactionMessageId && messages) {
|
|
3061
|
+
const exact = inFlight.find((m) => {
|
|
3062
|
+
const reply = findAssistantReplyAfter(messages, m.opencodeMessageId);
|
|
3063
|
+
return reply != null && messageIdOf(reply) === interactionMessageId;
|
|
1817
3064
|
});
|
|
3065
|
+
if (exact) return exact;
|
|
3066
|
+
}
|
|
3067
|
+
const byOldest = (a, b) => a.dispatchedAt - b.dispatchedAt;
|
|
3068
|
+
if (messages) {
|
|
3069
|
+
const runningPerSnapshot = inFlight.filter(
|
|
3070
|
+
(m) => messageRunState(messages, m.opencodeMessageId) === "running"
|
|
3071
|
+
);
|
|
3072
|
+
if (runningPerSnapshot.length > 0) {
|
|
3073
|
+
return runningPerSnapshot.sort(byOldest)[0];
|
|
3074
|
+
}
|
|
1818
3075
|
}
|
|
3076
|
+
const startedRunning = inFlight.filter((m) => m.started);
|
|
3077
|
+
if (startedRunning.length > 0) {
|
|
3078
|
+
return startedRunning.sort(byOldest)[0];
|
|
3079
|
+
}
|
|
3080
|
+
return inFlight.sort(byOldest)[0];
|
|
1819
3081
|
}
|
|
1820
3082
|
// -------------------------------------------------------------------------
|
|
1821
3083
|
// Evident API calls (combinedAuth thread routes)
|
|
@@ -1849,49 +3111,182 @@ var ChannelDriver = class {
|
|
|
1849
3111
|
}
|
|
1850
3112
|
return await res.json();
|
|
1851
3113
|
}
|
|
1852
|
-
|
|
3114
|
+
/**
|
|
3115
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
3116
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
3117
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
3118
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
3119
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
3120
|
+
* `opencode_session_id`, routing).
|
|
3121
|
+
*
|
|
3122
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
3123
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
3124
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
3125
|
+
* next tick retries.
|
|
3126
|
+
*/
|
|
3127
|
+
async getProcessingMessages() {
|
|
3128
|
+
const res = await this.fetchImpl(
|
|
3129
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
3130
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
3131
|
+
);
|
|
3132
|
+
this.assertAuth(res, "fetching processing messages");
|
|
3133
|
+
if (!res.ok) {
|
|
3134
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
3135
|
+
}
|
|
3136
|
+
const data = await res.json();
|
|
3137
|
+
let messages = data.messages ?? [];
|
|
3138
|
+
if (this.conversationFilter) {
|
|
3139
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
3140
|
+
}
|
|
3141
|
+
return messages;
|
|
3142
|
+
}
|
|
3143
|
+
/**
|
|
3144
|
+
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
3145
|
+
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
3146
|
+
* opencode_session_id}` → `notifyMessageStarted` (hourglass→runner swap +
|
|
3147
|
+
* deep-linked "View in Evident" notice).
|
|
3148
|
+
*
|
|
3149
|
+
* Return/throw contract (consumed by the watcher's swap-to-running guard):
|
|
3150
|
+
* - returns `true` → the server transitioned the row to processing;
|
|
3151
|
+
* - returns `false` → the server gave a DEFINITIVE "already-processing"
|
|
3152
|
+
* answer (a non-retryable, non-auth status — e.g. a
|
|
3153
|
+
* conflict because a duplicate already transitioned it),
|
|
3154
|
+
* so the caller treats it as already-started and does NOT
|
|
3155
|
+
* retry;
|
|
3156
|
+
* - throws `ChannelAuthError` on 401/403 (terminal auth failure);
|
|
3157
|
+
* - throws on a TRANSIENT failure (retryable 5xx/429 status, or a
|
|
3158
|
+
* network-level error from `fetch`) — i.e. NO definitive server response —
|
|
3159
|
+
* so the caller leaves the message un-started and retries the swap on the
|
|
3160
|
+
* next tick.
|
|
3161
|
+
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
3162
|
+
* retry vehicle for the swap-to-running.
|
|
3163
|
+
*/
|
|
3164
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
1853
3165
|
const res = await this.fetchImpl(
|
|
1854
3166
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
1855
3167
|
{
|
|
1856
3168
|
method: "PATCH",
|
|
1857
3169
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1858
|
-
body: JSON.stringify({
|
|
3170
|
+
body: JSON.stringify({
|
|
3171
|
+
status: "processing",
|
|
3172
|
+
opencode_session_id: sessionId,
|
|
3173
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3174
|
+
})
|
|
1859
3175
|
}
|
|
1860
3176
|
);
|
|
1861
3177
|
this.assertAuth(res, "marking message as processing");
|
|
1862
|
-
|
|
3178
|
+
if (res.ok) return true;
|
|
3179
|
+
if (isRetryableStatus(res.status)) {
|
|
3180
|
+
throw new Error(`marking message as processing: HTTP ${res.status}`);
|
|
3181
|
+
}
|
|
3182
|
+
return false;
|
|
1863
3183
|
}
|
|
1864
3184
|
/**
|
|
1865
|
-
* EXISTING combinedAuth completion route — idempotent
|
|
1866
|
-
*
|
|
3185
|
+
* EXISTING combinedAuth completion route — idempotent (WI-CHAN-2). `PATCH
|
|
3186
|
+
* .../messages/:id {status:'done', opencode_session_id}`. The server's
|
|
1867
3187
|
* `queued_conversation_messages.status`/`processed_at` gate makes a re-call
|
|
1868
|
-
* for an already-`done` message a no-op (no double Slack post).
|
|
3188
|
+
* for an already-`done` message a no-op (no double Slack post). Fired by the
|
|
3189
|
+
* watcher on per-message completion (Task 3.4) — no `confirmCompletion`
|
|
3190
|
+
* round-trip (we already observed completion via the message list).
|
|
3191
|
+
*
|
|
3192
|
+
* SINGLE ATTEMPT (no in-call `callWithRetry` backoff). The per-session watcher
|
|
3193
|
+
* services its in-flight messages SEQUENTIALLY within a tick
|
|
3194
|
+
* (`runWatcherLoop` → `serviceInFlightMessage`), so a long multi-attempt
|
|
3195
|
+
* backoff here would BLOCK sibling messages in the SAME session/tick: while
|
|
3196
|
+
* message A's done PATCH burned its internal retries, message B could not be
|
|
3197
|
+
* swapped to running even though opencode had already started it. Instead this
|
|
3198
|
+
* does ONE PATCH and surfaces the SAME outcome contract the watcher's markDone
|
|
3199
|
+
* handler already relies on, leaning on the per-tick retry across ticks
|
|
3200
|
+
* (bounded by `inFlight.deadline`) rather than an in-call retry:
|
|
3201
|
+
* - resolves (`void`) → the server transitioned the row to done
|
|
3202
|
+
* (or idempotently confirmed already-done);
|
|
3203
|
+
* - throws `ChannelAuthError` → 401/403 (terminal auth failure → loop
|
|
3204
|
+
* cleanup, Finding 1);
|
|
3205
|
+
* - throws `ChannelTerminalError`→ non-retryable, non-auth 4xx (will never
|
|
3206
|
+
* succeed → straight to the cron, Finding 4);
|
|
3207
|
+
* - throws a plain `Error` → TRANSIENT 5xx/429 or a network-level error
|
|
3208
|
+
* (no definitive server response → the
|
|
3209
|
+
* watcher retries next tick within the
|
|
3210
|
+
* deadline, Finding 4).
|
|
1869
3211
|
*/
|
|
1870
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3212
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
3213
|
+
const res = await this.fetchImpl(
|
|
3214
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
3215
|
+
{
|
|
3216
|
+
method: "PATCH",
|
|
3217
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3218
|
+
body: JSON.stringify({
|
|
3219
|
+
status: "done",
|
|
3220
|
+
opencode_session_id: sessionId,
|
|
3221
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3222
|
+
})
|
|
3223
|
+
}
|
|
3224
|
+
);
|
|
3225
|
+
this.assertAuth(res, "marking message as done");
|
|
3226
|
+
if (res.ok) return;
|
|
3227
|
+
if (isRetryableStatus(res.status)) {
|
|
3228
|
+
throw new Error(`marking message as done: HTTP ${res.status}`);
|
|
3229
|
+
}
|
|
3230
|
+
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
3231
|
+
}
|
|
3232
|
+
/**
|
|
3233
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3234
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3235
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3236
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3237
|
+
* failure reason reaches the channel.
|
|
3238
|
+
*/
|
|
3239
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3240
|
+
const body = { status: "failed" };
|
|
3241
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3242
|
+
if (error2 !== void 0) body.error = error2;
|
|
1871
3243
|
await this.callWithRetry(
|
|
1872
|
-
"marking message as
|
|
3244
|
+
"marking message as failed",
|
|
1873
3245
|
() => this.fetchImpl(
|
|
1874
3246
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
1875
3247
|
{
|
|
1876
3248
|
method: "PATCH",
|
|
1877
3249
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1878
|
-
body: JSON.stringify(
|
|
3250
|
+
body: JSON.stringify(body)
|
|
1879
3251
|
}
|
|
1880
3252
|
)
|
|
1881
3253
|
);
|
|
1882
3254
|
}
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
3255
|
+
/**
|
|
3256
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3257
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3258
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3259
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3260
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3261
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3262
|
+
* context (no silent catch, per development-workflow).
|
|
3263
|
+
*/
|
|
3264
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3265
|
+
try {
|
|
3266
|
+
const res = await this.fetchImpl(
|
|
3267
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
1888
3268
|
{
|
|
1889
|
-
method: "
|
|
3269
|
+
method: "POST",
|
|
1890
3270
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1891
|
-
body: JSON.stringify({
|
|
3271
|
+
body: JSON.stringify({ signal, ...extra })
|
|
1892
3272
|
}
|
|
1893
|
-
)
|
|
1894
|
-
|
|
3273
|
+
);
|
|
3274
|
+
if (!res.ok) {
|
|
3275
|
+
this.log({
|
|
3276
|
+
level: "error",
|
|
3277
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3278
|
+
conversation_id: conversationId,
|
|
3279
|
+
message_id: messageId
|
|
3280
|
+
});
|
|
3281
|
+
}
|
|
3282
|
+
} catch (err) {
|
|
3283
|
+
this.log({
|
|
3284
|
+
level: "error",
|
|
3285
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3286
|
+
conversation_id: conversationId,
|
|
3287
|
+
message_id: messageId
|
|
3288
|
+
});
|
|
3289
|
+
}
|
|
1895
3290
|
}
|
|
1896
3291
|
async persistSession(conversationId, sessionId) {
|
|
1897
3292
|
const res = await this.fetchImpl(
|
|
@@ -1906,10 +3301,17 @@ var ChannelDriver = class {
|
|
|
1906
3301
|
}
|
|
1907
3302
|
/**
|
|
1908
3303
|
* EXISTING combinedAuth interaction route (WI-CHAN-3) — idempotent + retried.
|
|
1909
|
-
* `POST .../interactive-event {type, data}`. The server
|
|
1910
|
-
* interaction and posts a link to the proxied opencode-web
|
|
3304
|
+
* `POST .../interactive-event {type, data, source_message_id?}`. The server
|
|
3305
|
+
* persists the interaction and posts a link to the proxied opencode-web
|
|
3306
|
+
* conversation, @mentioning the user who triggered THIS message's turn.
|
|
3307
|
+
*
|
|
3308
|
+
* WI-3 / WI-4 contract: `source_message_id` is the PAUSED message's own Slack
|
|
3309
|
+
* ts (`message.source_message_id`). The server resolves the @mention from that
|
|
3310
|
+
* message's user FIRST (falling back to the old "newest processing" precedence
|
|
3311
|
+
* only when absent), so the correct person is mentioned under concurrency. It
|
|
3312
|
+
* is OPTIONAL for back-compat with older clients / legacy rows.
|
|
1911
3313
|
*/
|
|
1912
|
-
async reportInteraction(conversationId, type, data) {
|
|
3314
|
+
async reportInteraction(conversationId, type, data, sourceMessageId) {
|
|
1913
3315
|
try {
|
|
1914
3316
|
await this.callWithRetry(
|
|
1915
3317
|
"reporting interactive event",
|
|
@@ -1918,7 +3320,9 @@ var ChannelDriver = class {
|
|
|
1918
3320
|
{
|
|
1919
3321
|
method: "POST",
|
|
1920
3322
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1921
|
-
body: JSON.stringify(
|
|
3323
|
+
body: JSON.stringify(
|
|
3324
|
+
sourceMessageId ? { type, data, source_message_id: sourceMessageId } : { type, data }
|
|
3325
|
+
)
|
|
1922
3326
|
}
|
|
1923
3327
|
)
|
|
1924
3328
|
);
|
|
@@ -1927,6 +3331,7 @@ var ChannelDriver = class {
|
|
|
1927
3331
|
message: `${type} surfaced to channel (id: ${data.id.slice(0, 8)})`,
|
|
1928
3332
|
conversation_id: conversationId
|
|
1929
3333
|
});
|
|
3334
|
+
return true;
|
|
1930
3335
|
} catch (err) {
|
|
1931
3336
|
if (err instanceof ChannelAuthError) throw err;
|
|
1932
3337
|
this.log({
|
|
@@ -1934,6 +3339,7 @@ var ChannelDriver = class {
|
|
|
1934
3339
|
message: `Failed to surface ${type}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1935
3340
|
conversation_id: conversationId
|
|
1936
3341
|
});
|
|
3342
|
+
return false;
|
|
1937
3343
|
}
|
|
1938
3344
|
}
|
|
1939
3345
|
// -------------------------------------------------------------------------
|
|
@@ -1972,8 +3378,9 @@ var ChannelDriver = class {
|
|
|
1972
3378
|
await this.sleep(backoffDelay(attempt, this.retry));
|
|
1973
3379
|
continue;
|
|
1974
3380
|
}
|
|
3381
|
+
break;
|
|
1975
3382
|
}
|
|
1976
|
-
throw new
|
|
3383
|
+
throw new ChannelTerminalError(`${context}: HTTP ${res.status}`, res.status);
|
|
1977
3384
|
}
|
|
1978
3385
|
throw lastError instanceof Error ? lastError : new Error(`${context}: exhausted retries`);
|
|
1979
3386
|
}
|
|
@@ -2151,6 +3558,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
2151
3558
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
2152
3559
|
}
|
|
2153
3560
|
}
|
|
3561
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3562
|
+
const apiUrl = getApiUrlConfig();
|
|
3563
|
+
try {
|
|
3564
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3565
|
+
method: "POST",
|
|
3566
|
+
headers: { Authorization: authHeader }
|
|
3567
|
+
});
|
|
3568
|
+
if (!response.ok) {
|
|
3569
|
+
const serverMessage = await readErrorMessage(response);
|
|
3570
|
+
return {
|
|
3571
|
+
ok: false,
|
|
3572
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3573
|
+
};
|
|
3574
|
+
}
|
|
3575
|
+
return { ok: true };
|
|
3576
|
+
} catch (error2) {
|
|
3577
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3578
|
+
}
|
|
3579
|
+
}
|
|
2154
3580
|
async function getAgentInfo(agentId, authHeader) {
|
|
2155
3581
|
const apiUrl = getApiUrlConfig();
|
|
2156
3582
|
try {
|
|
@@ -2196,7 +3622,9 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2196
3622
|
// src/commands/run.ts
|
|
2197
3623
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2198
3624
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2199
|
-
|
|
3625
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3626
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3627
|
+
function log2(state, message, isError = false) {
|
|
2200
3628
|
if (state.json) {
|
|
2201
3629
|
console.log(
|
|
2202
3630
|
JSON.stringify({
|
|
@@ -2221,9 +3649,9 @@ function logActivity(state, entry) {
|
|
|
2221
3649
|
}
|
|
2222
3650
|
if (!state.interactive) {
|
|
2223
3651
|
if (entry.type === "error") {
|
|
2224
|
-
|
|
3652
|
+
log2(state, entry.error ?? "Unknown error", true);
|
|
2225
3653
|
} else if (entry.type === "info" && entry.message) {
|
|
2226
|
-
|
|
3654
|
+
log2(state, entry.message);
|
|
2227
3655
|
}
|
|
2228
3656
|
}
|
|
2229
3657
|
}
|
|
@@ -2309,6 +3737,7 @@ async function handleAuthError(state, error2) {
|
|
|
2309
3737
|
}
|
|
2310
3738
|
async function driveChannels(state, driver) {
|
|
2311
3739
|
let idlePolls = 0;
|
|
3740
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2312
3741
|
while (state.running) {
|
|
2313
3742
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2314
3743
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2318,9 +3747,11 @@ async function driveChannels(state, driver) {
|
|
|
2318
3747
|
try {
|
|
2319
3748
|
const processed = await driver.drainPending();
|
|
2320
3749
|
state.messageCount += processed;
|
|
2321
|
-
|
|
3750
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
3751
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
3752
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2322
3753
|
idlePolls = 0;
|
|
2323
|
-
if (state.interactive) displayStatus(state);
|
|
3754
|
+
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2324
3755
|
} else if (state.idleTimeout !== null) {
|
|
2325
3756
|
idlePolls++;
|
|
2326
3757
|
if (idlePolls === 1) {
|
|
@@ -2347,7 +3778,7 @@ async function driveChannels(state, driver) {
|
|
|
2347
3778
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
2348
3779
|
if (state.interactive) displayStatus(state);
|
|
2349
3780
|
}
|
|
2350
|
-
await new Promise((
|
|
3781
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
2351
3782
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
2352
3783
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
2353
3784
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -2358,8 +3789,122 @@ async function driveChannels(state, driver) {
|
|
|
2358
3789
|
}
|
|
2359
3790
|
}
|
|
2360
3791
|
}
|
|
2361
|
-
|
|
3792
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
3793
|
+
async function runSweep(state, driver, config2) {
|
|
3794
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
3795
|
+
try {
|
|
3796
|
+
const sessions = await listSessions(state.port);
|
|
3797
|
+
if (sessions === null) {
|
|
3798
|
+
logActivity(state, {
|
|
3799
|
+
type: "info",
|
|
3800
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
3801
|
+
});
|
|
3802
|
+
return;
|
|
3803
|
+
}
|
|
3804
|
+
const toDelete = selectSessionsToDelete(
|
|
3805
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
3806
|
+
{
|
|
3807
|
+
maxAgeMs: config2.maxAgeMs,
|
|
3808
|
+
maxCount: config2.maxCount,
|
|
3809
|
+
nowMs: Date.now(),
|
|
3810
|
+
protectedIds: driver.protectedSessionIds()
|
|
3811
|
+
}
|
|
3812
|
+
);
|
|
3813
|
+
const protectedNow = driver.protectedSessionIds();
|
|
3814
|
+
let deleted = 0;
|
|
3815
|
+
let failed = 0;
|
|
3816
|
+
let skippedNewlyActive = 0;
|
|
3817
|
+
for (const id of toDelete) {
|
|
3818
|
+
if (protectedNow.has(id)) {
|
|
3819
|
+
skippedNewlyActive++;
|
|
3820
|
+
logActivity(state, {
|
|
3821
|
+
type: "info",
|
|
3822
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
3823
|
+
});
|
|
3824
|
+
continue;
|
|
3825
|
+
}
|
|
3826
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
3827
|
+
else failed++;
|
|
3828
|
+
}
|
|
3829
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
3830
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
3831
|
+
logActivity(state, {
|
|
3832
|
+
type: "info",
|
|
3833
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
3834
|
+
});
|
|
3835
|
+
} catch (error2) {
|
|
3836
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3837
|
+
logActivity(state, {
|
|
3838
|
+
type: "error",
|
|
3839
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
3840
|
+
});
|
|
3841
|
+
}
|
|
3842
|
+
}
|
|
3843
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
3844
|
+
const config2 = resolveSessionCleanupConfig(
|
|
3845
|
+
{
|
|
3846
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
3847
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
3848
|
+
interval: options.sessionCleanupInterval
|
|
3849
|
+
},
|
|
3850
|
+
process.env
|
|
3851
|
+
);
|
|
3852
|
+
for (const warning2 of config2.warnings) {
|
|
3853
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
3854
|
+
}
|
|
3855
|
+
if (!config2.enabled) return;
|
|
3856
|
+
logActivity(state, {
|
|
3857
|
+
type: "info",
|
|
3858
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
3859
|
+
});
|
|
3860
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
3861
|
+
const firstSweep = setTimeout(
|
|
3862
|
+
() => void runSweep(state, driver, config2),
|
|
3863
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
3864
|
+
);
|
|
3865
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
3866
|
+
}
|
|
3867
|
+
async function notifyOffline(state) {
|
|
3868
|
+
if (!state.agentId || !state.authHeader) return;
|
|
3869
|
+
if (!state.connected) {
|
|
3870
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
3871
|
+
return;
|
|
3872
|
+
}
|
|
3873
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
3874
|
+
if (result.ok) {
|
|
3875
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
3876
|
+
} else {
|
|
3877
|
+
logActivity(state, {
|
|
3878
|
+
type: "error",
|
|
3879
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
3880
|
+
});
|
|
3881
|
+
if (state.interactive) displayStatus(state);
|
|
3882
|
+
}
|
|
3883
|
+
}
|
|
3884
|
+
async function cleanup(state, opts = {}) {
|
|
2362
3885
|
state.running = false;
|
|
3886
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
3887
|
+
clearInterval(timer);
|
|
3888
|
+
clearTimeout(timer);
|
|
3889
|
+
}
|
|
3890
|
+
state.sessionCleanupTimers = [];
|
|
3891
|
+
if (opts.graceful && state.channelDriver) {
|
|
3892
|
+
state.channelDriver.stop();
|
|
3893
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
3894
|
+
if (state.interactive) {
|
|
3895
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
3896
|
+
displayStatus(state);
|
|
3897
|
+
}
|
|
3898
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
3899
|
+
if (!settled) {
|
|
3900
|
+
logActivity(state, {
|
|
3901
|
+
type: "info",
|
|
3902
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
3903
|
+
});
|
|
3904
|
+
if (state.interactive) displayStatus(state);
|
|
3905
|
+
}
|
|
3906
|
+
}
|
|
3907
|
+
await notifyOffline(state);
|
|
2363
3908
|
if (state.connection) {
|
|
2364
3909
|
state.connection.close();
|
|
2365
3910
|
state.connection = null;
|
|
@@ -2370,7 +3915,7 @@ async function cleanup(state) {
|
|
|
2370
3915
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2371
3916
|
displayStatus(state);
|
|
2372
3917
|
} else {
|
|
2373
|
-
|
|
3918
|
+
log2(state, "Stopped OpenCode process");
|
|
2374
3919
|
}
|
|
2375
3920
|
state.opencodeProcess = null;
|
|
2376
3921
|
}
|
|
@@ -2390,26 +3935,32 @@ async function run(options) {
|
|
|
2390
3935
|
opencodeVersion: null,
|
|
2391
3936
|
opencodeProcess: null,
|
|
2392
3937
|
connection: null,
|
|
3938
|
+
channelDriver: null,
|
|
2393
3939
|
running: true,
|
|
3940
|
+
shuttingDown: false,
|
|
2394
3941
|
activityLog: [],
|
|
2395
3942
|
messageCount: 0,
|
|
3943
|
+
lastProxiedActivityAt: null,
|
|
3944
|
+
sessionCleanupTimers: [],
|
|
2396
3945
|
authHeader: ""
|
|
2397
3946
|
};
|
|
2398
3947
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2399
|
-
|
|
3948
|
+
log2(
|
|
2400
3949
|
state,
|
|
2401
3950
|
"Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
2402
3951
|
false
|
|
2403
3952
|
);
|
|
2404
3953
|
}
|
|
2405
3954
|
const handleSignal = async () => {
|
|
3955
|
+
if (state.shuttingDown) return;
|
|
3956
|
+
state.shuttingDown = true;
|
|
2406
3957
|
if (state.interactive) {
|
|
2407
3958
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2408
3959
|
displayStatus(state);
|
|
2409
3960
|
} else {
|
|
2410
|
-
|
|
3961
|
+
log2(state, "Shutting down...");
|
|
2411
3962
|
}
|
|
2412
|
-
await cleanup(state);
|
|
3963
|
+
await cleanup(state, { graceful: true });
|
|
2413
3964
|
await shutdownTelemetry();
|
|
2414
3965
|
process.exit(0);
|
|
2415
3966
|
};
|
|
@@ -2440,7 +3991,7 @@ async function run(options) {
|
|
|
2440
3991
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2441
3992
|
if (resolved.agent_id) {
|
|
2442
3993
|
state.agentId = resolved.agent_id;
|
|
2443
|
-
|
|
3994
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2444
3995
|
if (state.interactive && !state.json) {
|
|
2445
3996
|
logActivity(state, {
|
|
2446
3997
|
type: "info",
|
|
@@ -2503,14 +4054,21 @@ async function run(options) {
|
|
|
2503
4054
|
port: state.port,
|
|
2504
4055
|
interactive: state.interactive,
|
|
2505
4056
|
agentId: state.agentId,
|
|
2506
|
-
log: (message) =>
|
|
4057
|
+
log: (message) => log2(state, message)
|
|
2507
4058
|
});
|
|
2508
4059
|
state.port = oc.port;
|
|
2509
4060
|
state.opencodeProcess = oc.process;
|
|
2510
4061
|
state.opencodeVersion = oc.version;
|
|
2511
4062
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2512
|
-
const
|
|
2513
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
4063
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
4064
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
4065
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
4066
|
+
if (versionWarning) {
|
|
4067
|
+
log2(state, versionWarning, false);
|
|
4068
|
+
if (state.interactive && !state.json) {
|
|
4069
|
+
logActivity(state, { type: "info", message: versionWarning });
|
|
4070
|
+
}
|
|
4071
|
+
}
|
|
2514
4072
|
} catch (error2) {
|
|
2515
4073
|
ocSpinner?.fail(error2.message);
|
|
2516
4074
|
throw error2;
|
|
@@ -2522,12 +4080,14 @@ async function run(options) {
|
|
|
2522
4080
|
apiUrl: getApiUrlConfig(),
|
|
2523
4081
|
getAuthHeader: () => state.authHeader,
|
|
2524
4082
|
conversationFilter: state.conversationFilter,
|
|
4083
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
2525
4084
|
log: (entry) => logActivity(state, {
|
|
2526
4085
|
type: entry.level === "error" ? "error" : "info",
|
|
2527
4086
|
message: entry.message,
|
|
2528
4087
|
error: entry.level === "error" ? entry.message : void 0
|
|
2529
4088
|
})
|
|
2530
4089
|
});
|
|
4090
|
+
state.channelDriver = channelDriver;
|
|
2531
4091
|
const connection = new RunnerConnection({
|
|
2532
4092
|
agentId: state.agentId,
|
|
2533
4093
|
getAuthHeader: () => state.authHeader,
|
|
@@ -2541,7 +4101,11 @@ async function run(options) {
|
|
|
2541
4101
|
type: "info",
|
|
2542
4102
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2543
4103
|
});
|
|
2544
|
-
emitAgentConnected(state.agentId, {
|
|
4104
|
+
emitAgentConnected(state.agentId, {
|
|
4105
|
+
port: state.port,
|
|
4106
|
+
cli_version: getCliVersion(),
|
|
4107
|
+
opencode_version: state.opencodeVersion
|
|
4108
|
+
});
|
|
2545
4109
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2546
4110
|
if (state.interactive) displayStatus(state);
|
|
2547
4111
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2575,9 +4139,14 @@ async function run(options) {
|
|
|
2575
4139
|
logActivity(state, { type: "error", error: error2 });
|
|
2576
4140
|
if (state.interactive) displayStatus(state);
|
|
2577
4141
|
},
|
|
2578
|
-
// Web traffic is proxied transparently;
|
|
4142
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
4143
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
4144
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
4145
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
4146
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2579
4147
|
onResponse: () => {
|
|
2580
4148
|
state.opencodeConnected = true;
|
|
4149
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2581
4150
|
},
|
|
2582
4151
|
// A channel message was queued and the api-worker pinged us over the
|
|
2583
4152
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2615,10 +4184,12 @@ async function run(options) {
|
|
|
2615
4184
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
2616
4185
|
throw error2;
|
|
2617
4186
|
}
|
|
4187
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
2618
4188
|
if (!interactive || state.json) {
|
|
2619
|
-
|
|
4189
|
+
log2(state, "Driving channel messages...");
|
|
2620
4190
|
}
|
|
2621
4191
|
await driveChannels(state, channelDriver);
|
|
4192
|
+
if (state.shuttingDown) return;
|
|
2622
4193
|
await cleanup(state);
|
|
2623
4194
|
if (state.json) {
|
|
2624
4195
|
console.log(
|
|
@@ -2628,11 +4199,12 @@ async function run(options) {
|
|
|
2628
4199
|
})
|
|
2629
4200
|
);
|
|
2630
4201
|
} else if (!interactive) {
|
|
2631
|
-
|
|
4202
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2632
4203
|
}
|
|
2633
4204
|
await shutdownTelemetry();
|
|
2634
4205
|
process.exit(0);
|
|
2635
4206
|
} catch (error2) {
|
|
4207
|
+
if (state.shuttingDown) return;
|
|
2636
4208
|
await cleanup(state);
|
|
2637
4209
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
2638
4210
|
if (state.json) {
|
|
@@ -2650,8 +4222,9 @@ async function run(options) {
|
|
|
2650
4222
|
}
|
|
2651
4223
|
|
|
2652
4224
|
// src/index.ts
|
|
4225
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2653
4226
|
var program = new Command();
|
|
2654
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
4227
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2655
4228
|
"--endpoint <url>",
|
|
2656
4229
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2657
4230
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|
|
@@ -2666,7 +4239,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
2666
4239
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
2667
4240
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
2668
4241
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
2669
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4242
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4243
|
+
"--session-cleanup-max-age <duration>",
|
|
4244
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4245
|
+
).option(
|
|
4246
|
+
"--session-cleanup-max-count <n>",
|
|
4247
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4248
|
+
).option(
|
|
4249
|
+
"--session-cleanup-interval <duration>",
|
|
4250
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4251
|
+
).action(
|
|
2670
4252
|
(options) => {
|
|
2671
4253
|
run({
|
|
2672
4254
|
agent: options.agent,
|
|
@@ -2674,7 +4256,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
2674
4256
|
verbose: options.verbose,
|
|
2675
4257
|
conversation: options.conversation,
|
|
2676
4258
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
2677
|
-
json: options.json
|
|
4259
|
+
json: options.json,
|
|
4260
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4261
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4262
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4263
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
2678
4264
|
});
|
|
2679
4265
|
}
|
|
2680
4266
|
);
|