@evident-ai/cli 3.0.1-dev.28a3007 → 3.0.1-dev.291178f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/index.js +2145 -166
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -284,14 +285,14 @@ function blank() {
|
|
|
284
285
|
console.log();
|
|
285
286
|
}
|
|
286
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
287
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
288
289
|
process.stdout.write(chalk.dim(prompt));
|
|
289
290
|
const handler = () => {
|
|
290
291
|
process.stdin.removeListener("data", handler);
|
|
291
292
|
process.stdin.setRawMode?.(false);
|
|
292
293
|
process.stdin.pause();
|
|
293
294
|
console.log();
|
|
294
|
-
|
|
295
|
+
resolve2();
|
|
295
296
|
};
|
|
296
297
|
if (process.stdin.isTTY) {
|
|
297
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -301,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
301
302
|
});
|
|
302
303
|
}
|
|
303
304
|
function sleep(ms) {
|
|
304
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
305
306
|
}
|
|
306
307
|
|
|
307
308
|
// src/commands/login.ts
|
|
@@ -375,19 +376,19 @@ async function tokenLogin() {
|
|
|
375
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
376
377
|
blank();
|
|
377
378
|
process.stdout.write("Paste token: ");
|
|
378
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
379
380
|
let data = "";
|
|
380
381
|
process.stdin.setEncoding("utf8");
|
|
381
382
|
process.stdin.on("data", (chunk) => {
|
|
382
383
|
data += chunk;
|
|
383
384
|
});
|
|
384
385
|
process.stdin.on("end", () => {
|
|
385
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
386
387
|
});
|
|
387
388
|
if (process.stdin.isTTY) {
|
|
388
389
|
process.stdin.once("data", (chunk) => {
|
|
389
390
|
process.stdin.pause();
|
|
390
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
391
392
|
});
|
|
392
393
|
process.stdin.resume();
|
|
393
394
|
}
|
|
@@ -484,8 +485,34 @@ var TelemetryEventTypes = {
|
|
|
484
485
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
486
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
487
|
|
|
488
|
+
// ../../packages/types/src/logging/index.ts
|
|
489
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
|
+
function log(level, event, fields) {
|
|
491
|
+
const method = level === "debug" ? "log" : level;
|
|
492
|
+
try {
|
|
493
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
494
|
+
} catch (err) {
|
|
495
|
+
console.error(
|
|
496
|
+
"[evident] log_serialize_failed",
|
|
497
|
+
event,
|
|
498
|
+
err instanceof Error ? err.message : String(err)
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
function stripQuery(url) {
|
|
503
|
+
try {
|
|
504
|
+
return new URL(url).pathname;
|
|
505
|
+
} catch {
|
|
506
|
+
const q = url.indexOf("?");
|
|
507
|
+
return q === -1 ? url : url.slice(0, q);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
487
511
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
489
516
|
var eventBuffer = [];
|
|
490
517
|
var flushTimeout = null;
|
|
491
518
|
var isShuttingDown = false;
|
|
@@ -679,20 +706,20 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
679
706
|
if (health.healthy) {
|
|
680
707
|
return health;
|
|
681
708
|
}
|
|
682
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
683
710
|
}
|
|
684
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
685
712
|
}
|
|
686
713
|
|
|
687
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
688
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
689
|
-
function isQueueValidatedVersion(
|
|
690
|
-
if (!
|
|
691
|
-
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(
|
|
692
|
-
}
|
|
693
|
-
function buildOpenCodeVersionWarning(
|
|
694
|
-
if (isQueueValidatedVersion(
|
|
695
|
-
const detected =
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
716
|
+
function isQueueValidatedVersion(version2) {
|
|
717
|
+
if (!version2) return false;
|
|
718
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
719
|
+
}
|
|
720
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
721
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
722
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
696
723
|
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
697
724
|
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
698
725
|
}
|
|
@@ -1009,7 +1036,11 @@ function roleOf(m) {
|
|
|
1009
1036
|
}
|
|
1010
1037
|
function completedOf(m) {
|
|
1011
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1012
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1013
1044
|
}
|
|
1014
1045
|
function idOf(m) {
|
|
1015
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1023,6 +1054,108 @@ function parentIdOf(m) {
|
|
|
1023
1054
|
const infoParent = m.info?.parentID;
|
|
1024
1055
|
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1025
1056
|
}
|
|
1057
|
+
function finishOf(m) {
|
|
1058
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1059
|
+
if (typeof m.finish === "string") return m.finish;
|
|
1060
|
+
const infoFinish = m.info?.finish;
|
|
1061
|
+
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1062
|
+
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1070
|
+
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1026
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1027
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1028
1161
|
if (directory && directory.trim()) {
|
|
@@ -1040,10 +1173,113 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1040
1173
|
const data = await response.json();
|
|
1041
1174
|
return data.id;
|
|
1042
1175
|
}
|
|
1043
|
-
async function
|
|
1176
|
+
async function getModelAttachmentCapability(port, model) {
|
|
1177
|
+
try {
|
|
1178
|
+
const res = await fetch(`${opencodeBase(port)}/config/providers`);
|
|
1179
|
+
if (!res.ok) {
|
|
1180
|
+
console.error(
|
|
1181
|
+
`[getModelAttachmentCapability] GET /config/providers returned HTTP ${res.status} (port ${port})`
|
|
1182
|
+
);
|
|
1183
|
+
return null;
|
|
1184
|
+
}
|
|
1185
|
+
const body = await res.json();
|
|
1186
|
+
const providers = Array.isArray(body?.providers) ? body.providers : null;
|
|
1187
|
+
if (!providers) {
|
|
1188
|
+
console.error(
|
|
1189
|
+
`[getModelAttachmentCapability] GET /config/providers body had no providers array (port ${port})`
|
|
1190
|
+
);
|
|
1191
|
+
return null;
|
|
1192
|
+
}
|
|
1193
|
+
const slash = model ? model.indexOf("/") : -1;
|
|
1194
|
+
const providerId = slash > 0 ? model.slice(0, slash) : void 0;
|
|
1195
|
+
let modelId = slash > 0 ? model.slice(slash + 1) : void 0;
|
|
1196
|
+
const defaults2 = body?.default && typeof body.default === "object" ? body.default : void 0;
|
|
1197
|
+
let provider = providerId ? providers.find((p) => p?.id === providerId) : void 0;
|
|
1198
|
+
if (!provider && !providerId) {
|
|
1199
|
+
const defaultProviderIds = defaults2 ? Object.keys(defaults2) : [];
|
|
1200
|
+
if (defaultProviderIds.length === 1) {
|
|
1201
|
+
provider = providers.find((p) => p?.id === defaultProviderIds[0]);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
if (!provider || !provider.models) return null;
|
|
1205
|
+
if (!modelId && defaults2 && typeof provider.id === "string") {
|
|
1206
|
+
const def = defaults2[provider.id];
|
|
1207
|
+
if (typeof def === "string") modelId = def;
|
|
1208
|
+
}
|
|
1209
|
+
if (!modelId) {
|
|
1210
|
+
if (providerId) {
|
|
1211
|
+
const keys = Object.keys(provider.models);
|
|
1212
|
+
if (keys.length === 1) modelId = keys[0];
|
|
1213
|
+
}
|
|
1214
|
+
if (!modelId) return null;
|
|
1215
|
+
}
|
|
1216
|
+
const entry = provider.models[modelId];
|
|
1217
|
+
if (!entry || typeof entry !== "object") return null;
|
|
1218
|
+
return typeof entry.attachment === "boolean" ? entry.attachment : null;
|
|
1219
|
+
} catch (err) {
|
|
1220
|
+
console.error(
|
|
1221
|
+
`[getModelAttachmentCapability] GET /config/providers failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1222
|
+
);
|
|
1223
|
+
return null;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
async function buildFileParts(attachments, capable) {
|
|
1227
|
+
const outcomes = [];
|
|
1228
|
+
const parts = [];
|
|
1229
|
+
const capabilityUnknown = capable === null;
|
|
1230
|
+
if (capable !== true) {
|
|
1231
|
+
for (const a of attachments.inputs) {
|
|
1232
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "skipped" });
|
|
1233
|
+
}
|
|
1234
|
+
return { parts, outcomes, capabilityUnknown };
|
|
1235
|
+
}
|
|
1236
|
+
for (const a of attachments.inputs) {
|
|
1237
|
+
let dataUrl = null;
|
|
1238
|
+
try {
|
|
1239
|
+
dataUrl = await attachments.fetchDataUrl(a.index);
|
|
1240
|
+
} catch (err) {
|
|
1241
|
+
console.error(
|
|
1242
|
+
`[buildFileParts] attachment ${a.index} (${a.mime}) fetch threw \u2014 omitting: ${err instanceof Error ? err.message : String(err)}`
|
|
1243
|
+
);
|
|
1244
|
+
dataUrl = null;
|
|
1245
|
+
}
|
|
1246
|
+
if (dataUrl == null) {
|
|
1247
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "failed" });
|
|
1248
|
+
continue;
|
|
1249
|
+
}
|
|
1250
|
+
parts.push({
|
|
1251
|
+
type: "file",
|
|
1252
|
+
mime: a.mime,
|
|
1253
|
+
url: dataUrl,
|
|
1254
|
+
...a.filename ? { filename: a.filename } : {}
|
|
1255
|
+
});
|
|
1256
|
+
outcomes.push({ index: a.index, mime: a.mime, filename: a.filename, status: "sent" });
|
|
1257
|
+
}
|
|
1258
|
+
return { parts, outcomes, capabilityUnknown };
|
|
1259
|
+
}
|
|
1260
|
+
function messageText(m) {
|
|
1261
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1262
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1263
|
+
}
|
|
1264
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
1265
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1266
|
+
const knownUserIds = new Set(
|
|
1267
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1268
|
+
);
|
|
1269
|
+
const parts = [{ type: "text", text: content }];
|
|
1270
|
+
let pendingOutcomes = null;
|
|
1271
|
+
if (attachments && attachments.inputs.length > 0) {
|
|
1272
|
+
const capable = await getModelAttachmentCapability(port, options?.model);
|
|
1273
|
+
const {
|
|
1274
|
+
parts: fileParts,
|
|
1275
|
+
outcomes,
|
|
1276
|
+
capabilityUnknown
|
|
1277
|
+
} = await buildFileParts(attachments, capable);
|
|
1278
|
+
parts.push(...fileParts);
|
|
1279
|
+
if (attachments.onOutcomes) pendingOutcomes = { outcomes, capabilityUnknown };
|
|
1280
|
+
}
|
|
1044
1281
|
const body = {
|
|
1045
|
-
|
|
1046
|
-
parts: [{ type: "text", text: content }]
|
|
1282
|
+
parts
|
|
1047
1283
|
};
|
|
1048
1284
|
if (options?.agent) {
|
|
1049
1285
|
body.agent = options.agent;
|
|
@@ -1066,6 +1302,32 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1066
1302
|
const text = await res.text().catch(() => "");
|
|
1067
1303
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1068
1304
|
}
|
|
1305
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1306
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1307
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1308
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1309
|
+
if (after) {
|
|
1310
|
+
let best = null;
|
|
1311
|
+
for (const m of after) {
|
|
1312
|
+
if (roleOf(m) !== "user") continue;
|
|
1313
|
+
const id = idOf(m);
|
|
1314
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1315
|
+
if (messageText(m) !== content) continue;
|
|
1316
|
+
const created = createdOf(m) ?? 0;
|
|
1317
|
+
if (best === null || created > best.created) {
|
|
1318
|
+
best = { id, created };
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
if (best) {
|
|
1322
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
1323
|
+
return best.id;
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1327
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
return null;
|
|
1069
1331
|
}
|
|
1070
1332
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1071
1333
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1080,19 +1342,172 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1080
1342
|
}
|
|
1081
1343
|
return null;
|
|
1082
1344
|
}
|
|
1345
|
+
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1346
|
+
if (!messages || messages.length === 0) return null;
|
|
1347
|
+
let lastCorrelated = null;
|
|
1348
|
+
let lastNonErrored = null;
|
|
1349
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1350
|
+
const m = messages[i];
|
|
1351
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1352
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1353
|
+
if (errorOf(m) == null) {
|
|
1354
|
+
lastNonErrored = m;
|
|
1355
|
+
break;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1359
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1360
|
+
if (userIndex === -1) return null;
|
|
1361
|
+
let last = null;
|
|
1362
|
+
let lastOk = null;
|
|
1363
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1364
|
+
const role = roleOf(messages[i]);
|
|
1365
|
+
if (role === "user") break;
|
|
1366
|
+
if (role === "assistant") {
|
|
1367
|
+
last = messages[i];
|
|
1368
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
return lastOk ?? last;
|
|
1372
|
+
}
|
|
1083
1373
|
function messageRunState(messages, userMessageId) {
|
|
1084
1374
|
if (!messages || messages.length === 0) return "unknown";
|
|
1085
1375
|
const hasUser = messages.some((m) => idOf(m) === userMessageId);
|
|
1086
|
-
const reply =
|
|
1376
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1087
1377
|
if (!hasUser) {
|
|
1088
1378
|
if (!reply) return "unknown";
|
|
1089
1379
|
}
|
|
1090
1380
|
if (!reply) return "queued";
|
|
1091
|
-
|
|
1381
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1382
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1383
|
+
}
|
|
1384
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1385
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1386
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1387
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1388
|
+
}
|
|
1389
|
+
function messageError(messages, userMessageId) {
|
|
1390
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1391
|
+
const error2 = errorOf(reply);
|
|
1392
|
+
if (error2 == null) return null;
|
|
1393
|
+
if (typeof error2 === "string") return error2;
|
|
1394
|
+
if (typeof error2 === "object") {
|
|
1395
|
+
const e = error2;
|
|
1396
|
+
const dataMessage = e.data?.message;
|
|
1397
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1398
|
+
if (typeof e.message === "string") return e.message;
|
|
1399
|
+
}
|
|
1400
|
+
return "The agent run failed.";
|
|
1401
|
+
}
|
|
1402
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1403
|
+
if (!messages || messages.length === 0) return false;
|
|
1404
|
+
return messages.some(
|
|
1405
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1410
|
+
var DURATION_UNIT_MS = {
|
|
1411
|
+
s: 1e3,
|
|
1412
|
+
m: 60 * 1e3,
|
|
1413
|
+
h: 60 * 60 * 1e3,
|
|
1414
|
+
d: 24 * 60 * 60 * 1e3
|
|
1415
|
+
};
|
|
1416
|
+
function parseDurationMs(input) {
|
|
1417
|
+
const trimmed = input.trim();
|
|
1418
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1419
|
+
if (!match) {
|
|
1420
|
+
throw new Error(
|
|
1421
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
const value = Number(match[1]);
|
|
1425
|
+
if (value <= 0) {
|
|
1426
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1427
|
+
}
|
|
1428
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1429
|
+
}
|
|
1430
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1431
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1432
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1433
|
+
const ageEligible = (s) => {
|
|
1434
|
+
if (maxAgeMs === void 0) return false;
|
|
1435
|
+
if (s.lastActivityMs === null) return true;
|
|
1436
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1437
|
+
};
|
|
1438
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1439
|
+
if (maxCount !== void 0) {
|
|
1440
|
+
const byActivityDesc = [...sessions].sort(
|
|
1441
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1442
|
+
);
|
|
1443
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1444
|
+
countEligibleIds.add(s.id);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
const toDelete = [];
|
|
1448
|
+
for (const s of sessions) {
|
|
1449
|
+
if (protectedIds.has(s.id)) continue;
|
|
1450
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1451
|
+
toDelete.push(s.id);
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
return toDelete;
|
|
1455
|
+
}
|
|
1456
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1457
|
+
function resolve(flag, envValue, fallback) {
|
|
1458
|
+
return flag ?? envValue ?? fallback;
|
|
1459
|
+
}
|
|
1460
|
+
function parseMaxCount(input) {
|
|
1461
|
+
const trimmed = input.trim();
|
|
1462
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1463
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1464
|
+
}
|
|
1465
|
+
const value = Number(trimmed);
|
|
1466
|
+
if (value <= 0) {
|
|
1467
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1468
|
+
}
|
|
1469
|
+
return value;
|
|
1092
1470
|
}
|
|
1093
|
-
function
|
|
1094
|
-
const
|
|
1095
|
-
|
|
1471
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1472
|
+
const warnings = [];
|
|
1473
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1474
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1475
|
+
const intervalRaw = resolve(
|
|
1476
|
+
flags.interval,
|
|
1477
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1478
|
+
DEFAULT_INTERVAL
|
|
1479
|
+
);
|
|
1480
|
+
let maxAgeMs;
|
|
1481
|
+
if (maxAgeRaw !== void 0) {
|
|
1482
|
+
try {
|
|
1483
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1484
|
+
} catch (err) {
|
|
1485
|
+
warnings.push(
|
|
1486
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1487
|
+
);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
let maxCount;
|
|
1491
|
+
if (maxCountRaw !== void 0) {
|
|
1492
|
+
try {
|
|
1493
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1494
|
+
} catch (err) {
|
|
1495
|
+
warnings.push(
|
|
1496
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1497
|
+
);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
let intervalMs;
|
|
1501
|
+
try {
|
|
1502
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1503
|
+
} catch (err) {
|
|
1504
|
+
warnings.push(
|
|
1505
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1506
|
+
);
|
|
1507
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1508
|
+
}
|
|
1509
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1510
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1096
1511
|
}
|
|
1097
1512
|
|
|
1098
1513
|
// src/lib/tunnel/connection.ts
|
|
@@ -1163,24 +1578,34 @@ var StreamForwarder = class {
|
|
|
1163
1578
|
}
|
|
1164
1579
|
async handleOpen(frame) {
|
|
1165
1580
|
const { sid, method, path, headers, has_body } = frame;
|
|
1581
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1582
|
+
const startedAt = Date.now();
|
|
1166
1583
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1167
1584
|
this.callbacks.onDrainPing?.();
|
|
1168
1585
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1169
1586
|
this.send({ type: "res_end", sid });
|
|
1170
1587
|
return;
|
|
1171
1588
|
}
|
|
1589
|
+
if (process.env.DEBUG) {
|
|
1590
|
+
log("debug", "agent_request", {
|
|
1591
|
+
correlation_id: correlationId,
|
|
1592
|
+
sid,
|
|
1593
|
+
method,
|
|
1594
|
+
path: stripQuery(path)
|
|
1595
|
+
});
|
|
1596
|
+
}
|
|
1172
1597
|
const ac = new AbortController();
|
|
1173
1598
|
let bodyPromise;
|
|
1174
1599
|
let pushBody;
|
|
1175
1600
|
let endBody;
|
|
1176
1601
|
if (has_body) {
|
|
1177
1602
|
const chunks = [];
|
|
1178
|
-
bodyPromise = new Promise((
|
|
1603
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1179
1604
|
pushBody = (buf) => {
|
|
1180
1605
|
chunks.push(buf);
|
|
1181
1606
|
};
|
|
1182
1607
|
endBody = () => {
|
|
1183
|
-
|
|
1608
|
+
resolve2(Buffer.concat(chunks));
|
|
1184
1609
|
};
|
|
1185
1610
|
});
|
|
1186
1611
|
}
|
|
@@ -1215,6 +1640,14 @@ var StreamForwarder = class {
|
|
|
1215
1640
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1216
1641
|
});
|
|
1217
1642
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1643
|
+
if (process.env.DEBUG) {
|
|
1644
|
+
log("debug", "agent_response", {
|
|
1645
|
+
correlation_id: correlationId,
|
|
1646
|
+
sid,
|
|
1647
|
+
status: upstream.status,
|
|
1648
|
+
duration_ms: Date.now() - startedAt
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1218
1651
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1219
1652
|
try {
|
|
1220
1653
|
if (upstream.body) {
|
|
@@ -1290,7 +1723,7 @@ function connectTunnel(options) {
|
|
|
1290
1723
|
} = options;
|
|
1291
1724
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1292
1725
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1293
|
-
return new Promise((
|
|
1726
|
+
return new Promise((resolve2, reject) => {
|
|
1294
1727
|
const ws = new WebSocket2(url, {
|
|
1295
1728
|
headers: {
|
|
1296
1729
|
Authorization: authHeader
|
|
@@ -1355,7 +1788,7 @@ function connectTunnel(options) {
|
|
|
1355
1788
|
clearTimeout(connectionTimeout);
|
|
1356
1789
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1357
1790
|
onConnected?.(connectedAgentId);
|
|
1358
|
-
|
|
1791
|
+
resolve2({
|
|
1359
1792
|
ws,
|
|
1360
1793
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1361
1794
|
});
|
|
@@ -1477,6 +1910,17 @@ function messageIdOf(m) {
|
|
|
1477
1910
|
const infoId = m.info?.id;
|
|
1478
1911
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1479
1912
|
}
|
|
1913
|
+
function cleanImageMime(contentType) {
|
|
1914
|
+
if (!contentType) return null;
|
|
1915
|
+
const media = contentType.split(";")[0].trim().toLowerCase();
|
|
1916
|
+
return /^image\/[a-z0-9.+-]+$/.test(media) ? media : null;
|
|
1917
|
+
}
|
|
1918
|
+
var LOG_LEVELS = {
|
|
1919
|
+
debug: 0,
|
|
1920
|
+
info: 1,
|
|
1921
|
+
warn: 2,
|
|
1922
|
+
error: 3
|
|
1923
|
+
};
|
|
1480
1924
|
var DEFAULT_RETRY_POLICY = {
|
|
1481
1925
|
maxAttempts: 6,
|
|
1482
1926
|
baseDelayMs: 500,
|
|
@@ -1484,7 +1928,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1484
1928
|
};
|
|
1485
1929
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1486
1930
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1487
|
-
var
|
|
1931
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1932
|
+
var HEARTBEAT_MS = 6e4;
|
|
1933
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1934
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1488
1935
|
var ChannelAuthError = class extends Error {
|
|
1489
1936
|
constructor(message) {
|
|
1490
1937
|
super(message);
|
|
@@ -1519,10 +1966,18 @@ var ChannelDriver = class {
|
|
|
1519
1966
|
sleep;
|
|
1520
1967
|
pausedPollIntervalMs;
|
|
1521
1968
|
pausedMaxWaitMs;
|
|
1522
|
-
|
|
1969
|
+
stuckQueuedMs;
|
|
1523
1970
|
now;
|
|
1524
1971
|
/** Cache of conversationId → opencode sessionId. */
|
|
1525
1972
|
sessions = /* @__PURE__ */ new Map();
|
|
1973
|
+
/**
|
|
1974
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1975
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1976
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1977
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1978
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1979
|
+
*/
|
|
1980
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1526
1981
|
/**
|
|
1527
1982
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1528
1983
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1539,6 +1994,72 @@ var ChannelDriver = class {
|
|
|
1539
1994
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1540
1995
|
*/
|
|
1541
1996
|
dispatched = /* @__PURE__ */ new Set();
|
|
1997
|
+
/**
|
|
1998
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1999
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
2000
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
2001
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
2002
|
+
* the processing list.
|
|
2003
|
+
*/
|
|
2004
|
+
readopted = /* @__PURE__ */ new Set();
|
|
2005
|
+
/**
|
|
2006
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
2007
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
2008
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
2009
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
2010
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
2011
|
+
*
|
|
2012
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
2013
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
2014
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
2015
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
2016
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
2017
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
2018
|
+
*/
|
|
2019
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
2020
|
+
/**
|
|
2021
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
2022
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
2023
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
2024
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
2025
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
2026
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
2027
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
2028
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
2029
|
+
*/
|
|
2030
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
2031
|
+
/**
|
|
2032
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
2033
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
2034
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
2035
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
2036
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
2037
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
2038
|
+
*/
|
|
2039
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
2040
|
+
/**
|
|
2041
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
2042
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
2043
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
2044
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
2045
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
2046
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
2047
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
2048
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
2049
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
2050
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
2051
|
+
* so the NEXT tick may retry exactly once more).
|
|
2052
|
+
*/
|
|
2053
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
2054
|
+
/**
|
|
2055
|
+
* "Already signalled `attachments_skipped` for this Evident message id" (#376).
|
|
2056
|
+
* The in-thread skip note is an OUTCOME, so it must fire AT MOST ONCE per message
|
|
2057
|
+
* — never re-post on a re-dispatch of the same row (`forceReadoptRun` or the
|
|
2058
|
+
* next-tick null-id retry both re-run `sendPromptAsync`, which re-fires
|
|
2059
|
+
* `onOutcomes`). Mirrors `readoptPollUnresolvedSignalled`: a local dedup on the
|
|
2060
|
+
* outcome, not the dispatch. Not cleared (a message is signalled once for life).
|
|
2061
|
+
*/
|
|
2062
|
+
attachmentsSkippedSignalled = /* @__PURE__ */ new Set();
|
|
1542
2063
|
/**
|
|
1543
2064
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1544
2065
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1546,8 +2067,43 @@ var ChannelDriver = class {
|
|
|
1546
2067
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1547
2068
|
*/
|
|
1548
2069
|
opencodeDirectory = void 0;
|
|
2070
|
+
/**
|
|
2071
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
2072
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
2073
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
2074
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
2075
|
+
* question/permission can be attributed to the watched session's subtree
|
|
2076
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
2077
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
2078
|
+
*/
|
|
2079
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
2080
|
+
/**
|
|
2081
|
+
* Per-session OpenCode title cache (#310), keyed by sessionId. Only a resolved
|
|
2082
|
+
* NON-EMPTY name is stored (terminal — a real session name won't later un-name),
|
|
2083
|
+
* so we do NOT re-GET `/session/:id` every tick. A missing entry = not yet
|
|
2084
|
+
* resolved OR resolved-but-still-empty → re-fetch on next need, since OpenCode
|
|
2085
|
+
* names sessions asynchronously mid-turn. Driver-level (not per-watcher) so both
|
|
2086
|
+
* the watcher completion path AND the restart-recovery re-adopt path (which has
|
|
2087
|
+
* no watcher) can resolve the title.
|
|
2088
|
+
*/
|
|
2089
|
+
sessionTitles = /* @__PURE__ */ new Map();
|
|
1549
2090
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1550
2091
|
draining = false;
|
|
2092
|
+
/**
|
|
2093
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
2094
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
2095
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
2096
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
2097
|
+
*/
|
|
2098
|
+
activeDrain = null;
|
|
2099
|
+
/**
|
|
2100
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
2101
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
2102
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
2103
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
2104
|
+
* and stops opencode.
|
|
2105
|
+
*/
|
|
2106
|
+
stopped = false;
|
|
1551
2107
|
constructor(config2) {
|
|
1552
2108
|
this.agentId = config2.agentId;
|
|
1553
2109
|
this.port = config2.port;
|
|
@@ -1561,16 +2117,13 @@ var ChannelDriver = class {
|
|
|
1561
2117
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1562
2118
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1563
2119
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1564
|
-
this.
|
|
2120
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1565
2121
|
this.now = config2.now ?? (() => Date.now());
|
|
1566
2122
|
}
|
|
1567
2123
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
1568
2124
|
get opencodeBase() {
|
|
1569
2125
|
return `http://127.0.0.1:${this.port}`;
|
|
1570
2126
|
}
|
|
1571
|
-
// -------------------------------------------------------------------------
|
|
1572
|
-
// Public API
|
|
1573
|
-
// -------------------------------------------------------------------------
|
|
1574
2127
|
/**
|
|
1575
2128
|
* Drain all pending channel conversations once: poll → dispatch → register.
|
|
1576
2129
|
* Called on tunnel `connected` (WI-CHAN-4) and on each poll tick by `run.ts`.
|
|
@@ -1579,8 +2132,21 @@ var ChannelDriver = class {
|
|
|
1579
2132
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1580
2133
|
*/
|
|
1581
2134
|
async drainPending() {
|
|
2135
|
+
if (this.stopped) return 0;
|
|
1582
2136
|
if (this.draining) return 0;
|
|
1583
2137
|
this.draining = true;
|
|
2138
|
+
const run2 = this.runDrain();
|
|
2139
|
+
this.activeDrain = run2.then(
|
|
2140
|
+
() => {
|
|
2141
|
+
this.activeDrain = null;
|
|
2142
|
+
},
|
|
2143
|
+
() => {
|
|
2144
|
+
this.activeDrain = null;
|
|
2145
|
+
}
|
|
2146
|
+
);
|
|
2147
|
+
return run2;
|
|
2148
|
+
}
|
|
2149
|
+
async runDrain() {
|
|
1584
2150
|
let dispatched = 0;
|
|
1585
2151
|
try {
|
|
1586
2152
|
const conversations = await this.getPendingConversations();
|
|
@@ -1592,8 +2158,10 @@ var ChannelDriver = class {
|
|
|
1592
2158
|
});
|
|
1593
2159
|
}
|
|
1594
2160
|
for (const conv of conversations) {
|
|
2161
|
+
if (this.stopped) break;
|
|
1595
2162
|
dispatched += await this.processConversation(conv);
|
|
1596
2163
|
}
|
|
2164
|
+
await this.readoptProcessing();
|
|
1597
2165
|
} finally {
|
|
1598
2166
|
this.draining = false;
|
|
1599
2167
|
}
|
|
@@ -1611,6 +2179,73 @@ var ChannelDriver = class {
|
|
|
1611
2179
|
}
|
|
1612
2180
|
return false;
|
|
1613
2181
|
}
|
|
2182
|
+
/**
|
|
2183
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2184
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2185
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2186
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2187
|
+
*
|
|
2188
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2189
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2190
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2191
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2192
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2193
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2194
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2195
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2196
|
+
*/
|
|
2197
|
+
protectedSessionIds() {
|
|
2198
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2199
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2200
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2201
|
+
}
|
|
2202
|
+
return ids;
|
|
2203
|
+
}
|
|
2204
|
+
/**
|
|
2205
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2206
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2207
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2208
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2209
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2210
|
+
*/
|
|
2211
|
+
stop() {
|
|
2212
|
+
this.stopped = true;
|
|
2213
|
+
}
|
|
2214
|
+
/**
|
|
2215
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2216
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2217
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2218
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2219
|
+
*
|
|
2220
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2221
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2222
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2223
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2224
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2225
|
+
* next runner start (ADR-0046).
|
|
2226
|
+
*
|
|
2227
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2228
|
+
* timeout elapsed with work still in flight.
|
|
2229
|
+
*/
|
|
2230
|
+
async waitForInFlight(timeoutMs) {
|
|
2231
|
+
const deadline = this.now() + timeoutMs;
|
|
2232
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2233
|
+
if (this.activeDrain) {
|
|
2234
|
+
let drainSettled = false;
|
|
2235
|
+
void this.activeDrain.then(() => {
|
|
2236
|
+
drainSettled = true;
|
|
2237
|
+
});
|
|
2238
|
+
while (!drainSettled) {
|
|
2239
|
+
if (this.now() >= deadline) return false;
|
|
2240
|
+
await this.sleep(step);
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
while (this.hasInFlightWatchers()) {
|
|
2244
|
+
if (this.now() >= deadline) return false;
|
|
2245
|
+
await this.sleep(step);
|
|
2246
|
+
}
|
|
2247
|
+
return true;
|
|
2248
|
+
}
|
|
1614
2249
|
/**
|
|
1615
2250
|
* Await all outstanding per-session watchers (WI-3).
|
|
1616
2251
|
*
|
|
@@ -1630,9 +2265,7 @@ var ChannelDriver = class {
|
|
|
1630
2265
|
if (!stillLive) return;
|
|
1631
2266
|
}
|
|
1632
2267
|
}
|
|
1633
|
-
// -------------------------------------------------------------------------
|
|
1634
2268
|
// Conversation processing (WI-3 — async dispatch)
|
|
1635
|
-
// -------------------------------------------------------------------------
|
|
1636
2269
|
/**
|
|
1637
2270
|
* Dispatch each pending message for a conversation to opencode's native queue
|
|
1638
2271
|
* via `prompt_async` (Task 3.2) and register it with the conversation's
|
|
@@ -1645,15 +2278,18 @@ var ChannelDriver = class {
|
|
|
1645
2278
|
const sessionId = await this.ensureSession(conv);
|
|
1646
2279
|
const messages = await this.getPendingMessages(conv.id);
|
|
1647
2280
|
let dispatched = 0;
|
|
2281
|
+
let skippedAlreadyDispatched = 0;
|
|
1648
2282
|
for (const message of messages) {
|
|
2283
|
+
if (this.stopped) break;
|
|
1649
2284
|
if (this.dispatched.has(message.id)) {
|
|
2285
|
+
skippedAlreadyDispatched += 1;
|
|
1650
2286
|
continue;
|
|
1651
2287
|
}
|
|
1652
|
-
const opencodeMessageId = opencodeMessageIdFor(message.id);
|
|
1653
2288
|
const options = {
|
|
1654
2289
|
agent: message.opencode_agent ?? void 0,
|
|
1655
2290
|
model: message.opencode_model ?? void 0
|
|
1656
2291
|
};
|
|
2292
|
+
let opencodeMessageId;
|
|
1657
2293
|
try {
|
|
1658
2294
|
this.log({
|
|
1659
2295
|
level: "info",
|
|
@@ -1661,10 +2297,24 @@ var ChannelDriver = class {
|
|
|
1661
2297
|
conversation_id: conv.id,
|
|
1662
2298
|
message_id: message.id
|
|
1663
2299
|
});
|
|
1664
|
-
|
|
2300
|
+
const sendAttachments = this.buildSendAttachments(conv, message);
|
|
2301
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2302
|
+
sessionId,
|
|
2303
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options, sendAttachments)
|
|
2304
|
+
);
|
|
1665
2305
|
} catch (err) {
|
|
1666
2306
|
if (err instanceof ChannelAuthError) throw err;
|
|
1667
2307
|
this.dispatched.delete(message.id);
|
|
2308
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2309
|
+
this.sessions.delete(conv.id);
|
|
2310
|
+
this.log({
|
|
2311
|
+
level: "warn",
|
|
2312
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2313
|
+
conversation_id: conv.id,
|
|
2314
|
+
message_id: message.id
|
|
2315
|
+
});
|
|
2316
|
+
break;
|
|
2317
|
+
}
|
|
1668
2318
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1669
2319
|
});
|
|
1670
2320
|
this.log({
|
|
@@ -1675,24 +2325,58 @@ var ChannelDriver = class {
|
|
|
1675
2325
|
});
|
|
1676
2326
|
continue;
|
|
1677
2327
|
}
|
|
2328
|
+
if (opencodeMessageId === null) {
|
|
2329
|
+
this.log({
|
|
2330
|
+
level: "warn",
|
|
2331
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2332
|
+
conversation_id: conv.id,
|
|
2333
|
+
message_id: message.id
|
|
2334
|
+
});
|
|
2335
|
+
continue;
|
|
2336
|
+
}
|
|
1678
2337
|
this.dispatched.add(message.id);
|
|
1679
2338
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1680
2339
|
dispatched += 1;
|
|
2340
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
2341
|
+
}
|
|
2342
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
2343
|
+
this.log({
|
|
2344
|
+
level: "warn",
|
|
2345
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
2346
|
+
conversation_id: conv.id
|
|
2347
|
+
});
|
|
1681
2348
|
}
|
|
1682
2349
|
this.ensureWatcherRunning(sessionId);
|
|
1683
2350
|
return dispatched;
|
|
1684
2351
|
}
|
|
1685
2352
|
async ensureSession(conv) {
|
|
1686
|
-
const
|
|
1687
|
-
if (
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
2353
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2354
|
+
if (bound) {
|
|
2355
|
+
const exists = await sessionExists(this.port, bound);
|
|
2356
|
+
if (exists === false) {
|
|
2357
|
+
this.log({
|
|
2358
|
+
level: "debug",
|
|
2359
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2360
|
+
conversation_id: conv.id
|
|
2361
|
+
});
|
|
2362
|
+
this.sessions.delete(conv.id);
|
|
2363
|
+
return this.createAndBindSession(conv.id);
|
|
2364
|
+
}
|
|
2365
|
+
this.sessions.set(conv.id, bound);
|
|
2366
|
+
return bound;
|
|
1691
2367
|
}
|
|
2368
|
+
return this.createAndBindSession(conv.id);
|
|
2369
|
+
}
|
|
2370
|
+
/**
|
|
2371
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2372
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2373
|
+
* self-heal recreate path in `ensureSession`.
|
|
2374
|
+
*/
|
|
2375
|
+
async createAndBindSession(conversationId) {
|
|
1692
2376
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1693
2377
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1694
|
-
this.sessions.set(
|
|
1695
|
-
await this.persistSession(
|
|
2378
|
+
this.sessions.set(conversationId, sessionId);
|
|
2379
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1696
2380
|
});
|
|
1697
2381
|
return sessionId;
|
|
1698
2382
|
}
|
|
@@ -1706,51 +2390,250 @@ var ChannelDriver = class {
|
|
|
1706
2390
|
this.opencodeDirectory = await getOpenCodeDirectory(this.port);
|
|
1707
2391
|
if (!this.opencodeDirectory) {
|
|
1708
2392
|
this.log({
|
|
1709
|
-
level: "
|
|
2393
|
+
level: "warn",
|
|
1710
2394
|
message: "Could not determine opencode directory (GET /path) \u2014 new sessions may not appear in opencode web"
|
|
1711
2395
|
});
|
|
1712
2396
|
}
|
|
1713
2397
|
return this.opencodeDirectory;
|
|
1714
2398
|
}
|
|
1715
|
-
// -------------------------------------------------------------------------
|
|
1716
2399
|
// Per-session watcher (WI-3)
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
message,
|
|
1736
|
-
dispatchedAt: now,
|
|
1737
|
-
deadline: now + this.pausedMaxWaitMs,
|
|
1738
|
-
started: false,
|
|
1739
|
-
done: false
|
|
1740
|
-
});
|
|
2400
|
+
/**
|
|
2401
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2402
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2403
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2404
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2405
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2406
|
+
*/
|
|
2407
|
+
dispatchLocked(sessionId, fn) {
|
|
2408
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2409
|
+
const run2 = prior.then(fn, fn);
|
|
2410
|
+
this.sessionDispatchLocks.set(
|
|
2411
|
+
sessionId,
|
|
2412
|
+
run2.then(
|
|
2413
|
+
() => void 0,
|
|
2414
|
+
() => void 0
|
|
2415
|
+
)
|
|
2416
|
+
);
|
|
2417
|
+
return run2;
|
|
1741
2418
|
}
|
|
2419
|
+
// Inbound image attachments (#255, WI-8)
|
|
1742
2420
|
/**
|
|
1743
|
-
*
|
|
1744
|
-
*
|
|
1745
|
-
*
|
|
1746
|
-
*
|
|
1747
|
-
*
|
|
2421
|
+
* Build the `SendAttachmentsInput` for a message's inbound images, or
|
|
2422
|
+
* `undefined` when the message has none (so a text-only turn is unchanged).
|
|
2423
|
+
*
|
|
2424
|
+
* The driver OWNS the two channel-facing concerns the session module cannot:
|
|
2425
|
+
* - the AUTHENTICATED byte fetch through Evident's WI-6 endpoint
|
|
2426
|
+
* (`fetchAttachmentDataUrl`), using the SAME `getAuthHeader()` as every
|
|
2427
|
+
* other combinedAuth callback — the CLI NEVER talks to Slack directly;
|
|
2428
|
+
* - the in-thread SKIP NOTE (`signalAttachmentsSkipped`) posted over the
|
|
2429
|
+
* existing callback surface when any image was skipped/failed.
|
|
2430
|
+
* `sendPromptAsync` applies the capability gate + appends the `file` parts and
|
|
2431
|
+
* reports outcomes back via `onOutcomes`.
|
|
1748
2432
|
*/
|
|
1749
|
-
|
|
1750
|
-
const
|
|
1751
|
-
if (!
|
|
1752
|
-
|
|
1753
|
-
|
|
2433
|
+
buildSendAttachments(conv, message) {
|
|
2434
|
+
const refs = message.attachments;
|
|
2435
|
+
if (!refs || refs.length === 0) return void 0;
|
|
2436
|
+
return {
|
|
2437
|
+
inputs: refs.map((a, index) => ({
|
|
2438
|
+
index,
|
|
2439
|
+
mime: a.mime,
|
|
2440
|
+
...a.filename ? { filename: a.filename } : {}
|
|
2441
|
+
})),
|
|
2442
|
+
fetchDataUrl: (index) => this.fetchAttachmentDataUrl(message.id, index, refs[index].mime),
|
|
2443
|
+
onOutcomes: ({ outcomes, capabilityUnknown }) => this.signalAttachmentsSkipped(conv.id, message.id, outcomes, capabilityUnknown)
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2446
|
+
/**
|
|
2447
|
+
* Fetch ONE inbound image's bytes through Evident's WI-6 endpoint
|
|
2448
|
+
* (`GET {apiUrl}/agents/{agentId}/attachments/{messageId}/{index}`) using the
|
|
2449
|
+
* existing authenticated fetch, and base64-encode into a
|
|
2450
|
+
* `data:<mime>;base64,<…>` URL for the opencode `file` part's `url`.
|
|
2451
|
+
*
|
|
2452
|
+
* The endpoint streams the source bytes verbatim (200), or returns 404
|
|
2453
|
+
* (not-owned / out-of-range / deleted-at-source / workspace gone) / 413
|
|
2454
|
+
* (over-cap). On ANY non-2xx or thrown failure we return `null` so the caller
|
|
2455
|
+
* OMITS that one image and the text turn still sends — NEVER throws the turn.
|
|
2456
|
+
* Failures are logged with context (no silent swallow).
|
|
2457
|
+
*/
|
|
2458
|
+
async fetchAttachmentDataUrl(messageId, index, mime) {
|
|
2459
|
+
try {
|
|
2460
|
+
const res = await this.fetchImpl(
|
|
2461
|
+
`${this.apiUrl}/agents/${this.agentId}/attachments/${messageId}/${index}`,
|
|
2462
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2463
|
+
);
|
|
2464
|
+
if (!res.ok) {
|
|
2465
|
+
this.log({
|
|
2466
|
+
level: "error",
|
|
2467
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index} returned HTTP ${res.status} \u2014 omitting this image (text turn proceeds)`,
|
|
2468
|
+
message_id: messageId
|
|
2469
|
+
});
|
|
2470
|
+
return null;
|
|
2471
|
+
}
|
|
2472
|
+
const buf = await res.arrayBuffer();
|
|
2473
|
+
const base64 = Buffer.from(buf).toString("base64");
|
|
2474
|
+
const dataMime = cleanImageMime(res.headers.get("content-type")) || mime;
|
|
2475
|
+
return `data:${dataMime};base64,${base64}`;
|
|
2476
|
+
} catch (err) {
|
|
2477
|
+
this.log({
|
|
2478
|
+
level: "error",
|
|
2479
|
+
message: `Attachment fetch for message ${messageId.slice(0, 8)} index ${index} failed \u2014 omitting this image (text turn proceeds): ${err instanceof Error ? err.message : String(err)}`,
|
|
2480
|
+
message_id: messageId
|
|
2481
|
+
});
|
|
2482
|
+
return null;
|
|
2483
|
+
}
|
|
2484
|
+
}
|
|
2485
|
+
/**
|
|
2486
|
+
* On any skipped/failed image, post an in-thread note to Evident over the
|
|
2487
|
+
* EXISTING combinedAuth callback surface — the CLI NEVER posts to Slack directly.
|
|
2488
|
+
* Evident routes the note to source via `conversation.deliver`.
|
|
2489
|
+
*
|
|
2490
|
+
* The `POST .../messages/:id/signal` route accepts `attachments_skipped` (in
|
|
2491
|
+
* `messageSignalSchema`) and turns it into an in-thread note delivered through
|
|
2492
|
+
* `conversation.deliver` (e.g. "N image(s) couldn't be forwarded"), so the note
|
|
2493
|
+
* reaches the channel.
|
|
2494
|
+
*
|
|
2495
|
+
* Fire-and-forget: never throws into the send/tick (logs its own failure).
|
|
2496
|
+
*/
|
|
2497
|
+
signalAttachmentsSkipped(conversationId, messageId, outcomes, capabilityUnknown) {
|
|
2498
|
+
const skipped = outcomes.filter((o) => o.status === "skipped").length;
|
|
2499
|
+
const failed = outcomes.filter((o) => o.status === "failed").length;
|
|
2500
|
+
if (skipped === 0 && failed === 0) return;
|
|
2501
|
+
if (this.attachmentsSkippedSignalled.has(messageId)) return;
|
|
2502
|
+
this.attachmentsSkippedSignalled.add(messageId);
|
|
2503
|
+
const skippedReason = capabilityUnknown ? "unknown" : "unsupported";
|
|
2504
|
+
this.log({
|
|
2505
|
+
level: "info",
|
|
2506
|
+
message: `Message ${messageId.slice(0, 8)}: ${skipped} image(s) skipped (${capabilityUnknown ? "capability was unreadable \u2014 failed open to text-only" : "model not attachment-capable"}), ${failed} image(s) unavailable (deleted-at-source or fetch failure) \u2014 noting to Evident`,
|
|
2507
|
+
conversation_id: conversationId,
|
|
2508
|
+
message_id: messageId
|
|
2509
|
+
});
|
|
2510
|
+
void this.postSignal(conversationId, messageId, "attachments_skipped", {
|
|
2511
|
+
skipped,
|
|
2512
|
+
failed,
|
|
2513
|
+
...skipped > 0 ? { skipped_reason: skippedReason } : {}
|
|
2514
|
+
});
|
|
2515
|
+
}
|
|
2516
|
+
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
2517
|
+
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
2518
|
+
let watcher = this.watchers.get(sessionId);
|
|
2519
|
+
if (!watcher) {
|
|
2520
|
+
watcher = {
|
|
2521
|
+
conv,
|
|
2522
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2523
|
+
loop: null,
|
|
2524
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2525
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2526
|
+
lastGoodPollAt: this.now(),
|
|
2527
|
+
hadUsablePoll: false
|
|
2528
|
+
};
|
|
2529
|
+
this.watchers.set(sessionId, watcher);
|
|
2530
|
+
}
|
|
2531
|
+
const now = this.now();
|
|
2532
|
+
watcher.inFlight.set(message.id, {
|
|
2533
|
+
evidentMessageId: message.id,
|
|
2534
|
+
opencodeMessageId,
|
|
2535
|
+
message,
|
|
2536
|
+
dispatchedAt: now,
|
|
2537
|
+
processingAnchorMs: now,
|
|
2538
|
+
deadline: now + this.pausedMaxWaitMs,
|
|
2539
|
+
started: false,
|
|
2540
|
+
done: false,
|
|
2541
|
+
stuckReported: false,
|
|
2542
|
+
lastAliveAt: 0,
|
|
2543
|
+
aliveInFlight: false,
|
|
2544
|
+
awaitingHumanLatched: false,
|
|
2545
|
+
pausedOnQuestion: false,
|
|
2546
|
+
pausedOnPermission: false,
|
|
2547
|
+
pausedClearConfirmed: false,
|
|
2548
|
+
pausedInFlight: false,
|
|
2549
|
+
deliveryDeadlineAnchored: false
|
|
2550
|
+
});
|
|
2551
|
+
}
|
|
2552
|
+
/**
|
|
2553
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2554
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2555
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2556
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2557
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2558
|
+
*
|
|
2559
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2560
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2561
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2562
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2563
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2564
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2565
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2566
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2567
|
+
* (only the appear-guard uses it).
|
|
2568
|
+
*
|
|
2569
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2570
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
2571
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
2572
|
+
*
|
|
2573
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
2574
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
2575
|
+
* fire from the watcher's normal branches.
|
|
2576
|
+
*/
|
|
2577
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2578
|
+
let watcher = this.watchers.get(sessionId);
|
|
2579
|
+
if (!watcher) {
|
|
2580
|
+
watcher = {
|
|
2581
|
+
conv,
|
|
2582
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2583
|
+
loop: null,
|
|
2584
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2585
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2586
|
+
lastGoodPollAt: this.now(),
|
|
2587
|
+
hadUsablePoll: false
|
|
2588
|
+
};
|
|
2589
|
+
this.watchers.set(sessionId, watcher);
|
|
2590
|
+
}
|
|
2591
|
+
watcher.inFlight.set(message.id, {
|
|
2592
|
+
evidentMessageId: message.id,
|
|
2593
|
+
opencodeMessageId,
|
|
2594
|
+
message,
|
|
2595
|
+
dispatchedAt: this.now(),
|
|
2596
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2597
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2598
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2599
|
+
processingAnchorMs: processedAtMs,
|
|
2600
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2601
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2602
|
+
started: true,
|
|
2603
|
+
done: false,
|
|
2604
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2605
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2606
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2607
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2608
|
+
// (#210/#220 observability).
|
|
2609
|
+
stuckReported: false,
|
|
2610
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2611
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2612
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2613
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2614
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2615
|
+
lastAliveAt: 0,
|
|
2616
|
+
aliveInFlight: false,
|
|
2617
|
+
awaitingHumanLatched: false,
|
|
2618
|
+
pausedOnQuestion: false,
|
|
2619
|
+
pausedOnPermission: false,
|
|
2620
|
+
pausedClearConfirmed: false,
|
|
2621
|
+
pausedInFlight: false,
|
|
2622
|
+
deliveryDeadlineAnchored: false
|
|
2623
|
+
});
|
|
2624
|
+
}
|
|
2625
|
+
/**
|
|
2626
|
+
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
2627
|
+
* work and is not already running. Single-flight per session. The loop is
|
|
2628
|
+
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
2629
|
+
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
2630
|
+
* stays as the safety net.
|
|
2631
|
+
*/
|
|
2632
|
+
ensureWatcherRunning(sessionId) {
|
|
2633
|
+
const watcher = this.watchers.get(sessionId);
|
|
2634
|
+
if (!watcher) return;
|
|
2635
|
+
if (watcher.loop) return;
|
|
2636
|
+
if (watcher.inFlight.size === 0) {
|
|
1754
2637
|
this.watchers.delete(sessionId);
|
|
1755
2638
|
return;
|
|
1756
2639
|
}
|
|
@@ -1787,12 +2670,30 @@ var ChannelDriver = class {
|
|
|
1787
2670
|
messages = Array.isArray(body) ? body : null;
|
|
1788
2671
|
}
|
|
1789
2672
|
} catch {
|
|
1790
|
-
continue;
|
|
1791
2673
|
}
|
|
2674
|
+
if (messages != null && messages.length > 0) {
|
|
2675
|
+
watcher.lastGoodPollAt = this.now();
|
|
2676
|
+
watcher.hadUsablePoll = true;
|
|
2677
|
+
} else {
|
|
2678
|
+
const emptyButReachable = messages != null;
|
|
2679
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2680
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2681
|
+
continue;
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
1792
2685
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
1793
|
-
await this.serviceInFlightMessage(
|
|
2686
|
+
await this.serviceInFlightMessage(
|
|
2687
|
+
sessionId,
|
|
2688
|
+
watcher,
|
|
2689
|
+
inFlight,
|
|
2690
|
+
messages,
|
|
2691
|
+
openQuestions,
|
|
2692
|
+
openPermissions,
|
|
2693
|
+
questionsPolledOk,
|
|
2694
|
+
permissionsPolledOk
|
|
2695
|
+
);
|
|
1794
2696
|
}
|
|
1795
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
1796
2697
|
}
|
|
1797
2698
|
} catch (err) {
|
|
1798
2699
|
if (err instanceof ChannelAuthError) {
|
|
@@ -1802,6 +2703,7 @@ var ChannelDriver = class {
|
|
|
1802
2703
|
conversation_id: watcher.conv.id
|
|
1803
2704
|
});
|
|
1804
2705
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2706
|
+
this.readopted.delete(evidentMessageId);
|
|
1805
2707
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1806
2708
|
}
|
|
1807
2709
|
return;
|
|
@@ -1813,23 +2715,55 @@ var ChannelDriver = class {
|
|
|
1813
2715
|
});
|
|
1814
2716
|
}
|
|
1815
2717
|
}
|
|
2718
|
+
/**
|
|
2719
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2720
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2721
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2722
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2723
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2724
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2725
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2726
|
+
* or past now, so a still-ample window is left untouched.
|
|
2727
|
+
*/
|
|
2728
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2729
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2730
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2731
|
+
if (this.now() >= inFlight.deadline) {
|
|
2732
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2733
|
+
}
|
|
2734
|
+
}
|
|
1816
2735
|
/**
|
|
1817
2736
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
1818
2737
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
1819
2738
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
1820
2739
|
* in-flight set on completion or timeout.
|
|
1821
2740
|
*/
|
|
1822
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2741
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
1823
2742
|
const conv = watcher.conv;
|
|
1824
2743
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1825
|
-
|
|
2744
|
+
const id = inFlight.evidentMessageId;
|
|
2745
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2746
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2747
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2748
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2749
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2750
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2751
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2752
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2753
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
1826
2754
|
let claimed;
|
|
1827
2755
|
try {
|
|
1828
|
-
claimed = await this.markProcessing(
|
|
2756
|
+
claimed = await this.markProcessing(
|
|
2757
|
+
conv.id,
|
|
2758
|
+
inFlight.evidentMessageId,
|
|
2759
|
+
sessionId,
|
|
2760
|
+
inFlight.opencodeMessageId,
|
|
2761
|
+
title
|
|
2762
|
+
);
|
|
1829
2763
|
} catch (err) {
|
|
1830
2764
|
if (err instanceof ChannelAuthError) throw err;
|
|
1831
2765
|
this.log({
|
|
1832
|
-
level: "
|
|
2766
|
+
level: "warn",
|
|
1833
2767
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
1834
2768
|
conversation_id: conv.id,
|
|
1835
2769
|
message_id: inFlight.evidentMessageId
|
|
@@ -1839,7 +2773,7 @@ var ChannelDriver = class {
|
|
|
1839
2773
|
inFlight.started = true;
|
|
1840
2774
|
if (!claimed) {
|
|
1841
2775
|
this.log({
|
|
1842
|
-
level: "
|
|
2776
|
+
level: "debug",
|
|
1843
2777
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
1844
2778
|
conversation_id: conv.id,
|
|
1845
2779
|
message_id: inFlight.evidentMessageId
|
|
@@ -1847,6 +2781,7 @@ var ChannelDriver = class {
|
|
|
1847
2781
|
}
|
|
1848
2782
|
}
|
|
1849
2783
|
if (state === "done") {
|
|
2784
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
1850
2785
|
if (!inFlight.done) {
|
|
1851
2786
|
this.log({
|
|
1852
2787
|
level: "info",
|
|
@@ -1854,13 +2789,20 @@ var ChannelDriver = class {
|
|
|
1854
2789
|
conversation_id: conv.id,
|
|
1855
2790
|
message_id: inFlight.evidentMessageId
|
|
1856
2791
|
});
|
|
2792
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
1857
2793
|
try {
|
|
1858
|
-
await this.markDone(
|
|
2794
|
+
await this.markDone(
|
|
2795
|
+
conv.id,
|
|
2796
|
+
inFlight.evidentMessageId,
|
|
2797
|
+
sessionId,
|
|
2798
|
+
inFlight.opencodeMessageId,
|
|
2799
|
+
title
|
|
2800
|
+
);
|
|
1859
2801
|
} catch (err) {
|
|
1860
2802
|
if (err instanceof ChannelAuthError) throw err;
|
|
1861
2803
|
if (err instanceof ChannelTerminalError) {
|
|
1862
2804
|
this.log({
|
|
1863
|
-
level: "
|
|
2805
|
+
level: "warn",
|
|
1864
2806
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
1865
2807
|
conversation_id: conv.id,
|
|
1866
2808
|
message_id: inFlight.evidentMessageId
|
|
@@ -1870,7 +2812,7 @@ var ChannelDriver = class {
|
|
|
1870
2812
|
}
|
|
1871
2813
|
if (this.now() >= inFlight.deadline) {
|
|
1872
2814
|
this.log({
|
|
1873
|
-
level: "
|
|
2815
|
+
level: "warn",
|
|
1874
2816
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
1875
2817
|
conversation_id: conv.id,
|
|
1876
2818
|
message_id: inFlight.evidentMessageId
|
|
@@ -1879,7 +2821,7 @@ var ChannelDriver = class {
|
|
|
1879
2821
|
return;
|
|
1880
2822
|
}
|
|
1881
2823
|
this.log({
|
|
1882
|
-
level: "
|
|
2824
|
+
level: "warn",
|
|
1883
2825
|
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
1884
2826
|
conversation_id: conv.id,
|
|
1885
2827
|
message_id: inFlight.evidentMessageId
|
|
@@ -1891,60 +2833,583 @@ var ChannelDriver = class {
|
|
|
1891
2833
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1892
2834
|
return;
|
|
1893
2835
|
}
|
|
1894
|
-
if (state === "
|
|
1895
|
-
|
|
1896
|
-
|
|
2836
|
+
if (state === "failed") {
|
|
2837
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2838
|
+
if (!inFlight.done) {
|
|
2839
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2840
|
+
this.log({
|
|
2841
|
+
level: "error",
|
|
2842
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2843
|
+
conversation_id: conv.id,
|
|
2844
|
+
message_id: inFlight.evidentMessageId
|
|
2845
|
+
});
|
|
2846
|
+
try {
|
|
2847
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2848
|
+
} catch (err) {
|
|
2849
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2850
|
+
if (err instanceof ChannelTerminalError) {
|
|
2851
|
+
this.log({
|
|
2852
|
+
level: "warn",
|
|
2853
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2854
|
+
conversation_id: conv.id,
|
|
2855
|
+
message_id: inFlight.evidentMessageId
|
|
2856
|
+
});
|
|
2857
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2858
|
+
return;
|
|
2859
|
+
}
|
|
2860
|
+
if (this.now() >= inFlight.deadline) {
|
|
2861
|
+
this.log({
|
|
2862
|
+
level: "warn",
|
|
2863
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2864
|
+
conversation_id: conv.id,
|
|
2865
|
+
message_id: inFlight.evidentMessageId
|
|
2866
|
+
});
|
|
2867
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2868
|
+
return;
|
|
2869
|
+
}
|
|
2870
|
+
this.log({
|
|
2871
|
+
level: "warn",
|
|
2872
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2873
|
+
conversation_id: conv.id,
|
|
2874
|
+
message_id: inFlight.evidentMessageId
|
|
2875
|
+
});
|
|
2876
|
+
return;
|
|
2877
|
+
}
|
|
2878
|
+
inFlight.done = true;
|
|
2879
|
+
}
|
|
2880
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2881
|
+
return;
|
|
2882
|
+
}
|
|
2883
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2884
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2885
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2886
|
+
inFlight.stuckReported = true;
|
|
2887
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2888
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2889
|
+
});
|
|
2890
|
+
}
|
|
2891
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2892
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2893
|
+
this.log({
|
|
2894
|
+
level: "warn",
|
|
2895
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2896
|
+
conversation_id: conv.id,
|
|
2897
|
+
message_id: inFlight.evidentMessageId
|
|
2898
|
+
});
|
|
2899
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2900
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2901
|
+
});
|
|
2902
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2903
|
+
return;
|
|
2904
|
+
}
|
|
2905
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2906
|
+
inFlight.aliveInFlight = true;
|
|
2907
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2908
|
+
inFlight.aliveInFlight = false;
|
|
2909
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2910
|
+
});
|
|
2911
|
+
}
|
|
2912
|
+
if (awaitingHuman) {
|
|
2913
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2914
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2915
|
+
inFlight.awaitingHumanLatched = true;
|
|
2916
|
+
}
|
|
2917
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2918
|
+
inFlight.pausedInFlight = true;
|
|
2919
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2920
|
+
inFlight.pausedInFlight = false;
|
|
2921
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2922
|
+
});
|
|
1897
2923
|
}
|
|
2924
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2925
|
+
inFlight.awaitingHumanLatched = false;
|
|
2926
|
+
inFlight.pausedOnQuestion = false;
|
|
2927
|
+
inFlight.pausedOnPermission = false;
|
|
2928
|
+
inFlight.pausedClearConfirmed = false;
|
|
1898
2929
|
}
|
|
1899
|
-
|
|
2930
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2931
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2932
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2933
|
+
);
|
|
2934
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2935
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
1900
2936
|
this.log({
|
|
1901
|
-
level: "
|
|
2937
|
+
level: "debug",
|
|
1902
2938
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
1903
2939
|
conversation_id: conv.id,
|
|
1904
2940
|
message_id: inFlight.evidentMessageId
|
|
1905
2941
|
});
|
|
2942
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2943
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2944
|
+
});
|
|
1906
2945
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1907
2946
|
}
|
|
1908
2947
|
}
|
|
2948
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2949
|
+
/**
|
|
2950
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2951
|
+
*
|
|
2952
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2953
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2954
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2955
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2956
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2957
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2958
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2959
|
+
*
|
|
2960
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2961
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2962
|
+
*/
|
|
2963
|
+
async readoptProcessing() {
|
|
2964
|
+
const rows = await this.getProcessingMessages();
|
|
2965
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2966
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2967
|
+
for (const id of [
|
|
2968
|
+
...this.dontRedispatch,
|
|
2969
|
+
...this.doneUndeliverable,
|
|
2970
|
+
...this.readoptPollUnresolvedSignalled
|
|
2971
|
+
]) {
|
|
2972
|
+
if (!stillProcessing.has(id)) {
|
|
2973
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2974
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2975
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2976
|
+
if (cleared || clearedUndeliverable) {
|
|
2977
|
+
this.log({
|
|
2978
|
+
level: "debug",
|
|
2979
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2980
|
+
message_id: id
|
|
2981
|
+
});
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
}
|
|
2986
|
+
if (rows.length === 0) return;
|
|
2987
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2988
|
+
for (const row of rows) {
|
|
2989
|
+
if (!row.opencode_session_id) {
|
|
2990
|
+
this.log({
|
|
2991
|
+
level: "warn",
|
|
2992
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2993
|
+
conversation_id: row.conversation_id,
|
|
2994
|
+
message_id: row.id
|
|
2995
|
+
});
|
|
2996
|
+
continue;
|
|
2997
|
+
}
|
|
2998
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2999
|
+
list.push(row);
|
|
3000
|
+
bySession.set(row.opencode_session_id, list);
|
|
3001
|
+
}
|
|
3002
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
3003
|
+
let messages;
|
|
3004
|
+
try {
|
|
3005
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
3006
|
+
if (!res.ok) {
|
|
3007
|
+
this.log({
|
|
3008
|
+
level: "warn",
|
|
3009
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
3010
|
+
});
|
|
3011
|
+
continue;
|
|
3012
|
+
}
|
|
3013
|
+
const body = await res.json();
|
|
3014
|
+
if (!Array.isArray(body)) {
|
|
3015
|
+
this.log({
|
|
3016
|
+
level: "warn",
|
|
3017
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
3018
|
+
});
|
|
3019
|
+
continue;
|
|
3020
|
+
}
|
|
3021
|
+
messages = body;
|
|
3022
|
+
} catch (err) {
|
|
3023
|
+
this.log({
|
|
3024
|
+
level: "warn",
|
|
3025
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
3026
|
+
});
|
|
3027
|
+
continue;
|
|
3028
|
+
}
|
|
3029
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
3030
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
3031
|
+
for (const row of sessionRows) {
|
|
3032
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
3033
|
+
}
|
|
3034
|
+
}
|
|
3035
|
+
}
|
|
3036
|
+
/**
|
|
3037
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
3038
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
3039
|
+
*
|
|
3040
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
3041
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
3042
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
3043
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
3044
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
3045
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
3046
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
3047
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
3048
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
3049
|
+
* tracking the stored id so the reply correlates by it;
|
|
3050
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
3051
|
+
*
|
|
3052
|
+
* Only `ChannelAuthError` propagates.
|
|
3053
|
+
*/
|
|
3054
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
3055
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
3056
|
+
this.log({
|
|
3057
|
+
level: "debug",
|
|
3058
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
3059
|
+
conversation_id: row.conversation_id,
|
|
3060
|
+
message_id: row.id
|
|
3061
|
+
});
|
|
3062
|
+
return;
|
|
3063
|
+
}
|
|
3064
|
+
const ocId = row.opencode_message_id;
|
|
3065
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
3066
|
+
if (state === "done") {
|
|
3067
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
3068
|
+
this.log({
|
|
3069
|
+
level: "debug",
|
|
3070
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
3071
|
+
conversation_id: row.conversation_id,
|
|
3072
|
+
message_id: row.id
|
|
3073
|
+
});
|
|
3074
|
+
return;
|
|
3075
|
+
}
|
|
3076
|
+
this.log({
|
|
3077
|
+
level: "info",
|
|
3078
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
3079
|
+
conversation_id: row.conversation_id,
|
|
3080
|
+
message_id: row.id
|
|
3081
|
+
});
|
|
3082
|
+
try {
|
|
3083
|
+
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
3084
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId, title);
|
|
3085
|
+
} catch (err) {
|
|
3086
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3087
|
+
if (err instanceof ChannelTerminalError) {
|
|
3088
|
+
this.doneUndeliverable.add(row.id);
|
|
3089
|
+
this.log({
|
|
3090
|
+
level: "warn",
|
|
3091
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
3092
|
+
conversation_id: row.conversation_id,
|
|
3093
|
+
message_id: row.id
|
|
3094
|
+
});
|
|
3095
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
3096
|
+
return;
|
|
3097
|
+
}
|
|
3098
|
+
this.log({
|
|
3099
|
+
level: "warn",
|
|
3100
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3101
|
+
conversation_id: row.conversation_id,
|
|
3102
|
+
message_id: row.id
|
|
3103
|
+
});
|
|
3104
|
+
return;
|
|
3105
|
+
}
|
|
3106
|
+
this.dontRedispatch.delete(row.id);
|
|
3107
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
3108
|
+
return;
|
|
3109
|
+
}
|
|
3110
|
+
if (state === "failed") {
|
|
3111
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3112
|
+
this.log({
|
|
3113
|
+
level: "error",
|
|
3114
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
3115
|
+
conversation_id: row.conversation_id,
|
|
3116
|
+
message_id: row.id
|
|
3117
|
+
});
|
|
3118
|
+
try {
|
|
3119
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
3120
|
+
} catch (err) {
|
|
3121
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3122
|
+
if (err instanceof ChannelTerminalError) {
|
|
3123
|
+
this.doneUndeliverable.add(row.id);
|
|
3124
|
+
this.log({
|
|
3125
|
+
level: "warn",
|
|
3126
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
3127
|
+
conversation_id: row.conversation_id,
|
|
3128
|
+
message_id: row.id
|
|
3129
|
+
});
|
|
3130
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
3131
|
+
return;
|
|
3132
|
+
}
|
|
3133
|
+
this.log({
|
|
3134
|
+
level: "warn",
|
|
3135
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3136
|
+
conversation_id: row.conversation_id,
|
|
3137
|
+
message_id: row.id
|
|
3138
|
+
});
|
|
3139
|
+
return;
|
|
3140
|
+
}
|
|
3141
|
+
this.dontRedispatch.delete(row.id);
|
|
3142
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
3143
|
+
return;
|
|
3144
|
+
}
|
|
3145
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
3146
|
+
this.log({
|
|
3147
|
+
level: "debug",
|
|
3148
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
3149
|
+
conversation_id: row.conversation_id,
|
|
3150
|
+
message_id: row.id
|
|
3151
|
+
});
|
|
3152
|
+
return;
|
|
3153
|
+
}
|
|
3154
|
+
let statusReadableOngoing = null;
|
|
3155
|
+
if (state === "running" && ocId) {
|
|
3156
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
3157
|
+
const shape = this.replyCompletionShape(reply);
|
|
3158
|
+
const ongoing = sessionOngoing;
|
|
3159
|
+
statusReadableOngoing = ongoing;
|
|
3160
|
+
if (ongoing === false) {
|
|
3161
|
+
this.log({
|
|
3162
|
+
level: "info",
|
|
3163
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
3164
|
+
conversation_id: row.conversation_id,
|
|
3165
|
+
message_id: row.id
|
|
3166
|
+
});
|
|
3167
|
+
await this.forceReadoptRun(sessionId, row);
|
|
3168
|
+
return;
|
|
3169
|
+
}
|
|
3170
|
+
if (ongoing === true) {
|
|
3171
|
+
this.log({
|
|
3172
|
+
level: "debug",
|
|
3173
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
3174
|
+
conversation_id: row.conversation_id,
|
|
3175
|
+
message_id: row.id
|
|
3176
|
+
});
|
|
3177
|
+
} else {
|
|
3178
|
+
if (shape === "b1") {
|
|
3179
|
+
this.log({
|
|
3180
|
+
level: "debug",
|
|
3181
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
3182
|
+
conversation_id: row.conversation_id,
|
|
3183
|
+
message_id: row.id
|
|
3184
|
+
});
|
|
3185
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
3186
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
3187
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
3188
|
+
}
|
|
3189
|
+
return;
|
|
3190
|
+
}
|
|
3191
|
+
this.log({
|
|
3192
|
+
level: "debug",
|
|
3193
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
3194
|
+
conversation_id: row.conversation_id,
|
|
3195
|
+
message_id: row.id
|
|
3196
|
+
});
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
3200
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
3201
|
+
if (descendantAlive === true) {
|
|
3202
|
+
this.log({
|
|
3203
|
+
level: "debug",
|
|
3204
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
3205
|
+
conversation_id: row.conversation_id,
|
|
3206
|
+
message_id: row.id
|
|
3207
|
+
});
|
|
3208
|
+
} else {
|
|
3209
|
+
this.log({
|
|
3210
|
+
level: "info",
|
|
3211
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
3212
|
+
conversation_id: row.conversation_id,
|
|
3213
|
+
message_id: row.id
|
|
3214
|
+
});
|
|
3215
|
+
await this.forceReadoptRun(sessionId, row);
|
|
3216
|
+
return;
|
|
3217
|
+
}
|
|
3218
|
+
}
|
|
3219
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
3220
|
+
const conv = this.convForRow(sessionId, row);
|
|
3221
|
+
const message = this.queuedMessageForRow(row);
|
|
3222
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
3223
|
+
this.dispatched.add(row.id);
|
|
3224
|
+
this.readopted.add(row.id);
|
|
3225
|
+
this.ensureWatcherRunning(sessionId);
|
|
3226
|
+
this.log({
|
|
3227
|
+
level: "debug",
|
|
3228
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
3229
|
+
conversation_id: row.conversation_id,
|
|
3230
|
+
message_id: row.id
|
|
3231
|
+
});
|
|
3232
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
3233
|
+
return;
|
|
3234
|
+
}
|
|
3235
|
+
await this.forceReadoptRun(sessionId, row);
|
|
3236
|
+
}
|
|
1909
3237
|
/**
|
|
1910
|
-
* Re-dispatch
|
|
1911
|
-
*
|
|
1912
|
-
*
|
|
1913
|
-
*
|
|
3238
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
3239
|
+
*
|
|
3240
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
3241
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
3242
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
3243
|
+
* correlates server-side.
|
|
3244
|
+
*
|
|
3245
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3246
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3247
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3248
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3249
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3250
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3251
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3252
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3253
|
+
* may retry exactly once more).
|
|
3254
|
+
*
|
|
3255
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
3256
|
+
* `processed_at` (Invariant 1).
|
|
1914
3257
|
*/
|
|
1915
|
-
async
|
|
3258
|
+
async forceReadoptRun(sessionId, row) {
|
|
3259
|
+
if (this.stopped) {
|
|
3260
|
+
this.log({
|
|
3261
|
+
level: "debug",
|
|
3262
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3263
|
+
conversation_id: row.conversation_id,
|
|
3264
|
+
message_id: row.id
|
|
3265
|
+
});
|
|
3266
|
+
return;
|
|
3267
|
+
}
|
|
3268
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3269
|
+
this.log({
|
|
3270
|
+
level: "debug",
|
|
3271
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3272
|
+
conversation_id: row.conversation_id,
|
|
3273
|
+
message_id: row.id
|
|
3274
|
+
});
|
|
3275
|
+
return;
|
|
3276
|
+
}
|
|
3277
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
3278
|
+
this.dontRedispatch.add(row.id);
|
|
3279
|
+
this.log({
|
|
3280
|
+
level: "debug",
|
|
3281
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
3282
|
+
conversation_id: row.conversation_id,
|
|
3283
|
+
message_id: row.id
|
|
3284
|
+
});
|
|
3285
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
3286
|
+
return;
|
|
3287
|
+
}
|
|
1916
3288
|
const options = {
|
|
1917
|
-
agent:
|
|
1918
|
-
model:
|
|
3289
|
+
agent: row.opencode_agent ?? void 0,
|
|
3290
|
+
model: row.opencode_model ?? void 0
|
|
1919
3291
|
};
|
|
1920
3292
|
this.log({
|
|
1921
3293
|
level: "info",
|
|
1922
|
-
message: `
|
|
1923
|
-
|
|
3294
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
3295
|
+
conversation_id: row.conversation_id,
|
|
3296
|
+
message_id: row.id
|
|
1924
3297
|
});
|
|
3298
|
+
this.awaitingReadopt.add(row.id);
|
|
3299
|
+
const readoptConv = this.convForRow(sessionId, row);
|
|
3300
|
+
const readoptMessage = this.queuedMessageForRow(row);
|
|
3301
|
+
const sendAttachments = this.buildSendAttachments(readoptConv, readoptMessage);
|
|
3302
|
+
let ocId;
|
|
1925
3303
|
try {
|
|
1926
|
-
await
|
|
1927
|
-
this.port,
|
|
3304
|
+
ocId = await this.dispatchLocked(
|
|
1928
3305
|
sessionId,
|
|
1929
|
-
|
|
1930
|
-
options,
|
|
1931
|
-
inFlight.opencodeMessageId
|
|
3306
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options, sendAttachments)
|
|
1932
3307
|
);
|
|
1933
3308
|
} catch (err) {
|
|
3309
|
+
this.awaitingReadopt.delete(row.id);
|
|
3310
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
1934
3311
|
this.log({
|
|
1935
|
-
level: "
|
|
1936
|
-
message: `Re-dispatch failed for message ${
|
|
1937
|
-
|
|
3312
|
+
level: "warn",
|
|
3313
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3314
|
+
conversation_id: row.conversation_id,
|
|
3315
|
+
message_id: row.id
|
|
3316
|
+
});
|
|
3317
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3318
|
+
return;
|
|
3319
|
+
}
|
|
3320
|
+
if (ocId === null) {
|
|
3321
|
+
this.awaitingReadopt.delete(row.id);
|
|
3322
|
+
this.log({
|
|
3323
|
+
level: "warn",
|
|
3324
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3325
|
+
conversation_id: row.conversation_id,
|
|
3326
|
+
message_id: row.id
|
|
1938
3327
|
});
|
|
3328
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3329
|
+
return;
|
|
1939
3330
|
}
|
|
1940
|
-
|
|
3331
|
+
this.registerReadopted(readoptConv, sessionId, readoptMessage, ocId, this.processedAtMs(row));
|
|
3332
|
+
this.dispatched.add(row.id);
|
|
3333
|
+
this.readopted.add(row.id);
|
|
3334
|
+
this.awaitingReadopt.delete(row.id);
|
|
3335
|
+
this.ensureWatcherRunning(sessionId);
|
|
3336
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
3337
|
+
}
|
|
3338
|
+
/**
|
|
3339
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
3340
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
3341
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
3342
|
+
*/
|
|
3343
|
+
isTracked(sessionId, evidentMessageId) {
|
|
3344
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
3345
|
+
const watcher = this.watchers.get(sessionId);
|
|
3346
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
3347
|
+
}
|
|
3348
|
+
/**
|
|
3349
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
3350
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
3351
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
3352
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
3353
|
+
* intended, which is worth surfacing.
|
|
3354
|
+
*/
|
|
3355
|
+
processedAtMs(row) {
|
|
3356
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
3357
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
3358
|
+
this.log({
|
|
3359
|
+
level: "error",
|
|
3360
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
3361
|
+
conversation_id: row.conversation_id,
|
|
3362
|
+
message_id: row.id
|
|
3363
|
+
});
|
|
3364
|
+
return this.now();
|
|
3365
|
+
}
|
|
3366
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
3367
|
+
convForRow(sessionId, row) {
|
|
3368
|
+
return {
|
|
3369
|
+
id: row.conversation_id,
|
|
3370
|
+
agent_id: this.agentId,
|
|
3371
|
+
opencode_session_id: sessionId,
|
|
3372
|
+
pending_message_count: 0,
|
|
3373
|
+
oldest_pending_at: row.processed_at
|
|
3374
|
+
};
|
|
3375
|
+
}
|
|
3376
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
3377
|
+
queuedMessageForRow(row) {
|
|
3378
|
+
return {
|
|
3379
|
+
id: row.id,
|
|
3380
|
+
content: row.content,
|
|
3381
|
+
status: "processing",
|
|
3382
|
+
opencode_agent: row.opencode_agent,
|
|
3383
|
+
opencode_model: row.opencode_model,
|
|
3384
|
+
source_message_id: row.source_message_id,
|
|
3385
|
+
slack_user_id: row.slack_user_id,
|
|
3386
|
+
attachments: row.attachments ?? null
|
|
3387
|
+
};
|
|
1941
3388
|
}
|
|
1942
3389
|
/**
|
|
1943
3390
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
1944
3391
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
1945
3392
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
3393
|
+
*
|
|
3394
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
3395
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
3396
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
3397
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
3398
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
3399
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
3400
|
+
* the done branch still delivers it (Bugbot #202).
|
|
1946
3401
|
*/
|
|
1947
3402
|
removeInFlight(watcher, evidentMessageId) {
|
|
3403
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
3404
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
3405
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
3406
|
+
this.log({
|
|
3407
|
+
level: "debug",
|
|
3408
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
3409
|
+
conversation_id: watcher.conv.id,
|
|
3410
|
+
message_id: evidentMessageId
|
|
3411
|
+
});
|
|
3412
|
+
}
|
|
1948
3413
|
watcher.inFlight.delete(evidentMessageId);
|
|
1949
3414
|
this.dispatched.delete(evidentMessageId);
|
|
1950
3415
|
}
|
|
@@ -1961,21 +3426,41 @@ var ChannelDriver = class {
|
|
|
1961
3426
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
1962
3427
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
1963
3428
|
* oldest running message.
|
|
3429
|
+
*
|
|
3430
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3431
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3432
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3433
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3434
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3435
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3436
|
+
* even after it was already surfaced to the channel.
|
|
1964
3437
|
*/
|
|
1965
3438
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3439
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3440
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3441
|
+
let questionsPolledOk = true;
|
|
3442
|
+
let permissionsPolledOk = true;
|
|
1966
3443
|
let questions = [];
|
|
1967
3444
|
try {
|
|
1968
3445
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
1969
3446
|
if (res.ok) {
|
|
1970
3447
|
const body = await res.json();
|
|
1971
|
-
|
|
3448
|
+
if (Array.isArray(body)) {
|
|
3449
|
+
questions = body;
|
|
3450
|
+
} else {
|
|
3451
|
+
questionsPolledOk = false;
|
|
3452
|
+
}
|
|
3453
|
+
} else {
|
|
3454
|
+
questionsPolledOk = false;
|
|
1972
3455
|
}
|
|
1973
3456
|
} catch {
|
|
3457
|
+
questionsPolledOk = false;
|
|
1974
3458
|
}
|
|
1975
3459
|
for (const q of questions) {
|
|
1976
|
-
if (q.sessionID
|
|
1977
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3460
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
1978
3461
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3462
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3463
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
1979
3464
|
const reported = await this.reportInteraction(
|
|
1980
3465
|
watcher.conv.id,
|
|
1981
3466
|
"question",
|
|
@@ -1989,14 +3474,22 @@ var ChannelDriver = class {
|
|
|
1989
3474
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
1990
3475
|
if (res.ok) {
|
|
1991
3476
|
const body = await res.json();
|
|
1992
|
-
|
|
3477
|
+
if (Array.isArray(body)) {
|
|
3478
|
+
permissions = body;
|
|
3479
|
+
} else {
|
|
3480
|
+
permissionsPolledOk = false;
|
|
3481
|
+
}
|
|
3482
|
+
} else {
|
|
3483
|
+
permissionsPolledOk = false;
|
|
1993
3484
|
}
|
|
1994
3485
|
} catch {
|
|
3486
|
+
permissionsPolledOk = false;
|
|
1995
3487
|
}
|
|
1996
3488
|
for (const p of permissions) {
|
|
1997
|
-
if (p.sessionID
|
|
1998
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3489
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
1999
3490
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3491
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3492
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2000
3493
|
const reported = await this.reportInteraction(
|
|
2001
3494
|
watcher.conv.id,
|
|
2002
3495
|
"permission",
|
|
@@ -2005,6 +3498,173 @@ var ChannelDriver = class {
|
|
|
2005
3498
|
);
|
|
2006
3499
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2007
3500
|
}
|
|
3501
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3502
|
+
}
|
|
3503
|
+
/**
|
|
3504
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3505
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3506
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3507
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3508
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3509
|
+
*
|
|
3510
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3511
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3512
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3513
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3514
|
+
*/
|
|
3515
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3516
|
+
let current = sessionId;
|
|
3517
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3518
|
+
if (current === rootSessionId) return true;
|
|
3519
|
+
const parent = await this.resolveSessionParent(current);
|
|
3520
|
+
if (parent === null || parent === void 0) return false;
|
|
3521
|
+
current = parent;
|
|
3522
|
+
}
|
|
3523
|
+
return false;
|
|
3524
|
+
}
|
|
3525
|
+
/**
|
|
3526
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3527
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3528
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3529
|
+
* caching a wrong answer — the next tick retries).
|
|
3530
|
+
*/
|
|
3531
|
+
async resolveSessionParent(sessionId) {
|
|
3532
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3533
|
+
if (cached !== void 0) return cached;
|
|
3534
|
+
let parent = void 0;
|
|
3535
|
+
try {
|
|
3536
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3537
|
+
if (res.ok) {
|
|
3538
|
+
const body = await res.json();
|
|
3539
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3540
|
+
}
|
|
3541
|
+
} catch {
|
|
3542
|
+
parent = void 0;
|
|
3543
|
+
}
|
|
3544
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3545
|
+
return parent;
|
|
3546
|
+
}
|
|
3547
|
+
/**
|
|
3548
|
+
* Resolve (and cache in `sessionTitles`) the OpenCode session TITLE (#310) so the
|
|
3549
|
+
* status PATCH can carry it into the "Live sessions" list. Driver-level cache so
|
|
3550
|
+
* BOTH the watcher completion path and the restart-recovery re-adopt path (which
|
|
3551
|
+
* has no watcher) can use it. `conversationId` is passed only for log context.
|
|
3552
|
+
* Best-effort:
|
|
3553
|
+
* - a resolved NON-EMPTY title is cached and terminal (a real session name
|
|
3554
|
+
* won't later un-name), so we do NOT re-GET `/session/:id` every tick;
|
|
3555
|
+
* - while the title is still absent/empty we do NOT latch it — OpenCode names
|
|
3556
|
+
* sessions asynchronously mid-turn, so an early call (e.g. at `processing`)
|
|
3557
|
+
* must leave the cache unresolved and re-fetch on the next need so a later
|
|
3558
|
+
* call (e.g. at `done`) picks up the name assigned in the meantime. Such a
|
|
3559
|
+
* call returns `null` (omit the title on THIS PATCH) without caching;
|
|
3560
|
+
* - a failed request likewise leaves the cache unresolved (retry next need)
|
|
3561
|
+
* and returns `null` — it must NEVER throw or block completion.
|
|
3562
|
+
* A failure is logged with agent/session context (no silent catch).
|
|
3563
|
+
*/
|
|
3564
|
+
async resolveSessionTitle(sessionId, conversationId) {
|
|
3565
|
+
const cached = this.sessionTitles.get(sessionId);
|
|
3566
|
+
if (cached != null) return cached;
|
|
3567
|
+
try {
|
|
3568
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3569
|
+
if (res.ok) {
|
|
3570
|
+
const body = await res.json();
|
|
3571
|
+
const title = body && typeof body.title === "string" ? body.title.trim() : "";
|
|
3572
|
+
if (title.length > 0) {
|
|
3573
|
+
this.sessionTitles.set(sessionId, title);
|
|
3574
|
+
return title;
|
|
3575
|
+
}
|
|
3576
|
+
return null;
|
|
3577
|
+
}
|
|
3578
|
+
this.log({
|
|
3579
|
+
level: "debug",
|
|
3580
|
+
message: `Session title fetch for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) returned HTTP ${res.status} \u2014 omitting title`,
|
|
3581
|
+
conversation_id: conversationId
|
|
3582
|
+
});
|
|
3583
|
+
} catch (err) {
|
|
3584
|
+
this.log({
|
|
3585
|
+
level: "debug",
|
|
3586
|
+
message: `Best-effort session title fetch failed for session ${sessionId.slice(0, 8)} (agent ${this.agentId.slice(0, 8)}) \u2014 omitting title: ${err instanceof Error ? err.message : String(err)}`,
|
|
3587
|
+
conversation_id: conversationId
|
|
3588
|
+
});
|
|
3589
|
+
}
|
|
3590
|
+
return null;
|
|
3591
|
+
}
|
|
3592
|
+
/**
|
|
3593
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3594
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3595
|
+
*
|
|
3596
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3597
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3598
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3599
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3600
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3601
|
+
* provably in flight at the exact moment of recovery.
|
|
3602
|
+
*
|
|
3603
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3604
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3605
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3606
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3607
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3608
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3609
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3610
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3611
|
+
*
|
|
3612
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3613
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3614
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3615
|
+
* case), OR no descendant is found at all.
|
|
3616
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3617
|
+
*
|
|
3618
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3619
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3620
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3621
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3622
|
+
* true/false/null faithfully.
|
|
3623
|
+
*
|
|
3624
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3625
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3626
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3627
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3628
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3629
|
+
* `SessionStatus` only.
|
|
3630
|
+
*/
|
|
3631
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3632
|
+
const sessions = await listSessions(this.port);
|
|
3633
|
+
if (!sessions) {
|
|
3634
|
+
this.log({
|
|
3635
|
+
level: "warn",
|
|
3636
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3637
|
+
});
|
|
3638
|
+
return null;
|
|
3639
|
+
}
|
|
3640
|
+
for (const candidate of sessions) {
|
|
3641
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3642
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3643
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3644
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3645
|
+
return true;
|
|
3646
|
+
}
|
|
3647
|
+
}
|
|
3648
|
+
return false;
|
|
3649
|
+
}
|
|
3650
|
+
/**
|
|
3651
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3652
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3653
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3654
|
+
* the aborted-in-flight production bug after a restart.
|
|
3655
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3656
|
+
* (the sub-agent preamble — #253's shape).
|
|
3657
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3658
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3659
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3660
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3661
|
+
*/
|
|
3662
|
+
replyCompletionShape(reply) {
|
|
3663
|
+
if (!reply) return "other";
|
|
3664
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3665
|
+
if (completed == null) return "b1";
|
|
3666
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3667
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2008
3668
|
}
|
|
2009
3669
|
/**
|
|
2010
3670
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -2056,9 +3716,7 @@ var ChannelDriver = class {
|
|
|
2056
3716
|
}
|
|
2057
3717
|
return inFlight.sort(byOldest)[0];
|
|
2058
3718
|
}
|
|
2059
|
-
// -------------------------------------------------------------------------
|
|
2060
3719
|
// Evident API calls (combinedAuth thread routes)
|
|
2061
|
-
// -------------------------------------------------------------------------
|
|
2062
3720
|
async getPendingConversations() {
|
|
2063
3721
|
const res = await this.fetchImpl(
|
|
2064
3722
|
`${this.apiUrl}/agents/${this.agentId}/conversations/pending`,
|
|
@@ -2088,6 +3746,35 @@ var ChannelDriver = class {
|
|
|
2088
3746
|
}
|
|
2089
3747
|
return await res.json();
|
|
2090
3748
|
}
|
|
3749
|
+
/**
|
|
3750
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
3751
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
3752
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
3753
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
3754
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
3755
|
+
* `opencode_session_id`, routing).
|
|
3756
|
+
*
|
|
3757
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
3758
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
3759
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
3760
|
+
* next tick retries.
|
|
3761
|
+
*/
|
|
3762
|
+
async getProcessingMessages() {
|
|
3763
|
+
const res = await this.fetchImpl(
|
|
3764
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
3765
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
3766
|
+
);
|
|
3767
|
+
this.assertAuth(res, "fetching processing messages");
|
|
3768
|
+
if (!res.ok) {
|
|
3769
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
3770
|
+
}
|
|
3771
|
+
const data = await res.json();
|
|
3772
|
+
let messages = data.messages ?? [];
|
|
3773
|
+
if (this.conversationFilter) {
|
|
3774
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
3775
|
+
}
|
|
3776
|
+
return messages;
|
|
3777
|
+
}
|
|
2091
3778
|
/**
|
|
2092
3779
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2093
3780
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2109,13 +3796,18 @@ var ChannelDriver = class {
|
|
|
2109
3796
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2110
3797
|
* retry vehicle for the swap-to-running.
|
|
2111
3798
|
*/
|
|
2112
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
3799
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId, title) {
|
|
2113
3800
|
const res = await this.fetchImpl(
|
|
2114
3801
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2115
3802
|
{
|
|
2116
3803
|
method: "PATCH",
|
|
2117
3804
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2118
|
-
body: JSON.stringify({
|
|
3805
|
+
body: JSON.stringify({
|
|
3806
|
+
status: "processing",
|
|
3807
|
+
opencode_session_id: sessionId,
|
|
3808
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
3809
|
+
...title ? { title } : {}
|
|
3810
|
+
})
|
|
2119
3811
|
}
|
|
2120
3812
|
);
|
|
2121
3813
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2153,13 +3845,18 @@ var ChannelDriver = class {
|
|
|
2153
3845
|
* watcher retries next tick within the
|
|
2154
3846
|
* deadline, Finding 4).
|
|
2155
3847
|
*/
|
|
2156
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3848
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId, title) {
|
|
2157
3849
|
const res = await this.fetchImpl(
|
|
2158
3850
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2159
3851
|
{
|
|
2160
3852
|
method: "PATCH",
|
|
2161
3853
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2162
|
-
body: JSON.stringify({
|
|
3854
|
+
body: JSON.stringify({
|
|
3855
|
+
status: "done",
|
|
3856
|
+
opencode_session_id: sessionId,
|
|
3857
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
3858
|
+
...title ? { title } : {}
|
|
3859
|
+
})
|
|
2163
3860
|
}
|
|
2164
3861
|
);
|
|
2165
3862
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2169,7 +3866,17 @@ var ChannelDriver = class {
|
|
|
2169
3866
|
}
|
|
2170
3867
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2171
3868
|
}
|
|
2172
|
-
|
|
3869
|
+
/**
|
|
3870
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3871
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3872
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3873
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3874
|
+
* failure reason reaches the channel.
|
|
3875
|
+
*/
|
|
3876
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3877
|
+
const body = { status: "failed" };
|
|
3878
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3879
|
+
if (error2 !== void 0) body.error = error2;
|
|
2173
3880
|
await this.callWithRetry(
|
|
2174
3881
|
"marking message as failed",
|
|
2175
3882
|
() => this.fetchImpl(
|
|
@@ -2177,11 +3884,56 @@ var ChannelDriver = class {
|
|
|
2177
3884
|
{
|
|
2178
3885
|
method: "PATCH",
|
|
2179
3886
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2180
|
-
body: JSON.stringify(
|
|
3887
|
+
body: JSON.stringify(body)
|
|
2181
3888
|
}
|
|
2182
3889
|
)
|
|
2183
3890
|
);
|
|
2184
3891
|
}
|
|
3892
|
+
/**
|
|
3893
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3894
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3895
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3896
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3897
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3898
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3899
|
+
* context (no silent catch, per development-workflow).
|
|
3900
|
+
*
|
|
3901
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3902
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3903
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3904
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3905
|
+
* leaves liveness").
|
|
3906
|
+
*/
|
|
3907
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3908
|
+
try {
|
|
3909
|
+
const res = await this.fetchImpl(
|
|
3910
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
3911
|
+
{
|
|
3912
|
+
method: "POST",
|
|
3913
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3914
|
+
body: JSON.stringify({ signal, ...extra })
|
|
3915
|
+
}
|
|
3916
|
+
);
|
|
3917
|
+
if (!res.ok) {
|
|
3918
|
+
this.log({
|
|
3919
|
+
level: "warn",
|
|
3920
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3921
|
+
conversation_id: conversationId,
|
|
3922
|
+
message_id: messageId
|
|
3923
|
+
});
|
|
3924
|
+
return false;
|
|
3925
|
+
}
|
|
3926
|
+
return true;
|
|
3927
|
+
} catch (err) {
|
|
3928
|
+
this.log({
|
|
3929
|
+
level: "warn",
|
|
3930
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3931
|
+
conversation_id: conversationId,
|
|
3932
|
+
message_id: messageId
|
|
3933
|
+
});
|
|
3934
|
+
return false;
|
|
3935
|
+
}
|
|
3936
|
+
}
|
|
2185
3937
|
async persistSession(conversationId, sessionId) {
|
|
2186
3938
|
const res = await this.fetchImpl(
|
|
2187
3939
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2236,9 +3988,7 @@ var ChannelDriver = class {
|
|
|
2236
3988
|
return false;
|
|
2237
3989
|
}
|
|
2238
3990
|
}
|
|
2239
|
-
// -------------------------------------------------------------------------
|
|
2240
3991
|
// Retry wrapper
|
|
2241
|
-
// -------------------------------------------------------------------------
|
|
2242
3992
|
/**
|
|
2243
3993
|
* Invoke an Evident API call, retrying on transient failures (5xx / 429 /
|
|
2244
3994
|
* network errors) with exponential backoff + jitter (capped). Auth failures
|
|
@@ -2452,6 +4202,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
2452
4202
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
2453
4203
|
}
|
|
2454
4204
|
}
|
|
4205
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
4206
|
+
const apiUrl = getApiUrlConfig();
|
|
4207
|
+
try {
|
|
4208
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
4209
|
+
method: "POST",
|
|
4210
|
+
headers: { Authorization: authHeader }
|
|
4211
|
+
});
|
|
4212
|
+
if (!response.ok) {
|
|
4213
|
+
const serverMessage = await readErrorMessage(response);
|
|
4214
|
+
return {
|
|
4215
|
+
ok: false,
|
|
4216
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4217
|
+
};
|
|
4218
|
+
}
|
|
4219
|
+
return { ok: true };
|
|
4220
|
+
} catch (error2) {
|
|
4221
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
4222
|
+
}
|
|
4223
|
+
}
|
|
2455
4224
|
async function getAgentInfo(agentId, authHeader) {
|
|
2456
4225
|
const apiUrl = getApiUrlConfig();
|
|
2457
4226
|
try {
|
|
@@ -2497,23 +4266,55 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2497
4266
|
// src/commands/run.ts
|
|
2498
4267
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2499
4268
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2500
|
-
|
|
4269
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4270
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
4271
|
+
function resolveLogLevel(options) {
|
|
4272
|
+
const accepted = Object.keys(LOG_LEVELS);
|
|
4273
|
+
const validate = (value, source) => {
|
|
4274
|
+
const normalized = value.trim().toLowerCase();
|
|
4275
|
+
if (!accepted.includes(normalized)) {
|
|
4276
|
+
throw new Error(
|
|
4277
|
+
`Invalid log level "${value}"${source}; expected one of ${accepted.join(", ")}`
|
|
4278
|
+
);
|
|
4279
|
+
}
|
|
4280
|
+
return normalized;
|
|
4281
|
+
};
|
|
4282
|
+
if (options.logLevel !== void 0) {
|
|
4283
|
+
return validate(options.logLevel, " (--log-level)");
|
|
4284
|
+
}
|
|
4285
|
+
if (options.verbose) {
|
|
4286
|
+
return "debug";
|
|
4287
|
+
}
|
|
4288
|
+
const env = process.env.EVIDENT_LOG_LEVEL;
|
|
4289
|
+
if (env !== void 0 && env !== "") {
|
|
4290
|
+
return validate(env, " (EVIDENT_LOG_LEVEL)");
|
|
4291
|
+
}
|
|
4292
|
+
return "info";
|
|
4293
|
+
}
|
|
4294
|
+
function meetsThreshold(state, level) {
|
|
4295
|
+
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
4296
|
+
}
|
|
4297
|
+
function log2(state, message, level = "info") {
|
|
4298
|
+
if (!meetsThreshold(state, level)) return;
|
|
2501
4299
|
if (state.json) {
|
|
2502
4300
|
console.log(
|
|
2503
4301
|
JSON.stringify({
|
|
2504
4302
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2505
|
-
level
|
|
4303
|
+
level,
|
|
2506
4304
|
message
|
|
2507
4305
|
})
|
|
2508
4306
|
);
|
|
2509
4307
|
} else if (!state.interactive) {
|
|
2510
|
-
const prefix =
|
|
4308
|
+
const prefix = level === "error" ? chalk6.red("\u2717") : level === "warn" ? chalk6.yellow("!") : level === "debug" ? chalk6.dim("\xB7") : chalk6.green("\u2022");
|
|
2511
4309
|
console.log(`${prefix} ${message}`);
|
|
2512
4310
|
}
|
|
2513
4311
|
}
|
|
2514
4312
|
function logActivity(state, entry) {
|
|
4313
|
+
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
4314
|
+
if (!meetsThreshold(state, level)) return;
|
|
2515
4315
|
const fullEntry = {
|
|
2516
4316
|
...entry,
|
|
4317
|
+
level,
|
|
2517
4318
|
timestamp: /* @__PURE__ */ new Date()
|
|
2518
4319
|
};
|
|
2519
4320
|
state.activityLog.push(fullEntry);
|
|
@@ -2522,9 +4323,9 @@ function logActivity(state, entry) {
|
|
|
2522
4323
|
}
|
|
2523
4324
|
if (!state.interactive) {
|
|
2524
4325
|
if (entry.type === "error") {
|
|
2525
|
-
|
|
2526
|
-
} else if (entry.
|
|
2527
|
-
|
|
4326
|
+
log2(state, entry.error ?? "Unknown error", level);
|
|
4327
|
+
} else if (entry.message) {
|
|
4328
|
+
log2(state, entry.message, level);
|
|
2528
4329
|
}
|
|
2529
4330
|
}
|
|
2530
4331
|
}
|
|
@@ -2610,6 +4411,7 @@ async function handleAuthError(state, error2) {
|
|
|
2610
4411
|
}
|
|
2611
4412
|
async function driveChannels(state, driver) {
|
|
2612
4413
|
let idlePolls = 0;
|
|
4414
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2613
4415
|
while (state.running) {
|
|
2614
4416
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2615
4417
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2619,7 +4421,9 @@ async function driveChannels(state, driver) {
|
|
|
2619
4421
|
try {
|
|
2620
4422
|
const processed = await driver.drainPending();
|
|
2621
4423
|
state.messageCount += processed;
|
|
2622
|
-
|
|
4424
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
4425
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
4426
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2623
4427
|
idlePolls = 0;
|
|
2624
4428
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2625
4429
|
} else if (state.idleTimeout !== null) {
|
|
@@ -2648,7 +4452,7 @@ async function driveChannels(state, driver) {
|
|
|
2648
4452
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
2649
4453
|
if (state.interactive) displayStatus(state);
|
|
2650
4454
|
}
|
|
2651
|
-
await new Promise((
|
|
4455
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
2652
4456
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
2653
4457
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
2654
4458
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -2659,8 +4463,122 @@ async function driveChannels(state, driver) {
|
|
|
2659
4463
|
}
|
|
2660
4464
|
}
|
|
2661
4465
|
}
|
|
2662
|
-
|
|
4466
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4467
|
+
async function runSweep(state, driver, config2) {
|
|
4468
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4469
|
+
try {
|
|
4470
|
+
const sessions = await listSessions(state.port);
|
|
4471
|
+
if (sessions === null) {
|
|
4472
|
+
logActivity(state, {
|
|
4473
|
+
type: "info",
|
|
4474
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4475
|
+
});
|
|
4476
|
+
return;
|
|
4477
|
+
}
|
|
4478
|
+
const toDelete = selectSessionsToDelete(
|
|
4479
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4480
|
+
{
|
|
4481
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4482
|
+
maxCount: config2.maxCount,
|
|
4483
|
+
nowMs: Date.now(),
|
|
4484
|
+
protectedIds: driver.protectedSessionIds()
|
|
4485
|
+
}
|
|
4486
|
+
);
|
|
4487
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4488
|
+
let deleted = 0;
|
|
4489
|
+
let failed = 0;
|
|
4490
|
+
let skippedNewlyActive = 0;
|
|
4491
|
+
for (const id of toDelete) {
|
|
4492
|
+
if (protectedNow.has(id)) {
|
|
4493
|
+
skippedNewlyActive++;
|
|
4494
|
+
logActivity(state, {
|
|
4495
|
+
type: "info",
|
|
4496
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4497
|
+
});
|
|
4498
|
+
continue;
|
|
4499
|
+
}
|
|
4500
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4501
|
+
else failed++;
|
|
4502
|
+
}
|
|
4503
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4504
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4505
|
+
logActivity(state, {
|
|
4506
|
+
type: "info",
|
|
4507
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4508
|
+
});
|
|
4509
|
+
} catch (error2) {
|
|
4510
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4511
|
+
logActivity(state, {
|
|
4512
|
+
type: "error",
|
|
4513
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4514
|
+
});
|
|
4515
|
+
}
|
|
4516
|
+
}
|
|
4517
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4518
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4519
|
+
{
|
|
4520
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4521
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4522
|
+
interval: options.sessionCleanupInterval
|
|
4523
|
+
},
|
|
4524
|
+
process.env
|
|
4525
|
+
);
|
|
4526
|
+
for (const warning2 of config2.warnings) {
|
|
4527
|
+
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
4528
|
+
}
|
|
4529
|
+
if (!config2.enabled) return;
|
|
4530
|
+
logActivity(state, {
|
|
4531
|
+
type: "info",
|
|
4532
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4533
|
+
});
|
|
4534
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4535
|
+
const firstSweep = setTimeout(
|
|
4536
|
+
() => void runSweep(state, driver, config2),
|
|
4537
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4538
|
+
);
|
|
4539
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4540
|
+
}
|
|
4541
|
+
async function notifyOffline(state) {
|
|
4542
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4543
|
+
if (!state.connected) {
|
|
4544
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4545
|
+
return;
|
|
4546
|
+
}
|
|
4547
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4548
|
+
if (result.ok) {
|
|
4549
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4550
|
+
} else {
|
|
4551
|
+
logActivity(state, {
|
|
4552
|
+
type: "error",
|
|
4553
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4554
|
+
});
|
|
4555
|
+
if (state.interactive) displayStatus(state);
|
|
4556
|
+
}
|
|
4557
|
+
}
|
|
4558
|
+
async function cleanup(state, opts = {}) {
|
|
2663
4559
|
state.running = false;
|
|
4560
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4561
|
+
clearInterval(timer);
|
|
4562
|
+
clearTimeout(timer);
|
|
4563
|
+
}
|
|
4564
|
+
state.sessionCleanupTimers = [];
|
|
4565
|
+
if (opts.graceful && state.channelDriver) {
|
|
4566
|
+
state.channelDriver.stop();
|
|
4567
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4568
|
+
if (state.interactive) {
|
|
4569
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4570
|
+
displayStatus(state);
|
|
4571
|
+
}
|
|
4572
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4573
|
+
if (!settled) {
|
|
4574
|
+
logActivity(state, {
|
|
4575
|
+
type: "info",
|
|
4576
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4577
|
+
});
|
|
4578
|
+
if (state.interactive) displayStatus(state);
|
|
4579
|
+
}
|
|
4580
|
+
}
|
|
4581
|
+
await notifyOffline(state);
|
|
2664
4582
|
if (state.connection) {
|
|
2665
4583
|
state.connection.close();
|
|
2666
4584
|
state.connection = null;
|
|
@@ -2671,13 +4589,27 @@ async function cleanup(state) {
|
|
|
2671
4589
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2672
4590
|
displayStatus(state);
|
|
2673
4591
|
} else {
|
|
2674
|
-
|
|
4592
|
+
log2(state, "Stopped OpenCode process");
|
|
2675
4593
|
}
|
|
2676
4594
|
state.opencodeProcess = null;
|
|
2677
4595
|
}
|
|
2678
4596
|
}
|
|
2679
4597
|
async function run(options) {
|
|
2680
4598
|
const interactive = isInteractive(options.json);
|
|
4599
|
+
let logLevel;
|
|
4600
|
+
try {
|
|
4601
|
+
logLevel = resolveLogLevel(options);
|
|
4602
|
+
} catch (error2) {
|
|
4603
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4604
|
+
if (options.json) {
|
|
4605
|
+
console.log(JSON.stringify({ status: "error", error: message }));
|
|
4606
|
+
} else {
|
|
4607
|
+
printError(message);
|
|
4608
|
+
}
|
|
4609
|
+
await shutdownTelemetry();
|
|
4610
|
+
process.exit(1);
|
|
4611
|
+
return;
|
|
4612
|
+
}
|
|
2681
4613
|
const state = {
|
|
2682
4614
|
agentId: options.agent || "",
|
|
2683
4615
|
agentName: null,
|
|
@@ -2686,31 +4618,38 @@ async function run(options) {
|
|
|
2686
4618
|
idleTimeout: options.idleTimeout ?? null,
|
|
2687
4619
|
json: options.json ?? false,
|
|
2688
4620
|
interactive,
|
|
4621
|
+
logLevel,
|
|
2689
4622
|
connected: false,
|
|
2690
4623
|
opencodeConnected: false,
|
|
2691
4624
|
opencodeVersion: null,
|
|
2692
4625
|
opencodeProcess: null,
|
|
2693
4626
|
connection: null,
|
|
4627
|
+
channelDriver: null,
|
|
2694
4628
|
running: true,
|
|
4629
|
+
shuttingDown: false,
|
|
2695
4630
|
activityLog: [],
|
|
2696
4631
|
messageCount: 0,
|
|
4632
|
+
lastProxiedActivityAt: null,
|
|
4633
|
+
sessionCleanupTimers: [],
|
|
2697
4634
|
authHeader: ""
|
|
2698
4635
|
};
|
|
2699
4636
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2700
|
-
|
|
4637
|
+
log2(
|
|
2701
4638
|
state,
|
|
2702
|
-
"
|
|
2703
|
-
|
|
4639
|
+
"No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
4640
|
+
"warn"
|
|
2704
4641
|
);
|
|
2705
4642
|
}
|
|
2706
4643
|
const handleSignal = async () => {
|
|
4644
|
+
if (state.shuttingDown) return;
|
|
4645
|
+
state.shuttingDown = true;
|
|
2707
4646
|
if (state.interactive) {
|
|
2708
4647
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2709
4648
|
displayStatus(state);
|
|
2710
4649
|
} else {
|
|
2711
|
-
|
|
4650
|
+
log2(state, "Shutting down...");
|
|
2712
4651
|
}
|
|
2713
|
-
await cleanup(state);
|
|
4652
|
+
await cleanup(state, { graceful: true });
|
|
2714
4653
|
await shutdownTelemetry();
|
|
2715
4654
|
process.exit(0);
|
|
2716
4655
|
};
|
|
@@ -2741,7 +4680,7 @@ async function run(options) {
|
|
|
2741
4680
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2742
4681
|
if (resolved.agent_id) {
|
|
2743
4682
|
state.agentId = resolved.agent_id;
|
|
2744
|
-
|
|
4683
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2745
4684
|
if (state.interactive && !state.json) {
|
|
2746
4685
|
logActivity(state, {
|
|
2747
4686
|
type: "info",
|
|
@@ -2804,19 +4743,19 @@ async function run(options) {
|
|
|
2804
4743
|
port: state.port,
|
|
2805
4744
|
interactive: state.interactive,
|
|
2806
4745
|
agentId: state.agentId,
|
|
2807
|
-
log: (message) =>
|
|
4746
|
+
log: (message) => log2(state, message)
|
|
2808
4747
|
});
|
|
2809
4748
|
state.port = oc.port;
|
|
2810
4749
|
state.opencodeProcess = oc.process;
|
|
2811
4750
|
state.opencodeVersion = oc.version;
|
|
2812
4751
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2813
|
-
const
|
|
2814
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
4752
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
4753
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
2815
4754
|
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
2816
4755
|
if (versionWarning) {
|
|
2817
|
-
|
|
4756
|
+
log2(state, versionWarning, "warn");
|
|
2818
4757
|
if (state.interactive && !state.json) {
|
|
2819
|
-
logActivity(state, { type: "info", message: versionWarning });
|
|
4758
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
2820
4759
|
}
|
|
2821
4760
|
}
|
|
2822
4761
|
} catch (error2) {
|
|
@@ -2830,12 +4769,20 @@ async function run(options) {
|
|
|
2830
4769
|
apiUrl: getApiUrlConfig(),
|
|
2831
4770
|
getAuthHeader: () => state.authHeader,
|
|
2832
4771
|
conversationFilter: state.conversationFilter,
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
4772
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
4773
|
+
log: (entry) => (
|
|
4774
|
+
// Thread the driver's real level straight through so `debug`/`warn`
|
|
4775
|
+
// survive the sink filter (they no longer collapse to info). `type`
|
|
4776
|
+
// stays the coarse error/non-error split the activity log renders with.
|
|
4777
|
+
logActivity(state, {
|
|
4778
|
+
type: entry.level === "error" ? "error" : "info",
|
|
4779
|
+
level: entry.level,
|
|
4780
|
+
message: entry.message,
|
|
4781
|
+
error: entry.level === "error" ? entry.message : void 0
|
|
4782
|
+
})
|
|
4783
|
+
)
|
|
2838
4784
|
});
|
|
4785
|
+
state.channelDriver = channelDriver;
|
|
2839
4786
|
const connection = new RunnerConnection({
|
|
2840
4787
|
agentId: state.agentId,
|
|
2841
4788
|
getAuthHeader: () => state.authHeader,
|
|
@@ -2849,7 +4796,11 @@ async function run(options) {
|
|
|
2849
4796
|
type: "info",
|
|
2850
4797
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2851
4798
|
});
|
|
2852
|
-
emitAgentConnected(state.agentId, {
|
|
4799
|
+
emitAgentConnected(state.agentId, {
|
|
4800
|
+
port: state.port,
|
|
4801
|
+
cli_version: getCliVersion(),
|
|
4802
|
+
opencode_version: state.opencodeVersion
|
|
4803
|
+
});
|
|
2853
4804
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2854
4805
|
if (state.interactive) displayStatus(state);
|
|
2855
4806
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2883,9 +4834,14 @@ async function run(options) {
|
|
|
2883
4834
|
logActivity(state, { type: "error", error: error2 });
|
|
2884
4835
|
if (state.interactive) displayStatus(state);
|
|
2885
4836
|
},
|
|
2886
|
-
// Web traffic is proxied transparently;
|
|
4837
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
4838
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
4839
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
4840
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
4841
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2887
4842
|
onResponse: () => {
|
|
2888
4843
|
state.opencodeConnected = true;
|
|
4844
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2889
4845
|
},
|
|
2890
4846
|
// A channel message was queued and the api-worker pinged us over the
|
|
2891
4847
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2923,10 +4879,12 @@ async function run(options) {
|
|
|
2923
4879
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
2924
4880
|
throw error2;
|
|
2925
4881
|
}
|
|
4882
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
2926
4883
|
if (!interactive || state.json) {
|
|
2927
|
-
|
|
4884
|
+
log2(state, "Driving channel messages...");
|
|
2928
4885
|
}
|
|
2929
4886
|
await driveChannels(state, channelDriver);
|
|
4887
|
+
if (state.shuttingDown) return;
|
|
2930
4888
|
await cleanup(state);
|
|
2931
4889
|
if (state.json) {
|
|
2932
4890
|
console.log(
|
|
@@ -2936,11 +4894,12 @@ async function run(options) {
|
|
|
2936
4894
|
})
|
|
2937
4895
|
);
|
|
2938
4896
|
} else if (!interactive) {
|
|
2939
|
-
|
|
4897
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2940
4898
|
}
|
|
2941
4899
|
await shutdownTelemetry();
|
|
2942
4900
|
process.exit(0);
|
|
2943
4901
|
} catch (error2) {
|
|
4902
|
+
if (state.shuttingDown) return;
|
|
2944
4903
|
await cleanup(state);
|
|
2945
4904
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
2946
4905
|
if (state.json) {
|
|
@@ -2958,8 +4917,9 @@ async function run(options) {
|
|
|
2958
4917
|
}
|
|
2959
4918
|
|
|
2960
4919
|
// src/index.ts
|
|
4920
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2961
4921
|
var program = new Command();
|
|
2962
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
4922
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2963
4923
|
"--endpoint <url>",
|
|
2964
4924
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2965
4925
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|
|
@@ -2974,15 +4934,34 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
2974
4934
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
2975
4935
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
2976
4936
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
2977
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
4937
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
4938
|
+
"--log-level <level>",
|
|
4939
|
+
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
4940
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4941
|
+
"--session-cleanup-max-age <duration>",
|
|
4942
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4943
|
+
).option(
|
|
4944
|
+
"--session-cleanup-max-count <n>",
|
|
4945
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4946
|
+
).option(
|
|
4947
|
+
"--session-cleanup-interval <duration>",
|
|
4948
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4949
|
+
).action(
|
|
2978
4950
|
(options) => {
|
|
2979
4951
|
run({
|
|
2980
4952
|
agent: options.agent,
|
|
2981
4953
|
port: parseInt(options.port, 10),
|
|
4954
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
4955
|
+
// resolveLogLevel (flag > -v > EVIDENT_LOG_LEVEL > info).
|
|
4956
|
+
logLevel: options.logLevel,
|
|
2982
4957
|
verbose: options.verbose,
|
|
2983
4958
|
conversation: options.conversation,
|
|
2984
4959
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
2985
|
-
json: options.json
|
|
4960
|
+
json: options.json,
|
|
4961
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4962
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4963
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4964
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
2986
4965
|
});
|
|
2987
4966
|
}
|
|
2988
4967
|
);
|