@evident-ai/cli 3.0.1-dev.379acd9 → 3.0.1-dev.39949ec
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1079 -339
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve2();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -471,12 +471,6 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
-
// ../../packages/types/src/opencode/index.ts
|
|
475
|
-
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
-
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
-
return `msg_${sanitized}`;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
474
|
// ../../packages/types/src/telemetry/index.ts
|
|
481
475
|
var TelemetryEventTypes = {
|
|
482
476
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -712,13 +706,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
712
706
|
if (health.healthy) {
|
|
713
707
|
return health;
|
|
714
708
|
}
|
|
715
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
716
710
|
}
|
|
717
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
718
712
|
}
|
|
719
713
|
|
|
720
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
721
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
722
716
|
function isQueueValidatedVersion(version2) {
|
|
723
717
|
if (!version2) return false;
|
|
724
718
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1074,6 +1068,94 @@ function isAssistantInFlight(m) {
|
|
|
1074
1068
|
if (completedOf(m) == null) return true;
|
|
1075
1069
|
return finishOf(m) === "tool-calls";
|
|
1076
1070
|
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1077
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1078
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1079
1161
|
if (directory && directory.trim()) {
|
|
@@ -1091,9 +1173,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1091
1173
|
const data = await response.json();
|
|
1092
1174
|
return data.id;
|
|
1093
1175
|
}
|
|
1094
|
-
|
|
1176
|
+
function messageText(m) {
|
|
1177
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
+
}
|
|
1180
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
+
const knownUserIds = new Set(
|
|
1183
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
+
);
|
|
1095
1185
|
const body = {
|
|
1096
|
-
messageID: messageId,
|
|
1097
1186
|
parts: [{ type: "text", text: content }]
|
|
1098
1187
|
};
|
|
1099
1188
|
if (options?.agent) {
|
|
@@ -1117,6 +1206,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1117
1206
|
const text = await res.text().catch(() => "");
|
|
1118
1207
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1119
1208
|
}
|
|
1209
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
+
if (after) {
|
|
1214
|
+
let best = null;
|
|
1215
|
+
for (const m of after) {
|
|
1216
|
+
if (roleOf(m) !== "user") continue;
|
|
1217
|
+
const id = idOf(m);
|
|
1218
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
+
if (messageText(m) !== content) continue;
|
|
1220
|
+
const created = createdOf(m) ?? 0;
|
|
1221
|
+
if (best === null || created > best.created) {
|
|
1222
|
+
best = { id, created };
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (best) return best.id;
|
|
1226
|
+
}
|
|
1227
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1120
1232
|
}
|
|
1121
1233
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1122
1234
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1170,6 +1282,11 @@ function messageRunState(messages, userMessageId) {
|
|
|
1170
1282
|
if (isAssistantInFlight(reply)) return "running";
|
|
1171
1283
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1172
1284
|
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1173
1290
|
function messageError(messages, userMessageId) {
|
|
1174
1291
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1175
1292
|
const error2 = errorOf(reply);
|
|
@@ -1189,30 +1306,109 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1189
1306
|
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1190
1307
|
);
|
|
1191
1308
|
}
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1309
|
+
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1201
1346
|
}
|
|
1202
1347
|
}
|
|
1203
|
-
|
|
1204
|
-
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
return toDelete;
|
|
1205
1356
|
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1208
1360
|
}
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1214
1369
|
}
|
|
1215
|
-
return
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
+
}
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1216
1412
|
}
|
|
1217
1413
|
|
|
1218
1414
|
// src/lib/tunnel/connection.ts
|
|
@@ -1291,24 +1487,26 @@ var StreamForwarder = class {
|
|
|
1291
1487
|
this.send({ type: "res_end", sid });
|
|
1292
1488
|
return;
|
|
1293
1489
|
}
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1300
1498
|
const ac = new AbortController();
|
|
1301
1499
|
let bodyPromise;
|
|
1302
1500
|
let pushBody;
|
|
1303
1501
|
let endBody;
|
|
1304
1502
|
if (has_body) {
|
|
1305
1503
|
const chunks = [];
|
|
1306
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1307
1505
|
pushBody = (buf) => {
|
|
1308
1506
|
chunks.push(buf);
|
|
1309
1507
|
};
|
|
1310
1508
|
endBody = () => {
|
|
1311
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1312
1510
|
};
|
|
1313
1511
|
});
|
|
1314
1512
|
}
|
|
@@ -1343,12 +1541,14 @@ var StreamForwarder = class {
|
|
|
1343
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1344
1542
|
});
|
|
1345
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1352
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1353
1553
|
try {
|
|
1354
1554
|
if (upstream.body) {
|
|
@@ -1424,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1424
1624
|
} = options;
|
|
1425
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1426
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1427
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1428
1628
|
const ws = new WebSocket2(url, {
|
|
1429
1629
|
headers: {
|
|
1430
1630
|
Authorization: authHeader
|
|
@@ -1489,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1489
1689
|
clearTimeout(connectionTimeout);
|
|
1490
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1491
1691
|
onConnected?.(connectedAgentId);
|
|
1492
|
-
|
|
1692
|
+
resolve2({
|
|
1493
1693
|
ws,
|
|
1494
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1495
1695
|
});
|
|
@@ -1618,10 +1818,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1618
1818
|
};
|
|
1619
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1620
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1621
|
-
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1622
1821
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1623
|
-
var
|
|
1624
|
-
var
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1625
1825
|
var ChannelAuthError = class extends Error {
|
|
1626
1826
|
constructor(message) {
|
|
1627
1827
|
super(message);
|
|
@@ -1656,13 +1856,18 @@ var ChannelDriver = class {
|
|
|
1656
1856
|
sleep;
|
|
1657
1857
|
pausedPollIntervalMs;
|
|
1658
1858
|
pausedMaxWaitMs;
|
|
1659
|
-
dispatchConfirmMs;
|
|
1660
1859
|
stuckQueuedMs;
|
|
1661
|
-
stuckQueuedRedriveMax;
|
|
1662
|
-
settleMs;
|
|
1663
1860
|
now;
|
|
1664
1861
|
/** Cache of conversationId → opencode sessionId. */
|
|
1665
1862
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
+
/**
|
|
1864
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
+
*/
|
|
1870
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1666
1871
|
/**
|
|
1667
1872
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1668
1873
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1713,6 +1918,29 @@ var ChannelDriver = class {
|
|
|
1713
1918
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1714
1919
|
*/
|
|
1715
1920
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
+
*/
|
|
1929
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1930
|
+
/**
|
|
1931
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1932
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1933
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1934
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1935
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1936
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1937
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1938
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1939
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1940
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1941
|
+
* so the NEXT tick may retry exactly once more).
|
|
1942
|
+
*/
|
|
1943
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1716
1944
|
/**
|
|
1717
1945
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1718
1946
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1720,8 +1948,33 @@ var ChannelDriver = class {
|
|
|
1720
1948
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1721
1949
|
*/
|
|
1722
1950
|
opencodeDirectory = void 0;
|
|
1951
|
+
/**
|
|
1952
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1953
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1954
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1955
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1956
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1957
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1958
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1959
|
+
*/
|
|
1960
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1723
1961
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1724
1962
|
draining = false;
|
|
1963
|
+
/**
|
|
1964
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1965
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1966
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1967
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1968
|
+
*/
|
|
1969
|
+
activeDrain = null;
|
|
1970
|
+
/**
|
|
1971
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1972
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1973
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1974
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1975
|
+
* and stops opencode.
|
|
1976
|
+
*/
|
|
1977
|
+
stopped = false;
|
|
1725
1978
|
constructor(config2) {
|
|
1726
1979
|
this.agentId = config2.agentId;
|
|
1727
1980
|
this.port = config2.port;
|
|
@@ -1735,10 +1988,7 @@ var ChannelDriver = class {
|
|
|
1735
1988
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1736
1989
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1737
1990
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1738
|
-
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1739
1991
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1740
|
-
this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
|
|
1741
|
-
this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
|
|
1742
1992
|
this.now = config2.now ?? (() => Date.now());
|
|
1743
1993
|
}
|
|
1744
1994
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1756,8 +2006,21 @@ var ChannelDriver = class {
|
|
|
1756
2006
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1757
2007
|
*/
|
|
1758
2008
|
async drainPending() {
|
|
2009
|
+
if (this.stopped) return 0;
|
|
1759
2010
|
if (this.draining) return 0;
|
|
1760
2011
|
this.draining = true;
|
|
2012
|
+
const run2 = this.runDrain();
|
|
2013
|
+
this.activeDrain = run2.then(
|
|
2014
|
+
() => {
|
|
2015
|
+
this.activeDrain = null;
|
|
2016
|
+
},
|
|
2017
|
+
() => {
|
|
2018
|
+
this.activeDrain = null;
|
|
2019
|
+
}
|
|
2020
|
+
);
|
|
2021
|
+
return run2;
|
|
2022
|
+
}
|
|
2023
|
+
async runDrain() {
|
|
1761
2024
|
let dispatched = 0;
|
|
1762
2025
|
try {
|
|
1763
2026
|
const conversations = await this.getPendingConversations();
|
|
@@ -1769,6 +2032,7 @@ var ChannelDriver = class {
|
|
|
1769
2032
|
});
|
|
1770
2033
|
}
|
|
1771
2034
|
for (const conv of conversations) {
|
|
2035
|
+
if (this.stopped) break;
|
|
1772
2036
|
dispatched += await this.processConversation(conv);
|
|
1773
2037
|
}
|
|
1774
2038
|
await this.readoptProcessing();
|
|
@@ -1789,6 +2053,73 @@ var ChannelDriver = class {
|
|
|
1789
2053
|
}
|
|
1790
2054
|
return false;
|
|
1791
2055
|
}
|
|
2056
|
+
/**
|
|
2057
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2058
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2059
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2060
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2061
|
+
*
|
|
2062
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2063
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2064
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2065
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2066
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2067
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2068
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2069
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2070
|
+
*/
|
|
2071
|
+
protectedSessionIds() {
|
|
2072
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2073
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2074
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2075
|
+
}
|
|
2076
|
+
return ids;
|
|
2077
|
+
}
|
|
2078
|
+
/**
|
|
2079
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2080
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2081
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2082
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2083
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2084
|
+
*/
|
|
2085
|
+
stop() {
|
|
2086
|
+
this.stopped = true;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2090
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2091
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2092
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2093
|
+
*
|
|
2094
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2095
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2096
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2097
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2098
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2099
|
+
* next runner start (ADR-0046).
|
|
2100
|
+
*
|
|
2101
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2102
|
+
* timeout elapsed with work still in flight.
|
|
2103
|
+
*/
|
|
2104
|
+
async waitForInFlight(timeoutMs) {
|
|
2105
|
+
const deadline = this.now() + timeoutMs;
|
|
2106
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2107
|
+
if (this.activeDrain) {
|
|
2108
|
+
let drainSettled = false;
|
|
2109
|
+
void this.activeDrain.then(() => {
|
|
2110
|
+
drainSettled = true;
|
|
2111
|
+
});
|
|
2112
|
+
while (!drainSettled) {
|
|
2113
|
+
if (this.now() >= deadline) return false;
|
|
2114
|
+
await this.sleep(step);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
while (this.hasInFlightWatchers()) {
|
|
2118
|
+
if (this.now() >= deadline) return false;
|
|
2119
|
+
await this.sleep(step);
|
|
2120
|
+
}
|
|
2121
|
+
return true;
|
|
2122
|
+
}
|
|
1792
2123
|
/**
|
|
1793
2124
|
* Await all outstanding per-session watchers (WI-3).
|
|
1794
2125
|
*
|
|
@@ -1825,15 +2156,16 @@ var ChannelDriver = class {
|
|
|
1825
2156
|
let dispatched = 0;
|
|
1826
2157
|
let skippedAlreadyDispatched = 0;
|
|
1827
2158
|
for (const message of messages) {
|
|
2159
|
+
if (this.stopped) break;
|
|
1828
2160
|
if (this.dispatched.has(message.id)) {
|
|
1829
2161
|
skippedAlreadyDispatched += 1;
|
|
1830
2162
|
continue;
|
|
1831
2163
|
}
|
|
1832
|
-
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1833
2164
|
const options = {
|
|
1834
2165
|
agent: message.opencode_agent ?? void 0,
|
|
1835
2166
|
model: message.opencode_model ?? void 0
|
|
1836
2167
|
};
|
|
2168
|
+
let opencodeMessageId;
|
|
1837
2169
|
try {
|
|
1838
2170
|
this.log({
|
|
1839
2171
|
level: "info",
|
|
@@ -1841,10 +2173,23 @@ var ChannelDriver = class {
|
|
|
1841
2173
|
conversation_id: conv.id,
|
|
1842
2174
|
message_id: message.id
|
|
1843
2175
|
});
|
|
1844
|
-
await
|
|
2176
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2177
|
+
sessionId,
|
|
2178
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2179
|
+
);
|
|
1845
2180
|
} catch (err) {
|
|
1846
2181
|
if (err instanceof ChannelAuthError) throw err;
|
|
1847
2182
|
this.dispatched.delete(message.id);
|
|
2183
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2184
|
+
this.sessions.delete(conv.id);
|
|
2185
|
+
this.log({
|
|
2186
|
+
level: "info",
|
|
2187
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2188
|
+
conversation_id: conv.id,
|
|
2189
|
+
message_id: message.id
|
|
2190
|
+
});
|
|
2191
|
+
break;
|
|
2192
|
+
}
|
|
1848
2193
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1849
2194
|
});
|
|
1850
2195
|
this.log({
|
|
@@ -1855,6 +2200,15 @@ var ChannelDriver = class {
|
|
|
1855
2200
|
});
|
|
1856
2201
|
continue;
|
|
1857
2202
|
}
|
|
2203
|
+
if (opencodeMessageId === null) {
|
|
2204
|
+
this.log({
|
|
2205
|
+
level: "error",
|
|
2206
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2207
|
+
conversation_id: conv.id,
|
|
2208
|
+
message_id: message.id
|
|
2209
|
+
});
|
|
2210
|
+
continue;
|
|
2211
|
+
}
|
|
1858
2212
|
this.dispatched.add(message.id);
|
|
1859
2213
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1860
2214
|
dispatched += 1;
|
|
@@ -1871,16 +2225,33 @@ var ChannelDriver = class {
|
|
|
1871
2225
|
return dispatched;
|
|
1872
2226
|
}
|
|
1873
2227
|
async ensureSession(conv) {
|
|
1874
|
-
const
|
|
1875
|
-
if (
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
2228
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2229
|
+
if (bound) {
|
|
2230
|
+
const exists = await sessionExists(this.port, bound);
|
|
2231
|
+
if (exists === false) {
|
|
2232
|
+
this.log({
|
|
2233
|
+
level: "info",
|
|
2234
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2235
|
+
conversation_id: conv.id
|
|
2236
|
+
});
|
|
2237
|
+
this.sessions.delete(conv.id);
|
|
2238
|
+
return this.createAndBindSession(conv.id);
|
|
2239
|
+
}
|
|
2240
|
+
this.sessions.set(conv.id, bound);
|
|
2241
|
+
return bound;
|
|
1879
2242
|
}
|
|
2243
|
+
return this.createAndBindSession(conv.id);
|
|
2244
|
+
}
|
|
2245
|
+
/**
|
|
2246
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2247
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2248
|
+
* self-heal recreate path in `ensureSession`.
|
|
2249
|
+
*/
|
|
2250
|
+
async createAndBindSession(conversationId) {
|
|
1880
2251
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1881
2252
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1882
|
-
this.sessions.set(
|
|
1883
|
-
await this.persistSession(
|
|
2253
|
+
this.sessions.set(conversationId, sessionId);
|
|
2254
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1884
2255
|
});
|
|
1885
2256
|
return sessionId;
|
|
1886
2257
|
}
|
|
@@ -1903,6 +2274,25 @@ var ChannelDriver = class {
|
|
|
1903
2274
|
// -------------------------------------------------------------------------
|
|
1904
2275
|
// Per-session watcher (WI-3)
|
|
1905
2276
|
// -------------------------------------------------------------------------
|
|
2277
|
+
/**
|
|
2278
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2279
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2280
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2281
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2282
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2283
|
+
*/
|
|
2284
|
+
dispatchLocked(sessionId, fn) {
|
|
2285
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2286
|
+
const run2 = prior.then(fn, fn);
|
|
2287
|
+
this.sessionDispatchLocks.set(
|
|
2288
|
+
sessionId,
|
|
2289
|
+
run2.then(
|
|
2290
|
+
() => void 0,
|
|
2291
|
+
() => void 0
|
|
2292
|
+
)
|
|
2293
|
+
);
|
|
2294
|
+
return run2;
|
|
2295
|
+
}
|
|
1906
2296
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1907
2297
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1908
2298
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1912,7 +2302,9 @@ var ChannelDriver = class {
|
|
|
1912
2302
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1913
2303
|
loop: null,
|
|
1914
2304
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1915
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2305
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2306
|
+
lastGoodPollAt: this.now(),
|
|
2307
|
+
hadUsablePoll: false
|
|
1916
2308
|
};
|
|
1917
2309
|
this.watchers.set(sessionId, watcher);
|
|
1918
2310
|
}
|
|
@@ -1922,24 +2314,37 @@ var ChannelDriver = class {
|
|
|
1922
2314
|
opencodeMessageId,
|
|
1923
2315
|
message,
|
|
1924
2316
|
dispatchedAt: now,
|
|
2317
|
+
processingAnchorMs: now,
|
|
1925
2318
|
deadline: now + this.pausedMaxWaitMs,
|
|
1926
2319
|
started: false,
|
|
1927
2320
|
done: false,
|
|
1928
2321
|
stuckReported: false,
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
2322
|
+
lastAliveAt: 0,
|
|
2323
|
+
aliveInFlight: false,
|
|
2324
|
+
awaitingHumanLatched: false,
|
|
2325
|
+
pausedOnQuestion: false,
|
|
2326
|
+
pausedOnPermission: false,
|
|
2327
|
+
pausedClearConfirmed: false,
|
|
2328
|
+
pausedInFlight: false,
|
|
2329
|
+
deliveryDeadlineAnchored: false
|
|
1933
2330
|
});
|
|
1934
2331
|
}
|
|
1935
2332
|
/**
|
|
1936
2333
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1937
2334
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1938
2335
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1939
|
-
* `now
|
|
1940
|
-
* (10 min after `processed_at
|
|
1941
|
-
*
|
|
1942
|
-
*
|
|
2336
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2337
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2338
|
+
*
|
|
2339
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2340
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2341
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2342
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2343
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2344
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2345
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2346
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2347
|
+
* (only the appear-guard uses it).
|
|
1943
2348
|
*
|
|
1944
2349
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1945
2350
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -1949,7 +2354,7 @@ var ChannelDriver = class {
|
|
|
1949
2354
|
* server already flipped to `processing`; the running/done transitions still
|
|
1950
2355
|
* fire from the watcher's normal branches.
|
|
1951
2356
|
*/
|
|
1952
|
-
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs
|
|
2357
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
1953
2358
|
let watcher = this.watchers.get(sessionId);
|
|
1954
2359
|
if (!watcher) {
|
|
1955
2360
|
watcher = {
|
|
@@ -1957,7 +2362,9 @@ var ChannelDriver = class {
|
|
|
1957
2362
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1958
2363
|
loop: null,
|
|
1959
2364
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1960
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2365
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2366
|
+
lastGoodPollAt: this.now(),
|
|
2367
|
+
hadUsablePoll: false
|
|
1961
2368
|
};
|
|
1962
2369
|
this.watchers.set(sessionId, watcher);
|
|
1963
2370
|
}
|
|
@@ -1966,22 +2373,33 @@ var ChannelDriver = class {
|
|
|
1966
2373
|
opencodeMessageId,
|
|
1967
2374
|
message,
|
|
1968
2375
|
dispatchedAt: this.now(),
|
|
2376
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2377
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2378
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2379
|
+
processingAnchorMs: processedAtMs,
|
|
1969
2380
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1970
2381
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1971
2382
|
started: true,
|
|
1972
2383
|
done: false,
|
|
1973
2384
|
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1974
|
-
// AND the stuck-queued observer
|
|
1975
|
-
//
|
|
1976
|
-
//
|
|
1977
|
-
// (
|
|
1978
|
-
// re-driven by the same path (ADR-0046 §c deferred the live-session redrive
|
|
1979
|
-
// to here) — `redriveAttempts` starts fresh so it gets the full budget.
|
|
2385
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2386
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2387
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2388
|
+
// (#210/#220 observability).
|
|
1980
2389
|
stuckReported: false,
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
2390
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2391
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2392
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2393
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2394
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2395
|
+
lastAliveAt: 0,
|
|
2396
|
+
aliveInFlight: false,
|
|
2397
|
+
awaitingHumanLatched: false,
|
|
2398
|
+
pausedOnQuestion: false,
|
|
2399
|
+
pausedOnPermission: false,
|
|
2400
|
+
pausedClearConfirmed: false,
|
|
2401
|
+
pausedInFlight: false,
|
|
2402
|
+
deliveryDeadlineAnchored: false
|
|
1985
2403
|
});
|
|
1986
2404
|
}
|
|
1987
2405
|
/**
|
|
@@ -2032,12 +2450,30 @@ var ChannelDriver = class {
|
|
|
2032
2450
|
messages = Array.isArray(body) ? body : null;
|
|
2033
2451
|
}
|
|
2034
2452
|
} catch {
|
|
2035
|
-
continue;
|
|
2036
2453
|
}
|
|
2454
|
+
if (messages != null && messages.length > 0) {
|
|
2455
|
+
watcher.lastGoodPollAt = this.now();
|
|
2456
|
+
watcher.hadUsablePoll = true;
|
|
2457
|
+
} else {
|
|
2458
|
+
const emptyButReachable = messages != null;
|
|
2459
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2460
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2461
|
+
continue;
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2037
2465
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2038
|
-
await this.serviceInFlightMessage(
|
|
2466
|
+
await this.serviceInFlightMessage(
|
|
2467
|
+
sessionId,
|
|
2468
|
+
watcher,
|
|
2469
|
+
inFlight,
|
|
2470
|
+
messages,
|
|
2471
|
+
openQuestions,
|
|
2472
|
+
openPermissions,
|
|
2473
|
+
questionsPolledOk,
|
|
2474
|
+
permissionsPolledOk
|
|
2475
|
+
);
|
|
2039
2476
|
}
|
|
2040
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
2041
2477
|
}
|
|
2042
2478
|
} catch (err) {
|
|
2043
2479
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2059,15 +2495,40 @@ var ChannelDriver = class {
|
|
|
2059
2495
|
});
|
|
2060
2496
|
}
|
|
2061
2497
|
}
|
|
2498
|
+
/**
|
|
2499
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2500
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2501
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2502
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2503
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2504
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2505
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2506
|
+
* or past now, so a still-ample window is left untouched.
|
|
2507
|
+
*/
|
|
2508
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2509
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2510
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2511
|
+
if (this.now() >= inFlight.deadline) {
|
|
2512
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2062
2515
|
/**
|
|
2063
2516
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2064
2517
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2065
2518
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2066
2519
|
* in-flight set on completion or timeout.
|
|
2067
2520
|
*/
|
|
2068
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2521
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
2069
2522
|
const conv = watcher.conv;
|
|
2070
2523
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2524
|
+
const id = inFlight.evidentMessageId;
|
|
2525
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2526
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2527
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2528
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2529
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2530
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2531
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2071
2532
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2072
2533
|
let claimed;
|
|
2073
2534
|
try {
|
|
@@ -2075,7 +2536,7 @@ var ChannelDriver = class {
|
|
|
2075
2536
|
conv.id,
|
|
2076
2537
|
inFlight.evidentMessageId,
|
|
2077
2538
|
sessionId,
|
|
2078
|
-
inFlight.
|
|
2539
|
+
inFlight.opencodeMessageId
|
|
2079
2540
|
);
|
|
2080
2541
|
} catch (err) {
|
|
2081
2542
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2098,6 +2559,7 @@ var ChannelDriver = class {
|
|
|
2098
2559
|
}
|
|
2099
2560
|
}
|
|
2100
2561
|
if (state === "done") {
|
|
2562
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2101
2563
|
if (!inFlight.done) {
|
|
2102
2564
|
this.log({
|
|
2103
2565
|
level: "info",
|
|
@@ -2110,7 +2572,7 @@ var ChannelDriver = class {
|
|
|
2110
2572
|
conv.id,
|
|
2111
2573
|
inFlight.evidentMessageId,
|
|
2112
2574
|
sessionId,
|
|
2113
|
-
inFlight.
|
|
2575
|
+
inFlight.opencodeMessageId
|
|
2114
2576
|
);
|
|
2115
2577
|
} catch (err) {
|
|
2116
2578
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2148,6 +2610,7 @@ var ChannelDriver = class {
|
|
|
2148
2610
|
return;
|
|
2149
2611
|
}
|
|
2150
2612
|
if (state === "failed") {
|
|
2613
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2151
2614
|
if (!inFlight.done) {
|
|
2152
2615
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2153
2616
|
this.log({
|
|
@@ -2193,215 +2656,70 @@ var ChannelDriver = class {
|
|
|
2193
2656
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2194
2657
|
return;
|
|
2195
2658
|
}
|
|
2196
|
-
if (state === "unknown") {
|
|
2197
|
-
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
2198
|
-
await this.redispatchInFlight(conv.id, sessionId, inFlight);
|
|
2199
|
-
}
|
|
2200
|
-
}
|
|
2201
2659
|
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2202
2660
|
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2203
|
-
if (state === "queued" && pastStuckBound && sessionIdle) {
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
});
|
|
2209
|
-
}
|
|
2210
|
-
const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
|
|
2211
|
-
if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
|
|
2212
|
-
const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
|
|
2213
|
-
if (removed) return;
|
|
2214
|
-
}
|
|
2661
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2662
|
+
inFlight.stuckReported = true;
|
|
2663
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2664
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2665
|
+
});
|
|
2215
2666
|
}
|
|
2216
|
-
|
|
2667
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2668
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2217
2669
|
this.log({
|
|
2218
|
-
level: "
|
|
2219
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2670
|
+
level: "error",
|
|
2671
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2220
2672
|
conversation_id: conv.id,
|
|
2221
2673
|
message_id: inFlight.evidentMessageId
|
|
2222
2674
|
});
|
|
2223
2675
|
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2224
|
-
watched_for_ms: this.now() - inFlight.
|
|
2676
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2225
2677
|
});
|
|
2226
2678
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2679
|
+
return;
|
|
2227
2680
|
}
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
* the dispatch timestamp so the guard doesn't immediately fire again.
|
|
2234
|
-
*/
|
|
2235
|
-
async redispatchInFlight(conversationId, sessionId, inFlight) {
|
|
2236
|
-
const options = {
|
|
2237
|
-
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2238
|
-
model: inFlight.message.opencode_model ?? void 0
|
|
2239
|
-
};
|
|
2240
|
-
this.log({
|
|
2241
|
-
level: "info",
|
|
2242
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
|
|
2243
|
-
message_id: inFlight.evidentMessageId
|
|
2244
|
-
});
|
|
2245
|
-
void this.postSignal(conversationId, inFlight.evidentMessageId, "redispatched");
|
|
2246
|
-
try {
|
|
2247
|
-
await sendPromptAsync(
|
|
2248
|
-
this.port,
|
|
2249
|
-
sessionId,
|
|
2250
|
-
inFlight.message.content,
|
|
2251
|
-
options,
|
|
2252
|
-
inFlight.opencodeMessageId
|
|
2253
|
-
);
|
|
2254
|
-
} catch (err) {
|
|
2255
|
-
this.log({
|
|
2256
|
-
level: "error",
|
|
2257
|
-
message: `Re-dispatch failed for message ${inFlight.evidentMessageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2258
|
-
message_id: inFlight.evidentMessageId
|
|
2681
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2682
|
+
inFlight.aliveInFlight = true;
|
|
2683
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2684
|
+
inFlight.aliveInFlight = false;
|
|
2685
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2259
2686
|
});
|
|
2260
2687
|
}
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
* the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
|
|
2272
|
-
* runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
|
|
2273
|
-
* turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
|
|
2274
|
-
* (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
|
|
2275
|
-
* dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
|
|
2276
|
-
*
|
|
2277
|
-
* So we re-`prompt_async` into the SAME opencode session (opencode-web does
|
|
2278
|
-
* exactly this and it works — UPDATE 2 in the investigation doc), preserving the
|
|
2279
|
-
* conversation's history/continuity: we do NOT create a fresh session and we do
|
|
2280
|
-
* NOT overwrite the conversation's `opencode_session_id`. Only the opencode
|
|
2281
|
-
* user-message id changes — to a fresh native id per attempt (opencode's
|
|
2282
|
-
* caller-supplied id dedup is global+permanent, so each attempt needs a never-
|
|
2283
|
-
* seen id).
|
|
2284
|
-
*
|
|
2285
|
-
* Reply correlation is PRESERVED: the native id is random and NOT re-derivable
|
|
2286
|
-
* from the row id, so the runner carries it to the server (on markProcessing/
|
|
2287
|
-
* markDone via `redriveOpencodeMessageId`), which persists it on the row and
|
|
2288
|
-
* correlates the reply by THAT id (`conversation-notification.ts`). A normal,
|
|
2289
|
-
* never-re-driven message still correlates by the derived stable id — unchanged.
|
|
2290
|
-
*
|
|
2291
|
-
* Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
|
|
2292
|
-
* FAILED (existing channel failure affordance) so the user is TOLD it could not
|
|
2293
|
-
* be answered rather than left silent, and it is removed from the in-flight set.
|
|
2294
|
-
*
|
|
2295
|
-
* @returns true if the message LEFT this watcher's in-flight set (gave up +
|
|
2296
|
-
* marked failed) — the caller then stops servicing it this tick. Returns false
|
|
2297
|
-
* when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
|
|
2298
|
-
* send failed and the next window retries).
|
|
2299
|
-
*/
|
|
2300
|
-
async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
|
|
2301
|
-
if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
|
|
2302
|
-
return true;
|
|
2303
|
-
}
|
|
2304
|
-
if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
|
|
2305
|
-
this.log({
|
|
2306
|
-
level: "error",
|
|
2307
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
|
|
2308
|
-
conversation_id: watcher.conv.id,
|
|
2309
|
-
message_id: inFlight.evidentMessageId
|
|
2310
|
-
});
|
|
2311
|
-
try {
|
|
2312
|
-
await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
|
|
2313
|
-
} catch (err) {
|
|
2314
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2315
|
-
this.log({
|
|
2316
|
-
level: "error",
|
|
2317
|
-
message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2318
|
-
conversation_id: watcher.conv.id,
|
|
2319
|
-
message_id: inFlight.evidentMessageId
|
|
2688
|
+
if (awaitingHuman) {
|
|
2689
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2690
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2691
|
+
inFlight.awaitingHumanLatched = true;
|
|
2692
|
+
}
|
|
2693
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2694
|
+
inFlight.pausedInFlight = true;
|
|
2695
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2696
|
+
inFlight.pausedInFlight = false;
|
|
2697
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2320
2698
|
});
|
|
2321
2699
|
}
|
|
2322
|
-
|
|
2323
|
-
|
|
2700
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2701
|
+
inFlight.awaitingHumanLatched = false;
|
|
2702
|
+
inFlight.pausedOnQuestion = false;
|
|
2703
|
+
inFlight.pausedOnPermission = false;
|
|
2704
|
+
inFlight.pausedClearConfirmed = false;
|
|
2324
2705
|
}
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
level: "info",
|
|
2329
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
|
|
2330
|
-
conversation_id: watcher.conv.id,
|
|
2331
|
-
message_id: inFlight.evidentMessageId
|
|
2332
|
-
});
|
|
2333
|
-
void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
|
|
2334
|
-
redrive_attempt: inFlight.redriveAttempts
|
|
2335
|
-
});
|
|
2336
|
-
const options = {
|
|
2337
|
-
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2338
|
-
model: inFlight.message.opencode_model ?? void 0
|
|
2339
|
-
};
|
|
2340
|
-
const nativeId = nativeOpencodeMessageId();
|
|
2341
|
-
try {
|
|
2342
|
-
await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
|
|
2343
|
-
} catch (err) {
|
|
2344
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2345
|
-
inFlight.redriveAttempts -= 1;
|
|
2346
|
-
this.log({
|
|
2347
|
-
level: "error",
|
|
2348
|
-
message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
|
|
2349
|
-
conversation_id: watcher.conv.id,
|
|
2350
|
-
message_id: inFlight.evidentMessageId
|
|
2351
|
-
});
|
|
2352
|
-
return false;
|
|
2353
|
-
}
|
|
2354
|
-
inFlight.opencodeMessageId = nativeId;
|
|
2355
|
-
inFlight.redriveOpencodeMessageId = nativeId;
|
|
2356
|
-
inFlight.attemptedOpencodeMessageIds.push(nativeId);
|
|
2357
|
-
inFlight.started = false;
|
|
2358
|
-
return false;
|
|
2359
|
-
}
|
|
2360
|
-
/**
|
|
2361
|
-
* Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
|
|
2362
|
-
* driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
|
|
2363
|
-
* in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
|
|
2364
|
-
* earlier re-drive attempt eventually produced is delivered) instead of
|
|
2365
|
-
* re-driving again or marking failed. Idempotent + guarded like the watcher's
|
|
2366
|
-
* done branch. `@returns` true when the message left the in-flight set (delivered
|
|
2367
|
-
* or terminally-undeliverable) — the caller then stops servicing it this tick.
|
|
2368
|
-
*
|
|
2369
|
-
* The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
|
|
2370
|
-
* branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
|
|
2371
|
-
* polls. `messageRunState === 'done'` means a completed, non-errored correlated
|
|
2372
|
-
* reply exists for that id.
|
|
2373
|
-
*/
|
|
2374
|
-
async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
|
|
2375
|
-
const completedId = inFlight.attemptedOpencodeMessageIds.find(
|
|
2376
|
-
(id) => messageRunState(messages, id) === "done"
|
|
2706
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2707
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2708
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2377
2709
|
);
|
|
2378
|
-
|
|
2379
|
-
if (inFlight.
|
|
2380
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2381
|
-
return true;
|
|
2382
|
-
}
|
|
2383
|
-
this.log({
|
|
2384
|
-
level: "info",
|
|
2385
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
|
|
2386
|
-
conversation_id: watcher.conv.id,
|
|
2387
|
-
message_id: inFlight.evidentMessageId
|
|
2388
|
-
});
|
|
2389
|
-
try {
|
|
2390
|
-
await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
|
|
2391
|
-
} catch (err) {
|
|
2392
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2710
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2711
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2393
2712
|
this.log({
|
|
2394
|
-
level: "
|
|
2395
|
-
message: `
|
|
2396
|
-
conversation_id:
|
|
2713
|
+
level: "info",
|
|
2714
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2715
|
+
conversation_id: conv.id,
|
|
2397
2716
|
message_id: inFlight.evidentMessageId
|
|
2398
2717
|
});
|
|
2718
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2719
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2720
|
+
});
|
|
2399
2721
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2400
|
-
return true;
|
|
2401
2722
|
}
|
|
2402
|
-
inFlight.done = true;
|
|
2403
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2404
|
-
return true;
|
|
2405
2723
|
}
|
|
2406
2724
|
// -------------------------------------------------------------------------
|
|
2407
2725
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -2422,12 +2740,17 @@ var ChannelDriver = class {
|
|
|
2422
2740
|
*/
|
|
2423
2741
|
async readoptProcessing() {
|
|
2424
2742
|
const rows = await this.getProcessingMessages();
|
|
2425
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2743
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2426
2744
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2427
|
-
for (const id of [
|
|
2745
|
+
for (const id of [
|
|
2746
|
+
...this.dontRedispatch,
|
|
2747
|
+
...this.doneUndeliverable,
|
|
2748
|
+
...this.readoptPollUnresolvedSignalled
|
|
2749
|
+
]) {
|
|
2428
2750
|
if (!stillProcessing.has(id)) {
|
|
2429
2751
|
const cleared = this.dontRedispatch.delete(id);
|
|
2430
2752
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2753
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2431
2754
|
if (cleared || clearedUndeliverable) {
|
|
2432
2755
|
this.log({
|
|
2433
2756
|
level: "info",
|
|
@@ -2481,8 +2804,10 @@ var ChannelDriver = class {
|
|
|
2481
2804
|
});
|
|
2482
2805
|
continue;
|
|
2483
2806
|
}
|
|
2807
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2808
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2484
2809
|
for (const row of sessionRows) {
|
|
2485
|
-
await this.readoptOne(sessionId, row, messages);
|
|
2810
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2486
2811
|
}
|
|
2487
2812
|
}
|
|
2488
2813
|
}
|
|
@@ -2490,23 +2815,21 @@ var ChannelDriver = class {
|
|
|
2490
2815
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2491
2816
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2492
2817
|
*
|
|
2493
|
-
* Branches on `messageRunState(messages,
|
|
2494
|
-
*
|
|
2495
|
-
* (
|
|
2496
|
-
*
|
|
2497
|
-
* re-
|
|
2498
|
-
* of looking `queued` under the (never-run) stable id and being re-driven AGAIN
|
|
2499
|
-
* (Bugbot #217 Finding 3 — the duplicate-turn bug):
|
|
2818
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2819
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2820
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2821
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2822
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2500
2823
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2501
2824
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2502
2825
|
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2503
2826
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2504
|
-
* tracking the
|
|
2505
|
-
* - `unknown
|
|
2827
|
+
* tracking the stored id so the reply correlates by it;
|
|
2828
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2506
2829
|
*
|
|
2507
2830
|
* Only `ChannelAuthError` propagates.
|
|
2508
2831
|
*/
|
|
2509
|
-
async readoptOne(sessionId, row, messages) {
|
|
2832
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2510
2833
|
if (this.isTracked(sessionId, row.id)) {
|
|
2511
2834
|
this.log({
|
|
2512
2835
|
level: "info",
|
|
@@ -2516,10 +2839,8 @@ var ChannelDriver = class {
|
|
|
2516
2839
|
});
|
|
2517
2840
|
return;
|
|
2518
2841
|
}
|
|
2519
|
-
const
|
|
2520
|
-
const
|
|
2521
|
-
const ocId = nativeRedriveId ?? stableId;
|
|
2522
|
-
const state = messageRunState(messages, ocId);
|
|
2842
|
+
const ocId = row.opencode_message_id;
|
|
2843
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2523
2844
|
if (state === "done") {
|
|
2524
2845
|
if (this.doneUndeliverable.has(row.id)) {
|
|
2525
2846
|
this.log({
|
|
@@ -2537,7 +2858,7 @@ var ChannelDriver = class {
|
|
|
2537
2858
|
message_id: row.id
|
|
2538
2859
|
});
|
|
2539
2860
|
try {
|
|
2540
|
-
await this.markDone(row.conversation_id, row.id, sessionId,
|
|
2861
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2541
2862
|
} catch (err) {
|
|
2542
2863
|
if (err instanceof ChannelAuthError) throw err;
|
|
2543
2864
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -2548,6 +2869,7 @@ var ChannelDriver = class {
|
|
|
2548
2869
|
conversation_id: row.conversation_id,
|
|
2549
2870
|
message_id: row.id
|
|
2550
2871
|
});
|
|
2872
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2551
2873
|
return;
|
|
2552
2874
|
}
|
|
2553
2875
|
this.log({
|
|
@@ -2559,10 +2881,11 @@ var ChannelDriver = class {
|
|
|
2559
2881
|
return;
|
|
2560
2882
|
}
|
|
2561
2883
|
this.dontRedispatch.delete(row.id);
|
|
2884
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2562
2885
|
return;
|
|
2563
2886
|
}
|
|
2564
2887
|
if (state === "failed") {
|
|
2565
|
-
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2888
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2566
2889
|
this.log({
|
|
2567
2890
|
level: "error",
|
|
2568
2891
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -2581,6 +2904,7 @@ var ChannelDriver = class {
|
|
|
2581
2904
|
conversation_id: row.conversation_id,
|
|
2582
2905
|
message_id: row.id
|
|
2583
2906
|
});
|
|
2907
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2584
2908
|
return;
|
|
2585
2909
|
}
|
|
2586
2910
|
this.log({
|
|
@@ -2592,6 +2916,7 @@ var ChannelDriver = class {
|
|
|
2592
2916
|
return;
|
|
2593
2917
|
}
|
|
2594
2918
|
this.dontRedispatch.delete(row.id);
|
|
2919
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2595
2920
|
return;
|
|
2596
2921
|
}
|
|
2597
2922
|
if (this.dontRedispatch.has(row.id)) {
|
|
@@ -2603,48 +2928,129 @@ var ChannelDriver = class {
|
|
|
2603
2928
|
});
|
|
2604
2929
|
return;
|
|
2605
2930
|
}
|
|
2606
|
-
|
|
2931
|
+
let statusReadableOngoing = null;
|
|
2932
|
+
if (state === "running" && ocId) {
|
|
2933
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2934
|
+
const shape = this.replyCompletionShape(reply);
|
|
2935
|
+
const ongoing = sessionOngoing;
|
|
2936
|
+
statusReadableOngoing = ongoing;
|
|
2937
|
+
if (ongoing === false) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2945
|
+
return;
|
|
2946
|
+
}
|
|
2947
|
+
if (ongoing === true) {
|
|
2948
|
+
this.log({
|
|
2949
|
+
level: "info",
|
|
2950
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2951
|
+
conversation_id: row.conversation_id,
|
|
2952
|
+
message_id: row.id
|
|
2953
|
+
});
|
|
2954
|
+
} else {
|
|
2955
|
+
if (shape === "b1") {
|
|
2956
|
+
this.log({
|
|
2957
|
+
level: "info",
|
|
2958
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2959
|
+
conversation_id: row.conversation_id,
|
|
2960
|
+
message_id: row.id
|
|
2961
|
+
});
|
|
2962
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2963
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2964
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2965
|
+
}
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
this.log({
|
|
2969
|
+
level: "info",
|
|
2970
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2971
|
+
conversation_id: row.conversation_id,
|
|
2972
|
+
message_id: row.id
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2977
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2978
|
+
if (descendantAlive === true) {
|
|
2979
|
+
this.log({
|
|
2980
|
+
level: "info",
|
|
2981
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2982
|
+
conversation_id: row.conversation_id,
|
|
2983
|
+
message_id: row.id
|
|
2984
|
+
});
|
|
2985
|
+
} else {
|
|
2986
|
+
this.log({
|
|
2987
|
+
level: "info",
|
|
2988
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2989
|
+
conversation_id: row.conversation_id,
|
|
2990
|
+
message_id: row.id
|
|
2991
|
+
});
|
|
2992
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2607
2997
|
const conv = this.convForRow(sessionId, row);
|
|
2608
2998
|
const message = this.queuedMessageForRow(row);
|
|
2609
|
-
this.registerReadopted(
|
|
2610
|
-
conv,
|
|
2611
|
-
sessionId,
|
|
2612
|
-
message,
|
|
2613
|
-
ocId,
|
|
2614
|
-
this.processedAtMs(row),
|
|
2615
|
-
nativeRedriveId
|
|
2616
|
-
);
|
|
2999
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2617
3000
|
this.dispatched.add(row.id);
|
|
2618
3001
|
this.readopted.add(row.id);
|
|
2619
3002
|
this.ensureWatcherRunning(sessionId);
|
|
2620
3003
|
this.log({
|
|
2621
3004
|
level: "info",
|
|
2622
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (
|
|
3005
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2623
3006
|
conversation_id: row.conversation_id,
|
|
2624
3007
|
message_id: row.id
|
|
2625
3008
|
});
|
|
3009
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
2626
3010
|
return;
|
|
2627
3011
|
}
|
|
2628
3012
|
await this.forceReadoptRun(sessionId, row);
|
|
2629
3013
|
}
|
|
2630
3014
|
/**
|
|
2631
|
-
* Re-dispatch an orphaned (`unknown
|
|
3015
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2632
3016
|
*
|
|
2633
|
-
*
|
|
2634
|
-
*
|
|
2635
|
-
*
|
|
2636
|
-
*
|
|
2637
|
-
* (which hangs off the stable id) is found and delivered. The residual
|
|
2638
|
-
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2639
|
-
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2640
|
-
* to swallow the duplicate.
|
|
3017
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
3018
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
3019
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
3020
|
+
* correlates server-side.
|
|
2641
3021
|
*
|
|
2642
|
-
*
|
|
2643
|
-
*
|
|
3022
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3023
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3024
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3025
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3026
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3027
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3028
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3029
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3030
|
+
* may retry exactly once more).
|
|
3031
|
+
*
|
|
3032
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2644
3033
|
* `processed_at` (Invariant 1).
|
|
2645
3034
|
*/
|
|
2646
3035
|
async forceReadoptRun(sessionId, row) {
|
|
2647
|
-
|
|
3036
|
+
if (this.stopped) {
|
|
3037
|
+
this.log({
|
|
3038
|
+
level: "info",
|
|
3039
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3040
|
+
conversation_id: row.conversation_id,
|
|
3041
|
+
message_id: row.id
|
|
3042
|
+
});
|
|
3043
|
+
return;
|
|
3044
|
+
}
|
|
3045
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3046
|
+
this.log({
|
|
3047
|
+
level: "info",
|
|
3048
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3049
|
+
conversation_id: row.conversation_id,
|
|
3050
|
+
message_id: row.id
|
|
3051
|
+
});
|
|
3052
|
+
return;
|
|
3053
|
+
}
|
|
2648
3054
|
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2649
3055
|
this.dontRedispatch.add(row.id);
|
|
2650
3056
|
this.log({
|
|
@@ -2653,6 +3059,7 @@ var ChannelDriver = class {
|
|
|
2653
3059
|
conversation_id: row.conversation_id,
|
|
2654
3060
|
message_id: row.id
|
|
2655
3061
|
});
|
|
3062
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
2656
3063
|
return;
|
|
2657
3064
|
}
|
|
2658
3065
|
const options = {
|
|
@@ -2661,13 +3068,19 @@ var ChannelDriver = class {
|
|
|
2661
3068
|
};
|
|
2662
3069
|
this.log({
|
|
2663
3070
|
level: "info",
|
|
2664
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching
|
|
3071
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2665
3072
|
conversation_id: row.conversation_id,
|
|
2666
3073
|
message_id: row.id
|
|
2667
3074
|
});
|
|
3075
|
+
this.awaitingReadopt.add(row.id);
|
|
3076
|
+
let ocId;
|
|
2668
3077
|
try {
|
|
2669
|
-
await
|
|
3078
|
+
ocId = await this.dispatchLocked(
|
|
3079
|
+
sessionId,
|
|
3080
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
3081
|
+
);
|
|
2670
3082
|
} catch (err) {
|
|
3083
|
+
this.awaitingReadopt.delete(row.id);
|
|
2671
3084
|
if (err instanceof ChannelAuthError) throw err;
|
|
2672
3085
|
this.log({
|
|
2673
3086
|
level: "error",
|
|
@@ -2675,6 +3088,18 @@ var ChannelDriver = class {
|
|
|
2675
3088
|
conversation_id: row.conversation_id,
|
|
2676
3089
|
message_id: row.id
|
|
2677
3090
|
});
|
|
3091
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3092
|
+
return;
|
|
3093
|
+
}
|
|
3094
|
+
if (ocId === null) {
|
|
3095
|
+
this.awaitingReadopt.delete(row.id);
|
|
3096
|
+
this.log({
|
|
3097
|
+
level: "error",
|
|
3098
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3099
|
+
conversation_id: row.conversation_id,
|
|
3100
|
+
message_id: row.id
|
|
3101
|
+
});
|
|
3102
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2678
3103
|
return;
|
|
2679
3104
|
}
|
|
2680
3105
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -2682,7 +3107,9 @@ var ChannelDriver = class {
|
|
|
2682
3107
|
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2683
3108
|
this.dispatched.add(row.id);
|
|
2684
3109
|
this.readopted.add(row.id);
|
|
3110
|
+
this.awaitingReadopt.delete(row.id);
|
|
2685
3111
|
this.ensureWatcherRunning(sessionId);
|
|
3112
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
2686
3113
|
}
|
|
2687
3114
|
/**
|
|
2688
3115
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -2774,21 +3201,41 @@ var ChannelDriver = class {
|
|
|
2774
3201
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2775
3202
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2776
3203
|
* oldest running message.
|
|
3204
|
+
*
|
|
3205
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3206
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3207
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3208
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3209
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3210
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3211
|
+
* even after it was already surfaced to the channel.
|
|
2777
3212
|
*/
|
|
2778
3213
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3214
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3215
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3216
|
+
let questionsPolledOk = true;
|
|
3217
|
+
let permissionsPolledOk = true;
|
|
2779
3218
|
let questions = [];
|
|
2780
3219
|
try {
|
|
2781
3220
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2782
3221
|
if (res.ok) {
|
|
2783
3222
|
const body = await res.json();
|
|
2784
|
-
|
|
3223
|
+
if (Array.isArray(body)) {
|
|
3224
|
+
questions = body;
|
|
3225
|
+
} else {
|
|
3226
|
+
questionsPolledOk = false;
|
|
3227
|
+
}
|
|
3228
|
+
} else {
|
|
3229
|
+
questionsPolledOk = false;
|
|
2785
3230
|
}
|
|
2786
3231
|
} catch {
|
|
3232
|
+
questionsPolledOk = false;
|
|
2787
3233
|
}
|
|
2788
3234
|
for (const q of questions) {
|
|
2789
|
-
if (q.sessionID
|
|
2790
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3235
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2791
3236
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3237
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3238
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2792
3239
|
const reported = await this.reportInteraction(
|
|
2793
3240
|
watcher.conv.id,
|
|
2794
3241
|
"question",
|
|
@@ -2802,14 +3249,22 @@ var ChannelDriver = class {
|
|
|
2802
3249
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2803
3250
|
if (res.ok) {
|
|
2804
3251
|
const body = await res.json();
|
|
2805
|
-
|
|
3252
|
+
if (Array.isArray(body)) {
|
|
3253
|
+
permissions = body;
|
|
3254
|
+
} else {
|
|
3255
|
+
permissionsPolledOk = false;
|
|
3256
|
+
}
|
|
3257
|
+
} else {
|
|
3258
|
+
permissionsPolledOk = false;
|
|
2806
3259
|
}
|
|
2807
3260
|
} catch {
|
|
3261
|
+
permissionsPolledOk = false;
|
|
2808
3262
|
}
|
|
2809
3263
|
for (const p of permissions) {
|
|
2810
|
-
if (p.sessionID
|
|
2811
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3264
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2812
3265
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3266
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3267
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2813
3268
|
const reported = await this.reportInteraction(
|
|
2814
3269
|
watcher.conv.id,
|
|
2815
3270
|
"permission",
|
|
@@ -2818,6 +3273,128 @@ var ChannelDriver = class {
|
|
|
2818
3273
|
);
|
|
2819
3274
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2820
3275
|
}
|
|
3276
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3277
|
+
}
|
|
3278
|
+
/**
|
|
3279
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3280
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3281
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3282
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3283
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3284
|
+
*
|
|
3285
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3286
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3287
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3288
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3289
|
+
*/
|
|
3290
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3291
|
+
let current = sessionId;
|
|
3292
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3293
|
+
if (current === rootSessionId) return true;
|
|
3294
|
+
const parent = await this.resolveSessionParent(current);
|
|
3295
|
+
if (parent === null || parent === void 0) return false;
|
|
3296
|
+
current = parent;
|
|
3297
|
+
}
|
|
3298
|
+
return false;
|
|
3299
|
+
}
|
|
3300
|
+
/**
|
|
3301
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3302
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3303
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3304
|
+
* caching a wrong answer — the next tick retries).
|
|
3305
|
+
*/
|
|
3306
|
+
async resolveSessionParent(sessionId) {
|
|
3307
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3308
|
+
if (cached !== void 0) return cached;
|
|
3309
|
+
let parent = void 0;
|
|
3310
|
+
try {
|
|
3311
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3312
|
+
if (res.ok) {
|
|
3313
|
+
const body = await res.json();
|
|
3314
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3315
|
+
}
|
|
3316
|
+
} catch {
|
|
3317
|
+
parent = void 0;
|
|
3318
|
+
}
|
|
3319
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3320
|
+
return parent;
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3324
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3325
|
+
*
|
|
3326
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3327
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3328
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3329
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3330
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3331
|
+
* provably in flight at the exact moment of recovery.
|
|
3332
|
+
*
|
|
3333
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3334
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3335
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3336
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3337
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3338
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3339
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3340
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3341
|
+
*
|
|
3342
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3343
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3344
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3345
|
+
* case), OR no descendant is found at all.
|
|
3346
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3347
|
+
*
|
|
3348
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3349
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3350
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3351
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3352
|
+
* true/false/null faithfully.
|
|
3353
|
+
*
|
|
3354
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3355
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3356
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3357
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3358
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3359
|
+
* `SessionStatus` only.
|
|
3360
|
+
*/
|
|
3361
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3362
|
+
const sessions = await listSessions(this.port);
|
|
3363
|
+
if (!sessions) {
|
|
3364
|
+
this.log({
|
|
3365
|
+
level: "error",
|
|
3366
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3367
|
+
});
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
for (const candidate of sessions) {
|
|
3371
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3372
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3373
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3374
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3375
|
+
return true;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
return false;
|
|
3379
|
+
}
|
|
3380
|
+
/**
|
|
3381
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3382
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3383
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3384
|
+
* the aborted-in-flight production bug after a restart.
|
|
3385
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3386
|
+
* (the sub-agent preamble — #253's shape).
|
|
3387
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3388
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3389
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3390
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3391
|
+
*/
|
|
3392
|
+
replyCompletionShape(reply) {
|
|
3393
|
+
if (!reply) return "other";
|
|
3394
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3395
|
+
if (completed == null) return "b1";
|
|
3396
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3397
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2821
3398
|
}
|
|
2822
3399
|
/**
|
|
2823
3400
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -3050,6 +3627,12 @@ var ChannelDriver = class {
|
|
|
3050
3627
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3051
3628
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3052
3629
|
* context (no silent catch, per development-workflow).
|
|
3630
|
+
*
|
|
3631
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3632
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3633
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3634
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3635
|
+
* leaves liveness").
|
|
3053
3636
|
*/
|
|
3054
3637
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
3055
3638
|
try {
|
|
@@ -3068,7 +3651,9 @@ var ChannelDriver = class {
|
|
|
3068
3651
|
conversation_id: conversationId,
|
|
3069
3652
|
message_id: messageId
|
|
3070
3653
|
});
|
|
3654
|
+
return false;
|
|
3071
3655
|
}
|
|
3656
|
+
return true;
|
|
3072
3657
|
} catch (err) {
|
|
3073
3658
|
this.log({
|
|
3074
3659
|
level: "error",
|
|
@@ -3076,6 +3661,7 @@ var ChannelDriver = class {
|
|
|
3076
3661
|
conversation_id: conversationId,
|
|
3077
3662
|
message_id: messageId
|
|
3078
3663
|
});
|
|
3664
|
+
return false;
|
|
3079
3665
|
}
|
|
3080
3666
|
}
|
|
3081
3667
|
async persistSession(conversationId, sessionId) {
|
|
@@ -3348,6 +3934,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3348
3934
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
3349
3935
|
}
|
|
3350
3936
|
}
|
|
3937
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3938
|
+
const apiUrl = getApiUrlConfig();
|
|
3939
|
+
try {
|
|
3940
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3941
|
+
method: "POST",
|
|
3942
|
+
headers: { Authorization: authHeader }
|
|
3943
|
+
});
|
|
3944
|
+
if (!response.ok) {
|
|
3945
|
+
const serverMessage = await readErrorMessage(response);
|
|
3946
|
+
return {
|
|
3947
|
+
ok: false,
|
|
3948
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3949
|
+
};
|
|
3950
|
+
}
|
|
3951
|
+
return { ok: true };
|
|
3952
|
+
} catch (error2) {
|
|
3953
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3351
3956
|
async function getAgentInfo(agentId, authHeader) {
|
|
3352
3957
|
const apiUrl = getApiUrlConfig();
|
|
3353
3958
|
try {
|
|
@@ -3394,7 +3999,7 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3394
3999
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
3395
4000
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
3396
4001
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3397
|
-
var
|
|
4002
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3398
4003
|
function log2(state, message, isError = false) {
|
|
3399
4004
|
if (state.json) {
|
|
3400
4005
|
console.log(
|
|
@@ -3549,7 +4154,7 @@ async function driveChannels(state, driver) {
|
|
|
3549
4154
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
3550
4155
|
if (state.interactive) displayStatus(state);
|
|
3551
4156
|
}
|
|
3552
|
-
await new Promise((
|
|
4157
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
3553
4158
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
3554
4159
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
3555
4160
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -3560,8 +4165,122 @@ async function driveChannels(state, driver) {
|
|
|
3560
4165
|
}
|
|
3561
4166
|
}
|
|
3562
4167
|
}
|
|
3563
|
-
|
|
4168
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4169
|
+
async function runSweep(state, driver, config2) {
|
|
4170
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4171
|
+
try {
|
|
4172
|
+
const sessions = await listSessions(state.port);
|
|
4173
|
+
if (sessions === null) {
|
|
4174
|
+
logActivity(state, {
|
|
4175
|
+
type: "info",
|
|
4176
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4177
|
+
});
|
|
4178
|
+
return;
|
|
4179
|
+
}
|
|
4180
|
+
const toDelete = selectSessionsToDelete(
|
|
4181
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4182
|
+
{
|
|
4183
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4184
|
+
maxCount: config2.maxCount,
|
|
4185
|
+
nowMs: Date.now(),
|
|
4186
|
+
protectedIds: driver.protectedSessionIds()
|
|
4187
|
+
}
|
|
4188
|
+
);
|
|
4189
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4190
|
+
let deleted = 0;
|
|
4191
|
+
let failed = 0;
|
|
4192
|
+
let skippedNewlyActive = 0;
|
|
4193
|
+
for (const id of toDelete) {
|
|
4194
|
+
if (protectedNow.has(id)) {
|
|
4195
|
+
skippedNewlyActive++;
|
|
4196
|
+
logActivity(state, {
|
|
4197
|
+
type: "info",
|
|
4198
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4199
|
+
});
|
|
4200
|
+
continue;
|
|
4201
|
+
}
|
|
4202
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4203
|
+
else failed++;
|
|
4204
|
+
}
|
|
4205
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4206
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4207
|
+
logActivity(state, {
|
|
4208
|
+
type: "info",
|
|
4209
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4210
|
+
});
|
|
4211
|
+
} catch (error2) {
|
|
4212
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4213
|
+
logActivity(state, {
|
|
4214
|
+
type: "error",
|
|
4215
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4216
|
+
});
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4220
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4221
|
+
{
|
|
4222
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4223
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4224
|
+
interval: options.sessionCleanupInterval
|
|
4225
|
+
},
|
|
4226
|
+
process.env
|
|
4227
|
+
);
|
|
4228
|
+
for (const warning2 of config2.warnings) {
|
|
4229
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4230
|
+
}
|
|
4231
|
+
if (!config2.enabled) return;
|
|
4232
|
+
logActivity(state, {
|
|
4233
|
+
type: "info",
|
|
4234
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4235
|
+
});
|
|
4236
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4237
|
+
const firstSweep = setTimeout(
|
|
4238
|
+
() => void runSweep(state, driver, config2),
|
|
4239
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4240
|
+
);
|
|
4241
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4242
|
+
}
|
|
4243
|
+
async function notifyOffline(state) {
|
|
4244
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4245
|
+
if (!state.connected) {
|
|
4246
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4247
|
+
return;
|
|
4248
|
+
}
|
|
4249
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4250
|
+
if (result.ok) {
|
|
4251
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4252
|
+
} else {
|
|
4253
|
+
logActivity(state, {
|
|
4254
|
+
type: "error",
|
|
4255
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4256
|
+
});
|
|
4257
|
+
if (state.interactive) displayStatus(state);
|
|
4258
|
+
}
|
|
4259
|
+
}
|
|
4260
|
+
async function cleanup(state, opts = {}) {
|
|
3564
4261
|
state.running = false;
|
|
4262
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4263
|
+
clearInterval(timer);
|
|
4264
|
+
clearTimeout(timer);
|
|
4265
|
+
}
|
|
4266
|
+
state.sessionCleanupTimers = [];
|
|
4267
|
+
if (opts.graceful && state.channelDriver) {
|
|
4268
|
+
state.channelDriver.stop();
|
|
4269
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4270
|
+
if (state.interactive) {
|
|
4271
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4272
|
+
displayStatus(state);
|
|
4273
|
+
}
|
|
4274
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4275
|
+
if (!settled) {
|
|
4276
|
+
logActivity(state, {
|
|
4277
|
+
type: "info",
|
|
4278
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4279
|
+
});
|
|
4280
|
+
if (state.interactive) displayStatus(state);
|
|
4281
|
+
}
|
|
4282
|
+
}
|
|
4283
|
+
await notifyOffline(state);
|
|
3565
4284
|
if (state.connection) {
|
|
3566
4285
|
state.connection.close();
|
|
3567
4286
|
state.connection = null;
|
|
@@ -3592,10 +4311,13 @@ async function run(options) {
|
|
|
3592
4311
|
opencodeVersion: null,
|
|
3593
4312
|
opencodeProcess: null,
|
|
3594
4313
|
connection: null,
|
|
4314
|
+
channelDriver: null,
|
|
3595
4315
|
running: true,
|
|
4316
|
+
shuttingDown: false,
|
|
3596
4317
|
activityLog: [],
|
|
3597
4318
|
messageCount: 0,
|
|
3598
4319
|
lastProxiedActivityAt: null,
|
|
4320
|
+
sessionCleanupTimers: [],
|
|
3599
4321
|
authHeader: ""
|
|
3600
4322
|
};
|
|
3601
4323
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -3606,13 +4328,15 @@ async function run(options) {
|
|
|
3606
4328
|
);
|
|
3607
4329
|
}
|
|
3608
4330
|
const handleSignal = async () => {
|
|
4331
|
+
if (state.shuttingDown) return;
|
|
4332
|
+
state.shuttingDown = true;
|
|
3609
4333
|
if (state.interactive) {
|
|
3610
4334
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
3611
4335
|
displayStatus(state);
|
|
3612
4336
|
} else {
|
|
3613
4337
|
log2(state, "Shutting down...");
|
|
3614
4338
|
}
|
|
3615
|
-
await cleanup(state);
|
|
4339
|
+
await cleanup(state, { graceful: true });
|
|
3616
4340
|
await shutdownTelemetry();
|
|
3617
4341
|
process.exit(0);
|
|
3618
4342
|
};
|
|
@@ -3733,13 +4457,13 @@ async function run(options) {
|
|
|
3733
4457
|
getAuthHeader: () => state.authHeader,
|
|
3734
4458
|
conversationFilter: state.conversationFilter,
|
|
3735
4459
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3736
|
-
settleMs: CHANNEL_SETTLE_MS,
|
|
3737
4460
|
log: (entry) => logActivity(state, {
|
|
3738
4461
|
type: entry.level === "error" ? "error" : "info",
|
|
3739
4462
|
message: entry.message,
|
|
3740
4463
|
error: entry.level === "error" ? entry.message : void 0
|
|
3741
4464
|
})
|
|
3742
4465
|
});
|
|
4466
|
+
state.channelDriver = channelDriver;
|
|
3743
4467
|
const connection = new RunnerConnection({
|
|
3744
4468
|
agentId: state.agentId,
|
|
3745
4469
|
getAuthHeader: () => state.authHeader,
|
|
@@ -3836,10 +4560,12 @@ async function run(options) {
|
|
|
3836
4560
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3837
4561
|
throw error2;
|
|
3838
4562
|
}
|
|
4563
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3839
4564
|
if (!interactive || state.json) {
|
|
3840
4565
|
log2(state, "Driving channel messages...");
|
|
3841
4566
|
}
|
|
3842
4567
|
await driveChannels(state, channelDriver);
|
|
4568
|
+
if (state.shuttingDown) return;
|
|
3843
4569
|
await cleanup(state);
|
|
3844
4570
|
if (state.json) {
|
|
3845
4571
|
console.log(
|
|
@@ -3854,6 +4580,7 @@ async function run(options) {
|
|
|
3854
4580
|
await shutdownTelemetry();
|
|
3855
4581
|
process.exit(0);
|
|
3856
4582
|
} catch (error2) {
|
|
4583
|
+
if (state.shuttingDown) return;
|
|
3857
4584
|
await cleanup(state);
|
|
3858
4585
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3859
4586
|
if (state.json) {
|
|
@@ -3888,7 +4615,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3888
4615
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3889
4616
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3890
4617
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3891
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4618
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4619
|
+
"--session-cleanup-max-age <duration>",
|
|
4620
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4621
|
+
).option(
|
|
4622
|
+
"--session-cleanup-max-count <n>",
|
|
4623
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4624
|
+
).option(
|
|
4625
|
+
"--session-cleanup-interval <duration>",
|
|
4626
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4627
|
+
).action(
|
|
3892
4628
|
(options) => {
|
|
3893
4629
|
run({
|
|
3894
4630
|
agent: options.agent,
|
|
@@ -3896,7 +4632,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3896
4632
|
verbose: options.verbose,
|
|
3897
4633
|
conversation: options.conversation,
|
|
3898
4634
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3899
|
-
json: options.json
|
|
4635
|
+
json: options.json,
|
|
4636
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4637
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4638
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4639
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3900
4640
|
});
|
|
3901
4641
|
}
|
|
3902
4642
|
);
|