@evident-ai/cli 3.0.1-dev.2950803 → 3.0.1-dev.379acd9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +339 -1079
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -471,6 +471,12 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
+
// ../../packages/types/src/opencode/index.ts
|
|
475
|
+
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
+
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
+
return `msg_${sanitized}`;
|
|
478
|
+
}
|
|
479
|
+
|
|
474
480
|
// ../../packages/types/src/telemetry/index.ts
|
|
475
481
|
var TelemetryEventTypes = {
|
|
476
482
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -706,13 +712,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
706
712
|
if (health.healthy) {
|
|
707
713
|
return health;
|
|
708
714
|
}
|
|
709
|
-
await new Promise((
|
|
715
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
710
716
|
}
|
|
711
717
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
712
718
|
}
|
|
713
719
|
|
|
714
720
|
// src/lib/opencode/opencode-version-gate.ts
|
|
715
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"
|
|
721
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
716
722
|
function isQueueValidatedVersion(version2) {
|
|
717
723
|
if (!version2) return false;
|
|
718
724
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1068,94 +1074,6 @@ function isAssistantInFlight(m) {
|
|
|
1068
1074
|
if (completedOf(m) == null) return true;
|
|
1069
1075
|
return finishOf(m) === "tool-calls";
|
|
1070
1076
|
}
|
|
1071
|
-
async function getSessionMessages(port, sessionId) {
|
|
1072
|
-
try {
|
|
1073
|
-
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
-
if (!res.ok) return null;
|
|
1075
|
-
const body = await res.json();
|
|
1076
|
-
return Array.isArray(body) ? body : null;
|
|
1077
|
-
} catch {
|
|
1078
|
-
return null;
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
function isSessionActivelyGenerating(messages) {
|
|
1082
|
-
if (!messages || messages.length === 0) return false;
|
|
1083
|
-
const last = messages[messages.length - 1];
|
|
1084
|
-
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
-
return completedOf(last) == null;
|
|
1086
|
-
}
|
|
1087
|
-
function sessionLastActivityMs(session) {
|
|
1088
|
-
const candidates = [
|
|
1089
|
-
session.time?.updated,
|
|
1090
|
-
session.time?.created,
|
|
1091
|
-
session.time_updated,
|
|
1092
|
-
session.time_created,
|
|
1093
|
-
session.updated,
|
|
1094
|
-
session.created
|
|
1095
|
-
];
|
|
1096
|
-
for (const c of candidates) {
|
|
1097
|
-
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
-
}
|
|
1099
|
-
return null;
|
|
1100
|
-
}
|
|
1101
|
-
async function listSessions(port) {
|
|
1102
|
-
try {
|
|
1103
|
-
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
-
if (!res.ok) return null;
|
|
1105
|
-
const body = await res.json();
|
|
1106
|
-
return Array.isArray(body) ? body : null;
|
|
1107
|
-
} catch {
|
|
1108
|
-
return null;
|
|
1109
|
-
}
|
|
1110
|
-
}
|
|
1111
|
-
async function deleteSession(port, id) {
|
|
1112
|
-
try {
|
|
1113
|
-
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
-
return res.status >= 200 && res.status < 300;
|
|
1115
|
-
} catch {
|
|
1116
|
-
return false;
|
|
1117
|
-
}
|
|
1118
|
-
}
|
|
1119
|
-
async function sessionExists(port, id) {
|
|
1120
|
-
try {
|
|
1121
|
-
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
-
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
-
if (res.status === 404) return false;
|
|
1124
|
-
return null;
|
|
1125
|
-
} catch {
|
|
1126
|
-
return null;
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
async function getSessionStatuses(port) {
|
|
1130
|
-
try {
|
|
1131
|
-
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
-
if (!res.ok) {
|
|
1133
|
-
console.error(
|
|
1134
|
-
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
-
);
|
|
1136
|
-
return null;
|
|
1137
|
-
}
|
|
1138
|
-
const body = await res.json();
|
|
1139
|
-
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
-
console.error(
|
|
1141
|
-
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
-
);
|
|
1143
|
-
return null;
|
|
1144
|
-
}
|
|
1145
|
-
return body;
|
|
1146
|
-
} catch (err) {
|
|
1147
|
-
console.error(
|
|
1148
|
-
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
-
);
|
|
1150
|
-
return null;
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
async function isSessionOngoing(port, id) {
|
|
1154
|
-
const map = await getSessionStatuses(port);
|
|
1155
|
-
if (map == null) return null;
|
|
1156
|
-
const entry = map[id];
|
|
1157
|
-
return entry != null && entry.type !== "idle";
|
|
1158
|
-
}
|
|
1159
1077
|
async function createOpenCodeSession(port, directory) {
|
|
1160
1078
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1161
1079
|
if (directory && directory.trim()) {
|
|
@@ -1173,16 +1091,9 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1173
1091
|
const data = await response.json();
|
|
1174
1092
|
return data.id;
|
|
1175
1093
|
}
|
|
1176
|
-
function
|
|
1177
|
-
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
-
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
-
}
|
|
1180
|
-
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
-
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
-
const knownUserIds = new Set(
|
|
1183
|
-
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
-
);
|
|
1094
|
+
async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
1185
1095
|
const body = {
|
|
1096
|
+
messageID: messageId,
|
|
1186
1097
|
parts: [{ type: "text", text: content }]
|
|
1187
1098
|
};
|
|
1188
1099
|
if (options?.agent) {
|
|
@@ -1206,29 +1117,6 @@ async function sendPromptAsync(port, sessionId, content, options) {
|
|
|
1206
1117
|
const text = await res.text().catch(() => "");
|
|
1207
1118
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1208
1119
|
}
|
|
1209
|
-
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
-
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
-
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
-
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
-
if (after) {
|
|
1214
|
-
let best = null;
|
|
1215
|
-
for (const m of after) {
|
|
1216
|
-
if (roleOf(m) !== "user") continue;
|
|
1217
|
-
const id = idOf(m);
|
|
1218
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
-
if (messageText(m) !== content) continue;
|
|
1220
|
-
const created = createdOf(m) ?? 0;
|
|
1221
|
-
if (best === null || created > best.created) {
|
|
1222
|
-
best = { id, created };
|
|
1223
|
-
}
|
|
1224
|
-
}
|
|
1225
|
-
if (best) return best.id;
|
|
1226
|
-
}
|
|
1227
|
-
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
-
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
-
}
|
|
1230
|
-
}
|
|
1231
|
-
return null;
|
|
1232
1120
|
}
|
|
1233
1121
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1234
1122
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1282,11 +1170,6 @@ function messageRunState(messages, userMessageId) {
|
|
|
1282
1170
|
if (isAssistantInFlight(reply)) return "running";
|
|
1283
1171
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1284
1172
|
}
|
|
1285
|
-
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
-
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
-
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
-
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
-
}
|
|
1290
1173
|
function messageError(messages, userMessageId) {
|
|
1291
1174
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1292
1175
|
const error2 = errorOf(reply);
|
|
@@ -1306,109 +1189,30 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1306
1189
|
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1307
1190
|
);
|
|
1308
1191
|
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
const trimmed = input.trim();
|
|
1319
|
-
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
-
if (!match) {
|
|
1321
|
-
throw new Error(
|
|
1322
|
-
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
-
);
|
|
1324
|
-
}
|
|
1325
|
-
const value = Number(match[1]);
|
|
1326
|
-
if (value <= 0) {
|
|
1327
|
-
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
-
}
|
|
1329
|
-
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
-
}
|
|
1331
|
-
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
-
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
-
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
-
const ageEligible = (s) => {
|
|
1335
|
-
if (maxAgeMs === void 0) return false;
|
|
1336
|
-
if (s.lastActivityMs === null) return true;
|
|
1337
|
-
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
-
};
|
|
1339
|
-
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
-
if (maxCount !== void 0) {
|
|
1341
|
-
const byActivityDesc = [...sessions].sort(
|
|
1342
|
-
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
-
);
|
|
1344
|
-
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
-
countEligibleIds.add(s.id);
|
|
1192
|
+
function isSessionSettled(messages, now, settleMs) {
|
|
1193
|
+
if (!messages || messages.length === 0) return true;
|
|
1194
|
+
let newestAssistantCreated = null;
|
|
1195
|
+
for (const m of messages) {
|
|
1196
|
+
if (roleOf(m) !== "assistant") continue;
|
|
1197
|
+
if (isAssistantInFlight(m)) return false;
|
|
1198
|
+
const created = createdOf(m);
|
|
1199
|
+
if (typeof created === "number" && (newestAssistantCreated === null || created > newestAssistantCreated)) {
|
|
1200
|
+
newestAssistantCreated = created;
|
|
1346
1201
|
}
|
|
1347
1202
|
}
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
if (protectedIds.has(s.id)) continue;
|
|
1351
|
-
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
-
toDelete.push(s.id);
|
|
1353
|
-
}
|
|
1354
|
-
}
|
|
1355
|
-
return toDelete;
|
|
1203
|
+
if (newestAssistantCreated === null) return true;
|
|
1204
|
+
return now - newestAssistantCreated >= settleMs;
|
|
1356
1205
|
}
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
return flag ?? envValue ?? fallback;
|
|
1206
|
+
function opencodeMessageIdFor2(queuedMessageId) {
|
|
1207
|
+
return opencodeMessageIdFor(queuedMessageId);
|
|
1360
1208
|
}
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
const value = Number(trimmed);
|
|
1367
|
-
if (value <= 0) {
|
|
1368
|
-
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1209
|
+
var NATIVE_ID_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
1210
|
+
function nativeOpencodeMessageId() {
|
|
1211
|
+
let id = "msg_";
|
|
1212
|
+
for (let i = 0; i < 24; i++) {
|
|
1213
|
+
id += NATIVE_ID_ALPHABET[Math.floor(Math.random() * NATIVE_ID_ALPHABET.length)];
|
|
1369
1214
|
}
|
|
1370
|
-
return
|
|
1371
|
-
}
|
|
1372
|
-
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
-
const warnings = [];
|
|
1374
|
-
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
-
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
-
const intervalRaw = resolve(
|
|
1377
|
-
flags.interval,
|
|
1378
|
-
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
-
DEFAULT_INTERVAL
|
|
1380
|
-
);
|
|
1381
|
-
let maxAgeMs;
|
|
1382
|
-
if (maxAgeRaw !== void 0) {
|
|
1383
|
-
try {
|
|
1384
|
-
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
-
} catch (err) {
|
|
1386
|
-
warnings.push(
|
|
1387
|
-
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
-
);
|
|
1389
|
-
}
|
|
1390
|
-
}
|
|
1391
|
-
let maxCount;
|
|
1392
|
-
if (maxCountRaw !== void 0) {
|
|
1393
|
-
try {
|
|
1394
|
-
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
-
} catch (err) {
|
|
1396
|
-
warnings.push(
|
|
1397
|
-
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
-
);
|
|
1399
|
-
}
|
|
1400
|
-
}
|
|
1401
|
-
let intervalMs;
|
|
1402
|
-
try {
|
|
1403
|
-
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
-
} catch (err) {
|
|
1405
|
-
warnings.push(
|
|
1406
|
-
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
-
);
|
|
1408
|
-
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1409
|
-
}
|
|
1410
|
-
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
-
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1215
|
+
return id;
|
|
1412
1216
|
}
|
|
1413
1217
|
|
|
1414
1218
|
// src/lib/tunnel/connection.ts
|
|
@@ -1487,26 +1291,24 @@ var StreamForwarder = class {
|
|
|
1487
1291
|
this.send({ type: "res_end", sid });
|
|
1488
1292
|
return;
|
|
1489
1293
|
}
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
});
|
|
1497
|
-
}
|
|
1294
|
+
log("info", "agent_request", {
|
|
1295
|
+
correlation_id: correlationId,
|
|
1296
|
+
sid,
|
|
1297
|
+
method,
|
|
1298
|
+
path: stripQuery(path)
|
|
1299
|
+
});
|
|
1498
1300
|
const ac = new AbortController();
|
|
1499
1301
|
let bodyPromise;
|
|
1500
1302
|
let pushBody;
|
|
1501
1303
|
let endBody;
|
|
1502
1304
|
if (has_body) {
|
|
1503
1305
|
const chunks = [];
|
|
1504
|
-
bodyPromise = new Promise((
|
|
1306
|
+
bodyPromise = new Promise((resolve) => {
|
|
1505
1307
|
pushBody = (buf) => {
|
|
1506
1308
|
chunks.push(buf);
|
|
1507
1309
|
};
|
|
1508
1310
|
endBody = () => {
|
|
1509
|
-
|
|
1311
|
+
resolve(Buffer.concat(chunks));
|
|
1510
1312
|
};
|
|
1511
1313
|
});
|
|
1512
1314
|
}
|
|
@@ -1541,14 +1343,12 @@ var StreamForwarder = class {
|
|
|
1541
1343
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1542
1344
|
});
|
|
1543
1345
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
});
|
|
1551
|
-
}
|
|
1346
|
+
log("info", "agent_response", {
|
|
1347
|
+
correlation_id: correlationId,
|
|
1348
|
+
sid,
|
|
1349
|
+
status: upstream.status,
|
|
1350
|
+
duration_ms: Date.now() - startedAt
|
|
1351
|
+
});
|
|
1552
1352
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1553
1353
|
try {
|
|
1554
1354
|
if (upstream.body) {
|
|
@@ -1624,7 +1424,7 @@ function connectTunnel(options) {
|
|
|
1624
1424
|
} = options;
|
|
1625
1425
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1626
1426
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1627
|
-
return new Promise((
|
|
1427
|
+
return new Promise((resolve, reject) => {
|
|
1628
1428
|
const ws = new WebSocket2(url, {
|
|
1629
1429
|
headers: {
|
|
1630
1430
|
Authorization: authHeader
|
|
@@ -1689,7 +1489,7 @@ function connectTunnel(options) {
|
|
|
1689
1489
|
clearTimeout(connectionTimeout);
|
|
1690
1490
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1691
1491
|
onConnected?.(connectedAgentId);
|
|
1692
|
-
|
|
1492
|
+
resolve({
|
|
1693
1493
|
ws,
|
|
1694
1494
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1695
1495
|
});
|
|
@@ -1818,10 +1618,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1818
1618
|
};
|
|
1819
1619
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1820
1620
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1621
|
+
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1821
1622
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
-
var
|
|
1823
|
-
var
|
|
1824
|
-
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1623
|
+
var DEFAULT_STUCK_QUEUED_REDRIVE_MAX = 3;
|
|
1624
|
+
var DEFAULT_SETTLE_MS = 3500;
|
|
1825
1625
|
var ChannelAuthError = class extends Error {
|
|
1826
1626
|
constructor(message) {
|
|
1827
1627
|
super(message);
|
|
@@ -1856,18 +1656,13 @@ var ChannelDriver = class {
|
|
|
1856
1656
|
sleep;
|
|
1857
1657
|
pausedPollIntervalMs;
|
|
1858
1658
|
pausedMaxWaitMs;
|
|
1659
|
+
dispatchConfirmMs;
|
|
1859
1660
|
stuckQueuedMs;
|
|
1661
|
+
stuckQueuedRedriveMax;
|
|
1662
|
+
settleMs;
|
|
1860
1663
|
now;
|
|
1861
1664
|
/** Cache of conversationId → opencode sessionId. */
|
|
1862
1665
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
-
/**
|
|
1864
|
-
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
-
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
-
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
-
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
-
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
-
*/
|
|
1870
|
-
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1871
1666
|
/**
|
|
1872
1667
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1873
1668
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1918,29 +1713,6 @@ var ChannelDriver = class {
|
|
|
1918
1713
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1919
1714
|
*/
|
|
1920
1715
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
-
/**
|
|
1922
|
-
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
-
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
-
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
-
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
-
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
-
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
-
*/
|
|
1929
|
-
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1930
|
-
/**
|
|
1931
|
-
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1932
|
-
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1933
|
-
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1934
|
-
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1935
|
-
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1936
|
-
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1937
|
-
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1938
|
-
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1939
|
-
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1940
|
-
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1941
|
-
* so the NEXT tick may retry exactly once more).
|
|
1942
|
-
*/
|
|
1943
|
-
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1944
1716
|
/**
|
|
1945
1717
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1946
1718
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1948,33 +1720,8 @@ var ChannelDriver = class {
|
|
|
1948
1720
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1949
1721
|
*/
|
|
1950
1722
|
opencodeDirectory = void 0;
|
|
1951
|
-
/**
|
|
1952
|
-
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1953
|
-
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1954
|
-
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1955
|
-
* (watched) session; we resolve this once per session so a child-session
|
|
1956
|
-
* question/permission can be attributed to the watched session's subtree
|
|
1957
|
-
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1958
|
-
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1959
|
-
*/
|
|
1960
|
-
sessionParents = /* @__PURE__ */ new Map();
|
|
1961
1723
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1962
1724
|
draining = false;
|
|
1963
|
-
/**
|
|
1964
|
-
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1965
|
-
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1966
|
-
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1967
|
-
* drain that entered before `stop()` still registers its watcher).
|
|
1968
|
-
*/
|
|
1969
|
-
activeDrain = null;
|
|
1970
|
-
/**
|
|
1971
|
-
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1972
|
-
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1973
|
-
* loops already running keep going so in-flight turns can finish and deliver
|
|
1974
|
-
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1975
|
-
* and stops opencode.
|
|
1976
|
-
*/
|
|
1977
|
-
stopped = false;
|
|
1978
1725
|
constructor(config2) {
|
|
1979
1726
|
this.agentId = config2.agentId;
|
|
1980
1727
|
this.port = config2.port;
|
|
@@ -1988,7 +1735,10 @@ var ChannelDriver = class {
|
|
|
1988
1735
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1989
1736
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1990
1737
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1738
|
+
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1991
1739
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1740
|
+
this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
|
|
1741
|
+
this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
|
|
1992
1742
|
this.now = config2.now ?? (() => Date.now());
|
|
1993
1743
|
}
|
|
1994
1744
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -2006,21 +1756,8 @@ var ChannelDriver = class {
|
|
|
2006
1756
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
2007
1757
|
*/
|
|
2008
1758
|
async drainPending() {
|
|
2009
|
-
if (this.stopped) return 0;
|
|
2010
1759
|
if (this.draining) return 0;
|
|
2011
1760
|
this.draining = true;
|
|
2012
|
-
const run2 = this.runDrain();
|
|
2013
|
-
this.activeDrain = run2.then(
|
|
2014
|
-
() => {
|
|
2015
|
-
this.activeDrain = null;
|
|
2016
|
-
},
|
|
2017
|
-
() => {
|
|
2018
|
-
this.activeDrain = null;
|
|
2019
|
-
}
|
|
2020
|
-
);
|
|
2021
|
-
return run2;
|
|
2022
|
-
}
|
|
2023
|
-
async runDrain() {
|
|
2024
1761
|
let dispatched = 0;
|
|
2025
1762
|
try {
|
|
2026
1763
|
const conversations = await this.getPendingConversations();
|
|
@@ -2032,7 +1769,6 @@ var ChannelDriver = class {
|
|
|
2032
1769
|
});
|
|
2033
1770
|
}
|
|
2034
1771
|
for (const conv of conversations) {
|
|
2035
|
-
if (this.stopped) break;
|
|
2036
1772
|
dispatched += await this.processConversation(conv);
|
|
2037
1773
|
}
|
|
2038
1774
|
await this.readoptProcessing();
|
|
@@ -2053,73 +1789,6 @@ var ChannelDriver = class {
|
|
|
2053
1789
|
}
|
|
2054
1790
|
return false;
|
|
2055
1791
|
}
|
|
2056
|
-
/**
|
|
2057
|
-
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2058
|
-
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2059
|
-
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2060
|
-
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2061
|
-
*
|
|
2062
|
-
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2063
|
-
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2064
|
-
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2065
|
-
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2066
|
-
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2067
|
-
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2068
|
-
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2069
|
-
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2070
|
-
*/
|
|
2071
|
-
protectedSessionIds() {
|
|
2072
|
-
const ids = /* @__PURE__ */ new Set();
|
|
2073
|
-
for (const [sessionId, watcher] of this.watchers) {
|
|
2074
|
-
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2075
|
-
}
|
|
2076
|
-
return ids;
|
|
2077
|
-
}
|
|
2078
|
-
/**
|
|
2079
|
-
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2080
|
-
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2081
|
-
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2082
|
-
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2083
|
-
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2084
|
-
*/
|
|
2085
|
-
stop() {
|
|
2086
|
-
this.stopped = true;
|
|
2087
|
-
}
|
|
2088
|
-
/**
|
|
2089
|
-
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2090
|
-
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2091
|
-
* window — is delivered before the process exits, instead of being cut off and
|
|
2092
|
-
* left for the ADR-0046 restart-recovery path.
|
|
2093
|
-
*
|
|
2094
|
-
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2095
|
-
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2096
|
-
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2097
|
-
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2098
|
-
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2099
|
-
* next runner start (ADR-0046).
|
|
2100
|
-
*
|
|
2101
|
-
* @returns true if all in-flight work settled within the window; false if the
|
|
2102
|
-
* timeout elapsed with work still in flight.
|
|
2103
|
-
*/
|
|
2104
|
-
async waitForInFlight(timeoutMs) {
|
|
2105
|
-
const deadline = this.now() + timeoutMs;
|
|
2106
|
-
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2107
|
-
if (this.activeDrain) {
|
|
2108
|
-
let drainSettled = false;
|
|
2109
|
-
void this.activeDrain.then(() => {
|
|
2110
|
-
drainSettled = true;
|
|
2111
|
-
});
|
|
2112
|
-
while (!drainSettled) {
|
|
2113
|
-
if (this.now() >= deadline) return false;
|
|
2114
|
-
await this.sleep(step);
|
|
2115
|
-
}
|
|
2116
|
-
}
|
|
2117
|
-
while (this.hasInFlightWatchers()) {
|
|
2118
|
-
if (this.now() >= deadline) return false;
|
|
2119
|
-
await this.sleep(step);
|
|
2120
|
-
}
|
|
2121
|
-
return true;
|
|
2122
|
-
}
|
|
2123
1792
|
/**
|
|
2124
1793
|
* Await all outstanding per-session watchers (WI-3).
|
|
2125
1794
|
*
|
|
@@ -2156,16 +1825,15 @@ var ChannelDriver = class {
|
|
|
2156
1825
|
let dispatched = 0;
|
|
2157
1826
|
let skippedAlreadyDispatched = 0;
|
|
2158
1827
|
for (const message of messages) {
|
|
2159
|
-
if (this.stopped) break;
|
|
2160
1828
|
if (this.dispatched.has(message.id)) {
|
|
2161
1829
|
skippedAlreadyDispatched += 1;
|
|
2162
1830
|
continue;
|
|
2163
1831
|
}
|
|
1832
|
+
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
2164
1833
|
const options = {
|
|
2165
1834
|
agent: message.opencode_agent ?? void 0,
|
|
2166
1835
|
model: message.opencode_model ?? void 0
|
|
2167
1836
|
};
|
|
2168
|
-
let opencodeMessageId;
|
|
2169
1837
|
try {
|
|
2170
1838
|
this.log({
|
|
2171
1839
|
level: "info",
|
|
@@ -2173,23 +1841,10 @@ var ChannelDriver = class {
|
|
|
2173
1841
|
conversation_id: conv.id,
|
|
2174
1842
|
message_id: message.id
|
|
2175
1843
|
});
|
|
2176
|
-
|
|
2177
|
-
sessionId,
|
|
2178
|
-
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2179
|
-
);
|
|
1844
|
+
await sendPromptAsync(this.port, sessionId, message.content, options, opencodeMessageId);
|
|
2180
1845
|
} catch (err) {
|
|
2181
1846
|
if (err instanceof ChannelAuthError) throw err;
|
|
2182
1847
|
this.dispatched.delete(message.id);
|
|
2183
|
-
if (await sessionExists(this.port, sessionId) === false) {
|
|
2184
|
-
this.sessions.delete(conv.id);
|
|
2185
|
-
this.log({
|
|
2186
|
-
level: "info",
|
|
2187
|
-
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2188
|
-
conversation_id: conv.id,
|
|
2189
|
-
message_id: message.id
|
|
2190
|
-
});
|
|
2191
|
-
break;
|
|
2192
|
-
}
|
|
2193
1848
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
2194
1849
|
});
|
|
2195
1850
|
this.log({
|
|
@@ -2200,15 +1855,6 @@ var ChannelDriver = class {
|
|
|
2200
1855
|
});
|
|
2201
1856
|
continue;
|
|
2202
1857
|
}
|
|
2203
|
-
if (opencodeMessageId === null) {
|
|
2204
|
-
this.log({
|
|
2205
|
-
level: "error",
|
|
2206
|
-
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2207
|
-
conversation_id: conv.id,
|
|
2208
|
-
message_id: message.id
|
|
2209
|
-
});
|
|
2210
|
-
continue;
|
|
2211
|
-
}
|
|
2212
1858
|
this.dispatched.add(message.id);
|
|
2213
1859
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
2214
1860
|
dispatched += 1;
|
|
@@ -2225,33 +1871,16 @@ var ChannelDriver = class {
|
|
|
2225
1871
|
return dispatched;
|
|
2226
1872
|
}
|
|
2227
1873
|
async ensureSession(conv) {
|
|
2228
|
-
const
|
|
2229
|
-
if (
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
level: "info",
|
|
2234
|
-
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2235
|
-
conversation_id: conv.id
|
|
2236
|
-
});
|
|
2237
|
-
this.sessions.delete(conv.id);
|
|
2238
|
-
return this.createAndBindSession(conv.id);
|
|
2239
|
-
}
|
|
2240
|
-
this.sessions.set(conv.id, bound);
|
|
2241
|
-
return bound;
|
|
1874
|
+
const cached = this.sessions.get(conv.id);
|
|
1875
|
+
if (cached) return cached;
|
|
1876
|
+
if (conv.opencode_session_id) {
|
|
1877
|
+
this.sessions.set(conv.id, conv.opencode_session_id);
|
|
1878
|
+
return conv.opencode_session_id;
|
|
2242
1879
|
}
|
|
2243
|
-
return this.createAndBindSession(conv.id);
|
|
2244
|
-
}
|
|
2245
|
-
/**
|
|
2246
|
-
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2247
|
-
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2248
|
-
* self-heal recreate path in `ensureSession`.
|
|
2249
|
-
*/
|
|
2250
|
-
async createAndBindSession(conversationId) {
|
|
2251
1880
|
const directory = await this.resolveOpenCodeDirectory();
|
|
2252
1881
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
2253
|
-
this.sessions.set(
|
|
2254
|
-
await this.persistSession(
|
|
1882
|
+
this.sessions.set(conv.id, sessionId);
|
|
1883
|
+
await this.persistSession(conv.id, sessionId).catch(() => {
|
|
2255
1884
|
});
|
|
2256
1885
|
return sessionId;
|
|
2257
1886
|
}
|
|
@@ -2274,25 +1903,6 @@ var ChannelDriver = class {
|
|
|
2274
1903
|
// -------------------------------------------------------------------------
|
|
2275
1904
|
// Per-session watcher (WI-3)
|
|
2276
1905
|
// -------------------------------------------------------------------------
|
|
2277
|
-
/**
|
|
2278
|
-
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2279
|
-
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2280
|
-
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2281
|
-
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2282
|
-
* (each dispatch reports its own outcome to its caller).
|
|
2283
|
-
*/
|
|
2284
|
-
dispatchLocked(sessionId, fn) {
|
|
2285
|
-
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2286
|
-
const run2 = prior.then(fn, fn);
|
|
2287
|
-
this.sessionDispatchLocks.set(
|
|
2288
|
-
sessionId,
|
|
2289
|
-
run2.then(
|
|
2290
|
-
() => void 0,
|
|
2291
|
-
() => void 0
|
|
2292
|
-
)
|
|
2293
|
-
);
|
|
2294
|
-
return run2;
|
|
2295
|
-
}
|
|
2296
1906
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
2297
1907
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
2298
1908
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -2302,9 +1912,7 @@ var ChannelDriver = class {
|
|
|
2302
1912
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2303
1913
|
loop: null,
|
|
2304
1914
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2305
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2306
|
-
lastGoodPollAt: this.now(),
|
|
2307
|
-
hadUsablePoll: false
|
|
1915
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2308
1916
|
};
|
|
2309
1917
|
this.watchers.set(sessionId, watcher);
|
|
2310
1918
|
}
|
|
@@ -2314,37 +1922,24 @@ var ChannelDriver = class {
|
|
|
2314
1922
|
opencodeMessageId,
|
|
2315
1923
|
message,
|
|
2316
1924
|
dispatchedAt: now,
|
|
2317
|
-
processingAnchorMs: now,
|
|
2318
1925
|
deadline: now + this.pausedMaxWaitMs,
|
|
2319
1926
|
started: false,
|
|
2320
1927
|
done: false,
|
|
2321
1928
|
stuckReported: false,
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
pausedOnPermission: false,
|
|
2327
|
-
pausedClearConfirmed: false,
|
|
2328
|
-
pausedInFlight: false,
|
|
2329
|
-
deliveryDeadlineAnchored: false
|
|
1929
|
+
redriveAttempts: 0,
|
|
1930
|
+
lastRedriveAt: null,
|
|
1931
|
+
redriveOpencodeMessageId: null,
|
|
1932
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
2330
1933
|
});
|
|
2331
1934
|
}
|
|
2332
1935
|
/**
|
|
2333
1936
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2334
1937
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2335
1938
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2336
|
-
* `now
|
|
2337
|
-
*
|
|
2338
|
-
*
|
|
2339
|
-
*
|
|
2340
|
-
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2341
|
-
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2342
|
-
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2343
|
-
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2344
|
-
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2345
|
-
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2346
|
-
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2347
|
-
* (only the appear-guard uses it).
|
|
1939
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1940
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1941
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1942
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
2348
1943
|
*
|
|
2349
1944
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2350
1945
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -2354,7 +1949,7 @@ var ChannelDriver = class {
|
|
|
2354
1949
|
* server already flipped to `processing`; the running/done transitions still
|
|
2355
1950
|
* fire from the watcher's normal branches.
|
|
2356
1951
|
*/
|
|
2357
|
-
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
1952
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs, redriveOpencodeMessageId = null) {
|
|
2358
1953
|
let watcher = this.watchers.get(sessionId);
|
|
2359
1954
|
if (!watcher) {
|
|
2360
1955
|
watcher = {
|
|
@@ -2362,9 +1957,7 @@ var ChannelDriver = class {
|
|
|
2362
1957
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2363
1958
|
loop: null,
|
|
2364
1959
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2365
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2366
|
-
lastGoodPollAt: this.now(),
|
|
2367
|
-
hadUsablePoll: false
|
|
1960
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2368
1961
|
};
|
|
2369
1962
|
this.watchers.set(sessionId, watcher);
|
|
2370
1963
|
}
|
|
@@ -2373,33 +1966,22 @@ var ChannelDriver = class {
|
|
|
2373
1966
|
opencodeMessageId,
|
|
2374
1967
|
message,
|
|
2375
1968
|
dispatchedAt: this.now(),
|
|
2376
|
-
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2377
|
-
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2378
|
-
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2379
|
-
processingAnchorMs: processedAtMs,
|
|
2380
1969
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2381
1970
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2382
1971
|
started: true,
|
|
2383
1972
|
done: false,
|
|
2384
1973
|
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2385
|
-
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it
|
|
2386
|
-
// on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
2387
|
-
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2388
|
-
// (#210
|
|
1974
|
+
// AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
|
|
1975
|
+
// gates on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
1976
|
+
// so a re-adopted row left wedged in `queued` still emits the signal once
|
|
1977
|
+
// (queued-followup-redrive, #210). A re-adopted `queued` wedge is ALSO
|
|
1978
|
+
// re-driven by the same path (ADR-0046 §c deferred the live-session redrive
|
|
1979
|
+
// to here) — `redriveAttempts` starts fresh so it gets the full budget.
|
|
2389
1980
|
stuckReported: false,
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2395
|
-
lastAliveAt: 0,
|
|
2396
|
-
aliveInFlight: false,
|
|
2397
|
-
awaitingHumanLatched: false,
|
|
2398
|
-
pausedOnQuestion: false,
|
|
2399
|
-
pausedOnPermission: false,
|
|
2400
|
-
pausedClearConfirmed: false,
|
|
2401
|
-
pausedInFlight: false,
|
|
2402
|
-
deliveryDeadlineAnchored: false
|
|
1981
|
+
redriveAttempts: 0,
|
|
1982
|
+
lastRedriveAt: null,
|
|
1983
|
+
redriveOpencodeMessageId,
|
|
1984
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
2403
1985
|
});
|
|
2404
1986
|
}
|
|
2405
1987
|
/**
|
|
@@ -2450,30 +2032,12 @@ var ChannelDriver = class {
|
|
|
2450
2032
|
messages = Array.isArray(body) ? body : null;
|
|
2451
2033
|
}
|
|
2452
2034
|
} catch {
|
|
2035
|
+
continue;
|
|
2453
2036
|
}
|
|
2454
|
-
if (messages != null && messages.length > 0) {
|
|
2455
|
-
watcher.lastGoodPollAt = this.now();
|
|
2456
|
-
watcher.hadUsablePoll = true;
|
|
2457
|
-
} else {
|
|
2458
|
-
const emptyButReachable = messages != null;
|
|
2459
|
-
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2460
|
-
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2461
|
-
continue;
|
|
2462
|
-
}
|
|
2463
|
-
}
|
|
2464
|
-
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2465
2037
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2466
|
-
await this.serviceInFlightMessage(
|
|
2467
|
-
sessionId,
|
|
2468
|
-
watcher,
|
|
2469
|
-
inFlight,
|
|
2470
|
-
messages,
|
|
2471
|
-
openQuestions,
|
|
2472
|
-
openPermissions,
|
|
2473
|
-
questionsPolledOk,
|
|
2474
|
-
permissionsPolledOk
|
|
2475
|
-
);
|
|
2038
|
+
await this.serviceInFlightMessage(sessionId, watcher, inFlight, messages);
|
|
2476
2039
|
}
|
|
2040
|
+
await this.pollInteractions(sessionId, watcher, messages);
|
|
2477
2041
|
}
|
|
2478
2042
|
} catch (err) {
|
|
2479
2043
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2495,40 +2059,15 @@ var ChannelDriver = class {
|
|
|
2495
2059
|
});
|
|
2496
2060
|
}
|
|
2497
2061
|
}
|
|
2498
|
-
/**
|
|
2499
|
-
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2500
|
-
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2501
|
-
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2502
|
-
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2503
|
-
* first transient PATCH failure — dropping the message before its reply lands
|
|
2504
|
-
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2505
|
-
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2506
|
-
* or past now, so a still-ample window is left untouched.
|
|
2507
|
-
*/
|
|
2508
|
-
anchorDeliveryDeadline(inFlight) {
|
|
2509
|
-
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2510
|
-
inFlight.deliveryDeadlineAnchored = true;
|
|
2511
|
-
if (this.now() >= inFlight.deadline) {
|
|
2512
|
-
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2513
|
-
}
|
|
2514
|
-
}
|
|
2515
2062
|
/**
|
|
2516
2063
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2517
2064
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2518
2065
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2519
2066
|
* in-flight set on completion or timeout.
|
|
2520
2067
|
*/
|
|
2521
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages
|
|
2068
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2522
2069
|
const conv = watcher.conv;
|
|
2523
2070
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2524
|
-
const id = inFlight.evidentMessageId;
|
|
2525
|
-
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2526
|
-
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2527
|
-
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2528
|
-
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2529
|
-
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2530
|
-
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2531
|
-
const awaitingHuman = observedOpen || latchedPaused;
|
|
2532
2071
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2533
2072
|
let claimed;
|
|
2534
2073
|
try {
|
|
@@ -2536,7 +2075,7 @@ var ChannelDriver = class {
|
|
|
2536
2075
|
conv.id,
|
|
2537
2076
|
inFlight.evidentMessageId,
|
|
2538
2077
|
sessionId,
|
|
2539
|
-
inFlight.
|
|
2078
|
+
inFlight.redriveOpencodeMessageId
|
|
2540
2079
|
);
|
|
2541
2080
|
} catch (err) {
|
|
2542
2081
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2559,7 +2098,6 @@ var ChannelDriver = class {
|
|
|
2559
2098
|
}
|
|
2560
2099
|
}
|
|
2561
2100
|
if (state === "done") {
|
|
2562
|
-
this.anchorDeliveryDeadline(inFlight);
|
|
2563
2101
|
if (!inFlight.done) {
|
|
2564
2102
|
this.log({
|
|
2565
2103
|
level: "info",
|
|
@@ -2572,7 +2110,7 @@ var ChannelDriver = class {
|
|
|
2572
2110
|
conv.id,
|
|
2573
2111
|
inFlight.evidentMessageId,
|
|
2574
2112
|
sessionId,
|
|
2575
|
-
inFlight.
|
|
2113
|
+
inFlight.redriveOpencodeMessageId
|
|
2576
2114
|
);
|
|
2577
2115
|
} catch (err) {
|
|
2578
2116
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2610,7 +2148,6 @@ var ChannelDriver = class {
|
|
|
2610
2148
|
return;
|
|
2611
2149
|
}
|
|
2612
2150
|
if (state === "failed") {
|
|
2613
|
-
this.anchorDeliveryDeadline(inFlight);
|
|
2614
2151
|
if (!inFlight.done) {
|
|
2615
2152
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2616
2153
|
this.log({
|
|
@@ -2656,70 +2193,215 @@ var ChannelDriver = class {
|
|
|
2656
2193
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2657
2194
|
return;
|
|
2658
2195
|
}
|
|
2196
|
+
if (state === "unknown") {
|
|
2197
|
+
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
2198
|
+
await this.redispatchInFlight(conv.id, sessionId, inFlight);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2659
2201
|
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2660
2202
|
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2661
|
-
if (state === "queued" && pastStuckBound && sessionIdle
|
|
2662
|
-
inFlight.stuckReported
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2203
|
+
if (state === "queued" && pastStuckBound && sessionIdle) {
|
|
2204
|
+
if (!inFlight.stuckReported) {
|
|
2205
|
+
inFlight.stuckReported = true;
|
|
2206
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2207
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
|
|
2211
|
+
if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
|
|
2212
|
+
const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
|
|
2213
|
+
if (removed) return;
|
|
2214
|
+
}
|
|
2666
2215
|
}
|
|
2667
|
-
|
|
2668
|
-
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2216
|
+
if (this.now() >= inFlight.deadline) {
|
|
2669
2217
|
this.log({
|
|
2670
|
-
level: "
|
|
2671
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2218
|
+
level: "info",
|
|
2219
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2672
2220
|
conversation_id: conv.id,
|
|
2673
2221
|
message_id: inFlight.evidentMessageId
|
|
2674
2222
|
});
|
|
2675
2223
|
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2676
|
-
watched_for_ms: this.now() - inFlight.
|
|
2224
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2677
2225
|
});
|
|
2678
2226
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2679
|
-
return;
|
|
2680
2227
|
}
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2228
|
+
}
|
|
2229
|
+
/**
|
|
2230
|
+
* Re-dispatch a message whose user row never appeared (idle-path guard). Safe:
|
|
2231
|
+
* opencode treats a duplicate caller-supplied `messageID` as idempotent (PoC
|
|
2232
|
+
* fact 9) — one user message + one reply even if the original DID land. Resets
|
|
2233
|
+
* the dispatch timestamp so the guard doesn't immediately fire again.
|
|
2234
|
+
*/
|
|
2235
|
+
async redispatchInFlight(conversationId, sessionId, inFlight) {
|
|
2236
|
+
const options = {
|
|
2237
|
+
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2238
|
+
model: inFlight.message.opencode_model ?? void 0
|
|
2239
|
+
};
|
|
2240
|
+
this.log({
|
|
2241
|
+
level: "info",
|
|
2242
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
|
|
2243
|
+
message_id: inFlight.evidentMessageId
|
|
2244
|
+
});
|
|
2245
|
+
void this.postSignal(conversationId, inFlight.evidentMessageId, "redispatched");
|
|
2246
|
+
try {
|
|
2247
|
+
await sendPromptAsync(
|
|
2248
|
+
this.port,
|
|
2249
|
+
sessionId,
|
|
2250
|
+
inFlight.message.content,
|
|
2251
|
+
options,
|
|
2252
|
+
inFlight.opencodeMessageId
|
|
2253
|
+
);
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
this.log({
|
|
2256
|
+
level: "error",
|
|
2257
|
+
message: `Re-dispatch failed for message ${inFlight.evidentMessageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2258
|
+
message_id: inFlight.evidentMessageId
|
|
2686
2259
|
});
|
|
2687
2260
|
}
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2261
|
+
inFlight.dispatchedAt = this.now();
|
|
2262
|
+
}
|
|
2263
|
+
/**
|
|
2264
|
+
* RE-DRIVE a stuck-`queued` follow-up so opencode actually runs it
|
|
2265
|
+
* (queued-followup-redrive; the fix for the 2026-07-18 dev incident).
|
|
2266
|
+
*
|
|
2267
|
+
* The wedge: a follow-up `prompt_async`'d mid-turn / in the post-turn settling
|
|
2268
|
+
* window is orphaned — its user message persists but the turn never runs
|
|
2269
|
+
* (`messageRunState === 'queued'`). Proven against real opencode 1.18.3 by an
|
|
2270
|
+
* isolated, interleaved experiment (`.harness/followup-send-mechanism-finding.md`):
|
|
2271
|
+
* the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
|
|
2272
|
+
* runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
|
|
2273
|
+
* turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
|
|
2274
|
+
* (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
|
|
2275
|
+
* dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
|
|
2276
|
+
*
|
|
2277
|
+
* So we re-`prompt_async` into the SAME opencode session (opencode-web does
|
|
2278
|
+
* exactly this and it works — UPDATE 2 in the investigation doc), preserving the
|
|
2279
|
+
* conversation's history/continuity: we do NOT create a fresh session and we do
|
|
2280
|
+
* NOT overwrite the conversation's `opencode_session_id`. Only the opencode
|
|
2281
|
+
* user-message id changes — to a fresh native id per attempt (opencode's
|
|
2282
|
+
* caller-supplied id dedup is global+permanent, so each attempt needs a never-
|
|
2283
|
+
* seen id).
|
|
2284
|
+
*
|
|
2285
|
+
* Reply correlation is PRESERVED: the native id is random and NOT re-derivable
|
|
2286
|
+
* from the row id, so the runner carries it to the server (on markProcessing/
|
|
2287
|
+
* markDone via `redriveOpencodeMessageId`), which persists it on the row and
|
|
2288
|
+
* correlates the reply by THAT id (`conversation-notification.ts`). A normal,
|
|
2289
|
+
* never-re-driven message still correlates by the derived stable id — unchanged.
|
|
2290
|
+
*
|
|
2291
|
+
* Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
|
|
2292
|
+
* FAILED (existing channel failure affordance) so the user is TOLD it could not
|
|
2293
|
+
* be answered rather than left silent, and it is removed from the in-flight set.
|
|
2294
|
+
*
|
|
2295
|
+
* @returns true if the message LEFT this watcher's in-flight set (gave up +
|
|
2296
|
+
* marked failed) — the caller then stops servicing it this tick. Returns false
|
|
2297
|
+
* when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
|
|
2298
|
+
* send failed and the next window retries).
|
|
2299
|
+
*/
|
|
2300
|
+
async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
|
|
2301
|
+
if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
|
|
2302
|
+
return true;
|
|
2303
|
+
}
|
|
2304
|
+
if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
|
|
2305
|
+
this.log({
|
|
2306
|
+
level: "error",
|
|
2307
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
|
|
2308
|
+
conversation_id: watcher.conv.id,
|
|
2309
|
+
message_id: inFlight.evidentMessageId
|
|
2310
|
+
});
|
|
2311
|
+
try {
|
|
2312
|
+
await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
|
|
2313
|
+
} catch (err) {
|
|
2314
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2315
|
+
this.log({
|
|
2316
|
+
level: "error",
|
|
2317
|
+
message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2318
|
+
conversation_id: watcher.conv.id,
|
|
2319
|
+
message_id: inFlight.evidentMessageId
|
|
2698
2320
|
});
|
|
2699
2321
|
}
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
inFlight.pausedOnQuestion = false;
|
|
2703
|
-
inFlight.pausedOnPermission = false;
|
|
2704
|
-
inFlight.pausedClearConfirmed = false;
|
|
2322
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2323
|
+
return true;
|
|
2705
2324
|
}
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2325
|
+
inFlight.redriveAttempts += 1;
|
|
2326
|
+
inFlight.lastRedriveAt = this.now();
|
|
2327
|
+
this.log({
|
|
2328
|
+
level: "info",
|
|
2329
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
|
|
2330
|
+
conversation_id: watcher.conv.id,
|
|
2331
|
+
message_id: inFlight.evidentMessageId
|
|
2332
|
+
});
|
|
2333
|
+
void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
|
|
2334
|
+
redrive_attempt: inFlight.redriveAttempts
|
|
2335
|
+
});
|
|
2336
|
+
const options = {
|
|
2337
|
+
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2338
|
+
model: inFlight.message.opencode_model ?? void 0
|
|
2339
|
+
};
|
|
2340
|
+
const nativeId = nativeOpencodeMessageId();
|
|
2341
|
+
try {
|
|
2342
|
+
await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
|
|
2343
|
+
} catch (err) {
|
|
2344
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2345
|
+
inFlight.redriveAttempts -= 1;
|
|
2712
2346
|
this.log({
|
|
2713
|
-
level: "
|
|
2714
|
-
message: `
|
|
2715
|
-
conversation_id: conv.id,
|
|
2347
|
+
level: "error",
|
|
2348
|
+
message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
|
|
2349
|
+
conversation_id: watcher.conv.id,
|
|
2716
2350
|
message_id: inFlight.evidentMessageId
|
|
2717
2351
|
});
|
|
2718
|
-
|
|
2719
|
-
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
inFlight.opencodeMessageId = nativeId;
|
|
2355
|
+
inFlight.redriveOpencodeMessageId = nativeId;
|
|
2356
|
+
inFlight.attemptedOpencodeMessageIds.push(nativeId);
|
|
2357
|
+
inFlight.started = false;
|
|
2358
|
+
return false;
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
|
|
2362
|
+
* driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
|
|
2363
|
+
* in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
|
|
2364
|
+
* earlier re-drive attempt eventually produced is delivered) instead of
|
|
2365
|
+
* re-driving again or marking failed. Idempotent + guarded like the watcher's
|
|
2366
|
+
* done branch. `@returns` true when the message left the in-flight set (delivered
|
|
2367
|
+
* or terminally-undeliverable) — the caller then stops servicing it this tick.
|
|
2368
|
+
*
|
|
2369
|
+
* The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
|
|
2370
|
+
* branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
|
|
2371
|
+
* polls. `messageRunState === 'done'` means a completed, non-errored correlated
|
|
2372
|
+
* reply exists for that id.
|
|
2373
|
+
*/
|
|
2374
|
+
async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
|
|
2375
|
+
const completedId = inFlight.attemptedOpencodeMessageIds.find(
|
|
2376
|
+
(id) => messageRunState(messages, id) === "done"
|
|
2377
|
+
);
|
|
2378
|
+
if (!completedId) return false;
|
|
2379
|
+
if (inFlight.done) {
|
|
2380
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2381
|
+
return true;
|
|
2382
|
+
}
|
|
2383
|
+
this.log({
|
|
2384
|
+
level: "info",
|
|
2385
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
|
|
2386
|
+
conversation_id: watcher.conv.id,
|
|
2387
|
+
message_id: inFlight.evidentMessageId
|
|
2388
|
+
});
|
|
2389
|
+
try {
|
|
2390
|
+
await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
|
|
2391
|
+
} catch (err) {
|
|
2392
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2393
|
+
this.log({
|
|
2394
|
+
level: "error",
|
|
2395
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done off an earlier re-drive attempt (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2396
|
+
conversation_id: watcher.conv.id,
|
|
2397
|
+
message_id: inFlight.evidentMessageId
|
|
2720
2398
|
});
|
|
2721
2399
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2400
|
+
return true;
|
|
2722
2401
|
}
|
|
2402
|
+
inFlight.done = true;
|
|
2403
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2404
|
+
return true;
|
|
2723
2405
|
}
|
|
2724
2406
|
// -------------------------------------------------------------------------
|
|
2725
2407
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -2740,17 +2422,12 @@ var ChannelDriver = class {
|
|
|
2740
2422
|
*/
|
|
2741
2423
|
async readoptProcessing() {
|
|
2742
2424
|
const rows = await this.getProcessingMessages();
|
|
2743
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0
|
|
2425
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2744
2426
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2745
|
-
for (const id of [
|
|
2746
|
-
...this.dontRedispatch,
|
|
2747
|
-
...this.doneUndeliverable,
|
|
2748
|
-
...this.readoptPollUnresolvedSignalled
|
|
2749
|
-
]) {
|
|
2427
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2750
2428
|
if (!stillProcessing.has(id)) {
|
|
2751
2429
|
const cleared = this.dontRedispatch.delete(id);
|
|
2752
2430
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2753
|
-
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2754
2431
|
if (cleared || clearedUndeliverable) {
|
|
2755
2432
|
this.log({
|
|
2756
2433
|
level: "info",
|
|
@@ -2804,10 +2481,8 @@ var ChannelDriver = class {
|
|
|
2804
2481
|
});
|
|
2805
2482
|
continue;
|
|
2806
2483
|
}
|
|
2807
|
-
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2808
|
-
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2809
2484
|
for (const row of sessionRows) {
|
|
2810
|
-
await this.readoptOne(sessionId, row, messages
|
|
2485
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2811
2486
|
}
|
|
2812
2487
|
}
|
|
2813
2488
|
}
|
|
@@ -2815,21 +2490,23 @@ var ChannelDriver = class {
|
|
|
2815
2490
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2816
2491
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2817
2492
|
*
|
|
2818
|
-
* Branches on `messageRunState(messages,
|
|
2819
|
-
*
|
|
2820
|
-
* (
|
|
2821
|
-
*
|
|
2822
|
-
* re-
|
|
2493
|
+
* Branches on `messageRunState(messages, effectiveId)`, where `effectiveId` is
|
|
2494
|
+
* the NATIVE id a prior lifetime's re-drive ran this row under
|
|
2495
|
+
* (`row.opencode_message_id`) if present, else the STABLE derived id
|
|
2496
|
+
* (`opencodeMessageIdFor(row.id)`). Consulting the native id is what makes a
|
|
2497
|
+
* re-driven row that ALREADY ran/completed resolve correctly on restart instead
|
|
2498
|
+
* of looking `queued` under the (never-run) stable id and being re-driven AGAIN
|
|
2499
|
+
* (Bugbot #217 Finding 3 — the duplicate-turn bug):
|
|
2823
2500
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2824
2501
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2825
2502
|
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2826
2503
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2827
|
-
* tracking the
|
|
2828
|
-
* - `unknown
|
|
2504
|
+
* tracking the effective id so the reply correlates by it;
|
|
2505
|
+
* - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
|
|
2829
2506
|
*
|
|
2830
2507
|
* Only `ChannelAuthError` propagates.
|
|
2831
2508
|
*/
|
|
2832
|
-
async readoptOne(sessionId, row, messages
|
|
2509
|
+
async readoptOne(sessionId, row, messages) {
|
|
2833
2510
|
if (this.isTracked(sessionId, row.id)) {
|
|
2834
2511
|
this.log({
|
|
2835
2512
|
level: "info",
|
|
@@ -2839,8 +2516,10 @@ var ChannelDriver = class {
|
|
|
2839
2516
|
});
|
|
2840
2517
|
return;
|
|
2841
2518
|
}
|
|
2842
|
-
const
|
|
2843
|
-
const
|
|
2519
|
+
const stableId = opencodeMessageIdFor2(row.id);
|
|
2520
|
+
const nativeRedriveId = row.opencode_message_id;
|
|
2521
|
+
const ocId = nativeRedriveId ?? stableId;
|
|
2522
|
+
const state = messageRunState(messages, ocId);
|
|
2844
2523
|
if (state === "done") {
|
|
2845
2524
|
if (this.doneUndeliverable.has(row.id)) {
|
|
2846
2525
|
this.log({
|
|
@@ -2858,7 +2537,7 @@ var ChannelDriver = class {
|
|
|
2858
2537
|
message_id: row.id
|
|
2859
2538
|
});
|
|
2860
2539
|
try {
|
|
2861
|
-
await this.markDone(row.conversation_id, row.id, sessionId,
|
|
2540
|
+
await this.markDone(row.conversation_id, row.id, sessionId, nativeRedriveId);
|
|
2862
2541
|
} catch (err) {
|
|
2863
2542
|
if (err instanceof ChannelAuthError) throw err;
|
|
2864
2543
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -2869,7 +2548,6 @@ var ChannelDriver = class {
|
|
|
2869
2548
|
conversation_id: row.conversation_id,
|
|
2870
2549
|
message_id: row.id
|
|
2871
2550
|
});
|
|
2872
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2873
2551
|
return;
|
|
2874
2552
|
}
|
|
2875
2553
|
this.log({
|
|
@@ -2881,11 +2559,10 @@ var ChannelDriver = class {
|
|
|
2881
2559
|
return;
|
|
2882
2560
|
}
|
|
2883
2561
|
this.dontRedispatch.delete(row.id);
|
|
2884
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2885
2562
|
return;
|
|
2886
2563
|
}
|
|
2887
2564
|
if (state === "failed") {
|
|
2888
|
-
const error2 = messageError(messages, ocId
|
|
2565
|
+
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2889
2566
|
this.log({
|
|
2890
2567
|
level: "error",
|
|
2891
2568
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -2904,7 +2581,6 @@ var ChannelDriver = class {
|
|
|
2904
2581
|
conversation_id: row.conversation_id,
|
|
2905
2582
|
message_id: row.id
|
|
2906
2583
|
});
|
|
2907
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2908
2584
|
return;
|
|
2909
2585
|
}
|
|
2910
2586
|
this.log({
|
|
@@ -2916,7 +2592,6 @@ var ChannelDriver = class {
|
|
|
2916
2592
|
return;
|
|
2917
2593
|
}
|
|
2918
2594
|
this.dontRedispatch.delete(row.id);
|
|
2919
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2920
2595
|
return;
|
|
2921
2596
|
}
|
|
2922
2597
|
if (this.dontRedispatch.has(row.id)) {
|
|
@@ -2928,129 +2603,48 @@ var ChannelDriver = class {
|
|
|
2928
2603
|
});
|
|
2929
2604
|
return;
|
|
2930
2605
|
}
|
|
2931
|
-
|
|
2932
|
-
if (state === "running" && ocId) {
|
|
2933
|
-
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2934
|
-
const shape = this.replyCompletionShape(reply);
|
|
2935
|
-
const ongoing = sessionOngoing;
|
|
2936
|
-
statusReadableOngoing = ongoing;
|
|
2937
|
-
if (ongoing === false) {
|
|
2938
|
-
this.log({
|
|
2939
|
-
level: "info",
|
|
2940
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2941
|
-
conversation_id: row.conversation_id,
|
|
2942
|
-
message_id: row.id
|
|
2943
|
-
});
|
|
2944
|
-
await this.forceReadoptRun(sessionId, row);
|
|
2945
|
-
return;
|
|
2946
|
-
}
|
|
2947
|
-
if (ongoing === true) {
|
|
2948
|
-
this.log({
|
|
2949
|
-
level: "info",
|
|
2950
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2951
|
-
conversation_id: row.conversation_id,
|
|
2952
|
-
message_id: row.id
|
|
2953
|
-
});
|
|
2954
|
-
} else {
|
|
2955
|
-
if (shape === "b1") {
|
|
2956
|
-
this.log({
|
|
2957
|
-
level: "info",
|
|
2958
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2959
|
-
conversation_id: row.conversation_id,
|
|
2960
|
-
message_id: row.id
|
|
2961
|
-
});
|
|
2962
|
-
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2963
|
-
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2964
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2965
|
-
}
|
|
2966
|
-
return;
|
|
2967
|
-
}
|
|
2968
|
-
this.log({
|
|
2969
|
-
level: "info",
|
|
2970
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2971
|
-
conversation_id: row.conversation_id,
|
|
2972
|
-
message_id: row.id
|
|
2973
|
-
});
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2977
|
-
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2978
|
-
if (descendantAlive === true) {
|
|
2979
|
-
this.log({
|
|
2980
|
-
level: "info",
|
|
2981
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2982
|
-
conversation_id: row.conversation_id,
|
|
2983
|
-
message_id: row.id
|
|
2984
|
-
});
|
|
2985
|
-
} else {
|
|
2986
|
-
this.log({
|
|
2987
|
-
level: "info",
|
|
2988
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2989
|
-
conversation_id: row.conversation_id,
|
|
2990
|
-
message_id: row.id
|
|
2991
|
-
});
|
|
2992
|
-
await this.forceReadoptRun(sessionId, row);
|
|
2993
|
-
return;
|
|
2994
|
-
}
|
|
2995
|
-
}
|
|
2996
|
-
if ((state === "running" || state === "queued") && ocId) {
|
|
2606
|
+
if (state === "running" || state === "queued") {
|
|
2997
2607
|
const conv = this.convForRow(sessionId, row);
|
|
2998
2608
|
const message = this.queuedMessageForRow(row);
|
|
2999
|
-
this.registerReadopted(
|
|
2609
|
+
this.registerReadopted(
|
|
2610
|
+
conv,
|
|
2611
|
+
sessionId,
|
|
2612
|
+
message,
|
|
2613
|
+
ocId,
|
|
2614
|
+
this.processedAtMs(row),
|
|
2615
|
+
nativeRedriveId
|
|
2616
|
+
);
|
|
3000
2617
|
this.dispatched.add(row.id);
|
|
3001
2618
|
this.readopted.add(row.id);
|
|
3002
2619
|
this.ensureWatcherRunning(sessionId);
|
|
3003
2620
|
this.log({
|
|
3004
2621
|
level: "info",
|
|
3005
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (
|
|
2622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (${nativeRedriveId ? "native re-drive id" : "stable id"}, no re-dispatch)`,
|
|
3006
2623
|
conversation_id: row.conversation_id,
|
|
3007
2624
|
message_id: row.id
|
|
3008
2625
|
});
|
|
3009
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
3010
2626
|
return;
|
|
3011
2627
|
}
|
|
3012
2628
|
await this.forceReadoptRun(sessionId, row);
|
|
3013
2629
|
}
|
|
3014
2630
|
/**
|
|
3015
|
-
* Re-dispatch an orphaned (`unknown
|
|
2631
|
+
* Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
|
|
3016
2632
|
*
|
|
3017
|
-
*
|
|
3018
|
-
*
|
|
3019
|
-
*
|
|
3020
|
-
*
|
|
2633
|
+
* The stable-id user message is absent from the session, so we (re-)dispatch with
|
|
2634
|
+
* the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
|
|
2635
|
+
* This is what keeps the reply correlatable: the server's completion
|
|
2636
|
+
* notification looks for the reply under the stable id, so the fresh turn's reply
|
|
2637
|
+
* (which hangs off the stable id) is found and delivered. The residual
|
|
2638
|
+
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2639
|
+
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2640
|
+
* to swallow the duplicate.
|
|
3021
2641
|
*
|
|
3022
|
-
*
|
|
3023
|
-
*
|
|
3024
|
-
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3025
|
-
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3026
|
-
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3027
|
-
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3028
|
-
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3029
|
-
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3030
|
-
* may retry exactly once more).
|
|
3031
|
-
*
|
|
3032
|
-
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2642
|
+
* `evidentMessageId = row.id` addresses the SERVER row; the stable
|
|
2643
|
+
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
3033
2644
|
* `processed_at` (Invariant 1).
|
|
3034
2645
|
*/
|
|
3035
2646
|
async forceReadoptRun(sessionId, row) {
|
|
3036
|
-
|
|
3037
|
-
this.log({
|
|
3038
|
-
level: "info",
|
|
3039
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3040
|
-
conversation_id: row.conversation_id,
|
|
3041
|
-
message_id: row.id
|
|
3042
|
-
});
|
|
3043
|
-
return;
|
|
3044
|
-
}
|
|
3045
|
-
if (this.awaitingReadopt.has(row.id)) {
|
|
3046
|
-
this.log({
|
|
3047
|
-
level: "info",
|
|
3048
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3049
|
-
conversation_id: row.conversation_id,
|
|
3050
|
-
message_id: row.id
|
|
3051
|
-
});
|
|
3052
|
-
return;
|
|
3053
|
-
}
|
|
2647
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
3054
2648
|
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
3055
2649
|
this.dontRedispatch.add(row.id);
|
|
3056
2650
|
this.log({
|
|
@@ -3059,7 +2653,6 @@ var ChannelDriver = class {
|
|
|
3059
2653
|
conversation_id: row.conversation_id,
|
|
3060
2654
|
message_id: row.id
|
|
3061
2655
|
});
|
|
3062
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
3063
2656
|
return;
|
|
3064
2657
|
}
|
|
3065
2658
|
const options = {
|
|
@@ -3068,19 +2661,13 @@ var ChannelDriver = class {
|
|
|
3068
2661
|
};
|
|
3069
2662
|
this.log({
|
|
3070
2663
|
level: "info",
|
|
3071
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching
|
|
2664
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
|
|
3072
2665
|
conversation_id: row.conversation_id,
|
|
3073
2666
|
message_id: row.id
|
|
3074
2667
|
});
|
|
3075
|
-
this.awaitingReadopt.add(row.id);
|
|
3076
|
-
let ocId;
|
|
3077
2668
|
try {
|
|
3078
|
-
|
|
3079
|
-
sessionId,
|
|
3080
|
-
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
3081
|
-
);
|
|
2669
|
+
await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
|
|
3082
2670
|
} catch (err) {
|
|
3083
|
-
this.awaitingReadopt.delete(row.id);
|
|
3084
2671
|
if (err instanceof ChannelAuthError) throw err;
|
|
3085
2672
|
this.log({
|
|
3086
2673
|
level: "error",
|
|
@@ -3088,18 +2675,6 @@ var ChannelDriver = class {
|
|
|
3088
2675
|
conversation_id: row.conversation_id,
|
|
3089
2676
|
message_id: row.id
|
|
3090
2677
|
});
|
|
3091
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3092
|
-
return;
|
|
3093
|
-
}
|
|
3094
|
-
if (ocId === null) {
|
|
3095
|
-
this.awaitingReadopt.delete(row.id);
|
|
3096
|
-
this.log({
|
|
3097
|
-
level: "error",
|
|
3098
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3099
|
-
conversation_id: row.conversation_id,
|
|
3100
|
-
message_id: row.id
|
|
3101
|
-
});
|
|
3102
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3103
2678
|
return;
|
|
3104
2679
|
}
|
|
3105
2680
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -3107,9 +2682,7 @@ var ChannelDriver = class {
|
|
|
3107
2682
|
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
3108
2683
|
this.dispatched.add(row.id);
|
|
3109
2684
|
this.readopted.add(row.id);
|
|
3110
|
-
this.awaitingReadopt.delete(row.id);
|
|
3111
2685
|
this.ensureWatcherRunning(sessionId);
|
|
3112
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
3113
2686
|
}
|
|
3114
2687
|
/**
|
|
3115
2688
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -3201,41 +2774,21 @@ var ChannelDriver = class {
|
|
|
3201
2774
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
3202
2775
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
3203
2776
|
* oldest running message.
|
|
3204
|
-
*
|
|
3205
|
-
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3206
|
-
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3207
|
-
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3208
|
-
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3209
|
-
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3210
|
-
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3211
|
-
* even after it was already surfaced to the channel.
|
|
3212
2777
|
*/
|
|
3213
2778
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3214
|
-
const openQuestions = /* @__PURE__ */ new Set();
|
|
3215
|
-
const openPermissions = /* @__PURE__ */ new Set();
|
|
3216
|
-
let questionsPolledOk = true;
|
|
3217
|
-
let permissionsPolledOk = true;
|
|
3218
2779
|
let questions = [];
|
|
3219
2780
|
try {
|
|
3220
2781
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
3221
2782
|
if (res.ok) {
|
|
3222
2783
|
const body = await res.json();
|
|
3223
|
-
|
|
3224
|
-
questions = body;
|
|
3225
|
-
} else {
|
|
3226
|
-
questionsPolledOk = false;
|
|
3227
|
-
}
|
|
3228
|
-
} else {
|
|
3229
|
-
questionsPolledOk = false;
|
|
2784
|
+
questions = Array.isArray(body) ? body : [];
|
|
3230
2785
|
}
|
|
3231
2786
|
} catch {
|
|
3232
|
-
questionsPolledOk = false;
|
|
3233
2787
|
}
|
|
3234
2788
|
for (const q of questions) {
|
|
3235
|
-
if (
|
|
3236
|
-
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3237
|
-
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
2789
|
+
if (q.sessionID !== sessionId) continue;
|
|
3238
2790
|
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2791
|
+
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3239
2792
|
const reported = await this.reportInteraction(
|
|
3240
2793
|
watcher.conv.id,
|
|
3241
2794
|
"question",
|
|
@@ -3249,22 +2802,14 @@ var ChannelDriver = class {
|
|
|
3249
2802
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
3250
2803
|
if (res.ok) {
|
|
3251
2804
|
const body = await res.json();
|
|
3252
|
-
|
|
3253
|
-
permissions = body;
|
|
3254
|
-
} else {
|
|
3255
|
-
permissionsPolledOk = false;
|
|
3256
|
-
}
|
|
3257
|
-
} else {
|
|
3258
|
-
permissionsPolledOk = false;
|
|
2805
|
+
permissions = Array.isArray(body) ? body : [];
|
|
3259
2806
|
}
|
|
3260
2807
|
} catch {
|
|
3261
|
-
permissionsPolledOk = false;
|
|
3262
2808
|
}
|
|
3263
2809
|
for (const p of permissions) {
|
|
3264
|
-
if (
|
|
3265
|
-
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3266
|
-
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
2810
|
+
if (p.sessionID !== sessionId) continue;
|
|
3267
2811
|
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2812
|
+
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3268
2813
|
const reported = await this.reportInteraction(
|
|
3269
2814
|
watcher.conv.id,
|
|
3270
2815
|
"permission",
|
|
@@ -3273,128 +2818,6 @@ var ChannelDriver = class {
|
|
|
3273
2818
|
);
|
|
3274
2819
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
3275
2820
|
}
|
|
3276
|
-
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3277
|
-
}
|
|
3278
|
-
/**
|
|
3279
|
-
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3280
|
-
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3281
|
-
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3282
|
-
* so their questions/permissions live under a different `sessionID` that must
|
|
3283
|
-
* still be attributed to the root conversation the watcher owns.
|
|
3284
|
-
*
|
|
3285
|
-
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3286
|
-
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3287
|
-
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3288
|
-
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3289
|
-
*/
|
|
3290
|
-
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3291
|
-
let current = sessionId;
|
|
3292
|
-
for (let depth = 0; current && depth < 32; depth++) {
|
|
3293
|
-
if (current === rootSessionId) return true;
|
|
3294
|
-
const parent = await this.resolveSessionParent(current);
|
|
3295
|
-
if (parent === null || parent === void 0) return false;
|
|
3296
|
-
current = parent;
|
|
3297
|
-
}
|
|
3298
|
-
return false;
|
|
3299
|
-
}
|
|
3300
|
-
/**
|
|
3301
|
-
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3302
|
-
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3303
|
-
* unreachable / the session can't be read (so the caller stops walking without
|
|
3304
|
-
* caching a wrong answer — the next tick retries).
|
|
3305
|
-
*/
|
|
3306
|
-
async resolveSessionParent(sessionId) {
|
|
3307
|
-
const cached = this.sessionParents.get(sessionId);
|
|
3308
|
-
if (cached !== void 0) return cached;
|
|
3309
|
-
let parent = void 0;
|
|
3310
|
-
try {
|
|
3311
|
-
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3312
|
-
if (res.ok) {
|
|
3313
|
-
const body = await res.json();
|
|
3314
|
-
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3315
|
-
}
|
|
3316
|
-
} catch {
|
|
3317
|
-
parent = void 0;
|
|
3318
|
-
}
|
|
3319
|
-
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3320
|
-
return parent;
|
|
3321
|
-
}
|
|
3322
|
-
/**
|
|
3323
|
-
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3324
|
-
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3325
|
-
*
|
|
3326
|
-
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3327
|
-
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3328
|
-
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3329
|
-
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3330
|
-
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3331
|
-
* provably in flight at the exact moment of recovery.
|
|
3332
|
-
*
|
|
3333
|
-
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3334
|
-
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3335
|
-
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3336
|
-
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3337
|
-
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3338
|
-
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3339
|
-
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3340
|
-
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3341
|
-
*
|
|
3342
|
-
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3343
|
-
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3344
|
-
* - `false` → descendants exist but none is actively generating (the restart
|
|
3345
|
-
* case), OR no descendant is found at all.
|
|
3346
|
-
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3347
|
-
*
|
|
3348
|
-
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3349
|
-
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3350
|
-
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3351
|
-
* longer exists". The inversion lives in the caller; this method just reports
|
|
3352
|
-
* true/false/null faithfully.
|
|
3353
|
-
*
|
|
3354
|
-
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3355
|
-
* (already proven by the existing child-session interaction tests, via
|
|
3356
|
-
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3357
|
-
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3358
|
-
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3359
|
-
* `SessionStatus` only.
|
|
3360
|
-
*/
|
|
3361
|
-
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3362
|
-
const sessions = await listSessions(this.port);
|
|
3363
|
-
if (!sessions) {
|
|
3364
|
-
this.log({
|
|
3365
|
-
level: "error",
|
|
3366
|
-
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3367
|
-
});
|
|
3368
|
-
return null;
|
|
3369
|
-
}
|
|
3370
|
-
for (const candidate of sessions) {
|
|
3371
|
-
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3372
|
-
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3373
|
-
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3374
|
-
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3375
|
-
return true;
|
|
3376
|
-
}
|
|
3377
|
-
}
|
|
3378
|
-
return false;
|
|
3379
|
-
}
|
|
3380
|
-
/**
|
|
3381
|
-
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3382
|
-
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3383
|
-
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3384
|
-
* the aborted-in-flight production bug after a restart.
|
|
3385
|
-
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3386
|
-
* (the sub-agent preamble — #253's shape).
|
|
3387
|
-
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3388
|
-
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3389
|
-
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3390
|
-
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3391
|
-
*/
|
|
3392
|
-
replyCompletionShape(reply) {
|
|
3393
|
-
if (!reply) return "other";
|
|
3394
|
-
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3395
|
-
if (completed == null) return "b1";
|
|
3396
|
-
const finish = reply.info?.finish ?? reply.finish;
|
|
3397
|
-
return finish === "tool-calls" ? "b2" : "other";
|
|
3398
2821
|
}
|
|
3399
2822
|
/**
|
|
3400
2823
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -3627,12 +3050,6 @@ var ChannelDriver = class {
|
|
|
3627
3050
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3628
3051
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3629
3052
|
* context (no silent catch, per development-workflow).
|
|
3630
|
-
*
|
|
3631
|
-
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3632
|
-
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3633
|
-
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3634
|
-
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3635
|
-
* leaves liveness").
|
|
3636
3053
|
*/
|
|
3637
3054
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
3638
3055
|
try {
|
|
@@ -3651,9 +3068,7 @@ var ChannelDriver = class {
|
|
|
3651
3068
|
conversation_id: conversationId,
|
|
3652
3069
|
message_id: messageId
|
|
3653
3070
|
});
|
|
3654
|
-
return false;
|
|
3655
3071
|
}
|
|
3656
|
-
return true;
|
|
3657
3072
|
} catch (err) {
|
|
3658
3073
|
this.log({
|
|
3659
3074
|
level: "error",
|
|
@@ -3661,7 +3076,6 @@ var ChannelDriver = class {
|
|
|
3661
3076
|
conversation_id: conversationId,
|
|
3662
3077
|
message_id: messageId
|
|
3663
3078
|
});
|
|
3664
|
-
return false;
|
|
3665
3079
|
}
|
|
3666
3080
|
}
|
|
3667
3081
|
async persistSession(conversationId, sessionId) {
|
|
@@ -3934,25 +3348,6 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3934
3348
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
3935
3349
|
}
|
|
3936
3350
|
}
|
|
3937
|
-
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3938
|
-
const apiUrl = getApiUrlConfig();
|
|
3939
|
-
try {
|
|
3940
|
-
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3941
|
-
method: "POST",
|
|
3942
|
-
headers: { Authorization: authHeader }
|
|
3943
|
-
});
|
|
3944
|
-
if (!response.ok) {
|
|
3945
|
-
const serverMessage = await readErrorMessage(response);
|
|
3946
|
-
return {
|
|
3947
|
-
ok: false,
|
|
3948
|
-
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3949
|
-
};
|
|
3950
|
-
}
|
|
3951
|
-
return { ok: true };
|
|
3952
|
-
} catch (error2) {
|
|
3953
|
-
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3954
|
-
}
|
|
3955
|
-
}
|
|
3956
3351
|
async function getAgentInfo(agentId, authHeader) {
|
|
3957
3352
|
const apiUrl = getApiUrlConfig();
|
|
3958
3353
|
try {
|
|
@@ -3999,7 +3394,7 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3999
3394
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
4000
3395
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
4001
3396
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4002
|
-
var
|
|
3397
|
+
var CHANNEL_SETTLE_MS = Number(process.env.EVIDENT_SETTLE_MS) || void 0;
|
|
4003
3398
|
function log2(state, message, isError = false) {
|
|
4004
3399
|
if (state.json) {
|
|
4005
3400
|
console.log(
|
|
@@ -4154,7 +3549,7 @@ async function driveChannels(state, driver) {
|
|
|
4154
3549
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
4155
3550
|
if (state.interactive) displayStatus(state);
|
|
4156
3551
|
}
|
|
4157
|
-
await new Promise((
|
|
3552
|
+
await new Promise((resolve) => setTimeout(resolve, CHANNEL_POLL_INTERVAL_MS));
|
|
4158
3553
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
4159
3554
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
4160
3555
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -4165,122 +3560,8 @@ async function driveChannels(state, driver) {
|
|
|
4165
3560
|
}
|
|
4166
3561
|
}
|
|
4167
3562
|
}
|
|
4168
|
-
|
|
4169
|
-
async function runSweep(state, driver, config2) {
|
|
4170
|
-
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4171
|
-
try {
|
|
4172
|
-
const sessions = await listSessions(state.port);
|
|
4173
|
-
if (sessions === null) {
|
|
4174
|
-
logActivity(state, {
|
|
4175
|
-
type: "info",
|
|
4176
|
-
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4177
|
-
});
|
|
4178
|
-
return;
|
|
4179
|
-
}
|
|
4180
|
-
const toDelete = selectSessionsToDelete(
|
|
4181
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4182
|
-
{
|
|
4183
|
-
maxAgeMs: config2.maxAgeMs,
|
|
4184
|
-
maxCount: config2.maxCount,
|
|
4185
|
-
nowMs: Date.now(),
|
|
4186
|
-
protectedIds: driver.protectedSessionIds()
|
|
4187
|
-
}
|
|
4188
|
-
);
|
|
4189
|
-
const protectedNow = driver.protectedSessionIds();
|
|
4190
|
-
let deleted = 0;
|
|
4191
|
-
let failed = 0;
|
|
4192
|
-
let skippedNewlyActive = 0;
|
|
4193
|
-
for (const id of toDelete) {
|
|
4194
|
-
if (protectedNow.has(id)) {
|
|
4195
|
-
skippedNewlyActive++;
|
|
4196
|
-
logActivity(state, {
|
|
4197
|
-
type: "info",
|
|
4198
|
-
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4199
|
-
});
|
|
4200
|
-
continue;
|
|
4201
|
-
}
|
|
4202
|
-
if (await deleteSession(state.port, id)) deleted++;
|
|
4203
|
-
else failed++;
|
|
4204
|
-
}
|
|
4205
|
-
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4206
|
-
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4207
|
-
logActivity(state, {
|
|
4208
|
-
type: "info",
|
|
4209
|
-
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4210
|
-
});
|
|
4211
|
-
} catch (error2) {
|
|
4212
|
-
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4213
|
-
logActivity(state, {
|
|
4214
|
-
type: "error",
|
|
4215
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4216
|
-
});
|
|
4217
|
-
}
|
|
4218
|
-
}
|
|
4219
|
-
function scheduleSessionCleanup(state, driver, options) {
|
|
4220
|
-
const config2 = resolveSessionCleanupConfig(
|
|
4221
|
-
{
|
|
4222
|
-
maxAge: options.sessionCleanupMaxAge,
|
|
4223
|
-
maxCount: options.sessionCleanupMaxCount,
|
|
4224
|
-
interval: options.sessionCleanupInterval
|
|
4225
|
-
},
|
|
4226
|
-
process.env
|
|
4227
|
-
);
|
|
4228
|
-
for (const warning2 of config2.warnings) {
|
|
4229
|
-
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4230
|
-
}
|
|
4231
|
-
if (!config2.enabled) return;
|
|
4232
|
-
logActivity(state, {
|
|
4233
|
-
type: "info",
|
|
4234
|
-
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4235
|
-
});
|
|
4236
|
-
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4237
|
-
const firstSweep = setTimeout(
|
|
4238
|
-
() => void runSweep(state, driver, config2),
|
|
4239
|
-
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4240
|
-
);
|
|
4241
|
-
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4242
|
-
}
|
|
4243
|
-
async function notifyOffline(state) {
|
|
4244
|
-
if (!state.agentId || !state.authHeader) return;
|
|
4245
|
-
if (!state.connected) {
|
|
4246
|
-
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4247
|
-
return;
|
|
4248
|
-
}
|
|
4249
|
-
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4250
|
-
if (result.ok) {
|
|
4251
|
-
log2(state, "Notified Evident the agent is going offline");
|
|
4252
|
-
} else {
|
|
4253
|
-
logActivity(state, {
|
|
4254
|
-
type: "error",
|
|
4255
|
-
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4256
|
-
});
|
|
4257
|
-
if (state.interactive) displayStatus(state);
|
|
4258
|
-
}
|
|
4259
|
-
}
|
|
4260
|
-
async function cleanup(state, opts = {}) {
|
|
3563
|
+
async function cleanup(state) {
|
|
4261
3564
|
state.running = false;
|
|
4262
|
-
for (const timer of state.sessionCleanupTimers) {
|
|
4263
|
-
clearInterval(timer);
|
|
4264
|
-
clearTimeout(timer);
|
|
4265
|
-
}
|
|
4266
|
-
state.sessionCleanupTimers = [];
|
|
4267
|
-
if (opts.graceful && state.channelDriver) {
|
|
4268
|
-
state.channelDriver.stop();
|
|
4269
|
-
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4270
|
-
if (state.interactive) {
|
|
4271
|
-
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4272
|
-
displayStatus(state);
|
|
4273
|
-
}
|
|
4274
|
-
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4275
|
-
if (!settled) {
|
|
4276
|
-
logActivity(state, {
|
|
4277
|
-
type: "info",
|
|
4278
|
-
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4279
|
-
});
|
|
4280
|
-
if (state.interactive) displayStatus(state);
|
|
4281
|
-
}
|
|
4282
|
-
}
|
|
4283
|
-
await notifyOffline(state);
|
|
4284
3565
|
if (state.connection) {
|
|
4285
3566
|
state.connection.close();
|
|
4286
3567
|
state.connection = null;
|
|
@@ -4311,13 +3592,10 @@ async function run(options) {
|
|
|
4311
3592
|
opencodeVersion: null,
|
|
4312
3593
|
opencodeProcess: null,
|
|
4313
3594
|
connection: null,
|
|
4314
|
-
channelDriver: null,
|
|
4315
3595
|
running: true,
|
|
4316
|
-
shuttingDown: false,
|
|
4317
3596
|
activityLog: [],
|
|
4318
3597
|
messageCount: 0,
|
|
4319
3598
|
lastProxiedActivityAt: null,
|
|
4320
|
-
sessionCleanupTimers: [],
|
|
4321
3599
|
authHeader: ""
|
|
4322
3600
|
};
|
|
4323
3601
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -4328,15 +3606,13 @@ async function run(options) {
|
|
|
4328
3606
|
);
|
|
4329
3607
|
}
|
|
4330
3608
|
const handleSignal = async () => {
|
|
4331
|
-
if (state.shuttingDown) return;
|
|
4332
|
-
state.shuttingDown = true;
|
|
4333
3609
|
if (state.interactive) {
|
|
4334
3610
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
4335
3611
|
displayStatus(state);
|
|
4336
3612
|
} else {
|
|
4337
3613
|
log2(state, "Shutting down...");
|
|
4338
3614
|
}
|
|
4339
|
-
await cleanup(state
|
|
3615
|
+
await cleanup(state);
|
|
4340
3616
|
await shutdownTelemetry();
|
|
4341
3617
|
process.exit(0);
|
|
4342
3618
|
};
|
|
@@ -4457,13 +3733,13 @@ async function run(options) {
|
|
|
4457
3733
|
getAuthHeader: () => state.authHeader,
|
|
4458
3734
|
conversationFilter: state.conversationFilter,
|
|
4459
3735
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3736
|
+
settleMs: CHANNEL_SETTLE_MS,
|
|
4460
3737
|
log: (entry) => logActivity(state, {
|
|
4461
3738
|
type: entry.level === "error" ? "error" : "info",
|
|
4462
3739
|
message: entry.message,
|
|
4463
3740
|
error: entry.level === "error" ? entry.message : void 0
|
|
4464
3741
|
})
|
|
4465
3742
|
});
|
|
4466
|
-
state.channelDriver = channelDriver;
|
|
4467
3743
|
const connection = new RunnerConnection({
|
|
4468
3744
|
agentId: state.agentId,
|
|
4469
3745
|
getAuthHeader: () => state.authHeader,
|
|
@@ -4560,12 +3836,10 @@ async function run(options) {
|
|
|
4560
3836
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
4561
3837
|
throw error2;
|
|
4562
3838
|
}
|
|
4563
|
-
scheduleSessionCleanup(state, channelDriver, options);
|
|
4564
3839
|
if (!interactive || state.json) {
|
|
4565
3840
|
log2(state, "Driving channel messages...");
|
|
4566
3841
|
}
|
|
4567
3842
|
await driveChannels(state, channelDriver);
|
|
4568
|
-
if (state.shuttingDown) return;
|
|
4569
3843
|
await cleanup(state);
|
|
4570
3844
|
if (state.json) {
|
|
4571
3845
|
console.log(
|
|
@@ -4580,7 +3854,6 @@ async function run(options) {
|
|
|
4580
3854
|
await shutdownTelemetry();
|
|
4581
3855
|
process.exit(0);
|
|
4582
3856
|
} catch (error2) {
|
|
4583
|
-
if (state.shuttingDown) return;
|
|
4584
3857
|
await cleanup(state);
|
|
4585
3858
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4586
3859
|
if (state.json) {
|
|
@@ -4615,16 +3888,7 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
4615
3888
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
4616
3889
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
4617
3890
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
4618
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4619
|
-
"--session-cleanup-max-age <duration>",
|
|
4620
|
-
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4621
|
-
).option(
|
|
4622
|
-
"--session-cleanup-max-count <n>",
|
|
4623
|
-
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4624
|
-
).option(
|
|
4625
|
-
"--session-cleanup-interval <duration>",
|
|
4626
|
-
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4627
|
-
).action(
|
|
3891
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").action(
|
|
4628
3892
|
(options) => {
|
|
4629
3893
|
run({
|
|
4630
3894
|
agent: options.agent,
|
|
@@ -4632,11 +3896,7 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
4632
3896
|
verbose: options.verbose,
|
|
4633
3897
|
conversation: options.conversation,
|
|
4634
3898
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
4635
|
-
json: options.json
|
|
4636
|
-
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4637
|
-
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4638
|
-
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4639
|
-
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3899
|
+
json: options.json
|
|
4640
3900
|
});
|
|
4641
3901
|
}
|
|
4642
3902
|
);
|