@evident-ai/cli 3.0.1-dev.a748d55 → 3.0.1-dev.ab867cb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1061 -337
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -285,14 +285,14 @@ function blank() {
|
|
|
285
285
|
console.log();
|
|
286
286
|
}
|
|
287
287
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
288
|
+
return new Promise((resolve2) => {
|
|
289
289
|
process.stdout.write(chalk.dim(prompt));
|
|
290
290
|
const handler = () => {
|
|
291
291
|
process.stdin.removeListener("data", handler);
|
|
292
292
|
process.stdin.setRawMode?.(false);
|
|
293
293
|
process.stdin.pause();
|
|
294
294
|
console.log();
|
|
295
|
-
|
|
295
|
+
resolve2();
|
|
296
296
|
};
|
|
297
297
|
if (process.stdin.isTTY) {
|
|
298
298
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +302,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
302
|
});
|
|
303
303
|
}
|
|
304
304
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
305
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
306
306
|
}
|
|
307
307
|
|
|
308
308
|
// src/commands/login.ts
|
|
@@ -376,19 +376,19 @@ async function tokenLogin() {
|
|
|
376
376
|
console.log("Visit your Evident dashboard to generate a CLI token.");
|
|
377
377
|
blank();
|
|
378
378
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
379
|
+
const token = await new Promise((resolve2) => {
|
|
380
380
|
let data = "";
|
|
381
381
|
process.stdin.setEncoding("utf8");
|
|
382
382
|
process.stdin.on("data", (chunk) => {
|
|
383
383
|
data += chunk;
|
|
384
384
|
});
|
|
385
385
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
386
|
+
resolve2(data.trim());
|
|
387
387
|
});
|
|
388
388
|
if (process.stdin.isTTY) {
|
|
389
389
|
process.stdin.once("data", (chunk) => {
|
|
390
390
|
process.stdin.pause();
|
|
391
|
-
|
|
391
|
+
resolve2(chunk.toString().trim());
|
|
392
392
|
});
|
|
393
393
|
process.stdin.resume();
|
|
394
394
|
}
|
|
@@ -471,12 +471,6 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
-
// ../../packages/types/src/opencode/index.ts
|
|
475
|
-
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
-
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
-
return `msg_${sanitized}`;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
474
|
// ../../packages/types/src/telemetry/index.ts
|
|
481
475
|
var TelemetryEventTypes = {
|
|
482
476
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -515,7 +509,10 @@ function stripQuery(url) {
|
|
|
515
509
|
}
|
|
516
510
|
|
|
517
511
|
// src/lib/telemetry.ts
|
|
518
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
519
516
|
var eventBuffer = [];
|
|
520
517
|
var flushTimeout = null;
|
|
521
518
|
var isShuttingDown = false;
|
|
@@ -709,13 +706,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
709
706
|
if (health.healthy) {
|
|
710
707
|
return health;
|
|
711
708
|
}
|
|
712
|
-
await new Promise((
|
|
709
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
713
710
|
}
|
|
714
711
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
715
712
|
}
|
|
716
713
|
|
|
717
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
718
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
719
716
|
function isQueueValidatedVersion(version2) {
|
|
720
717
|
if (!version2) return false;
|
|
721
718
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1071,6 +1068,94 @@ function isAssistantInFlight(m) {
|
|
|
1071
1068
|
if (completedOf(m) == null) return true;
|
|
1072
1069
|
return finishOf(m) === "tool-calls";
|
|
1073
1070
|
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1087
|
+
function sessionLastActivityMs(session) {
|
|
1088
|
+
const candidates = [
|
|
1089
|
+
session.time?.updated,
|
|
1090
|
+
session.time?.created,
|
|
1091
|
+
session.time_updated,
|
|
1092
|
+
session.time_created,
|
|
1093
|
+
session.updated,
|
|
1094
|
+
session.created
|
|
1095
|
+
];
|
|
1096
|
+
for (const c of candidates) {
|
|
1097
|
+
if (typeof c === "number" && Number.isFinite(c)) return c;
|
|
1098
|
+
}
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
async function listSessions(port) {
|
|
1102
|
+
try {
|
|
1103
|
+
const res = await fetch(`${opencodeBase(port)}/session`);
|
|
1104
|
+
if (!res.ok) return null;
|
|
1105
|
+
const body = await res.json();
|
|
1106
|
+
return Array.isArray(body) ? body : null;
|
|
1107
|
+
} catch {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
async function deleteSession(port, id) {
|
|
1112
|
+
try {
|
|
1113
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`, { method: "DELETE" });
|
|
1114
|
+
return res.status >= 200 && res.status < 300;
|
|
1115
|
+
} catch {
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function sessionExists(port, id) {
|
|
1120
|
+
try {
|
|
1121
|
+
const res = await fetch(`${opencodeBase(port)}/session/${id}`);
|
|
1122
|
+
if (res.status >= 200 && res.status < 300) return true;
|
|
1123
|
+
if (res.status === 404) return false;
|
|
1124
|
+
return null;
|
|
1125
|
+
} catch {
|
|
1126
|
+
return null;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1074
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1075
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1076
1161
|
if (directory && directory.trim()) {
|
|
@@ -1088,9 +1173,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1088
1173
|
const data = await response.json();
|
|
1089
1174
|
return data.id;
|
|
1090
1175
|
}
|
|
1091
|
-
|
|
1176
|
+
function messageText(m) {
|
|
1177
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1178
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1179
|
+
}
|
|
1180
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1181
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1182
|
+
const knownUserIds = new Set(
|
|
1183
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1184
|
+
);
|
|
1092
1185
|
const body = {
|
|
1093
|
-
messageID: messageId,
|
|
1094
1186
|
parts: [{ type: "text", text: content }]
|
|
1095
1187
|
};
|
|
1096
1188
|
if (options?.agent) {
|
|
@@ -1114,6 +1206,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1114
1206
|
const text = await res.text().catch(() => "");
|
|
1115
1207
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1116
1208
|
}
|
|
1209
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1210
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1211
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1212
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1213
|
+
if (after) {
|
|
1214
|
+
let best = null;
|
|
1215
|
+
for (const m of after) {
|
|
1216
|
+
if (roleOf(m) !== "user") continue;
|
|
1217
|
+
const id = idOf(m);
|
|
1218
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1219
|
+
if (messageText(m) !== content) continue;
|
|
1220
|
+
const created = createdOf(m) ?? 0;
|
|
1221
|
+
if (best === null || created > best.created) {
|
|
1222
|
+
best = { id, created };
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
if (best) return best.id;
|
|
1226
|
+
}
|
|
1227
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1228
|
+
await new Promise((resolve2) => setTimeout(resolve2, READ_BACK_DELAY_MS));
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1117
1232
|
}
|
|
1118
1233
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1119
1234
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1167,6 +1282,11 @@ function messageRunState(messages, userMessageId) {
|
|
|
1167
1282
|
if (isAssistantInFlight(reply)) return "running";
|
|
1168
1283
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1169
1284
|
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1170
1290
|
function messageError(messages, userMessageId) {
|
|
1171
1291
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1172
1292
|
const error2 = errorOf(reply);
|
|
@@ -1186,30 +1306,109 @@ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
|
1186
1306
|
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1187
1307
|
);
|
|
1188
1308
|
}
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1309
|
+
|
|
1310
|
+
// src/lib/opencode/session-cleanup.ts
|
|
1311
|
+
var DURATION_UNIT_MS = {
|
|
1312
|
+
s: 1e3,
|
|
1313
|
+
m: 60 * 1e3,
|
|
1314
|
+
h: 60 * 60 * 1e3,
|
|
1315
|
+
d: 24 * 60 * 60 * 1e3
|
|
1316
|
+
};
|
|
1317
|
+
function parseDurationMs(input) {
|
|
1318
|
+
const trimmed = input.trim();
|
|
1319
|
+
const match = /^(\d+)([smhd])$/.exec(trimmed);
|
|
1320
|
+
if (!match) {
|
|
1321
|
+
throw new Error(
|
|
1322
|
+
`Invalid duration "${input}": expected <number><unit> where unit is one of s, m, h, d (e.g. "7d", "24h", "30m", "90s").`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
const value = Number(match[1]);
|
|
1326
|
+
if (value <= 0) {
|
|
1327
|
+
throw new Error(`Invalid duration "${input}": must be a positive value.`);
|
|
1328
|
+
}
|
|
1329
|
+
return value * DURATION_UNIT_MS[match[2]];
|
|
1330
|
+
}
|
|
1331
|
+
function selectSessionsToDelete(sessions, opts) {
|
|
1332
|
+
const { maxAgeMs, maxCount, nowMs, protectedIds } = opts;
|
|
1333
|
+
if (maxAgeMs === void 0 && maxCount === void 0) return [];
|
|
1334
|
+
const ageEligible = (s) => {
|
|
1335
|
+
if (maxAgeMs === void 0) return false;
|
|
1336
|
+
if (s.lastActivityMs === null) return true;
|
|
1337
|
+
return nowMs - s.lastActivityMs > maxAgeMs;
|
|
1338
|
+
};
|
|
1339
|
+
const countEligibleIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
if (maxCount !== void 0) {
|
|
1341
|
+
const byActivityDesc = [...sessions].sort(
|
|
1342
|
+
(a, b) => (b.lastActivityMs ?? -Infinity) - (a.lastActivityMs ?? -Infinity)
|
|
1343
|
+
);
|
|
1344
|
+
for (const s of byActivityDesc.slice(maxCount)) {
|
|
1345
|
+
countEligibleIds.add(s.id);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const toDelete = [];
|
|
1349
|
+
for (const s of sessions) {
|
|
1350
|
+
if (protectedIds.has(s.id)) continue;
|
|
1351
|
+
if (ageEligible(s) || countEligibleIds.has(s.id)) {
|
|
1352
|
+
toDelete.push(s.id);
|
|
1198
1353
|
}
|
|
1199
1354
|
}
|
|
1200
|
-
|
|
1201
|
-
return now - newestAssistantCreated >= settleMs;
|
|
1355
|
+
return toDelete;
|
|
1202
1356
|
}
|
|
1203
|
-
|
|
1204
|
-
|
|
1357
|
+
var DEFAULT_INTERVAL = "1h";
|
|
1358
|
+
function resolve(flag, envValue, fallback) {
|
|
1359
|
+
return flag ?? envValue ?? fallback;
|
|
1205
1360
|
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1361
|
+
function parseMaxCount(input) {
|
|
1362
|
+
const trimmed = input.trim();
|
|
1363
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
1364
|
+
throw new Error(`Invalid max-count "${input}": expected a positive integer.`);
|
|
1365
|
+
}
|
|
1366
|
+
const value = Number(trimmed);
|
|
1367
|
+
if (value <= 0) {
|
|
1368
|
+
throw new Error(`Invalid max-count "${input}": must be greater than 0.`);
|
|
1369
|
+
}
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
1373
|
+
const warnings = [];
|
|
1374
|
+
const maxAgeRaw = resolve(flags.maxAge, env.EVIDENT_SESSION_CLEANUP_MAX_AGE);
|
|
1375
|
+
const maxCountRaw = resolve(flags.maxCount, env.EVIDENT_SESSION_CLEANUP_MAX_COUNT);
|
|
1376
|
+
const intervalRaw = resolve(
|
|
1377
|
+
flags.interval,
|
|
1378
|
+
env.EVIDENT_SESSION_CLEANUP_INTERVAL,
|
|
1379
|
+
DEFAULT_INTERVAL
|
|
1380
|
+
);
|
|
1381
|
+
let maxAgeMs;
|
|
1382
|
+
if (maxAgeRaw !== void 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
maxAgeMs = parseDurationMs(maxAgeRaw);
|
|
1385
|
+
} catch (err) {
|
|
1386
|
+
warnings.push(
|
|
1387
|
+
`Ignoring invalid --session-cleanup-max-age: ${err instanceof Error ? err.message : String(err)}`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
let maxCount;
|
|
1392
|
+
if (maxCountRaw !== void 0) {
|
|
1393
|
+
try {
|
|
1394
|
+
maxCount = parseMaxCount(maxCountRaw);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
warnings.push(
|
|
1397
|
+
`Ignoring invalid --session-cleanup-max-count: ${err instanceof Error ? err.message : String(err)}`
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
let intervalMs;
|
|
1402
|
+
try {
|
|
1403
|
+
intervalMs = parseDurationMs(intervalRaw ?? DEFAULT_INTERVAL);
|
|
1404
|
+
} catch (err) {
|
|
1405
|
+
warnings.push(
|
|
1406
|
+
`Ignoring invalid --session-cleanup-interval, using default ${DEFAULT_INTERVAL}: ${err instanceof Error ? err.message : String(err)}`
|
|
1407
|
+
);
|
|
1408
|
+
intervalMs = parseDurationMs(DEFAULT_INTERVAL);
|
|
1211
1409
|
}
|
|
1212
|
-
|
|
1410
|
+
const enabled = maxAgeMs !== void 0 || maxCount !== void 0;
|
|
1411
|
+
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
1213
1412
|
}
|
|
1214
1413
|
|
|
1215
1414
|
// src/lib/tunnel/connection.ts
|
|
@@ -1288,24 +1487,26 @@ var StreamForwarder = class {
|
|
|
1288
1487
|
this.send({ type: "res_end", sid });
|
|
1289
1488
|
return;
|
|
1290
1489
|
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1490
|
+
if (process.env.DEBUG) {
|
|
1491
|
+
log("debug", "agent_request", {
|
|
1492
|
+
correlation_id: correlationId,
|
|
1493
|
+
sid,
|
|
1494
|
+
method,
|
|
1495
|
+
path: stripQuery(path)
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1297
1498
|
const ac = new AbortController();
|
|
1298
1499
|
let bodyPromise;
|
|
1299
1500
|
let pushBody;
|
|
1300
1501
|
let endBody;
|
|
1301
1502
|
if (has_body) {
|
|
1302
1503
|
const chunks = [];
|
|
1303
|
-
bodyPromise = new Promise((
|
|
1504
|
+
bodyPromise = new Promise((resolve2) => {
|
|
1304
1505
|
pushBody = (buf) => {
|
|
1305
1506
|
chunks.push(buf);
|
|
1306
1507
|
};
|
|
1307
1508
|
endBody = () => {
|
|
1308
|
-
|
|
1509
|
+
resolve2(Buffer.concat(chunks));
|
|
1309
1510
|
};
|
|
1310
1511
|
});
|
|
1311
1512
|
}
|
|
@@ -1340,12 +1541,14 @@ var StreamForwarder = class {
|
|
|
1340
1541
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1341
1542
|
});
|
|
1342
1543
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1544
|
+
if (process.env.DEBUG) {
|
|
1545
|
+
log("debug", "agent_response", {
|
|
1546
|
+
correlation_id: correlationId,
|
|
1547
|
+
sid,
|
|
1548
|
+
status: upstream.status,
|
|
1549
|
+
duration_ms: Date.now() - startedAt
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1349
1552
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1350
1553
|
try {
|
|
1351
1554
|
if (upstream.body) {
|
|
@@ -1421,7 +1624,7 @@ function connectTunnel(options) {
|
|
|
1421
1624
|
} = options;
|
|
1422
1625
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1423
1626
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1424
|
-
return new Promise((
|
|
1627
|
+
return new Promise((resolve2, reject) => {
|
|
1425
1628
|
const ws = new WebSocket2(url, {
|
|
1426
1629
|
headers: {
|
|
1427
1630
|
Authorization: authHeader
|
|
@@ -1486,7 +1689,7 @@ function connectTunnel(options) {
|
|
|
1486
1689
|
clearTimeout(connectionTimeout);
|
|
1487
1690
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1488
1691
|
onConnected?.(connectedAgentId);
|
|
1489
|
-
|
|
1692
|
+
resolve2({
|
|
1490
1693
|
ws,
|
|
1491
1694
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1492
1695
|
});
|
|
@@ -1615,10 +1818,10 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1615
1818
|
};
|
|
1616
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1617
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1618
|
-
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1619
1821
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1620
|
-
var
|
|
1621
|
-
var
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1622
1825
|
var ChannelAuthError = class extends Error {
|
|
1623
1826
|
constructor(message) {
|
|
1624
1827
|
super(message);
|
|
@@ -1653,13 +1856,18 @@ var ChannelDriver = class {
|
|
|
1653
1856
|
sleep;
|
|
1654
1857
|
pausedPollIntervalMs;
|
|
1655
1858
|
pausedMaxWaitMs;
|
|
1656
|
-
dispatchConfirmMs;
|
|
1657
1859
|
stuckQueuedMs;
|
|
1658
|
-
stuckQueuedRedriveMax;
|
|
1659
|
-
settleMs;
|
|
1660
1860
|
now;
|
|
1661
1861
|
/** Cache of conversationId → opencode sessionId. */
|
|
1662
1862
|
sessions = /* @__PURE__ */ new Map();
|
|
1863
|
+
/**
|
|
1864
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1865
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1866
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1867
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1868
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1869
|
+
*/
|
|
1870
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1663
1871
|
/**
|
|
1664
1872
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1665
1873
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1710,6 +1918,20 @@ var ChannelDriver = class {
|
|
|
1710
1918
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1711
1919
|
*/
|
|
1712
1920
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1923
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1924
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1925
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1926
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1927
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1928
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1929
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1930
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1931
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1932
|
+
* so the NEXT tick may retry exactly once more).
|
|
1933
|
+
*/
|
|
1934
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1713
1935
|
/**
|
|
1714
1936
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1715
1937
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1717,8 +1939,33 @@ var ChannelDriver = class {
|
|
|
1717
1939
|
* not yet resolved; `null` = resolved-but-unavailable (don't keep retrying).
|
|
1718
1940
|
*/
|
|
1719
1941
|
opencodeDirectory = void 0;
|
|
1942
|
+
/**
|
|
1943
|
+
* Cache of opencode `sessionId → parentID` (its parent session, or `null` when
|
|
1944
|
+
* the session is a root with no parent). Sub-agents spawned via the `task` tool
|
|
1945
|
+
* run in CHILD sessions whose `parentID` chains up to the Evident-created
|
|
1946
|
+
* (watched) session; we resolve this once per session so a child-session
|
|
1947
|
+
* question/permission can be attributed to the watched session's subtree
|
|
1948
|
+
* (`sessionBelongsTo`) instead of being dropped by an exact-id filter. A missing
|
|
1949
|
+
* entry = not yet resolved; `null` = resolved root (stop walking).
|
|
1950
|
+
*/
|
|
1951
|
+
sessionParents = /* @__PURE__ */ new Map();
|
|
1720
1952
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
1721
1953
|
draining = false;
|
|
1954
|
+
/**
|
|
1955
|
+
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
1956
|
+
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
1957
|
+
* is about to dispatch is not missed by the `hasInFlightWatchers()` check (a
|
|
1958
|
+
* drain that entered before `stop()` still registers its watcher).
|
|
1959
|
+
*/
|
|
1960
|
+
activeDrain = null;
|
|
1961
|
+
/**
|
|
1962
|
+
* Set by `stop()` on graceful shutdown. Once stopped, `drainPending` no longer
|
|
1963
|
+
* dispatches NEW work (it returns 0 immediately) — but the per-session watcher
|
|
1964
|
+
* loops already running keep going so in-flight turns can finish and deliver
|
|
1965
|
+
* their reply. `run.ts` awaits `waitForInFlight()` before it closes the tunnel
|
|
1966
|
+
* and stops opencode.
|
|
1967
|
+
*/
|
|
1968
|
+
stopped = false;
|
|
1722
1969
|
constructor(config2) {
|
|
1723
1970
|
this.agentId = config2.agentId;
|
|
1724
1971
|
this.port = config2.port;
|
|
@@ -1732,10 +1979,7 @@ var ChannelDriver = class {
|
|
|
1732
1979
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1733
1980
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1734
1981
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1735
|
-
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1736
1982
|
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1737
|
-
this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
|
|
1738
|
-
this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
|
|
1739
1983
|
this.now = config2.now ?? (() => Date.now());
|
|
1740
1984
|
}
|
|
1741
1985
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1753,8 +1997,21 @@ var ChannelDriver = class {
|
|
|
1753
1997
|
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1754
1998
|
*/
|
|
1755
1999
|
async drainPending() {
|
|
2000
|
+
if (this.stopped) return 0;
|
|
1756
2001
|
if (this.draining) return 0;
|
|
1757
2002
|
this.draining = true;
|
|
2003
|
+
const run2 = this.runDrain();
|
|
2004
|
+
this.activeDrain = run2.then(
|
|
2005
|
+
() => {
|
|
2006
|
+
this.activeDrain = null;
|
|
2007
|
+
},
|
|
2008
|
+
() => {
|
|
2009
|
+
this.activeDrain = null;
|
|
2010
|
+
}
|
|
2011
|
+
);
|
|
2012
|
+
return run2;
|
|
2013
|
+
}
|
|
2014
|
+
async runDrain() {
|
|
1758
2015
|
let dispatched = 0;
|
|
1759
2016
|
try {
|
|
1760
2017
|
const conversations = await this.getPendingConversations();
|
|
@@ -1766,6 +2023,7 @@ var ChannelDriver = class {
|
|
|
1766
2023
|
});
|
|
1767
2024
|
}
|
|
1768
2025
|
for (const conv of conversations) {
|
|
2026
|
+
if (this.stopped) break;
|
|
1769
2027
|
dispatched += await this.processConversation(conv);
|
|
1770
2028
|
}
|
|
1771
2029
|
await this.readoptProcessing();
|
|
@@ -1786,6 +2044,73 @@ var ChannelDriver = class {
|
|
|
1786
2044
|
}
|
|
1787
2045
|
return false;
|
|
1788
2046
|
}
|
|
2047
|
+
/**
|
|
2048
|
+
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2049
|
+
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
2050
|
+
* `watchers` entry whose `inFlight` set is non-empty — the same predicate
|
|
2051
|
+
* `hasInFlightWatchers()` uses, lifted to return the ids.
|
|
2052
|
+
*
|
|
2053
|
+
* Deliberately does NOT include `this.sessions` (the permanent, never-pruned
|
|
2054
|
+
* conversation→session cache). Protecting every bound-but-idle session there
|
|
2055
|
+
* would shield nearly every session and defeat cleanup — AND it is unnecessary:
|
|
2056
|
+
* `ensureSession` is self-healing (it recreates a session whose id no longer
|
|
2057
|
+
* exists), so deleting an idle bound session is harmless — the conversation's
|
|
2058
|
+
* next turn transparently rebinds a fresh one. The only thing worth protecting
|
|
2059
|
+
* is a session with a turn ACTIVELY in flight right now: tearing that down
|
|
2060
|
+
* mid-turn would strand the running `prompt_async`. Idle sessions are fair game.
|
|
2061
|
+
*/
|
|
2062
|
+
protectedSessionIds() {
|
|
2063
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2064
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
2065
|
+
if (watcher.inFlight.size > 0) ids.add(sessionId);
|
|
2066
|
+
}
|
|
2067
|
+
return ids;
|
|
2068
|
+
}
|
|
2069
|
+
/**
|
|
2070
|
+
* Begin a graceful stop: stop accepting NEW channel work. Idempotent. After
|
|
2071
|
+
* this, `drainPending()` is a no-op (returns 0), so no new message is dispatched
|
|
2072
|
+
* — but the watcher loops already tracking in-flight turns keep running, so a
|
|
2073
|
+
* turn that has finished (or is about to) still fires `markDone` and delivers
|
|
2074
|
+
* its reply. Pair with `waitForInFlight()` to bound how long shutdown waits.
|
|
2075
|
+
*/
|
|
2076
|
+
stop() {
|
|
2077
|
+
this.stopped = true;
|
|
2078
|
+
}
|
|
2079
|
+
/**
|
|
2080
|
+
* Wait (up to `timeoutMs`) for in-flight watcher work to settle during a
|
|
2081
|
+
* graceful shutdown, so a turn whose reply is ready — or completes within the
|
|
2082
|
+
* window — is delivered before the process exits, instead of being cut off and
|
|
2083
|
+
* left for the ADR-0046 restart-recovery path.
|
|
2084
|
+
*
|
|
2085
|
+
* Bounded on purpose: the watcher's own give-up deadline is up to 10 minutes,
|
|
2086
|
+
* far longer than a shutdown grace period (e.g. Fargate's SIGTERM→SIGKILL
|
|
2087
|
+
* window). We poll `hasInFlightWatchers()` and return as soon as the in-flight
|
|
2088
|
+
* set empties OR the timeout elapses. Anything still in flight at the timeout is
|
|
2089
|
+
* safe to abandon — it stays `processing` server-side and is re-adopted on the
|
|
2090
|
+
* next runner start (ADR-0046).
|
|
2091
|
+
*
|
|
2092
|
+
* @returns true if all in-flight work settled within the window; false if the
|
|
2093
|
+
* timeout elapsed with work still in flight.
|
|
2094
|
+
*/
|
|
2095
|
+
async waitForInFlight(timeoutMs) {
|
|
2096
|
+
const deadline = this.now() + timeoutMs;
|
|
2097
|
+
const step = Math.min(this.pausedPollIntervalMs, 250);
|
|
2098
|
+
if (this.activeDrain) {
|
|
2099
|
+
let drainSettled = false;
|
|
2100
|
+
void this.activeDrain.then(() => {
|
|
2101
|
+
drainSettled = true;
|
|
2102
|
+
});
|
|
2103
|
+
while (!drainSettled) {
|
|
2104
|
+
if (this.now() >= deadline) return false;
|
|
2105
|
+
await this.sleep(step);
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
while (this.hasInFlightWatchers()) {
|
|
2109
|
+
if (this.now() >= deadline) return false;
|
|
2110
|
+
await this.sleep(step);
|
|
2111
|
+
}
|
|
2112
|
+
return true;
|
|
2113
|
+
}
|
|
1789
2114
|
/**
|
|
1790
2115
|
* Await all outstanding per-session watchers (WI-3).
|
|
1791
2116
|
*
|
|
@@ -1822,15 +2147,16 @@ var ChannelDriver = class {
|
|
|
1822
2147
|
let dispatched = 0;
|
|
1823
2148
|
let skippedAlreadyDispatched = 0;
|
|
1824
2149
|
for (const message of messages) {
|
|
2150
|
+
if (this.stopped) break;
|
|
1825
2151
|
if (this.dispatched.has(message.id)) {
|
|
1826
2152
|
skippedAlreadyDispatched += 1;
|
|
1827
2153
|
continue;
|
|
1828
2154
|
}
|
|
1829
|
-
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1830
2155
|
const options = {
|
|
1831
2156
|
agent: message.opencode_agent ?? void 0,
|
|
1832
2157
|
model: message.opencode_model ?? void 0
|
|
1833
2158
|
};
|
|
2159
|
+
let opencodeMessageId;
|
|
1834
2160
|
try {
|
|
1835
2161
|
this.log({
|
|
1836
2162
|
level: "info",
|
|
@@ -1838,10 +2164,23 @@ var ChannelDriver = class {
|
|
|
1838
2164
|
conversation_id: conv.id,
|
|
1839
2165
|
message_id: message.id
|
|
1840
2166
|
});
|
|
1841
|
-
await
|
|
2167
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
2168
|
+
sessionId,
|
|
2169
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
2170
|
+
);
|
|
1842
2171
|
} catch (err) {
|
|
1843
2172
|
if (err instanceof ChannelAuthError) throw err;
|
|
1844
2173
|
this.dispatched.delete(message.id);
|
|
2174
|
+
if (await sessionExists(this.port, sessionId) === false) {
|
|
2175
|
+
this.sessions.delete(conv.id);
|
|
2176
|
+
this.log({
|
|
2177
|
+
level: "info",
|
|
2178
|
+
message: `Message ${message.id.slice(0, 8)} dispatch hit a session (${sessionId.slice(0, 8)}) that was deleted mid-dispatch (cleanup race) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick (recreated then, in order). Already-dispatched turns keep their watcher.`,
|
|
2179
|
+
conversation_id: conv.id,
|
|
2180
|
+
message_id: message.id
|
|
2181
|
+
});
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
1845
2184
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1846
2185
|
});
|
|
1847
2186
|
this.log({
|
|
@@ -1852,6 +2191,15 @@ var ChannelDriver = class {
|
|
|
1852
2191
|
});
|
|
1853
2192
|
continue;
|
|
1854
2193
|
}
|
|
2194
|
+
if (opencodeMessageId === null) {
|
|
2195
|
+
this.log({
|
|
2196
|
+
level: "error",
|
|
2197
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
2198
|
+
conversation_id: conv.id,
|
|
2199
|
+
message_id: message.id
|
|
2200
|
+
});
|
|
2201
|
+
continue;
|
|
2202
|
+
}
|
|
1855
2203
|
this.dispatched.add(message.id);
|
|
1856
2204
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1857
2205
|
dispatched += 1;
|
|
@@ -1868,16 +2216,33 @@ var ChannelDriver = class {
|
|
|
1868
2216
|
return dispatched;
|
|
1869
2217
|
}
|
|
1870
2218
|
async ensureSession(conv) {
|
|
1871
|
-
const
|
|
1872
|
-
if (
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
2219
|
+
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
2220
|
+
if (bound) {
|
|
2221
|
+
const exists = await sessionExists(this.port, bound);
|
|
2222
|
+
if (exists === false) {
|
|
2223
|
+
this.log({
|
|
2224
|
+
level: "info",
|
|
2225
|
+
message: `OpenCode session ${bound} for conversation ${conv.id.slice(0, 8)} no longer exists (deleted or DB reset) \u2014 creating a fresh session and rebinding.`,
|
|
2226
|
+
conversation_id: conv.id
|
|
2227
|
+
});
|
|
2228
|
+
this.sessions.delete(conv.id);
|
|
2229
|
+
return this.createAndBindSession(conv.id);
|
|
2230
|
+
}
|
|
2231
|
+
this.sessions.set(conv.id, bound);
|
|
2232
|
+
return bound;
|
|
1876
2233
|
}
|
|
2234
|
+
return this.createAndBindSession(conv.id);
|
|
2235
|
+
}
|
|
2236
|
+
/**
|
|
2237
|
+
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
2238
|
+
* best-effort persist it server-side. Shared by the first-ever bind and the
|
|
2239
|
+
* self-heal recreate path in `ensureSession`.
|
|
2240
|
+
*/
|
|
2241
|
+
async createAndBindSession(conversationId) {
|
|
1877
2242
|
const directory = await this.resolveOpenCodeDirectory();
|
|
1878
2243
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
1879
|
-
this.sessions.set(
|
|
1880
|
-
await this.persistSession(
|
|
2244
|
+
this.sessions.set(conversationId, sessionId);
|
|
2245
|
+
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
1881
2246
|
});
|
|
1882
2247
|
return sessionId;
|
|
1883
2248
|
}
|
|
@@ -1900,6 +2265,25 @@ var ChannelDriver = class {
|
|
|
1900
2265
|
// -------------------------------------------------------------------------
|
|
1901
2266
|
// Per-session watcher (WI-3)
|
|
1902
2267
|
// -------------------------------------------------------------------------
|
|
2268
|
+
/**
|
|
2269
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
2270
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
2271
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
2272
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
2273
|
+
* (each dispatch reports its own outcome to its caller).
|
|
2274
|
+
*/
|
|
2275
|
+
dispatchLocked(sessionId, fn) {
|
|
2276
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
2277
|
+
const run2 = prior.then(fn, fn);
|
|
2278
|
+
this.sessionDispatchLocks.set(
|
|
2279
|
+
sessionId,
|
|
2280
|
+
run2.then(
|
|
2281
|
+
() => void 0,
|
|
2282
|
+
() => void 0
|
|
2283
|
+
)
|
|
2284
|
+
);
|
|
2285
|
+
return run2;
|
|
2286
|
+
}
|
|
1903
2287
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1904
2288
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1905
2289
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1909,7 +2293,9 @@ var ChannelDriver = class {
|
|
|
1909
2293
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1910
2294
|
loop: null,
|
|
1911
2295
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1912
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2296
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2297
|
+
lastGoodPollAt: this.now(),
|
|
2298
|
+
hadUsablePoll: false
|
|
1913
2299
|
};
|
|
1914
2300
|
this.watchers.set(sessionId, watcher);
|
|
1915
2301
|
}
|
|
@@ -1919,24 +2305,37 @@ var ChannelDriver = class {
|
|
|
1919
2305
|
opencodeMessageId,
|
|
1920
2306
|
message,
|
|
1921
2307
|
dispatchedAt: now,
|
|
2308
|
+
processingAnchorMs: now,
|
|
1922
2309
|
deadline: now + this.pausedMaxWaitMs,
|
|
1923
2310
|
started: false,
|
|
1924
2311
|
done: false,
|
|
1925
2312
|
stuckReported: false,
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
2313
|
+
lastAliveAt: 0,
|
|
2314
|
+
aliveInFlight: false,
|
|
2315
|
+
awaitingHumanLatched: false,
|
|
2316
|
+
pausedOnQuestion: false,
|
|
2317
|
+
pausedOnPermission: false,
|
|
2318
|
+
pausedClearConfirmed: false,
|
|
2319
|
+
pausedInFlight: false,
|
|
2320
|
+
deliveryDeadlineAnchored: false
|
|
1930
2321
|
});
|
|
1931
2322
|
}
|
|
1932
2323
|
/**
|
|
1933
2324
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1934
2325
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1935
2326
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1936
|
-
* `now
|
|
1937
|
-
* (10 min after `processed_at
|
|
1938
|
-
*
|
|
1939
|
-
*
|
|
2327
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2328
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2329
|
+
*
|
|
2330
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2331
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2332
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2333
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2334
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2335
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2336
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2337
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2338
|
+
* (only the appear-guard uses it).
|
|
1940
2339
|
*
|
|
1941
2340
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1942
2341
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -1946,7 +2345,7 @@ var ChannelDriver = class {
|
|
|
1946
2345
|
* server already flipped to `processing`; the running/done transitions still
|
|
1947
2346
|
* fire from the watcher's normal branches.
|
|
1948
2347
|
*/
|
|
1949
|
-
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs
|
|
2348
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
1950
2349
|
let watcher = this.watchers.get(sessionId);
|
|
1951
2350
|
if (!watcher) {
|
|
1952
2351
|
watcher = {
|
|
@@ -1954,7 +2353,9 @@ var ChannelDriver = class {
|
|
|
1954
2353
|
inFlight: /* @__PURE__ */ new Map(),
|
|
1955
2354
|
loop: null,
|
|
1956
2355
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1957
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2356
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2357
|
+
lastGoodPollAt: this.now(),
|
|
2358
|
+
hadUsablePoll: false
|
|
1958
2359
|
};
|
|
1959
2360
|
this.watchers.set(sessionId, watcher);
|
|
1960
2361
|
}
|
|
@@ -1963,22 +2364,33 @@ var ChannelDriver = class {
|
|
|
1963
2364
|
opencodeMessageId,
|
|
1964
2365
|
message,
|
|
1965
2366
|
dispatchedAt: this.now(),
|
|
2367
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2368
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2369
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2370
|
+
processingAnchorMs: processedAtMs,
|
|
1966
2371
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1967
2372
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1968
2373
|
started: true,
|
|
1969
2374
|
done: false,
|
|
1970
2375
|
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1971
|
-
// AND the stuck-queued observer
|
|
1972
|
-
//
|
|
1973
|
-
//
|
|
1974
|
-
// (
|
|
1975
|
-
// re-driven by the same path (ADR-0046 §c deferred the live-session redrive
|
|
1976
|
-
// to here) — `redriveAttempts` starts fresh so it gets the full budget.
|
|
2376
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2377
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2378
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2379
|
+
// (#210/#220 observability).
|
|
1977
2380
|
stuckReported: false,
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
2381
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2382
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2383
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2384
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2385
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2386
|
+
lastAliveAt: 0,
|
|
2387
|
+
aliveInFlight: false,
|
|
2388
|
+
awaitingHumanLatched: false,
|
|
2389
|
+
pausedOnQuestion: false,
|
|
2390
|
+
pausedOnPermission: false,
|
|
2391
|
+
pausedClearConfirmed: false,
|
|
2392
|
+
pausedInFlight: false,
|
|
2393
|
+
deliveryDeadlineAnchored: false
|
|
1982
2394
|
});
|
|
1983
2395
|
}
|
|
1984
2396
|
/**
|
|
@@ -2029,12 +2441,30 @@ var ChannelDriver = class {
|
|
|
2029
2441
|
messages = Array.isArray(body) ? body : null;
|
|
2030
2442
|
}
|
|
2031
2443
|
} catch {
|
|
2032
|
-
continue;
|
|
2033
2444
|
}
|
|
2445
|
+
if (messages != null && messages.length > 0) {
|
|
2446
|
+
watcher.lastGoodPollAt = this.now();
|
|
2447
|
+
watcher.hadUsablePoll = true;
|
|
2448
|
+
} else {
|
|
2449
|
+
const emptyButReachable = messages != null;
|
|
2450
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2451
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2452
|
+
continue;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2034
2456
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2035
|
-
await this.serviceInFlightMessage(
|
|
2457
|
+
await this.serviceInFlightMessage(
|
|
2458
|
+
sessionId,
|
|
2459
|
+
watcher,
|
|
2460
|
+
inFlight,
|
|
2461
|
+
messages,
|
|
2462
|
+
openQuestions,
|
|
2463
|
+
openPermissions,
|
|
2464
|
+
questionsPolledOk,
|
|
2465
|
+
permissionsPolledOk
|
|
2466
|
+
);
|
|
2036
2467
|
}
|
|
2037
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
2038
2468
|
}
|
|
2039
2469
|
} catch (err) {
|
|
2040
2470
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2056,15 +2486,40 @@ var ChannelDriver = class {
|
|
|
2056
2486
|
});
|
|
2057
2487
|
}
|
|
2058
2488
|
}
|
|
2489
|
+
/**
|
|
2490
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2491
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2492
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2493
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2494
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2495
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2496
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2497
|
+
* or past now, so a still-ample window is left untouched.
|
|
2498
|
+
*/
|
|
2499
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2500
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2501
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2502
|
+
if (this.now() >= inFlight.deadline) {
|
|
2503
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2059
2506
|
/**
|
|
2060
2507
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2061
2508
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2062
2509
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2063
2510
|
* in-flight set on completion or timeout.
|
|
2064
2511
|
*/
|
|
2065
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2512
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
2066
2513
|
const conv = watcher.conv;
|
|
2067
2514
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2515
|
+
const id = inFlight.evidentMessageId;
|
|
2516
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2517
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2518
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2519
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2520
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2521
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2522
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2068
2523
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2069
2524
|
let claimed;
|
|
2070
2525
|
try {
|
|
@@ -2072,7 +2527,7 @@ var ChannelDriver = class {
|
|
|
2072
2527
|
conv.id,
|
|
2073
2528
|
inFlight.evidentMessageId,
|
|
2074
2529
|
sessionId,
|
|
2075
|
-
inFlight.
|
|
2530
|
+
inFlight.opencodeMessageId
|
|
2076
2531
|
);
|
|
2077
2532
|
} catch (err) {
|
|
2078
2533
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2095,6 +2550,7 @@ var ChannelDriver = class {
|
|
|
2095
2550
|
}
|
|
2096
2551
|
}
|
|
2097
2552
|
if (state === "done") {
|
|
2553
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2098
2554
|
if (!inFlight.done) {
|
|
2099
2555
|
this.log({
|
|
2100
2556
|
level: "info",
|
|
@@ -2107,7 +2563,7 @@ var ChannelDriver = class {
|
|
|
2107
2563
|
conv.id,
|
|
2108
2564
|
inFlight.evidentMessageId,
|
|
2109
2565
|
sessionId,
|
|
2110
|
-
inFlight.
|
|
2566
|
+
inFlight.opencodeMessageId
|
|
2111
2567
|
);
|
|
2112
2568
|
} catch (err) {
|
|
2113
2569
|
if (err instanceof ChannelAuthError) throw err;
|
|
@@ -2145,6 +2601,7 @@ var ChannelDriver = class {
|
|
|
2145
2601
|
return;
|
|
2146
2602
|
}
|
|
2147
2603
|
if (state === "failed") {
|
|
2604
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2148
2605
|
if (!inFlight.done) {
|
|
2149
2606
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2150
2607
|
this.log({
|
|
@@ -2190,211 +2647,70 @@ var ChannelDriver = class {
|
|
|
2190
2647
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2191
2648
|
return;
|
|
2192
2649
|
}
|
|
2193
|
-
if (state === "unknown") {
|
|
2194
|
-
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
2195
|
-
await this.redispatchInFlight(sessionId, inFlight);
|
|
2196
|
-
}
|
|
2197
|
-
}
|
|
2198
2650
|
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2199
2651
|
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2200
|
-
if (state === "queued" && pastStuckBound && sessionIdle) {
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
});
|
|
2206
|
-
}
|
|
2207
|
-
const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
|
|
2208
|
-
if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
|
|
2209
|
-
const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
|
|
2210
|
-
if (removed) return;
|
|
2211
|
-
}
|
|
2652
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2653
|
+
inFlight.stuckReported = true;
|
|
2654
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2655
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2656
|
+
});
|
|
2212
2657
|
}
|
|
2213
|
-
|
|
2658
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2659
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2214
2660
|
this.log({
|
|
2215
|
-
level: "
|
|
2216
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
2661
|
+
level: "error",
|
|
2662
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2217
2663
|
conversation_id: conv.id,
|
|
2218
2664
|
message_id: inFlight.evidentMessageId
|
|
2219
2665
|
});
|
|
2666
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2667
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2668
|
+
});
|
|
2220
2669
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2670
|
+
return;
|
|
2221
2671
|
}
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
* the dispatch timestamp so the guard doesn't immediately fire again.
|
|
2228
|
-
*/
|
|
2229
|
-
async redispatchInFlight(sessionId, inFlight) {
|
|
2230
|
-
const options = {
|
|
2231
|
-
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2232
|
-
model: inFlight.message.opencode_model ?? void 0
|
|
2233
|
-
};
|
|
2234
|
-
this.log({
|
|
2235
|
-
level: "info",
|
|
2236
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
|
|
2237
|
-
message_id: inFlight.evidentMessageId
|
|
2238
|
-
});
|
|
2239
|
-
try {
|
|
2240
|
-
await sendPromptAsync(
|
|
2241
|
-
this.port,
|
|
2242
|
-
sessionId,
|
|
2243
|
-
inFlight.message.content,
|
|
2244
|
-
options,
|
|
2245
|
-
inFlight.opencodeMessageId
|
|
2246
|
-
);
|
|
2247
|
-
} catch (err) {
|
|
2248
|
-
this.log({
|
|
2249
|
-
level: "error",
|
|
2250
|
-
message: `Re-dispatch failed for message ${inFlight.evidentMessageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2251
|
-
message_id: inFlight.evidentMessageId
|
|
2672
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2673
|
+
inFlight.aliveInFlight = true;
|
|
2674
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2675
|
+
inFlight.aliveInFlight = false;
|
|
2676
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2252
2677
|
});
|
|
2253
2678
|
}
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
* the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
|
|
2265
|
-
* runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
|
|
2266
|
-
* turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
|
|
2267
|
-
* (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
|
|
2268
|
-
* dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
|
|
2269
|
-
*
|
|
2270
|
-
* So we re-`prompt_async` into the SAME opencode session (opencode-web does
|
|
2271
|
-
* exactly this and it works — UPDATE 2 in the investigation doc), preserving the
|
|
2272
|
-
* conversation's history/continuity: we do NOT create a fresh session and we do
|
|
2273
|
-
* NOT overwrite the conversation's `opencode_session_id`. Only the opencode
|
|
2274
|
-
* user-message id changes — to a fresh native id per attempt (opencode's
|
|
2275
|
-
* caller-supplied id dedup is global+permanent, so each attempt needs a never-
|
|
2276
|
-
* seen id).
|
|
2277
|
-
*
|
|
2278
|
-
* Reply correlation is PRESERVED: the native id is random and NOT re-derivable
|
|
2279
|
-
* from the row id, so the runner carries it to the server (on markProcessing/
|
|
2280
|
-
* markDone via `redriveOpencodeMessageId`), which persists it on the row and
|
|
2281
|
-
* correlates the reply by THAT id (`conversation-notification.ts`). A normal,
|
|
2282
|
-
* never-re-driven message still correlates by the derived stable id — unchanged.
|
|
2283
|
-
*
|
|
2284
|
-
* Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
|
|
2285
|
-
* FAILED (existing channel failure affordance) so the user is TOLD it could not
|
|
2286
|
-
* be answered rather than left silent, and it is removed from the in-flight set.
|
|
2287
|
-
*
|
|
2288
|
-
* @returns true if the message LEFT this watcher's in-flight set (gave up +
|
|
2289
|
-
* marked failed) — the caller then stops servicing it this tick. Returns false
|
|
2290
|
-
* when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
|
|
2291
|
-
* send failed and the next window retries).
|
|
2292
|
-
*/
|
|
2293
|
-
async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
|
|
2294
|
-
if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
|
|
2295
|
-
return true;
|
|
2296
|
-
}
|
|
2297
|
-
if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
|
|
2298
|
-
this.log({
|
|
2299
|
-
level: "error",
|
|
2300
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
|
|
2301
|
-
conversation_id: watcher.conv.id,
|
|
2302
|
-
message_id: inFlight.evidentMessageId
|
|
2303
|
-
});
|
|
2304
|
-
try {
|
|
2305
|
-
await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
|
|
2306
|
-
} catch (err) {
|
|
2307
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2308
|
-
this.log({
|
|
2309
|
-
level: "error",
|
|
2310
|
-
message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2311
|
-
conversation_id: watcher.conv.id,
|
|
2312
|
-
message_id: inFlight.evidentMessageId
|
|
2679
|
+
if (awaitingHuman) {
|
|
2680
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2681
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2682
|
+
inFlight.awaitingHumanLatched = true;
|
|
2683
|
+
}
|
|
2684
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2685
|
+
inFlight.pausedInFlight = true;
|
|
2686
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2687
|
+
inFlight.pausedInFlight = false;
|
|
2688
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2313
2689
|
});
|
|
2314
2690
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2691
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2692
|
+
inFlight.awaitingHumanLatched = false;
|
|
2693
|
+
inFlight.pausedOnQuestion = false;
|
|
2694
|
+
inFlight.pausedOnPermission = false;
|
|
2695
|
+
inFlight.pausedClearConfirmed = false;
|
|
2317
2696
|
}
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
level: "info",
|
|
2322
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
|
|
2323
|
-
conversation_id: watcher.conv.id,
|
|
2324
|
-
message_id: inFlight.evidentMessageId
|
|
2325
|
-
});
|
|
2326
|
-
void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
|
|
2327
|
-
redrive_attempt: inFlight.redriveAttempts
|
|
2328
|
-
});
|
|
2329
|
-
const options = {
|
|
2330
|
-
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2331
|
-
model: inFlight.message.opencode_model ?? void 0
|
|
2332
|
-
};
|
|
2333
|
-
const nativeId = nativeOpencodeMessageId();
|
|
2334
|
-
try {
|
|
2335
|
-
await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
|
|
2336
|
-
} catch (err) {
|
|
2337
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2338
|
-
inFlight.redriveAttempts -= 1;
|
|
2339
|
-
this.log({
|
|
2340
|
-
level: "error",
|
|
2341
|
-
message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
|
|
2342
|
-
conversation_id: watcher.conv.id,
|
|
2343
|
-
message_id: inFlight.evidentMessageId
|
|
2344
|
-
});
|
|
2345
|
-
return false;
|
|
2346
|
-
}
|
|
2347
|
-
inFlight.opencodeMessageId = nativeId;
|
|
2348
|
-
inFlight.redriveOpencodeMessageId = nativeId;
|
|
2349
|
-
inFlight.attemptedOpencodeMessageIds.push(nativeId);
|
|
2350
|
-
inFlight.started = false;
|
|
2351
|
-
return false;
|
|
2352
|
-
}
|
|
2353
|
-
/**
|
|
2354
|
-
* Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
|
|
2355
|
-
* driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
|
|
2356
|
-
* in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
|
|
2357
|
-
* earlier re-drive attempt eventually produced is delivered) instead of
|
|
2358
|
-
* re-driving again or marking failed. Idempotent + guarded like the watcher's
|
|
2359
|
-
* done branch. `@returns` true when the message left the in-flight set (delivered
|
|
2360
|
-
* or terminally-undeliverable) — the caller then stops servicing it this tick.
|
|
2361
|
-
*
|
|
2362
|
-
* The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
|
|
2363
|
-
* branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
|
|
2364
|
-
* polls. `messageRunState === 'done'` means a completed, non-errored correlated
|
|
2365
|
-
* reply exists for that id.
|
|
2366
|
-
*/
|
|
2367
|
-
async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
|
|
2368
|
-
const completedId = inFlight.attemptedOpencodeMessageIds.find(
|
|
2369
|
-
(id) => messageRunState(messages, id) === "done"
|
|
2697
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2698
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2699
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2370
2700
|
);
|
|
2371
|
-
|
|
2372
|
-
if (inFlight.
|
|
2373
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2374
|
-
return true;
|
|
2375
|
-
}
|
|
2376
|
-
this.log({
|
|
2377
|
-
level: "info",
|
|
2378
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
|
|
2379
|
-
conversation_id: watcher.conv.id,
|
|
2380
|
-
message_id: inFlight.evidentMessageId
|
|
2381
|
-
});
|
|
2382
|
-
try {
|
|
2383
|
-
await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
|
|
2384
|
-
} catch (err) {
|
|
2385
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2701
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2702
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2386
2703
|
this.log({
|
|
2387
|
-
level: "
|
|
2388
|
-
message: `
|
|
2389
|
-
conversation_id:
|
|
2704
|
+
level: "info",
|
|
2705
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2706
|
+
conversation_id: conv.id,
|
|
2390
2707
|
message_id: inFlight.evidentMessageId
|
|
2391
2708
|
});
|
|
2709
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2710
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2711
|
+
});
|
|
2392
2712
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2393
|
-
return true;
|
|
2394
2713
|
}
|
|
2395
|
-
inFlight.done = true;
|
|
2396
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2397
|
-
return true;
|
|
2398
2714
|
}
|
|
2399
2715
|
// -------------------------------------------------------------------------
|
|
2400
2716
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
@@ -2474,8 +2790,10 @@ var ChannelDriver = class {
|
|
|
2474
2790
|
});
|
|
2475
2791
|
continue;
|
|
2476
2792
|
}
|
|
2793
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2794
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2477
2795
|
for (const row of sessionRows) {
|
|
2478
|
-
await this.readoptOne(sessionId, row, messages);
|
|
2796
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2479
2797
|
}
|
|
2480
2798
|
}
|
|
2481
2799
|
}
|
|
@@ -2483,23 +2801,21 @@ var ChannelDriver = class {
|
|
|
2483
2801
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2484
2802
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2485
2803
|
*
|
|
2486
|
-
* Branches on `messageRunState(messages,
|
|
2487
|
-
*
|
|
2488
|
-
* (
|
|
2489
|
-
*
|
|
2490
|
-
* re-
|
|
2491
|
-
* of looking `queued` under the (never-run) stable id and being re-driven AGAIN
|
|
2492
|
-
* (Bugbot #217 Finding 3 — the duplicate-turn bug):
|
|
2804
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2805
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2806
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2807
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2808
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2493
2809
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2494
2810
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2495
2811
|
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2496
2812
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2497
|
-
* tracking the
|
|
2498
|
-
* - `unknown
|
|
2813
|
+
* tracking the stored id so the reply correlates by it;
|
|
2814
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2499
2815
|
*
|
|
2500
2816
|
* Only `ChannelAuthError` propagates.
|
|
2501
2817
|
*/
|
|
2502
|
-
async readoptOne(sessionId, row, messages) {
|
|
2818
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2503
2819
|
if (this.isTracked(sessionId, row.id)) {
|
|
2504
2820
|
this.log({
|
|
2505
2821
|
level: "info",
|
|
@@ -2509,10 +2825,8 @@ var ChannelDriver = class {
|
|
|
2509
2825
|
});
|
|
2510
2826
|
return;
|
|
2511
2827
|
}
|
|
2512
|
-
const
|
|
2513
|
-
const
|
|
2514
|
-
const ocId = nativeRedriveId ?? stableId;
|
|
2515
|
-
const state = messageRunState(messages, ocId);
|
|
2828
|
+
const ocId = row.opencode_message_id;
|
|
2829
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2516
2830
|
if (state === "done") {
|
|
2517
2831
|
if (this.doneUndeliverable.has(row.id)) {
|
|
2518
2832
|
this.log({
|
|
@@ -2530,7 +2844,7 @@ var ChannelDriver = class {
|
|
|
2530
2844
|
message_id: row.id
|
|
2531
2845
|
});
|
|
2532
2846
|
try {
|
|
2533
|
-
await this.markDone(row.conversation_id, row.id, sessionId,
|
|
2847
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2534
2848
|
} catch (err) {
|
|
2535
2849
|
if (err instanceof ChannelAuthError) throw err;
|
|
2536
2850
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -2555,7 +2869,7 @@ var ChannelDriver = class {
|
|
|
2555
2869
|
return;
|
|
2556
2870
|
}
|
|
2557
2871
|
if (state === "failed") {
|
|
2558
|
-
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2872
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2559
2873
|
this.log({
|
|
2560
2874
|
level: "error",
|
|
2561
2875
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -2596,23 +2910,77 @@ var ChannelDriver = class {
|
|
|
2596
2910
|
});
|
|
2597
2911
|
return;
|
|
2598
2912
|
}
|
|
2599
|
-
|
|
2913
|
+
let statusReadableOngoing = null;
|
|
2914
|
+
if (state === "running" && ocId) {
|
|
2915
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2916
|
+
const shape = this.replyCompletionShape(reply);
|
|
2917
|
+
const ongoing = sessionOngoing;
|
|
2918
|
+
statusReadableOngoing = ongoing;
|
|
2919
|
+
if (ongoing === false) {
|
|
2920
|
+
this.log({
|
|
2921
|
+
level: "info",
|
|
2922
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2923
|
+
conversation_id: row.conversation_id,
|
|
2924
|
+
message_id: row.id
|
|
2925
|
+
});
|
|
2926
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2927
|
+
return;
|
|
2928
|
+
}
|
|
2929
|
+
if (ongoing === true) {
|
|
2930
|
+
this.log({
|
|
2931
|
+
level: "info",
|
|
2932
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2933
|
+
conversation_id: row.conversation_id,
|
|
2934
|
+
message_id: row.id
|
|
2935
|
+
});
|
|
2936
|
+
} else {
|
|
2937
|
+
if (shape === "b1") {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
return;
|
|
2945
|
+
}
|
|
2946
|
+
this.log({
|
|
2947
|
+
level: "info",
|
|
2948
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2949
|
+
conversation_id: row.conversation_id,
|
|
2950
|
+
message_id: row.id
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2955
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2956
|
+
if (descendantAlive === true) {
|
|
2957
|
+
this.log({
|
|
2958
|
+
level: "info",
|
|
2959
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2960
|
+
conversation_id: row.conversation_id,
|
|
2961
|
+
message_id: row.id
|
|
2962
|
+
});
|
|
2963
|
+
} else {
|
|
2964
|
+
this.log({
|
|
2965
|
+
level: "info",
|
|
2966
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2967
|
+
conversation_id: row.conversation_id,
|
|
2968
|
+
message_id: row.id
|
|
2969
|
+
});
|
|
2970
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2971
|
+
return;
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2600
2975
|
const conv = this.convForRow(sessionId, row);
|
|
2601
2976
|
const message = this.queuedMessageForRow(row);
|
|
2602
|
-
this.registerReadopted(
|
|
2603
|
-
conv,
|
|
2604
|
-
sessionId,
|
|
2605
|
-
message,
|
|
2606
|
-
ocId,
|
|
2607
|
-
this.processedAtMs(row),
|
|
2608
|
-
nativeRedriveId
|
|
2609
|
-
);
|
|
2977
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2610
2978
|
this.dispatched.add(row.id);
|
|
2611
2979
|
this.readopted.add(row.id);
|
|
2612
2980
|
this.ensureWatcherRunning(sessionId);
|
|
2613
2981
|
this.log({
|
|
2614
2982
|
level: "info",
|
|
2615
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (
|
|
2983
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2616
2984
|
conversation_id: row.conversation_id,
|
|
2617
2985
|
message_id: row.id
|
|
2618
2986
|
});
|
|
@@ -2621,23 +2989,45 @@ var ChannelDriver = class {
|
|
|
2621
2989
|
await this.forceReadoptRun(sessionId, row);
|
|
2622
2990
|
}
|
|
2623
2991
|
/**
|
|
2624
|
-
* Re-dispatch an orphaned (`unknown
|
|
2992
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2993
|
+
*
|
|
2994
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
2995
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
2996
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
2997
|
+
* correlates server-side.
|
|
2625
2998
|
*
|
|
2626
|
-
*
|
|
2627
|
-
*
|
|
2628
|
-
*
|
|
2629
|
-
*
|
|
2630
|
-
*
|
|
2631
|
-
*
|
|
2632
|
-
*
|
|
2633
|
-
*
|
|
2999
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
3000
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
3001
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
3002
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
3003
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
3004
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
3005
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
3006
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
3007
|
+
* may retry exactly once more).
|
|
2634
3008
|
*
|
|
2635
|
-
* `evidentMessageId = row.id` addresses the SERVER row
|
|
2636
|
-
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
3009
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2637
3010
|
* `processed_at` (Invariant 1).
|
|
2638
3011
|
*/
|
|
2639
3012
|
async forceReadoptRun(sessionId, row) {
|
|
2640
|
-
|
|
3013
|
+
if (this.stopped) {
|
|
3014
|
+
this.log({
|
|
3015
|
+
level: "info",
|
|
3016
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but the runner is stopping \u2014 not starting a fresh turn; leaving for restart recovery`,
|
|
3017
|
+
conversation_id: row.conversation_id,
|
|
3018
|
+
message_id: row.id
|
|
3019
|
+
});
|
|
3020
|
+
return;
|
|
3021
|
+
}
|
|
3022
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
3023
|
+
this.log({
|
|
3024
|
+
level: "info",
|
|
3025
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
3026
|
+
conversation_id: row.conversation_id,
|
|
3027
|
+
message_id: row.id
|
|
3028
|
+
});
|
|
3029
|
+
return;
|
|
3030
|
+
}
|
|
2641
3031
|
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2642
3032
|
this.dontRedispatch.add(row.id);
|
|
2643
3033
|
this.log({
|
|
@@ -2654,13 +3044,19 @@ var ChannelDriver = class {
|
|
|
2654
3044
|
};
|
|
2655
3045
|
this.log({
|
|
2656
3046
|
level: "info",
|
|
2657
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching
|
|
3047
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2658
3048
|
conversation_id: row.conversation_id,
|
|
2659
3049
|
message_id: row.id
|
|
2660
3050
|
});
|
|
3051
|
+
this.awaitingReadopt.add(row.id);
|
|
3052
|
+
let ocId;
|
|
2661
3053
|
try {
|
|
2662
|
-
await
|
|
3054
|
+
ocId = await this.dispatchLocked(
|
|
3055
|
+
sessionId,
|
|
3056
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
3057
|
+
);
|
|
2663
3058
|
} catch (err) {
|
|
3059
|
+
this.awaitingReadopt.delete(row.id);
|
|
2664
3060
|
if (err instanceof ChannelAuthError) throw err;
|
|
2665
3061
|
this.log({
|
|
2666
3062
|
level: "error",
|
|
@@ -2670,11 +3066,22 @@ var ChannelDriver = class {
|
|
|
2670
3066
|
});
|
|
2671
3067
|
return;
|
|
2672
3068
|
}
|
|
3069
|
+
if (ocId === null) {
|
|
3070
|
+
this.awaitingReadopt.delete(row.id);
|
|
3071
|
+
this.log({
|
|
3072
|
+
level: "error",
|
|
3073
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
3074
|
+
conversation_id: row.conversation_id,
|
|
3075
|
+
message_id: row.id
|
|
3076
|
+
});
|
|
3077
|
+
return;
|
|
3078
|
+
}
|
|
2673
3079
|
const conv = this.convForRow(sessionId, row);
|
|
2674
3080
|
const message = this.queuedMessageForRow(row);
|
|
2675
3081
|
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2676
3082
|
this.dispatched.add(row.id);
|
|
2677
3083
|
this.readopted.add(row.id);
|
|
3084
|
+
this.awaitingReadopt.delete(row.id);
|
|
2678
3085
|
this.ensureWatcherRunning(sessionId);
|
|
2679
3086
|
}
|
|
2680
3087
|
/**
|
|
@@ -2767,21 +3174,41 @@ var ChannelDriver = class {
|
|
|
2767
3174
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2768
3175
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2769
3176
|
* oldest running message.
|
|
3177
|
+
*
|
|
3178
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3179
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3180
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3181
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3182
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3183
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3184
|
+
* even after it was already surfaced to the channel.
|
|
2770
3185
|
*/
|
|
2771
3186
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3187
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3188
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3189
|
+
let questionsPolledOk = true;
|
|
3190
|
+
let permissionsPolledOk = true;
|
|
2772
3191
|
let questions = [];
|
|
2773
3192
|
try {
|
|
2774
3193
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2775
3194
|
if (res.ok) {
|
|
2776
3195
|
const body = await res.json();
|
|
2777
|
-
|
|
3196
|
+
if (Array.isArray(body)) {
|
|
3197
|
+
questions = body;
|
|
3198
|
+
} else {
|
|
3199
|
+
questionsPolledOk = false;
|
|
3200
|
+
}
|
|
3201
|
+
} else {
|
|
3202
|
+
questionsPolledOk = false;
|
|
2778
3203
|
}
|
|
2779
3204
|
} catch {
|
|
3205
|
+
questionsPolledOk = false;
|
|
2780
3206
|
}
|
|
2781
3207
|
for (const q of questions) {
|
|
2782
|
-
if (q.sessionID
|
|
2783
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
3208
|
+
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2784
3209
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3210
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3211
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2785
3212
|
const reported = await this.reportInteraction(
|
|
2786
3213
|
watcher.conv.id,
|
|
2787
3214
|
"question",
|
|
@@ -2795,14 +3222,22 @@ var ChannelDriver = class {
|
|
|
2795
3222
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2796
3223
|
if (res.ok) {
|
|
2797
3224
|
const body = await res.json();
|
|
2798
|
-
|
|
3225
|
+
if (Array.isArray(body)) {
|
|
3226
|
+
permissions = body;
|
|
3227
|
+
} else {
|
|
3228
|
+
permissionsPolledOk = false;
|
|
3229
|
+
}
|
|
3230
|
+
} else {
|
|
3231
|
+
permissionsPolledOk = false;
|
|
2799
3232
|
}
|
|
2800
3233
|
} catch {
|
|
3234
|
+
permissionsPolledOk = false;
|
|
2801
3235
|
}
|
|
2802
3236
|
for (const p of permissions) {
|
|
2803
|
-
if (p.sessionID
|
|
2804
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
3237
|
+
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2805
3238
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3239
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3240
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2806
3241
|
const reported = await this.reportInteraction(
|
|
2807
3242
|
watcher.conv.id,
|
|
2808
3243
|
"permission",
|
|
@@ -2811,6 +3246,128 @@ var ChannelDriver = class {
|
|
|
2811
3246
|
);
|
|
2812
3247
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2813
3248
|
}
|
|
3249
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
3250
|
+
}
|
|
3251
|
+
/**
|
|
3252
|
+
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
3253
|
+
* i.e. its `parentID` chain (resolved via `GET /session/:id`) reaches the
|
|
3254
|
+
* watched root. Sub-agents spawned via the `task` tool run in child sessions,
|
|
3255
|
+
* so their questions/permissions live under a different `sessionID` that must
|
|
3256
|
+
* still be attributed to the root conversation the watcher owns.
|
|
3257
|
+
*
|
|
3258
|
+
* Parents are cached in `sessionParents` so we walk each session at most once;
|
|
3259
|
+
* a bounded depth cap guards against a cycle or a pathological chain, and any
|
|
3260
|
+
* fetch failure is treated as "not a descendant" (best-effort — the interaction
|
|
3261
|
+
* simply isn't surfaced this tick and is retried next tick once resolvable).
|
|
3262
|
+
*/
|
|
3263
|
+
async sessionBelongsTo(sessionId, rootSessionId) {
|
|
3264
|
+
let current = sessionId;
|
|
3265
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
3266
|
+
if (current === rootSessionId) return true;
|
|
3267
|
+
const parent = await this.resolveSessionParent(current);
|
|
3268
|
+
if (parent === null || parent === void 0) return false;
|
|
3269
|
+
current = parent;
|
|
3270
|
+
}
|
|
3271
|
+
return false;
|
|
3272
|
+
}
|
|
3273
|
+
/**
|
|
3274
|
+
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3275
|
+
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
3276
|
+
* unreachable / the session can't be read (so the caller stops walking without
|
|
3277
|
+
* caching a wrong answer — the next tick retries).
|
|
3278
|
+
*/
|
|
3279
|
+
async resolveSessionParent(sessionId) {
|
|
3280
|
+
const cached = this.sessionParents.get(sessionId);
|
|
3281
|
+
if (cached !== void 0) return cached;
|
|
3282
|
+
let parent = void 0;
|
|
3283
|
+
try {
|
|
3284
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}`);
|
|
3285
|
+
if (res.ok) {
|
|
3286
|
+
const body = await res.json();
|
|
3287
|
+
parent = body && typeof body.parentID === "string" ? body.parentID : null;
|
|
3288
|
+
}
|
|
3289
|
+
} catch {
|
|
3290
|
+
parent = void 0;
|
|
3291
|
+
}
|
|
3292
|
+
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3293
|
+
return parent;
|
|
3294
|
+
}
|
|
3295
|
+
/**
|
|
3296
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3297
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3298
|
+
*
|
|
3299
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3300
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3301
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3302
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3303
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3304
|
+
* provably in flight at the exact moment of recovery.
|
|
3305
|
+
*
|
|
3306
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3307
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3308
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3309
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3310
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3311
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3312
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3313
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3314
|
+
*
|
|
3315
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3316
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3317
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3318
|
+
* case), OR no descendant is found at all.
|
|
3319
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3320
|
+
*
|
|
3321
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3322
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3323
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3324
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3325
|
+
* true/false/null faithfully.
|
|
3326
|
+
*
|
|
3327
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3328
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3329
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3330
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3331
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3332
|
+
* `SessionStatus` only.
|
|
3333
|
+
*/
|
|
3334
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3335
|
+
const sessions = await listSessions(this.port);
|
|
3336
|
+
if (!sessions) {
|
|
3337
|
+
this.log({
|
|
3338
|
+
level: "error",
|
|
3339
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3340
|
+
});
|
|
3341
|
+
return null;
|
|
3342
|
+
}
|
|
3343
|
+
for (const candidate of sessions) {
|
|
3344
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3345
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3346
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3347
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3348
|
+
return true;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
return false;
|
|
3352
|
+
}
|
|
3353
|
+
/**
|
|
3354
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3355
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3356
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3357
|
+
* the aborted-in-flight production bug after a restart.
|
|
3358
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3359
|
+
* (the sub-agent preamble — #253's shape).
|
|
3360
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3361
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3362
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3363
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3364
|
+
*/
|
|
3365
|
+
replyCompletionShape(reply) {
|
|
3366
|
+
if (!reply) return "other";
|
|
3367
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3368
|
+
if (completed == null) return "b1";
|
|
3369
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3370
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
2814
3371
|
}
|
|
2815
3372
|
/**
|
|
2816
3373
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -3043,6 +3600,12 @@ var ChannelDriver = class {
|
|
|
3043
3600
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3044
3601
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3045
3602
|
* context (no silent catch, per development-workflow).
|
|
3603
|
+
*
|
|
3604
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3605
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3606
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3607
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3608
|
+
* leaves liveness").
|
|
3046
3609
|
*/
|
|
3047
3610
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
3048
3611
|
try {
|
|
@@ -3061,7 +3624,9 @@ var ChannelDriver = class {
|
|
|
3061
3624
|
conversation_id: conversationId,
|
|
3062
3625
|
message_id: messageId
|
|
3063
3626
|
});
|
|
3627
|
+
return false;
|
|
3064
3628
|
}
|
|
3629
|
+
return true;
|
|
3065
3630
|
} catch (err) {
|
|
3066
3631
|
this.log({
|
|
3067
3632
|
level: "error",
|
|
@@ -3069,6 +3634,7 @@ var ChannelDriver = class {
|
|
|
3069
3634
|
conversation_id: conversationId,
|
|
3070
3635
|
message_id: messageId
|
|
3071
3636
|
});
|
|
3637
|
+
return false;
|
|
3072
3638
|
}
|
|
3073
3639
|
}
|
|
3074
3640
|
async persistSession(conversationId, sessionId) {
|
|
@@ -3341,6 +3907,25 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
3341
3907
|
return { error: `Failed to resolve agent from key: ${message}` };
|
|
3342
3908
|
}
|
|
3343
3909
|
}
|
|
3910
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
3911
|
+
const apiUrl = getApiUrlConfig();
|
|
3912
|
+
try {
|
|
3913
|
+
const response = await fetch(`${apiUrl}/agents/${agentId}/disconnect`, {
|
|
3914
|
+
method: "POST",
|
|
3915
|
+
headers: { Authorization: authHeader }
|
|
3916
|
+
});
|
|
3917
|
+
if (!response.ok) {
|
|
3918
|
+
const serverMessage = await readErrorMessage(response);
|
|
3919
|
+
return {
|
|
3920
|
+
ok: false,
|
|
3921
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
3922
|
+
};
|
|
3923
|
+
}
|
|
3924
|
+
return { ok: true };
|
|
3925
|
+
} catch (error2) {
|
|
3926
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
3927
|
+
}
|
|
3928
|
+
}
|
|
3344
3929
|
async function getAgentInfo(agentId, authHeader) {
|
|
3345
3930
|
const apiUrl = getApiUrlConfig();
|
|
3346
3931
|
try {
|
|
@@ -3387,7 +3972,7 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
3387
3972
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
3388
3973
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
3389
3974
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3390
|
-
var
|
|
3975
|
+
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
3391
3976
|
function log2(state, message, isError = false) {
|
|
3392
3977
|
if (state.json) {
|
|
3393
3978
|
console.log(
|
|
@@ -3542,7 +4127,7 @@ async function driveChannels(state, driver) {
|
|
|
3542
4127
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
3543
4128
|
if (state.interactive) displayStatus(state);
|
|
3544
4129
|
}
|
|
3545
|
-
await new Promise((
|
|
4130
|
+
await new Promise((resolve2) => setTimeout(resolve2, CHANNEL_POLL_INTERVAL_MS));
|
|
3546
4131
|
if (state.idleTimeout !== null && idlePolls >= 2) {
|
|
3547
4132
|
const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
|
|
3548
4133
|
if (idleMs > state.idleTimeout * 1e3) {
|
|
@@ -3553,8 +4138,122 @@ async function driveChannels(state, driver) {
|
|
|
3553
4138
|
}
|
|
3554
4139
|
}
|
|
3555
4140
|
}
|
|
3556
|
-
|
|
4141
|
+
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4142
|
+
async function runSweep(state, driver, config2) {
|
|
4143
|
+
const mode = `age=${config2.maxAgeMs ?? "\u2014"} count=${config2.maxCount ?? "\u2014"}`;
|
|
4144
|
+
try {
|
|
4145
|
+
const sessions = await listSessions(state.port);
|
|
4146
|
+
if (sessions === null) {
|
|
4147
|
+
logActivity(state, {
|
|
4148
|
+
type: "info",
|
|
4149
|
+
message: `Session cleanup: could not list sessions (opencode unreachable); skipping this sweep (${mode})`
|
|
4150
|
+
});
|
|
4151
|
+
return;
|
|
4152
|
+
}
|
|
4153
|
+
const toDelete = selectSessionsToDelete(
|
|
4154
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4155
|
+
{
|
|
4156
|
+
maxAgeMs: config2.maxAgeMs,
|
|
4157
|
+
maxCount: config2.maxCount,
|
|
4158
|
+
nowMs: Date.now(),
|
|
4159
|
+
protectedIds: driver.protectedSessionIds()
|
|
4160
|
+
}
|
|
4161
|
+
);
|
|
4162
|
+
const protectedNow = driver.protectedSessionIds();
|
|
4163
|
+
let deleted = 0;
|
|
4164
|
+
let failed = 0;
|
|
4165
|
+
let skippedNewlyActive = 0;
|
|
4166
|
+
for (const id of toDelete) {
|
|
4167
|
+
if (protectedNow.has(id)) {
|
|
4168
|
+
skippedNewlyActive++;
|
|
4169
|
+
logActivity(state, {
|
|
4170
|
+
type: "info",
|
|
4171
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
4172
|
+
});
|
|
4173
|
+
continue;
|
|
4174
|
+
}
|
|
4175
|
+
if (await deleteSession(state.port, id)) deleted++;
|
|
4176
|
+
else failed++;
|
|
4177
|
+
}
|
|
4178
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
4179
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
4180
|
+
logActivity(state, {
|
|
4181
|
+
type: "info",
|
|
4182
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
4183
|
+
});
|
|
4184
|
+
} catch (error2) {
|
|
4185
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4186
|
+
logActivity(state, {
|
|
4187
|
+
type: "error",
|
|
4188
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`
|
|
4189
|
+
});
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
4192
|
+
function scheduleSessionCleanup(state, driver, options) {
|
|
4193
|
+
const config2 = resolveSessionCleanupConfig(
|
|
4194
|
+
{
|
|
4195
|
+
maxAge: options.sessionCleanupMaxAge,
|
|
4196
|
+
maxCount: options.sessionCleanupMaxCount,
|
|
4197
|
+
interval: options.sessionCleanupInterval
|
|
4198
|
+
},
|
|
4199
|
+
process.env
|
|
4200
|
+
);
|
|
4201
|
+
for (const warning2 of config2.warnings) {
|
|
4202
|
+
logActivity(state, { type: "info", message: `Session cleanup: ${warning2}` });
|
|
4203
|
+
}
|
|
4204
|
+
if (!config2.enabled) return;
|
|
4205
|
+
logActivity(state, {
|
|
4206
|
+
type: "info",
|
|
4207
|
+
message: `Session cleanup enabled (age=${config2.maxAgeMs ?? "\u2014"}, count=${config2.maxCount ?? "\u2014"}, interval=${config2.intervalMs}ms)`
|
|
4208
|
+
});
|
|
4209
|
+
const interval = setInterval(() => void runSweep(state, driver, config2), config2.intervalMs);
|
|
4210
|
+
const firstSweep = setTimeout(
|
|
4211
|
+
() => void runSweep(state, driver, config2),
|
|
4212
|
+
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4213
|
+
);
|
|
4214
|
+
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4215
|
+
}
|
|
4216
|
+
async function notifyOffline(state) {
|
|
4217
|
+
if (!state.agentId || !state.authHeader) return;
|
|
4218
|
+
if (!state.connected) {
|
|
4219
|
+
log2(state, "Skipping offline signal \u2014 this runner does not hold the live tunnel");
|
|
4220
|
+
return;
|
|
4221
|
+
}
|
|
4222
|
+
const result = await notifyAgentDisconnected(state.agentId, state.authHeader);
|
|
4223
|
+
if (result.ok) {
|
|
4224
|
+
log2(state, "Notified Evident the agent is going offline");
|
|
4225
|
+
} else {
|
|
4226
|
+
logActivity(state, {
|
|
4227
|
+
type: "error",
|
|
4228
|
+
error: `Could not notify Evident of offline status (relay will still report it): ${result.error}`
|
|
4229
|
+
});
|
|
4230
|
+
if (state.interactive) displayStatus(state);
|
|
4231
|
+
}
|
|
4232
|
+
}
|
|
4233
|
+
async function cleanup(state, opts = {}) {
|
|
3557
4234
|
state.running = false;
|
|
4235
|
+
for (const timer of state.sessionCleanupTimers) {
|
|
4236
|
+
clearInterval(timer);
|
|
4237
|
+
clearTimeout(timer);
|
|
4238
|
+
}
|
|
4239
|
+
state.sessionCleanupTimers = [];
|
|
4240
|
+
if (opts.graceful && state.channelDriver) {
|
|
4241
|
+
state.channelDriver.stop();
|
|
4242
|
+
log2(state, "Draining in-flight channel work before shutdown...");
|
|
4243
|
+
if (state.interactive) {
|
|
4244
|
+
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4245
|
+
displayStatus(state);
|
|
4246
|
+
}
|
|
4247
|
+
const settled = await state.channelDriver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS);
|
|
4248
|
+
if (!settled) {
|
|
4249
|
+
logActivity(state, {
|
|
4250
|
+
type: "info",
|
|
4251
|
+
message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
|
|
4252
|
+
});
|
|
4253
|
+
if (state.interactive) displayStatus(state);
|
|
4254
|
+
}
|
|
4255
|
+
}
|
|
4256
|
+
await notifyOffline(state);
|
|
3558
4257
|
if (state.connection) {
|
|
3559
4258
|
state.connection.close();
|
|
3560
4259
|
state.connection = null;
|
|
@@ -3585,10 +4284,13 @@ async function run(options) {
|
|
|
3585
4284
|
opencodeVersion: null,
|
|
3586
4285
|
opencodeProcess: null,
|
|
3587
4286
|
connection: null,
|
|
4287
|
+
channelDriver: null,
|
|
3588
4288
|
running: true,
|
|
4289
|
+
shuttingDown: false,
|
|
3589
4290
|
activityLog: [],
|
|
3590
4291
|
messageCount: 0,
|
|
3591
4292
|
lastProxiedActivityAt: null,
|
|
4293
|
+
sessionCleanupTimers: [],
|
|
3592
4294
|
authHeader: ""
|
|
3593
4295
|
};
|
|
3594
4296
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
@@ -3599,13 +4301,15 @@ async function run(options) {
|
|
|
3599
4301
|
);
|
|
3600
4302
|
}
|
|
3601
4303
|
const handleSignal = async () => {
|
|
4304
|
+
if (state.shuttingDown) return;
|
|
4305
|
+
state.shuttingDown = true;
|
|
3602
4306
|
if (state.interactive) {
|
|
3603
4307
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
3604
4308
|
displayStatus(state);
|
|
3605
4309
|
} else {
|
|
3606
4310
|
log2(state, "Shutting down...");
|
|
3607
4311
|
}
|
|
3608
|
-
await cleanup(state);
|
|
4312
|
+
await cleanup(state, { graceful: true });
|
|
3609
4313
|
await shutdownTelemetry();
|
|
3610
4314
|
process.exit(0);
|
|
3611
4315
|
};
|
|
@@ -3726,13 +4430,13 @@ async function run(options) {
|
|
|
3726
4430
|
getAuthHeader: () => state.authHeader,
|
|
3727
4431
|
conversationFilter: state.conversationFilter,
|
|
3728
4432
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3729
|
-
settleMs: CHANNEL_SETTLE_MS,
|
|
3730
4433
|
log: (entry) => logActivity(state, {
|
|
3731
4434
|
type: entry.level === "error" ? "error" : "info",
|
|
3732
4435
|
message: entry.message,
|
|
3733
4436
|
error: entry.level === "error" ? entry.message : void 0
|
|
3734
4437
|
})
|
|
3735
4438
|
});
|
|
4439
|
+
state.channelDriver = channelDriver;
|
|
3736
4440
|
const connection = new RunnerConnection({
|
|
3737
4441
|
agentId: state.agentId,
|
|
3738
4442
|
getAuthHeader: () => state.authHeader,
|
|
@@ -3746,7 +4450,11 @@ async function run(options) {
|
|
|
3746
4450
|
type: "info",
|
|
3747
4451
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
3748
4452
|
});
|
|
3749
|
-
emitAgentConnected(state.agentId, {
|
|
4453
|
+
emitAgentConnected(state.agentId, {
|
|
4454
|
+
port: state.port,
|
|
4455
|
+
cli_version: getCliVersion(),
|
|
4456
|
+
opencode_version: state.opencodeVersion
|
|
4457
|
+
});
|
|
3750
4458
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
3751
4459
|
if (state.interactive) displayStatus(state);
|
|
3752
4460
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -3825,10 +4533,12 @@ async function run(options) {
|
|
|
3825
4533
|
if (error2.message === "Unauthorized") tunnelSpinner?.fail("Unauthorized");
|
|
3826
4534
|
throw error2;
|
|
3827
4535
|
}
|
|
4536
|
+
scheduleSessionCleanup(state, channelDriver, options);
|
|
3828
4537
|
if (!interactive || state.json) {
|
|
3829
4538
|
log2(state, "Driving channel messages...");
|
|
3830
4539
|
}
|
|
3831
4540
|
await driveChannels(state, channelDriver);
|
|
4541
|
+
if (state.shuttingDown) return;
|
|
3832
4542
|
await cleanup(state);
|
|
3833
4543
|
if (state.json) {
|
|
3834
4544
|
console.log(
|
|
@@ -3843,6 +4553,7 @@ async function run(options) {
|
|
|
3843
4553
|
await shutdownTelemetry();
|
|
3844
4554
|
process.exit(0);
|
|
3845
4555
|
} catch (error2) {
|
|
4556
|
+
if (state.shuttingDown) return;
|
|
3846
4557
|
await cleanup(state);
|
|
3847
4558
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
3848
4559
|
if (state.json) {
|
|
@@ -3877,7 +4588,16 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
3877
4588
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
3878
4589
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
3879
4590
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
3880
|
-
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").
|
|
4591
|
+
program.command("run").description("Connect to Evident and process messages").option("-a, --agent [id]", "Agent ID to connect to (optional when EVIDENT_AGENT_KEY is set)").option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option("-v, --verbose", "Show detailed request/response information").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option("--json", "Output in JSON format").option(
|
|
4592
|
+
"--session-cleanup-max-age <duration>",
|
|
4593
|
+
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
4594
|
+
).option(
|
|
4595
|
+
"--session-cleanup-max-count <n>",
|
|
4596
|
+
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
4597
|
+
).option(
|
|
4598
|
+
"--session-cleanup-interval <duration>",
|
|
4599
|
+
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
4600
|
+
).action(
|
|
3881
4601
|
(options) => {
|
|
3882
4602
|
run({
|
|
3883
4603
|
agent: options.agent,
|
|
@@ -3885,7 +4605,11 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
3885
4605
|
verbose: options.verbose,
|
|
3886
4606
|
conversation: options.conversation,
|
|
3887
4607
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
3888
|
-
json: options.json
|
|
4608
|
+
json: options.json,
|
|
4609
|
+
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
4610
|
+
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
4611
|
+
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
4612
|
+
sessionCleanupInterval: options.sessionCleanupInterval
|
|
3889
4613
|
});
|
|
3890
4614
|
}
|
|
3891
4615
|
);
|