@evident-ai/cli 3.0.1-dev.ec55c96 → 3.0.1-dev.edba747
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1482 -263
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -470,6 +471,12 @@ import chalk6 from "chalk";
|
|
|
470
471
|
import ora3 from "ora";
|
|
471
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
472
473
|
|
|
474
|
+
// ../../packages/types/src/opencode/index.ts
|
|
475
|
+
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
+
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
+
return `msg_${sanitized}`;
|
|
478
|
+
}
|
|
479
|
+
|
|
473
480
|
// ../../packages/types/src/telemetry/index.ts
|
|
474
481
|
var TelemetryEventTypes = {
|
|
475
482
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -484,8 +491,34 @@ var TelemetryEventTypes = {
|
|
|
484
491
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
492
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
493
|
|
|
494
|
+
// ../../packages/types/src/logging/index.ts
|
|
495
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
496
|
+
function log(level, event, fields) {
|
|
497
|
+
const method = level === "debug" ? "log" : level;
|
|
498
|
+
try {
|
|
499
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
500
|
+
} catch (err) {
|
|
501
|
+
console.error(
|
|
502
|
+
"[evident] log_serialize_failed",
|
|
503
|
+
event,
|
|
504
|
+
err instanceof Error ? err.message : String(err)
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
function stripQuery(url) {
|
|
509
|
+
try {
|
|
510
|
+
return new URL(url).pathname;
|
|
511
|
+
} catch {
|
|
512
|
+
const q = url.indexOf("?");
|
|
513
|
+
return q === -1 ? url : url.slice(0, q);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
487
517
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
518
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
519
|
+
function getCliVersion() {
|
|
520
|
+
return CLI_VERSION;
|
|
521
|
+
}
|
|
489
522
|
var eventBuffer = [];
|
|
490
523
|
var flushTimeout = null;
|
|
491
524
|
var isShuttingDown = false;
|
|
@@ -684,6 +717,19 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
684
717
|
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
685
718
|
}
|
|
686
719
|
|
|
720
|
+
// src/lib/opencode/opencode-version-gate.ts
|
|
721
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
722
|
+
function isQueueValidatedVersion(version2) {
|
|
723
|
+
if (!version2) return false;
|
|
724
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
725
|
+
}
|
|
726
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
727
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
728
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
729
|
+
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
730
|
+
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
731
|
+
}
|
|
732
|
+
|
|
687
733
|
// src/lib/opencode/process.ts
|
|
688
734
|
import { execSync, spawn } from "child_process";
|
|
689
735
|
var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
|
|
@@ -996,23 +1042,37 @@ function roleOf(m) {
|
|
|
996
1042
|
}
|
|
997
1043
|
function completedOf(m) {
|
|
998
1044
|
if (!m || typeof m !== "object") return void 0;
|
|
999
|
-
return m.info?.time?.completed;
|
|
1045
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1000
1046
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
if (!res.ok) return null;
|
|
1005
|
-
const body = await res.json();
|
|
1006
|
-
return Array.isArray(body) ? body : null;
|
|
1007
|
-
} catch {
|
|
1008
|
-
return null;
|
|
1009
|
-
}
|
|
1047
|
+
function createdOf(m) {
|
|
1048
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1049
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1010
1050
|
}
|
|
1011
|
-
function
|
|
1012
|
-
if (!
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
return
|
|
1051
|
+
function idOf(m) {
|
|
1052
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1053
|
+
if (typeof m.id === "string") return m.id;
|
|
1054
|
+
const infoId = m.info?.id;
|
|
1055
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1056
|
+
}
|
|
1057
|
+
function parentIdOf(m) {
|
|
1058
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1059
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1060
|
+
const infoParent = m.info?.parentID;
|
|
1061
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1062
|
+
}
|
|
1063
|
+
function finishOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
if (typeof m.finish === "string") return m.finish;
|
|
1066
|
+
const infoFinish = m.info?.finish;
|
|
1067
|
+
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1068
|
+
}
|
|
1069
|
+
function errorOf(m) {
|
|
1070
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1071
|
+
return m.info?.error ?? m.error;
|
|
1072
|
+
}
|
|
1073
|
+
function isAssistantInFlight(m) {
|
|
1074
|
+
if (completedOf(m) == null) return true;
|
|
1075
|
+
return finishOf(m) === "tool-calls";
|
|
1016
1076
|
}
|
|
1017
1077
|
async function createOpenCodeSession(port, directory) {
|
|
1018
1078
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
@@ -1031,8 +1091,9 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1031
1091
|
const data = await response.json();
|
|
1032
1092
|
return data.id;
|
|
1033
1093
|
}
|
|
1034
|
-
async function
|
|
1094
|
+
async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
1035
1095
|
const body = {
|
|
1096
|
+
messageID: messageId,
|
|
1036
1097
|
parts: [{ type: "text", text: content }]
|
|
1037
1098
|
};
|
|
1038
1099
|
if (options?.agent) {
|
|
@@ -1047,79 +1108,111 @@ async function sendMessageToOpenCode(port, sessionId, content, options, hooks, m
|
|
|
1047
1108
|
};
|
|
1048
1109
|
}
|
|
1049
1110
|
}
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1111
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/prompt_async`, {
|
|
1112
|
+
method: "POST",
|
|
1113
|
+
headers: { "Content-Type": "application/json" },
|
|
1114
|
+
body: JSON.stringify(body)
|
|
1115
|
+
});
|
|
1116
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1117
|
+
const text = await res.text().catch(() => "");
|
|
1118
|
+
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1122
|
+
if (!messages || messages.length === 0) return null;
|
|
1123
|
+
const byParent = messages.find(
|
|
1124
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1125
|
+
);
|
|
1126
|
+
if (byParent) return byParent;
|
|
1127
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1128
|
+
if (userIndex === -1) return null;
|
|
1129
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1130
|
+
if (roleOf(messages[i]) === "assistant") return messages[i];
|
|
1131
|
+
}
|
|
1132
|
+
return null;
|
|
1133
|
+
}
|
|
1134
|
+
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1135
|
+
if (!messages || messages.length === 0) return null;
|
|
1136
|
+
let lastCorrelated = null;
|
|
1137
|
+
let lastNonErrored = null;
|
|
1138
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1139
|
+
const m = messages[i];
|
|
1140
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1141
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1142
|
+
if (errorOf(m) == null) {
|
|
1143
|
+
lastNonErrored = m;
|
|
1144
|
+
break;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1148
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1149
|
+
if (userIndex === -1) return null;
|
|
1150
|
+
let last = null;
|
|
1151
|
+
let lastOk = null;
|
|
1152
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1153
|
+
const role = roleOf(messages[i]);
|
|
1154
|
+
if (role === "user") break;
|
|
1155
|
+
if (role === "assistant") {
|
|
1156
|
+
last = messages[i];
|
|
1157
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
return lastOk ?? last;
|
|
1161
|
+
}
|
|
1162
|
+
function messageRunState(messages, userMessageId) {
|
|
1163
|
+
if (!messages || messages.length === 0) return "unknown";
|
|
1164
|
+
const hasUser = messages.some((m) => idOf(m) === userMessageId);
|
|
1165
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1166
|
+
if (!hasUser) {
|
|
1167
|
+
if (!reply) return "unknown";
|
|
1168
|
+
}
|
|
1169
|
+
if (!reply) return "queued";
|
|
1170
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1171
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1172
|
+
}
|
|
1173
|
+
function messageError(messages, userMessageId) {
|
|
1174
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1175
|
+
const error2 = errorOf(reply);
|
|
1176
|
+
if (error2 == null) return null;
|
|
1177
|
+
if (typeof error2 === "string") return error2;
|
|
1178
|
+
if (typeof error2 === "object") {
|
|
1179
|
+
const e = error2;
|
|
1180
|
+
const dataMessage = e.data?.message;
|
|
1181
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1182
|
+
if (typeof e.message === "string") return e.message;
|
|
1183
|
+
}
|
|
1184
|
+
return "The agent run failed.";
|
|
1185
|
+
}
|
|
1186
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1187
|
+
if (!messages || messages.length === 0) return false;
|
|
1188
|
+
return messages.some(
|
|
1189
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
function isSessionSettled(messages, now, settleMs) {
|
|
1193
|
+
if (!messages || messages.length === 0) return true;
|
|
1194
|
+
let newestAssistantCreated = null;
|
|
1195
|
+
for (const m of messages) {
|
|
1196
|
+
if (roleOf(m) !== "assistant") continue;
|
|
1197
|
+
if (isAssistantInFlight(m)) return false;
|
|
1198
|
+
const created = createdOf(m);
|
|
1199
|
+
if (typeof created === "number" && (newestAssistantCreated === null || created > newestAssistantCreated)) {
|
|
1200
|
+
newestAssistantCreated = created;
|
|
1119
1201
|
}
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
return
|
|
1202
|
+
}
|
|
1203
|
+
if (newestAssistantCreated === null) return true;
|
|
1204
|
+
return now - newestAssistantCreated >= settleMs;
|
|
1205
|
+
}
|
|
1206
|
+
function opencodeMessageIdFor2(queuedMessageId) {
|
|
1207
|
+
return opencodeMessageIdFor(queuedMessageId);
|
|
1208
|
+
}
|
|
1209
|
+
var NATIVE_ID_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
1210
|
+
function nativeOpencodeMessageId() {
|
|
1211
|
+
let id = "msg_";
|
|
1212
|
+
for (let i = 0; i < 24; i++) {
|
|
1213
|
+
id += NATIVE_ID_ALPHABET[Math.floor(Math.random() * NATIVE_ID_ALPHABET.length)];
|
|
1214
|
+
}
|
|
1215
|
+
return id;
|
|
1123
1216
|
}
|
|
1124
1217
|
|
|
1125
1218
|
// src/lib/tunnel/connection.ts
|
|
@@ -1190,12 +1283,20 @@ var StreamForwarder = class {
|
|
|
1190
1283
|
}
|
|
1191
1284
|
async handleOpen(frame) {
|
|
1192
1285
|
const { sid, method, path, headers, has_body } = frame;
|
|
1286
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1287
|
+
const startedAt = Date.now();
|
|
1193
1288
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1194
1289
|
this.callbacks.onDrainPing?.();
|
|
1195
1290
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1196
1291
|
this.send({ type: "res_end", sid });
|
|
1197
1292
|
return;
|
|
1198
1293
|
}
|
|
1294
|
+
log("info", "agent_request", {
|
|
1295
|
+
correlation_id: correlationId,
|
|
1296
|
+
sid,
|
|
1297
|
+
method,
|
|
1298
|
+
path: stripQuery(path)
|
|
1299
|
+
});
|
|
1199
1300
|
const ac = new AbortController();
|
|
1200
1301
|
let bodyPromise;
|
|
1201
1302
|
let pushBody;
|
|
@@ -1242,6 +1343,12 @@ var StreamForwarder = class {
|
|
|
1242
1343
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1243
1344
|
});
|
|
1244
1345
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1346
|
+
log("info", "agent_response", {
|
|
1347
|
+
correlation_id: correlationId,
|
|
1348
|
+
sid,
|
|
1349
|
+
status: upstream.status,
|
|
1350
|
+
duration_ms: Date.now() - startedAt
|
|
1351
|
+
});
|
|
1245
1352
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1246
1353
|
try {
|
|
1247
1354
|
if (upstream.body) {
|
|
@@ -1498,6 +1605,12 @@ var RunnerConnection = class {
|
|
|
1498
1605
|
};
|
|
1499
1606
|
|
|
1500
1607
|
// src/lib/channels/driver.ts
|
|
1608
|
+
function messageIdOf(m) {
|
|
1609
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1610
|
+
if (typeof m.id === "string") return m.id;
|
|
1611
|
+
const infoId = m.info?.id;
|
|
1612
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1613
|
+
}
|
|
1501
1614
|
var DEFAULT_RETRY_POLICY = {
|
|
1502
1615
|
maxAttempts: 6,
|
|
1503
1616
|
baseDelayMs: 500,
|
|
@@ -1505,12 +1618,24 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1505
1618
|
};
|
|
1506
1619
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1507
1620
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1621
|
+
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1622
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1623
|
+
var DEFAULT_STUCK_QUEUED_REDRIVE_MAX = 3;
|
|
1624
|
+
var DEFAULT_SETTLE_MS = 3500;
|
|
1508
1625
|
var ChannelAuthError = class extends Error {
|
|
1509
1626
|
constructor(message) {
|
|
1510
1627
|
super(message);
|
|
1511
1628
|
this.name = "ChannelAuthError";
|
|
1512
1629
|
}
|
|
1513
1630
|
};
|
|
1631
|
+
var ChannelTerminalError = class extends Error {
|
|
1632
|
+
status;
|
|
1633
|
+
constructor(message, status) {
|
|
1634
|
+
super(message);
|
|
1635
|
+
this.name = "ChannelTerminalError";
|
|
1636
|
+
this.status = status;
|
|
1637
|
+
}
|
|
1638
|
+
};
|
|
1514
1639
|
function backoffDelay(attempt, policy) {
|
|
1515
1640
|
const exp = policy.baseDelayMs * Math.pow(2, attempt);
|
|
1516
1641
|
const capped = Math.min(policy.maxDelayMs, exp);
|
|
@@ -1531,15 +1656,63 @@ var ChannelDriver = class {
|
|
|
1531
1656
|
sleep;
|
|
1532
1657
|
pausedPollIntervalMs;
|
|
1533
1658
|
pausedMaxWaitMs;
|
|
1659
|
+
dispatchConfirmMs;
|
|
1660
|
+
stuckQueuedMs;
|
|
1661
|
+
stuckQueuedRedriveMax;
|
|
1662
|
+
settleMs;
|
|
1663
|
+
now;
|
|
1534
1664
|
/** Cache of conversationId → opencode sessionId. */
|
|
1535
1665
|
sessions = /* @__PURE__ */ new Map();
|
|
1536
1666
|
/**
|
|
1537
|
-
*
|
|
1538
|
-
*
|
|
1539
|
-
*
|
|
1540
|
-
*
|
|
1667
|
+
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1668
|
+
* session: one polling loop services all of that session's in-flight messages.
|
|
1669
|
+
* A session entry exists while it has any in-flight (dispatched-but-not-done)
|
|
1670
|
+
* message; it is removed once its in-flight set empties.
|
|
1541
1671
|
*/
|
|
1542
1672
|
watchers = /* @__PURE__ */ new Map();
|
|
1673
|
+
/**
|
|
1674
|
+
* AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
|
|
1675
|
+
* dispatched and are still in-flight. A message in this set is never
|
|
1676
|
+
* re-`prompt_async`-ed by a subsequent poll tick while it is queued/running.
|
|
1677
|
+
* Backed by a stable minted opencode `messageID` whose duplicate re-enqueue is
|
|
1678
|
+
* idempotent on opencode (PoC fact 9) — so even if this set is lost on restart,
|
|
1679
|
+
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1680
|
+
*/
|
|
1681
|
+
dispatched = /* @__PURE__ */ new Set();
|
|
1682
|
+
/**
|
|
1683
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1684
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1685
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1686
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1687
|
+
* the processing list.
|
|
1688
|
+
*/
|
|
1689
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1690
|
+
/**
|
|
1691
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1692
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1693
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1694
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1695
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1696
|
+
*
|
|
1697
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1698
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1699
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1700
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1701
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1702
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1703
|
+
*/
|
|
1704
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1705
|
+
/**
|
|
1706
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1707
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1708
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1709
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1710
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1711
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1712
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1713
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1714
|
+
*/
|
|
1715
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1543
1716
|
/**
|
|
1544
1717
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1545
1718
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1562,6 +1735,11 @@ var ChannelDriver = class {
|
|
|
1562
1735
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1563
1736
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1564
1737
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1738
|
+
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1739
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1740
|
+
this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
|
|
1741
|
+
this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
|
|
1742
|
+
this.now = config2.now ?? (() => Date.now());
|
|
1565
1743
|
}
|
|
1566
1744
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
1567
1745
|
get opencodeBase() {
|
|
@@ -1571,16 +1749,16 @@ var ChannelDriver = class {
|
|
|
1571
1749
|
// Public API
|
|
1572
1750
|
// -------------------------------------------------------------------------
|
|
1573
1751
|
/**
|
|
1574
|
-
* Drain all pending channel conversations once: poll →
|
|
1752
|
+
* Drain all pending channel conversations once: poll → dispatch → register.
|
|
1575
1753
|
* Called on tunnel `connected` (WI-CHAN-4) and on each poll tick by `run.ts`.
|
|
1576
1754
|
* Re-entrant calls while a drain is in flight are skipped (return 0).
|
|
1577
1755
|
*
|
|
1578
|
-
* @returns the number of messages
|
|
1756
|
+
* @returns the number of messages NEWLY dispatched to opencode's native queue.
|
|
1579
1757
|
*/
|
|
1580
1758
|
async drainPending() {
|
|
1581
1759
|
if (this.draining) return 0;
|
|
1582
1760
|
this.draining = true;
|
|
1583
|
-
let
|
|
1761
|
+
let dispatched = 0;
|
|
1584
1762
|
try {
|
|
1585
1763
|
const conversations = await this.getPendingConversations();
|
|
1586
1764
|
if (conversations.length > 0) {
|
|
@@ -1591,95 +1769,106 @@ var ChannelDriver = class {
|
|
|
1591
1769
|
});
|
|
1592
1770
|
}
|
|
1593
1771
|
for (const conv of conversations) {
|
|
1594
|
-
|
|
1772
|
+
dispatched += await this.processConversation(conv);
|
|
1595
1773
|
}
|
|
1774
|
+
await this.readoptProcessing();
|
|
1596
1775
|
} finally {
|
|
1597
1776
|
this.draining = false;
|
|
1598
1777
|
}
|
|
1599
|
-
return
|
|
1778
|
+
return dispatched;
|
|
1600
1779
|
}
|
|
1601
1780
|
/**
|
|
1602
|
-
*
|
|
1781
|
+
* True while any per-session watcher has a non-empty in-flight dispatched set
|
|
1782
|
+
* (Task 3.7). `run.ts` treats this as NON-idle so `--idle-timeout` cannot exit
|
|
1783
|
+
* the process while a dispatched message is still queued/running — which would
|
|
1784
|
+
* kill the turn and orphan its reply.
|
|
1785
|
+
*/
|
|
1786
|
+
hasInFlightWatchers() {
|
|
1787
|
+
for (const watcher of this.watchers.values()) {
|
|
1788
|
+
if (watcher.inFlight.size > 0) return true;
|
|
1789
|
+
}
|
|
1790
|
+
return false;
|
|
1791
|
+
}
|
|
1792
|
+
/**
|
|
1793
|
+
* Await all outstanding per-session watchers (WI-3).
|
|
1603
1794
|
*
|
|
1604
|
-
* In production the
|
|
1605
|
-
* loop never blocks on them and process exit is not held up (the cron
|
|
1606
|
-
* any abandoned ones). This helper exists primarily for deterministic
|
|
1607
|
-
* that need to observe
|
|
1608
|
-
* after a non-blocking `drainPending`.
|
|
1795
|
+
* In production the watcher loops are deliberately started-not-awaited so the
|
|
1796
|
+
* drain loop never blocks on them and process exit is not held up (the cron
|
|
1797
|
+
* recovers any abandoned ones). This helper exists primarily for deterministic
|
|
1798
|
+
* tests that need to observe a watcher's effect (the `processing`/`done` PATCH
|
|
1799
|
+
* or its giving up) after a non-blocking `drainPending`. Watcher loops never
|
|
1800
|
+
* reject, so this resolves.
|
|
1609
1801
|
*/
|
|
1610
1802
|
async flushPausedWatchers() {
|
|
1611
|
-
|
|
1803
|
+
while (true) {
|
|
1804
|
+
const loops = [...this.watchers.values()].map((w) => w.loop).filter((l) => l != null);
|
|
1805
|
+
if (loops.length === 0) return;
|
|
1806
|
+
await Promise.all(loops);
|
|
1807
|
+
const stillLive = [...this.watchers.values()].some((w) => w.loop != null);
|
|
1808
|
+
if (!stillLive) return;
|
|
1809
|
+
}
|
|
1612
1810
|
}
|
|
1613
1811
|
// -------------------------------------------------------------------------
|
|
1614
|
-
// Conversation processing
|
|
1812
|
+
// Conversation processing (WI-3 — async dispatch)
|
|
1615
1813
|
// -------------------------------------------------------------------------
|
|
1814
|
+
/**
|
|
1815
|
+
* Dispatch each pending message for a conversation to opencode's native queue
|
|
1816
|
+
* via `prompt_async` (Task 3.2) and register it with the conversation's
|
|
1817
|
+
* per-session watcher. Does NOT block on the turn and does NOT call
|
|
1818
|
+
* `markProcessing` here — that fires from the watcher on running-start.
|
|
1819
|
+
*
|
|
1820
|
+
* @returns the count of messages NEWLY dispatched (not already in-flight).
|
|
1821
|
+
*/
|
|
1616
1822
|
async processConversation(conv) {
|
|
1617
1823
|
const sessionId = await this.ensureSession(conv);
|
|
1618
1824
|
const messages = await this.getPendingMessages(conv.id);
|
|
1619
|
-
let
|
|
1825
|
+
let dispatched = 0;
|
|
1826
|
+
let skippedAlreadyDispatched = 0;
|
|
1620
1827
|
for (const message of messages) {
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
this.log({
|
|
1624
|
-
level: "info",
|
|
1625
|
-
message: `Message ${message.id.slice(0, 8)} already claimed \u2014 skipping`,
|
|
1626
|
-
conversation_id: conv.id,
|
|
1627
|
-
message_id: message.id
|
|
1628
|
-
});
|
|
1828
|
+
if (this.dispatched.has(message.id)) {
|
|
1829
|
+
skippedAlreadyDispatched += 1;
|
|
1629
1830
|
continue;
|
|
1630
1831
|
}
|
|
1832
|
+
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1833
|
+
const options = {
|
|
1834
|
+
agent: message.opencode_agent ?? void 0,
|
|
1835
|
+
model: message.opencode_model ?? void 0
|
|
1836
|
+
};
|
|
1631
1837
|
try {
|
|
1632
1838
|
this.log({
|
|
1633
1839
|
level: "info",
|
|
1634
|
-
message: `
|
|
1635
|
-
conversation_id: conv.id,
|
|
1636
|
-
message_id: message.id
|
|
1637
|
-
});
|
|
1638
|
-
const result = await sendMessageToOpenCode(
|
|
1639
|
-
this.port,
|
|
1640
|
-
sessionId,
|
|
1641
|
-
message.content,
|
|
1642
|
-
{
|
|
1643
|
-
agent: message.opencode_agent ?? void 0,
|
|
1644
|
-
model: message.opencode_model ?? void 0
|
|
1645
|
-
},
|
|
1646
|
-
{
|
|
1647
|
-
onQuestion: (question) => this.reportInteraction(conv.id, "question", question),
|
|
1648
|
-
onPermission: (permission) => this.reportInteraction(conv.id, "permission", permission)
|
|
1649
|
-
}
|
|
1650
|
-
);
|
|
1651
|
-
if (result.awaitingInteraction) {
|
|
1652
|
-
this.log({
|
|
1653
|
-
level: "info",
|
|
1654
|
-
message: `Message ${message.id.slice(0, 8)} paused awaiting interaction \u2014 watching session for completion`,
|
|
1655
|
-
conversation_id: conv.id,
|
|
1656
|
-
message_id: message.id
|
|
1657
|
-
});
|
|
1658
|
-
this.startPausedWatcher(conv, message, sessionId);
|
|
1659
|
-
continue;
|
|
1660
|
-
}
|
|
1661
|
-
await this.confirmCompletion(sessionId);
|
|
1662
|
-
await this.markDone(conv.id, message.id, sessionId);
|
|
1663
|
-
processed += 1;
|
|
1664
|
-
this.log({
|
|
1665
|
-
level: "info",
|
|
1666
|
-
message: `Message ${message.id.slice(0, 8)} processed`,
|
|
1840
|
+
message: `Dispatching message ${message.id.slice(0, 8)} to OpenCode native queue (session ${sessionId.slice(0, 8)})`,
|
|
1667
1841
|
conversation_id: conv.id,
|
|
1668
1842
|
message_id: message.id
|
|
1669
1843
|
});
|
|
1844
|
+
await sendPromptAsync(this.port, sessionId, message.content, options, opencodeMessageId);
|
|
1670
1845
|
} catch (err) {
|
|
1671
1846
|
if (err instanceof ChannelAuthError) throw err;
|
|
1847
|
+
this.dispatched.delete(message.id);
|
|
1672
1848
|
await this.markFailed(conv.id, message.id).catch(() => {
|
|
1673
1849
|
});
|
|
1674
1850
|
this.log({
|
|
1675
1851
|
level: "error",
|
|
1676
|
-
message: `Message ${message.id.slice(0, 8)} failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1852
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1677
1853
|
conversation_id: conv.id,
|
|
1678
1854
|
message_id: message.id
|
|
1679
1855
|
});
|
|
1856
|
+
continue;
|
|
1680
1857
|
}
|
|
1858
|
+
this.dispatched.add(message.id);
|
|
1859
|
+
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1860
|
+
dispatched += 1;
|
|
1861
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1681
1862
|
}
|
|
1682
|
-
|
|
1863
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1864
|
+
this.log({
|
|
1865
|
+
level: "error",
|
|
1866
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
1867
|
+
conversation_id: conv.id
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
this.ensureWatcherRunning(sessionId);
|
|
1871
|
+
return dispatched;
|
|
1683
1872
|
}
|
|
1684
1873
|
async ensureSession(conv) {
|
|
1685
1874
|
const cached = this.sessions.get(conv.id);
|
|
@@ -1711,111 +1900,974 @@ var ChannelDriver = class {
|
|
|
1711
1900
|
}
|
|
1712
1901
|
return this.opencodeDirectory;
|
|
1713
1902
|
}
|
|
1903
|
+
// -------------------------------------------------------------------------
|
|
1904
|
+
// Per-session watcher (WI-3)
|
|
1905
|
+
// -------------------------------------------------------------------------
|
|
1906
|
+
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1907
|
+
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1908
|
+
let watcher = this.watchers.get(sessionId);
|
|
1909
|
+
if (!watcher) {
|
|
1910
|
+
watcher = {
|
|
1911
|
+
conv,
|
|
1912
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1913
|
+
loop: null,
|
|
1914
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1915
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
1916
|
+
};
|
|
1917
|
+
this.watchers.set(sessionId, watcher);
|
|
1918
|
+
}
|
|
1919
|
+
const now = this.now();
|
|
1920
|
+
watcher.inFlight.set(message.id, {
|
|
1921
|
+
evidentMessageId: message.id,
|
|
1922
|
+
opencodeMessageId,
|
|
1923
|
+
message,
|
|
1924
|
+
dispatchedAt: now,
|
|
1925
|
+
deadline: now + this.pausedMaxWaitMs,
|
|
1926
|
+
started: false,
|
|
1927
|
+
done: false,
|
|
1928
|
+
stuckReported: false,
|
|
1929
|
+
redriveAttempts: 0,
|
|
1930
|
+
lastRedriveAt: null,
|
|
1931
|
+
redriveOpencodeMessageId: null,
|
|
1932
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1714
1935
|
/**
|
|
1715
|
-
*
|
|
1716
|
-
*
|
|
1936
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1937
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1938
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1939
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1940
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1941
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1942
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1943
|
+
*
|
|
1944
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1945
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1946
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1717
1947
|
*
|
|
1718
|
-
*
|
|
1719
|
-
*
|
|
1720
|
-
*
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1948
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1949
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1950
|
+
* fire from the watcher's normal branches.
|
|
1951
|
+
*/
|
|
1952
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs, redriveOpencodeMessageId = null) {
|
|
1953
|
+
let watcher = this.watchers.get(sessionId);
|
|
1954
|
+
if (!watcher) {
|
|
1955
|
+
watcher = {
|
|
1956
|
+
conv,
|
|
1957
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1958
|
+
loop: null,
|
|
1959
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1960
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
1961
|
+
};
|
|
1962
|
+
this.watchers.set(sessionId, watcher);
|
|
1963
|
+
}
|
|
1964
|
+
watcher.inFlight.set(message.id, {
|
|
1965
|
+
evidentMessageId: message.id,
|
|
1966
|
+
opencodeMessageId,
|
|
1967
|
+
message,
|
|
1968
|
+
dispatchedAt: this.now(),
|
|
1969
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1970
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1971
|
+
started: true,
|
|
1972
|
+
done: false,
|
|
1973
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1974
|
+
// AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
|
|
1975
|
+
// gates on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
1976
|
+
// so a re-adopted row left wedged in `queued` still emits the signal once
|
|
1977
|
+
// (queued-followup-redrive, #210). A re-adopted `queued` wedge is ALSO
|
|
1978
|
+
// re-driven by the same path (ADR-0046 §c deferred the live-session redrive
|
|
1979
|
+
// to here) — `redriveAttempts` starts fresh so it gets the full budget.
|
|
1980
|
+
stuckReported: false,
|
|
1981
|
+
redriveAttempts: 0,
|
|
1982
|
+
lastRedriveAt: null,
|
|
1983
|
+
redriveOpencodeMessageId,
|
|
1984
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1987
|
+
/**
|
|
1988
|
+
* Start (but do NOT await) the per-session watcher loop if it has in-flight
|
|
1989
|
+
* work and is not already running. Single-flight per session. The loop is
|
|
1990
|
+
* tracked on the watcher and cleared when it settles; it never rejects (fully
|
|
1991
|
+
* guarded), so a failed poll/callback can never crash the run loop — the cron
|
|
1992
|
+
* stays as the safety net.
|
|
1993
|
+
*/
|
|
1994
|
+
ensureWatcherRunning(sessionId) {
|
|
1995
|
+
const watcher = this.watchers.get(sessionId);
|
|
1996
|
+
if (!watcher) return;
|
|
1997
|
+
if (watcher.loop) return;
|
|
1998
|
+
if (watcher.inFlight.size === 0) {
|
|
1999
|
+
this.watchers.delete(sessionId);
|
|
2000
|
+
return;
|
|
2001
|
+
}
|
|
2002
|
+
const loop = this.runWatcherLoop(sessionId, watcher).finally(() => {
|
|
2003
|
+
watcher.loop = null;
|
|
2004
|
+
if (watcher.inFlight.size === 0) {
|
|
2005
|
+
this.watchers.delete(sessionId);
|
|
2006
|
+
}
|
|
2007
|
+
});
|
|
2008
|
+
watcher.loop = loop;
|
|
2009
|
+
}
|
|
2010
|
+
/**
|
|
2011
|
+
* The per-session polling loop (WI-3). Once per tick it:
|
|
2012
|
+
* 1. polls `GET /session/:id/message` once and, per in-flight message,
|
|
2013
|
+
* computes `messageRunState` and fires markProcessing (queued→running) /
|
|
2014
|
+
* markDone (done) exactly once per transition;
|
|
2015
|
+
* 2. applies the idle-path re-dispatch guard (a dispatched message that never
|
|
2016
|
+
* APPEARS → re-dispatch — D1 obligation 2);
|
|
2017
|
+
* 3. polls `/question` + `/permission` (scoped to the session) and surfaces
|
|
2018
|
+
* NEW ones via `reportInteraction`, carrying the PAUSED message's own
|
|
2019
|
+
* `source_message_id`;
|
|
2020
|
+
* 4. drops messages that completed or timed out from the in-flight set.
|
|
2021
|
+
* Exits when the in-flight set empties. Never throws.
|
|
1725
2022
|
*/
|
|
1726
|
-
async
|
|
2023
|
+
async runWatcherLoop(sessionId, watcher) {
|
|
1727
2024
|
try {
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
2025
|
+
while (watcher.inFlight.size > 0) {
|
|
2026
|
+
await this.sleep(this.pausedPollIntervalMs);
|
|
2027
|
+
let messages = null;
|
|
2028
|
+
try {
|
|
2029
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2030
|
+
if (res.ok) {
|
|
2031
|
+
const body = await res.json();
|
|
2032
|
+
messages = Array.isArray(body) ? body : null;
|
|
2033
|
+
}
|
|
2034
|
+
} catch {
|
|
2035
|
+
continue;
|
|
2036
|
+
}
|
|
2037
|
+
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2038
|
+
await this.serviceInFlightMessage(sessionId, watcher, inFlight, messages);
|
|
2039
|
+
}
|
|
2040
|
+
await this.pollInteractions(sessionId, watcher, messages);
|
|
2041
|
+
}
|
|
2042
|
+
} catch (err) {
|
|
2043
|
+
if (err instanceof ChannelAuthError) {
|
|
2044
|
+
this.log({
|
|
2045
|
+
level: "error",
|
|
2046
|
+
message: `Session watcher aborted on auth failure for session ${sessionId.slice(0, 8)} \u2014 clearing in-flight state for re-drive after re-auth: ${err.message}`,
|
|
2047
|
+
conversation_id: watcher.conv.id
|
|
2048
|
+
});
|
|
2049
|
+
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2050
|
+
this.readopted.delete(evidentMessageId);
|
|
2051
|
+
this.removeInFlight(watcher, evidentMessageId);
|
|
2052
|
+
}
|
|
2053
|
+
return;
|
|
2054
|
+
}
|
|
2055
|
+
this.log({
|
|
2056
|
+
level: "error",
|
|
2057
|
+
message: `Session watcher failed for session ${sessionId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2058
|
+
conversation_id: watcher.conv.id
|
|
2059
|
+
});
|
|
2060
|
+
}
|
|
2061
|
+
}
|
|
2062
|
+
/**
|
|
2063
|
+
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2064
|
+
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2065
|
+
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2066
|
+
* in-flight set on completion or timeout.
|
|
2067
|
+
*/
|
|
2068
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2069
|
+
const conv = watcher.conv;
|
|
2070
|
+
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2071
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2072
|
+
let claimed;
|
|
2073
|
+
try {
|
|
2074
|
+
claimed = await this.markProcessing(
|
|
2075
|
+
conv.id,
|
|
2076
|
+
inFlight.evidentMessageId,
|
|
2077
|
+
sessionId,
|
|
2078
|
+
inFlight.redriveOpencodeMessageId
|
|
2079
|
+
);
|
|
2080
|
+
} catch (err) {
|
|
2081
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2082
|
+
this.log({
|
|
2083
|
+
level: "error",
|
|
2084
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2085
|
+
conversation_id: conv.id,
|
|
2086
|
+
message_id: inFlight.evidentMessageId
|
|
2087
|
+
});
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2090
|
+
inFlight.started = true;
|
|
2091
|
+
if (!claimed) {
|
|
1733
2092
|
this.log({
|
|
1734
2093
|
level: "info",
|
|
1735
|
-
message: `
|
|
2094
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
2095
|
+
conversation_id: conv.id,
|
|
2096
|
+
message_id: inFlight.evidentMessageId
|
|
1736
2097
|
});
|
|
1737
2098
|
}
|
|
1738
|
-
}
|
|
2099
|
+
}
|
|
2100
|
+
if (state === "done") {
|
|
2101
|
+
if (!inFlight.done) {
|
|
2102
|
+
this.log({
|
|
2103
|
+
level: "info",
|
|
2104
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
2105
|
+
conversation_id: conv.id,
|
|
2106
|
+
message_id: inFlight.evidentMessageId
|
|
2107
|
+
});
|
|
2108
|
+
try {
|
|
2109
|
+
await this.markDone(
|
|
2110
|
+
conv.id,
|
|
2111
|
+
inFlight.evidentMessageId,
|
|
2112
|
+
sessionId,
|
|
2113
|
+
inFlight.redriveOpencodeMessageId
|
|
2114
|
+
);
|
|
2115
|
+
} catch (err) {
|
|
2116
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2117
|
+
if (err instanceof ChannelTerminalError) {
|
|
2118
|
+
this.log({
|
|
2119
|
+
level: "error",
|
|
2120
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2121
|
+
conversation_id: conv.id,
|
|
2122
|
+
message_id: inFlight.evidentMessageId
|
|
2123
|
+
});
|
|
2124
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
if (this.now() >= inFlight.deadline) {
|
|
2128
|
+
this.log({
|
|
2129
|
+
level: "error",
|
|
2130
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2131
|
+
conversation_id: conv.id,
|
|
2132
|
+
message_id: inFlight.evidentMessageId
|
|
2133
|
+
});
|
|
2134
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2135
|
+
return;
|
|
2136
|
+
}
|
|
2137
|
+
this.log({
|
|
2138
|
+
level: "error",
|
|
2139
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2140
|
+
conversation_id: conv.id,
|
|
2141
|
+
message_id: inFlight.evidentMessageId
|
|
2142
|
+
});
|
|
2143
|
+
return;
|
|
2144
|
+
}
|
|
2145
|
+
inFlight.done = true;
|
|
2146
|
+
}
|
|
2147
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2148
|
+
return;
|
|
2149
|
+
}
|
|
2150
|
+
if (state === "failed") {
|
|
2151
|
+
if (!inFlight.done) {
|
|
2152
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2153
|
+
this.log({
|
|
2154
|
+
level: "error",
|
|
2155
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2156
|
+
conversation_id: conv.id,
|
|
2157
|
+
message_id: inFlight.evidentMessageId
|
|
2158
|
+
});
|
|
2159
|
+
try {
|
|
2160
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2161
|
+
} catch (err) {
|
|
2162
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2163
|
+
if (err instanceof ChannelTerminalError) {
|
|
2164
|
+
this.log({
|
|
2165
|
+
level: "error",
|
|
2166
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2167
|
+
conversation_id: conv.id,
|
|
2168
|
+
message_id: inFlight.evidentMessageId
|
|
2169
|
+
});
|
|
2170
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
if (this.now() >= inFlight.deadline) {
|
|
2174
|
+
this.log({
|
|
2175
|
+
level: "error",
|
|
2176
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2177
|
+
conversation_id: conv.id,
|
|
2178
|
+
message_id: inFlight.evidentMessageId
|
|
2179
|
+
});
|
|
2180
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2181
|
+
return;
|
|
2182
|
+
}
|
|
2183
|
+
this.log({
|
|
2184
|
+
level: "error",
|
|
2185
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2186
|
+
conversation_id: conv.id,
|
|
2187
|
+
message_id: inFlight.evidentMessageId
|
|
2188
|
+
});
|
|
2189
|
+
return;
|
|
2190
|
+
}
|
|
2191
|
+
inFlight.done = true;
|
|
2192
|
+
}
|
|
2193
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2194
|
+
return;
|
|
2195
|
+
}
|
|
2196
|
+
if (state === "unknown") {
|
|
2197
|
+
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
2198
|
+
await this.redispatchInFlight(conv.id, sessionId, inFlight);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2202
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2203
|
+
if (state === "queued" && pastStuckBound && sessionIdle) {
|
|
2204
|
+
if (!inFlight.stuckReported) {
|
|
2205
|
+
inFlight.stuckReported = true;
|
|
2206
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2207
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
|
|
2211
|
+
if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
|
|
2212
|
+
const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
|
|
2213
|
+
if (removed) return;
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
if (this.now() >= inFlight.deadline) {
|
|
2217
|
+
this.log({
|
|
2218
|
+
level: "info",
|
|
2219
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
2220
|
+
conversation_id: conv.id,
|
|
2221
|
+
message_id: inFlight.evidentMessageId
|
|
2222
|
+
});
|
|
2223
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2224
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2225
|
+
});
|
|
2226
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1739
2227
|
}
|
|
1740
2228
|
}
|
|
1741
|
-
// -------------------------------------------------------------------------
|
|
1742
|
-
// Paused-session watcher (WI-2-CLI)
|
|
1743
|
-
// -------------------------------------------------------------------------
|
|
1744
2229
|
/**
|
|
1745
|
-
*
|
|
1746
|
-
*
|
|
1747
|
-
*
|
|
1748
|
-
*
|
|
1749
|
-
* when it settles; it never rejects (the body is fully guarded), so a failed
|
|
1750
|
-
* poll/markDone can never crash the run loop — the cron stays as the safety net.
|
|
2230
|
+
* Re-dispatch a message whose user row never appeared (idle-path guard). Safe:
|
|
2231
|
+
* opencode treats a duplicate caller-supplied `messageID` as idempotent (PoC
|
|
2232
|
+
* fact 9) — one user message + one reply even if the original DID land. Resets
|
|
2233
|
+
* the dispatch timestamp so the guard doesn't immediately fire again.
|
|
1751
2234
|
*/
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
2235
|
+
async redispatchInFlight(conversationId, sessionId, inFlight) {
|
|
2236
|
+
const options = {
|
|
2237
|
+
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2238
|
+
model: inFlight.message.opencode_model ?? void 0
|
|
2239
|
+
};
|
|
2240
|
+
this.log({
|
|
2241
|
+
level: "info",
|
|
2242
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
|
|
2243
|
+
message_id: inFlight.evidentMessageId
|
|
1756
2244
|
});
|
|
1757
|
-
this.
|
|
2245
|
+
void this.postSignal(conversationId, inFlight.evidentMessageId, "redispatched");
|
|
2246
|
+
try {
|
|
2247
|
+
await sendPromptAsync(
|
|
2248
|
+
this.port,
|
|
2249
|
+
sessionId,
|
|
2250
|
+
inFlight.message.content,
|
|
2251
|
+
options,
|
|
2252
|
+
inFlight.opencodeMessageId
|
|
2253
|
+
);
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
this.log({
|
|
2256
|
+
level: "error",
|
|
2257
|
+
message: `Re-dispatch failed for message ${inFlight.evidentMessageId.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2258
|
+
message_id: inFlight.evidentMessageId
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
inFlight.dispatchedAt = this.now();
|
|
1758
2262
|
}
|
|
1759
2263
|
/**
|
|
1760
|
-
*
|
|
1761
|
-
*
|
|
1762
|
-
*
|
|
1763
|
-
*
|
|
1764
|
-
*
|
|
1765
|
-
*
|
|
1766
|
-
*
|
|
2264
|
+
* RE-DRIVE a stuck-`queued` follow-up so opencode actually runs it
|
|
2265
|
+
* (queued-followup-redrive; the fix for the 2026-07-18 dev incident).
|
|
2266
|
+
*
|
|
2267
|
+
* The wedge: a follow-up `prompt_async`'d mid-turn / in the post-turn settling
|
|
2268
|
+
* window is orphaned — its user message persists but the turn never runs
|
|
2269
|
+
* (`messageRunState === 'queued'`). Proven against real opencode 1.18.3 by an
|
|
2270
|
+
* isolated, interleaved experiment (`.harness/followup-send-mechanism-finding.md`):
|
|
2271
|
+
* the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
|
|
2272
|
+
* runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
|
|
2273
|
+
* turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
|
|
2274
|
+
* (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
|
|
2275
|
+
* dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
|
|
2276
|
+
*
|
|
2277
|
+
* So we re-`prompt_async` into the SAME opencode session (opencode-web does
|
|
2278
|
+
* exactly this and it works — UPDATE 2 in the investigation doc), preserving the
|
|
2279
|
+
* conversation's history/continuity: we do NOT create a fresh session and we do
|
|
2280
|
+
* NOT overwrite the conversation's `opencode_session_id`. Only the opencode
|
|
2281
|
+
* user-message id changes — to a fresh native id per attempt (opencode's
|
|
2282
|
+
* caller-supplied id dedup is global+permanent, so each attempt needs a never-
|
|
2283
|
+
* seen id).
|
|
1767
2284
|
*
|
|
1768
|
-
*
|
|
1769
|
-
*
|
|
1770
|
-
*
|
|
1771
|
-
* `
|
|
2285
|
+
* Reply correlation is PRESERVED: the native id is random and NOT re-derivable
|
|
2286
|
+
* from the row id, so the runner carries it to the server (on markProcessing/
|
|
2287
|
+
* markDone via `redriveOpencodeMessageId`), which persists it on the row and
|
|
2288
|
+
* correlates the reply by THAT id (`conversation-notification.ts`). A normal,
|
|
2289
|
+
* never-re-driven message still correlates by the derived stable id — unchanged.
|
|
2290
|
+
*
|
|
2291
|
+
* Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
|
|
2292
|
+
* FAILED (existing channel failure affordance) so the user is TOLD it could not
|
|
2293
|
+
* be answered rather than left silent, and it is removed from the in-flight set.
|
|
2294
|
+
*
|
|
2295
|
+
* @returns true if the message LEFT this watcher's in-flight set (gave up +
|
|
2296
|
+
* marked failed) — the caller then stops servicing it this tick. Returns false
|
|
2297
|
+
* when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
|
|
2298
|
+
* send failed and the next window retries).
|
|
2299
|
+
*/
|
|
2300
|
+
async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
|
|
2301
|
+
if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
|
|
2302
|
+
return true;
|
|
2303
|
+
}
|
|
2304
|
+
if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
|
|
2305
|
+
this.log({
|
|
2306
|
+
level: "error",
|
|
2307
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
|
|
2308
|
+
conversation_id: watcher.conv.id,
|
|
2309
|
+
message_id: inFlight.evidentMessageId
|
|
2310
|
+
});
|
|
2311
|
+
try {
|
|
2312
|
+
await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
|
|
2313
|
+
} catch (err) {
|
|
2314
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2315
|
+
this.log({
|
|
2316
|
+
level: "error",
|
|
2317
|
+
message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2318
|
+
conversation_id: watcher.conv.id,
|
|
2319
|
+
message_id: inFlight.evidentMessageId
|
|
2320
|
+
});
|
|
2321
|
+
}
|
|
2322
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2323
|
+
return true;
|
|
2324
|
+
}
|
|
2325
|
+
inFlight.redriveAttempts += 1;
|
|
2326
|
+
inFlight.lastRedriveAt = this.now();
|
|
2327
|
+
this.log({
|
|
2328
|
+
level: "info",
|
|
2329
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
|
|
2330
|
+
conversation_id: watcher.conv.id,
|
|
2331
|
+
message_id: inFlight.evidentMessageId
|
|
2332
|
+
});
|
|
2333
|
+
void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
|
|
2334
|
+
redrive_attempt: inFlight.redriveAttempts
|
|
2335
|
+
});
|
|
2336
|
+
const options = {
|
|
2337
|
+
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2338
|
+
model: inFlight.message.opencode_model ?? void 0
|
|
2339
|
+
};
|
|
2340
|
+
const nativeId = nativeOpencodeMessageId();
|
|
2341
|
+
try {
|
|
2342
|
+
await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
|
|
2343
|
+
} catch (err) {
|
|
2344
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2345
|
+
inFlight.redriveAttempts -= 1;
|
|
2346
|
+
this.log({
|
|
2347
|
+
level: "error",
|
|
2348
|
+
message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
|
|
2349
|
+
conversation_id: watcher.conv.id,
|
|
2350
|
+
message_id: inFlight.evidentMessageId
|
|
2351
|
+
});
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
inFlight.opencodeMessageId = nativeId;
|
|
2355
|
+
inFlight.redriveOpencodeMessageId = nativeId;
|
|
2356
|
+
inFlight.attemptedOpencodeMessageIds.push(nativeId);
|
|
2357
|
+
inFlight.started = false;
|
|
2358
|
+
return false;
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
|
|
2362
|
+
* driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
|
|
2363
|
+
* in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
|
|
2364
|
+
* earlier re-drive attempt eventually produced is delivered) instead of
|
|
2365
|
+
* re-driving again or marking failed. Idempotent + guarded like the watcher's
|
|
2366
|
+
* done branch. `@returns` true when the message left the in-flight set (delivered
|
|
2367
|
+
* or terminally-undeliverable) — the caller then stops servicing it this tick.
|
|
1772
2368
|
*
|
|
1773
|
-
*
|
|
1774
|
-
*
|
|
1775
|
-
*
|
|
1776
|
-
*
|
|
1777
|
-
* out of the run loop.
|
|
2369
|
+
* The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
|
|
2370
|
+
* branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
|
|
2371
|
+
* polls. `messageRunState === 'done'` means a completed, non-errored correlated
|
|
2372
|
+
* reply exists for that id.
|
|
1778
2373
|
*/
|
|
1779
|
-
async
|
|
1780
|
-
const
|
|
2374
|
+
async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
|
|
2375
|
+
const completedId = inFlight.attemptedOpencodeMessageIds.find(
|
|
2376
|
+
(id) => messageRunState(messages, id) === "done"
|
|
2377
|
+
);
|
|
2378
|
+
if (!completedId) return false;
|
|
2379
|
+
if (inFlight.done) {
|
|
2380
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2381
|
+
return true;
|
|
2382
|
+
}
|
|
2383
|
+
this.log({
|
|
2384
|
+
level: "info",
|
|
2385
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
|
|
2386
|
+
conversation_id: watcher.conv.id,
|
|
2387
|
+
message_id: inFlight.evidentMessageId
|
|
2388
|
+
});
|
|
1781
2389
|
try {
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
2390
|
+
await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
|
|
2391
|
+
} catch (err) {
|
|
2392
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2393
|
+
this.log({
|
|
2394
|
+
level: "error",
|
|
2395
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done off an earlier re-drive attempt (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2396
|
+
conversation_id: watcher.conv.id,
|
|
2397
|
+
message_id: inFlight.evidentMessageId
|
|
2398
|
+
});
|
|
2399
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2400
|
+
return true;
|
|
2401
|
+
}
|
|
2402
|
+
inFlight.done = true;
|
|
2403
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2404
|
+
return true;
|
|
2405
|
+
}
|
|
2406
|
+
// -------------------------------------------------------------------------
|
|
2407
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2408
|
+
// -------------------------------------------------------------------------
|
|
2409
|
+
/**
|
|
2410
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2411
|
+
*
|
|
2412
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2413
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2414
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2415
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2416
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2417
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2418
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2419
|
+
*
|
|
2420
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2421
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2422
|
+
*/
|
|
2423
|
+
async readoptProcessing() {
|
|
2424
|
+
const rows = await this.getProcessingMessages();
|
|
2425
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2426
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2427
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2428
|
+
if (!stillProcessing.has(id)) {
|
|
2429
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2430
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2431
|
+
if (cleared || clearedUndeliverable) {
|
|
2432
|
+
this.log({
|
|
2433
|
+
level: "info",
|
|
2434
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2435
|
+
message_id: id
|
|
2436
|
+
});
|
|
1791
2437
|
}
|
|
1792
|
-
}
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
if (rows.length === 0) return;
|
|
2442
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2443
|
+
for (const row of rows) {
|
|
2444
|
+
if (!row.opencode_session_id) {
|
|
2445
|
+
this.log({
|
|
2446
|
+
level: "error",
|
|
2447
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2448
|
+
conversation_id: row.conversation_id,
|
|
2449
|
+
message_id: row.id
|
|
2450
|
+
});
|
|
2451
|
+
continue;
|
|
2452
|
+
}
|
|
2453
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2454
|
+
list.push(row);
|
|
2455
|
+
bySession.set(row.opencode_session_id, list);
|
|
2456
|
+
}
|
|
2457
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2458
|
+
let messages;
|
|
2459
|
+
try {
|
|
2460
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2461
|
+
if (!res.ok) {
|
|
2462
|
+
this.log({
|
|
2463
|
+
level: "error",
|
|
2464
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2465
|
+
});
|
|
2466
|
+
continue;
|
|
2467
|
+
}
|
|
2468
|
+
const body = await res.json();
|
|
2469
|
+
if (!Array.isArray(body)) {
|
|
2470
|
+
this.log({
|
|
2471
|
+
level: "error",
|
|
2472
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2473
|
+
});
|
|
1793
2474
|
continue;
|
|
1794
2475
|
}
|
|
1795
|
-
|
|
2476
|
+
messages = body;
|
|
2477
|
+
} catch (err) {
|
|
2478
|
+
this.log({
|
|
2479
|
+
level: "error",
|
|
2480
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2481
|
+
});
|
|
2482
|
+
continue;
|
|
2483
|
+
}
|
|
2484
|
+
for (const row of sessionRows) {
|
|
2485
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
/**
|
|
2490
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2491
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2492
|
+
*
|
|
2493
|
+
* Branches on `messageRunState(messages, effectiveId)`, where `effectiveId` is
|
|
2494
|
+
* the NATIVE id a prior lifetime's re-drive ran this row under
|
|
2495
|
+
* (`row.opencode_message_id`) if present, else the STABLE derived id
|
|
2496
|
+
* (`opencodeMessageIdFor(row.id)`). Consulting the native id is what makes a
|
|
2497
|
+
* re-driven row that ALREADY ran/completed resolve correctly on restart instead
|
|
2498
|
+
* of looking `queued` under the (never-run) stable id and being re-driven AGAIN
|
|
2499
|
+
* (Bugbot #217 Finding 3 — the duplicate-turn bug):
|
|
2500
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2501
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2502
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2503
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2504
|
+
* tracking the effective id so the reply correlates by it;
|
|
2505
|
+
* - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
|
|
2506
|
+
*
|
|
2507
|
+
* Only `ChannelAuthError` propagates.
|
|
2508
|
+
*/
|
|
2509
|
+
async readoptOne(sessionId, row, messages) {
|
|
2510
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2511
|
+
this.log({
|
|
2512
|
+
level: "info",
|
|
2513
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2514
|
+
conversation_id: row.conversation_id,
|
|
2515
|
+
message_id: row.id
|
|
2516
|
+
});
|
|
2517
|
+
return;
|
|
2518
|
+
}
|
|
2519
|
+
const stableId = opencodeMessageIdFor2(row.id);
|
|
2520
|
+
const nativeRedriveId = row.opencode_message_id;
|
|
2521
|
+
const ocId = nativeRedriveId ?? stableId;
|
|
2522
|
+
const state = messageRunState(messages, ocId);
|
|
2523
|
+
if (state === "done") {
|
|
2524
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
1796
2525
|
this.log({
|
|
1797
2526
|
level: "info",
|
|
1798
|
-
message: `
|
|
1799
|
-
conversation_id:
|
|
1800
|
-
message_id:
|
|
2527
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2528
|
+
conversation_id: row.conversation_id,
|
|
2529
|
+
message_id: row.id
|
|
1801
2530
|
});
|
|
1802
|
-
await this.markDone(conv.id, message.id, sessionId);
|
|
1803
2531
|
return;
|
|
1804
2532
|
}
|
|
1805
2533
|
this.log({
|
|
1806
2534
|
level: "info",
|
|
1807
|
-
message: `
|
|
1808
|
-
conversation_id:
|
|
1809
|
-
message_id:
|
|
2535
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2536
|
+
conversation_id: row.conversation_id,
|
|
2537
|
+
message_id: row.id
|
|
1810
2538
|
});
|
|
2539
|
+
try {
|
|
2540
|
+
await this.markDone(row.conversation_id, row.id, sessionId, nativeRedriveId);
|
|
2541
|
+
} catch (err) {
|
|
2542
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2543
|
+
if (err instanceof ChannelTerminalError) {
|
|
2544
|
+
this.doneUndeliverable.add(row.id);
|
|
2545
|
+
this.log({
|
|
2546
|
+
level: "error",
|
|
2547
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2548
|
+
conversation_id: row.conversation_id,
|
|
2549
|
+
message_id: row.id
|
|
2550
|
+
});
|
|
2551
|
+
return;
|
|
2552
|
+
}
|
|
2553
|
+
this.log({
|
|
2554
|
+
level: "error",
|
|
2555
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2556
|
+
conversation_id: row.conversation_id,
|
|
2557
|
+
message_id: row.id
|
|
2558
|
+
});
|
|
2559
|
+
return;
|
|
2560
|
+
}
|
|
2561
|
+
this.dontRedispatch.delete(row.id);
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
if (state === "failed") {
|
|
2565
|
+
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2566
|
+
this.log({
|
|
2567
|
+
level: "error",
|
|
2568
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2569
|
+
conversation_id: row.conversation_id,
|
|
2570
|
+
message_id: row.id
|
|
2571
|
+
});
|
|
2572
|
+
try {
|
|
2573
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2574
|
+
} catch (err) {
|
|
2575
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2576
|
+
if (err instanceof ChannelTerminalError) {
|
|
2577
|
+
this.doneUndeliverable.add(row.id);
|
|
2578
|
+
this.log({
|
|
2579
|
+
level: "error",
|
|
2580
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2581
|
+
conversation_id: row.conversation_id,
|
|
2582
|
+
message_id: row.id
|
|
2583
|
+
});
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
this.log({
|
|
2587
|
+
level: "error",
|
|
2588
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2589
|
+
conversation_id: row.conversation_id,
|
|
2590
|
+
message_id: row.id
|
|
2591
|
+
});
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
this.dontRedispatch.delete(row.id);
|
|
2595
|
+
return;
|
|
2596
|
+
}
|
|
2597
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2598
|
+
this.log({
|
|
2599
|
+
level: "info",
|
|
2600
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2601
|
+
conversation_id: row.conversation_id,
|
|
2602
|
+
message_id: row.id
|
|
2603
|
+
});
|
|
2604
|
+
return;
|
|
2605
|
+
}
|
|
2606
|
+
if (state === "running" || state === "queued") {
|
|
2607
|
+
const conv = this.convForRow(sessionId, row);
|
|
2608
|
+
const message = this.queuedMessageForRow(row);
|
|
2609
|
+
this.registerReadopted(
|
|
2610
|
+
conv,
|
|
2611
|
+
sessionId,
|
|
2612
|
+
message,
|
|
2613
|
+
ocId,
|
|
2614
|
+
this.processedAtMs(row),
|
|
2615
|
+
nativeRedriveId
|
|
2616
|
+
);
|
|
2617
|
+
this.dispatched.add(row.id);
|
|
2618
|
+
this.readopted.add(row.id);
|
|
2619
|
+
this.ensureWatcherRunning(sessionId);
|
|
2620
|
+
this.log({
|
|
2621
|
+
level: "info",
|
|
2622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (${nativeRedriveId ? "native re-drive id" : "stable id"}, no re-dispatch)`,
|
|
2623
|
+
conversation_id: row.conversation_id,
|
|
2624
|
+
message_id: row.id
|
|
2625
|
+
});
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2629
|
+
}
|
|
2630
|
+
/**
|
|
2631
|
+
* Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
|
|
2632
|
+
*
|
|
2633
|
+
* The stable-id user message is absent from the session, so we (re-)dispatch with
|
|
2634
|
+
* the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
|
|
2635
|
+
* This is what keeps the reply correlatable: the server's completion
|
|
2636
|
+
* notification looks for the reply under the stable id, so the fresh turn's reply
|
|
2637
|
+
* (which hangs off the stable id) is found and delivered. The residual
|
|
2638
|
+
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2639
|
+
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2640
|
+
* to swallow the duplicate.
|
|
2641
|
+
*
|
|
2642
|
+
* `evidentMessageId = row.id` addresses the SERVER row; the stable
|
|
2643
|
+
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
2644
|
+
* `processed_at` (Invariant 1).
|
|
2645
|
+
*/
|
|
2646
|
+
async forceReadoptRun(sessionId, row) {
|
|
2647
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2648
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2649
|
+
this.dontRedispatch.add(row.id);
|
|
2650
|
+
this.log({
|
|
2651
|
+
level: "info",
|
|
2652
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2653
|
+
conversation_id: row.conversation_id,
|
|
2654
|
+
message_id: row.id
|
|
2655
|
+
});
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
const options = {
|
|
2659
|
+
agent: row.opencode_agent ?? void 0,
|
|
2660
|
+
model: row.opencode_model ?? void 0
|
|
2661
|
+
};
|
|
2662
|
+
this.log({
|
|
2663
|
+
level: "info",
|
|
2664
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
|
|
2665
|
+
conversation_id: row.conversation_id,
|
|
2666
|
+
message_id: row.id
|
|
2667
|
+
});
|
|
2668
|
+
try {
|
|
2669
|
+
await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
|
|
1811
2670
|
} catch (err) {
|
|
2671
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
1812
2672
|
this.log({
|
|
1813
2673
|
level: "error",
|
|
1814
|
-
message: `
|
|
1815
|
-
conversation_id:
|
|
1816
|
-
message_id:
|
|
2674
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2675
|
+
conversation_id: row.conversation_id,
|
|
2676
|
+
message_id: row.id
|
|
1817
2677
|
});
|
|
2678
|
+
return;
|
|
1818
2679
|
}
|
|
2680
|
+
const conv = this.convForRow(sessionId, row);
|
|
2681
|
+
const message = this.queuedMessageForRow(row);
|
|
2682
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2683
|
+
this.dispatched.add(row.id);
|
|
2684
|
+
this.readopted.add(row.id);
|
|
2685
|
+
this.ensureWatcherRunning(sessionId);
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2689
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2690
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2691
|
+
*/
|
|
2692
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2693
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2694
|
+
const watcher = this.watchers.get(sessionId);
|
|
2695
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2696
|
+
}
|
|
2697
|
+
/**
|
|
2698
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2699
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2700
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2701
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2702
|
+
* intended, which is worth surfacing.
|
|
2703
|
+
*/
|
|
2704
|
+
processedAtMs(row) {
|
|
2705
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2706
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2707
|
+
this.log({
|
|
2708
|
+
level: "error",
|
|
2709
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2710
|
+
conversation_id: row.conversation_id,
|
|
2711
|
+
message_id: row.id
|
|
2712
|
+
});
|
|
2713
|
+
return this.now();
|
|
2714
|
+
}
|
|
2715
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2716
|
+
convForRow(sessionId, row) {
|
|
2717
|
+
return {
|
|
2718
|
+
id: row.conversation_id,
|
|
2719
|
+
agent_id: this.agentId,
|
|
2720
|
+
opencode_session_id: sessionId,
|
|
2721
|
+
pending_message_count: 0,
|
|
2722
|
+
oldest_pending_at: row.processed_at
|
|
2723
|
+
};
|
|
2724
|
+
}
|
|
2725
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2726
|
+
queuedMessageForRow(row) {
|
|
2727
|
+
return {
|
|
2728
|
+
id: row.id,
|
|
2729
|
+
content: row.content,
|
|
2730
|
+
status: "processing",
|
|
2731
|
+
opencode_agent: row.opencode_agent,
|
|
2732
|
+
opencode_model: row.opencode_model,
|
|
2733
|
+
source_message_id: row.source_message_id,
|
|
2734
|
+
slack_user_id: row.slack_user_id
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
2737
|
+
/**
|
|
2738
|
+
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2739
|
+
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2740
|
+
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2741
|
+
*
|
|
2742
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2743
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2744
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2745
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2746
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2747
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2748
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2749
|
+
*/
|
|
2750
|
+
removeInFlight(watcher, evidentMessageId) {
|
|
2751
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2752
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2753
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2754
|
+
this.log({
|
|
2755
|
+
level: "info",
|
|
2756
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2757
|
+
conversation_id: watcher.conv.id,
|
|
2758
|
+
message_id: evidentMessageId
|
|
2759
|
+
});
|
|
2760
|
+
}
|
|
2761
|
+
watcher.inFlight.delete(evidentMessageId);
|
|
2762
|
+
this.dispatched.delete(evidentMessageId);
|
|
2763
|
+
}
|
|
2764
|
+
/**
|
|
2765
|
+
* Poll `/question` + `/permission` (scoped to the session) and surface NEW ones
|
|
2766
|
+
* via `reportInteraction` (Task 3.5), carrying the PAUSED message's own
|
|
2767
|
+
* `source_message_id` so the server @mentions the correct person under
|
|
2768
|
+
* concurrency. Dedups by interaction id across ticks (reused per-session sets).
|
|
2769
|
+
*
|
|
2770
|
+
* The interaction is attributed to the in-flight message it paused on. opencode
|
|
2771
|
+
* stamps a `messageID` on a permission (and `tool.messageID` on a question) =
|
|
2772
|
+
* the assistant message id, whose `parentID` is the user message id — but the
|
|
2773
|
+
* simplest robust attribution here is: the single in-flight message that is
|
|
2774
|
+
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2775
|
+
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2776
|
+
* oldest running message.
|
|
2777
|
+
*/
|
|
2778
|
+
async pollInteractions(sessionId, watcher, messages) {
|
|
2779
|
+
let questions = [];
|
|
2780
|
+
try {
|
|
2781
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2782
|
+
if (res.ok) {
|
|
2783
|
+
const body = await res.json();
|
|
2784
|
+
questions = Array.isArray(body) ? body : [];
|
|
2785
|
+
}
|
|
2786
|
+
} catch {
|
|
2787
|
+
}
|
|
2788
|
+
for (const q of questions) {
|
|
2789
|
+
if (q.sessionID !== sessionId) continue;
|
|
2790
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2791
|
+
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
2792
|
+
const reported = await this.reportInteraction(
|
|
2793
|
+
watcher.conv.id,
|
|
2794
|
+
"question",
|
|
2795
|
+
q,
|
|
2796
|
+
paused?.message.source_message_id ?? void 0
|
|
2797
|
+
);
|
|
2798
|
+
if (reported) watcher.reportedQuestions.add(q.id);
|
|
2799
|
+
}
|
|
2800
|
+
let permissions = [];
|
|
2801
|
+
try {
|
|
2802
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2803
|
+
if (res.ok) {
|
|
2804
|
+
const body = await res.json();
|
|
2805
|
+
permissions = Array.isArray(body) ? body : [];
|
|
2806
|
+
}
|
|
2807
|
+
} catch {
|
|
2808
|
+
}
|
|
2809
|
+
for (const p of permissions) {
|
|
2810
|
+
if (p.sessionID !== sessionId) continue;
|
|
2811
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2812
|
+
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
2813
|
+
const reported = await this.reportInteraction(
|
|
2814
|
+
watcher.conv.id,
|
|
2815
|
+
"permission",
|
|
2816
|
+
p,
|
|
2817
|
+
paused?.message.source_message_id ?? void 0
|
|
2818
|
+
);
|
|
2819
|
+
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
/**
|
|
2823
|
+
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
2824
|
+
*
|
|
2825
|
+
* The interaction carries `interactionMessageId` — the ASSISTANT message id
|
|
2826
|
+
* that raised it (a question's `tool.messageID` / a permission's `messageID`).
|
|
2827
|
+
* That assistant message is the reply to ONE of our minted user messages
|
|
2828
|
+
* (correlated by `parentID`, GATE-B). So when we have the tick's message
|
|
2829
|
+
* snapshot, we resolve each running in-flight message's correlated assistant
|
|
2830
|
+
* reply (`findAssistantReplyAfter`) and match its id against
|
|
2831
|
+
* `interactionMessageId` — giving an EXACT attribution even with several
|
|
2832
|
+
* messages in flight concurrently in one session.
|
|
2833
|
+
*
|
|
2834
|
+
* We fall back to the oldest running message ONLY when no exact match is
|
|
2835
|
+
* possible (the id is absent, the snapshot is missing, or the reply has not yet
|
|
2836
|
+
* been correlated). With a single running message either path is exact. Never
|
|
2837
|
+
* throws.
|
|
2838
|
+
*
|
|
2839
|
+
* Attribution must NOT depend on our own `started` PATCH flag: opencode can
|
|
2840
|
+
* START a turn AND raise a question/permission BEFORE our next tick fires
|
|
2841
|
+
* `markProcessing` (which sets `started`). Relying on `started` would leave the
|
|
2842
|
+
* running set empty in that window and let the server fall back to "newest
|
|
2843
|
+
* processing/pending" — possibly @mentioning a FOLLOW-UP author rather than the
|
|
2844
|
+
* person whose active turn actually paused. So we derive "running" from the
|
|
2845
|
+
* tick's `messages` snapshot via `messageRunState` instead.
|
|
2846
|
+
*/
|
|
2847
|
+
attributeInteraction(watcher, interactionMessageId, messages) {
|
|
2848
|
+
const inFlight = [...watcher.inFlight.values()].filter((m) => !m.done);
|
|
2849
|
+
if (inFlight.length === 0) return void 0;
|
|
2850
|
+
if (interactionMessageId && messages) {
|
|
2851
|
+
const exact = inFlight.find((m) => {
|
|
2852
|
+
const reply = findAssistantReplyAfter(messages, m.opencodeMessageId);
|
|
2853
|
+
return reply != null && messageIdOf(reply) === interactionMessageId;
|
|
2854
|
+
});
|
|
2855
|
+
if (exact) return exact;
|
|
2856
|
+
}
|
|
2857
|
+
const byOldest = (a, b) => a.dispatchedAt - b.dispatchedAt;
|
|
2858
|
+
if (messages) {
|
|
2859
|
+
const runningPerSnapshot = inFlight.filter(
|
|
2860
|
+
(m) => messageRunState(messages, m.opencodeMessageId) === "running"
|
|
2861
|
+
);
|
|
2862
|
+
if (runningPerSnapshot.length > 0) {
|
|
2863
|
+
return runningPerSnapshot.sort(byOldest)[0];
|
|
2864
|
+
}
|
|
2865
|
+
}
|
|
2866
|
+
const startedRunning = inFlight.filter((m) => m.started);
|
|
2867
|
+
if (startedRunning.length > 0) {
|
|
2868
|
+
return startedRunning.sort(byOldest)[0];
|
|
2869
|
+
}
|
|
2870
|
+
return inFlight.sort(byOldest)[0];
|
|
1819
2871
|
}
|
|
1820
2872
|
// -------------------------------------------------------------------------
|
|
1821
2873
|
// Evident API calls (combinedAuth thread routes)
|
|
@@ -1849,52 +2901,182 @@ var ChannelDriver = class {
|
|
|
1849
2901
|
}
|
|
1850
2902
|
return await res.json();
|
|
1851
2903
|
}
|
|
1852
|
-
|
|
2904
|
+
/**
|
|
2905
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2906
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2907
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2908
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2909
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2910
|
+
* `opencode_session_id`, routing).
|
|
2911
|
+
*
|
|
2912
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2913
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2914
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2915
|
+
* next tick retries.
|
|
2916
|
+
*/
|
|
2917
|
+
async getProcessingMessages() {
|
|
2918
|
+
const res = await this.fetchImpl(
|
|
2919
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2920
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2921
|
+
);
|
|
2922
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2923
|
+
if (!res.ok) {
|
|
2924
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2925
|
+
}
|
|
2926
|
+
const data = await res.json();
|
|
2927
|
+
let messages = data.messages ?? [];
|
|
2928
|
+
if (this.conversationFilter) {
|
|
2929
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2930
|
+
}
|
|
2931
|
+
return messages;
|
|
2932
|
+
}
|
|
2933
|
+
/**
|
|
2934
|
+
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2935
|
+
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
2936
|
+
* opencode_session_id}` → `notifyMessageStarted` (hourglass→runner swap +
|
|
2937
|
+
* deep-linked "View in Evident" notice).
|
|
2938
|
+
*
|
|
2939
|
+
* Return/throw contract (consumed by the watcher's swap-to-running guard):
|
|
2940
|
+
* - returns `true` → the server transitioned the row to processing;
|
|
2941
|
+
* - returns `false` → the server gave a DEFINITIVE "already-processing"
|
|
2942
|
+
* answer (a non-retryable, non-auth status — e.g. a
|
|
2943
|
+
* conflict because a duplicate already transitioned it),
|
|
2944
|
+
* so the caller treats it as already-started and does NOT
|
|
2945
|
+
* retry;
|
|
2946
|
+
* - throws `ChannelAuthError` on 401/403 (terminal auth failure);
|
|
2947
|
+
* - throws on a TRANSIENT failure (retryable 5xx/429 status, or a
|
|
2948
|
+
* network-level error from `fetch`) — i.e. NO definitive server response —
|
|
2949
|
+
* so the caller leaves the message un-started and retries the swap on the
|
|
2950
|
+
* next tick.
|
|
2951
|
+
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2952
|
+
* retry vehicle for the swap-to-running.
|
|
2953
|
+
*/
|
|
2954
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
1853
2955
|
const res = await this.fetchImpl(
|
|
1854
2956
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
1855
2957
|
{
|
|
1856
2958
|
method: "PATCH",
|
|
1857
2959
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
2960
|
+
body: JSON.stringify({
|
|
2961
|
+
status: "processing",
|
|
2962
|
+
opencode_session_id: sessionId,
|
|
2963
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2964
|
+
})
|
|
1862
2965
|
}
|
|
1863
2966
|
);
|
|
1864
2967
|
this.assertAuth(res, "marking message as processing");
|
|
1865
|
-
|
|
2968
|
+
if (res.ok) return true;
|
|
2969
|
+
if (isRetryableStatus(res.status)) {
|
|
2970
|
+
throw new Error(`marking message as processing: HTTP ${res.status}`);
|
|
2971
|
+
}
|
|
2972
|
+
return false;
|
|
1866
2973
|
}
|
|
1867
2974
|
/**
|
|
1868
|
-
* EXISTING combinedAuth completion route — idempotent
|
|
1869
|
-
*
|
|
2975
|
+
* EXISTING combinedAuth completion route — idempotent (WI-CHAN-2). `PATCH
|
|
2976
|
+
* .../messages/:id {status:'done', opencode_session_id}`. The server's
|
|
1870
2977
|
* `queued_conversation_messages.status`/`processed_at` gate makes a re-call
|
|
1871
|
-
* for an already-`done` message a no-op (no double Slack post).
|
|
2978
|
+
* for an already-`done` message a no-op (no double Slack post). Fired by the
|
|
2979
|
+
* watcher on per-message completion (Task 3.4) — no `confirmCompletion`
|
|
2980
|
+
* round-trip (we already observed completion via the message list).
|
|
2981
|
+
*
|
|
2982
|
+
* SINGLE ATTEMPT (no in-call `callWithRetry` backoff). The per-session watcher
|
|
2983
|
+
* services its in-flight messages SEQUENTIALLY within a tick
|
|
2984
|
+
* (`runWatcherLoop` → `serviceInFlightMessage`), so a long multi-attempt
|
|
2985
|
+
* backoff here would BLOCK sibling messages in the SAME session/tick: while
|
|
2986
|
+
* message A's done PATCH burned its internal retries, message B could not be
|
|
2987
|
+
* swapped to running even though opencode had already started it. Instead this
|
|
2988
|
+
* does ONE PATCH and surfaces the SAME outcome contract the watcher's markDone
|
|
2989
|
+
* handler already relies on, leaning on the per-tick retry across ticks
|
|
2990
|
+
* (bounded by `inFlight.deadline`) rather than an in-call retry:
|
|
2991
|
+
* - resolves (`void`) → the server transitioned the row to done
|
|
2992
|
+
* (or idempotently confirmed already-done);
|
|
2993
|
+
* - throws `ChannelAuthError` → 401/403 (terminal auth failure → loop
|
|
2994
|
+
* cleanup, Finding 1);
|
|
2995
|
+
* - throws `ChannelTerminalError`→ non-retryable, non-auth 4xx (will never
|
|
2996
|
+
* succeed → straight to the cron, Finding 4);
|
|
2997
|
+
* - throws a plain `Error` → TRANSIENT 5xx/429 or a network-level error
|
|
2998
|
+
* (no definitive server response → the
|
|
2999
|
+
* watcher retries next tick within the
|
|
3000
|
+
* deadline, Finding 4).
|
|
3001
|
+
*/
|
|
3002
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
3003
|
+
const res = await this.fetchImpl(
|
|
3004
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
3005
|
+
{
|
|
3006
|
+
method: "PATCH",
|
|
3007
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3008
|
+
body: JSON.stringify({
|
|
3009
|
+
status: "done",
|
|
3010
|
+
opencode_session_id: sessionId,
|
|
3011
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3012
|
+
})
|
|
3013
|
+
}
|
|
3014
|
+
);
|
|
3015
|
+
this.assertAuth(res, "marking message as done");
|
|
3016
|
+
if (res.ok) return;
|
|
3017
|
+
if (isRetryableStatus(res.status)) {
|
|
3018
|
+
throw new Error(`marking message as done: HTTP ${res.status}`);
|
|
3019
|
+
}
|
|
3020
|
+
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
3021
|
+
}
|
|
3022
|
+
/**
|
|
3023
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3024
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3025
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3026
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3027
|
+
* failure reason reaches the channel.
|
|
1872
3028
|
*/
|
|
1873
|
-
async
|
|
3029
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3030
|
+
const body = { status: "failed" };
|
|
3031
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3032
|
+
if (error2 !== void 0) body.error = error2;
|
|
1874
3033
|
await this.callWithRetry(
|
|
1875
|
-
"marking message as
|
|
3034
|
+
"marking message as failed",
|
|
1876
3035
|
() => this.fetchImpl(
|
|
1877
3036
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
1878
3037
|
{
|
|
1879
3038
|
method: "PATCH",
|
|
1880
3039
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1881
|
-
body: JSON.stringify(
|
|
3040
|
+
body: JSON.stringify(body)
|
|
1882
3041
|
}
|
|
1883
3042
|
)
|
|
1884
3043
|
);
|
|
1885
3044
|
}
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
3045
|
+
/**
|
|
3046
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3047
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3048
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3049
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3050
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3051
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3052
|
+
* context (no silent catch, per development-workflow).
|
|
3053
|
+
*/
|
|
3054
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3055
|
+
try {
|
|
3056
|
+
const res = await this.fetchImpl(
|
|
3057
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
1891
3058
|
{
|
|
1892
|
-
method: "
|
|
3059
|
+
method: "POST",
|
|
1893
3060
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1894
|
-
body: JSON.stringify({
|
|
3061
|
+
body: JSON.stringify({ signal, ...extra })
|
|
1895
3062
|
}
|
|
1896
|
-
)
|
|
1897
|
-
|
|
3063
|
+
);
|
|
3064
|
+
if (!res.ok) {
|
|
3065
|
+
this.log({
|
|
3066
|
+
level: "error",
|
|
3067
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3068
|
+
conversation_id: conversationId,
|
|
3069
|
+
message_id: messageId
|
|
3070
|
+
});
|
|
3071
|
+
}
|
|
3072
|
+
} catch (err) {
|
|
3073
|
+
this.log({
|
|
3074
|
+
level: "error",
|
|
3075
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3076
|
+
conversation_id: conversationId,
|
|
3077
|
+
message_id: messageId
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
1898
3080
|
}
|
|
1899
3081
|
async persistSession(conversationId, sessionId) {
|
|
1900
3082
|
const res = await this.fetchImpl(
|
|
@@ -1909,10 +3091,17 @@ var ChannelDriver = class {
|
|
|
1909
3091
|
}
|
|
1910
3092
|
/**
|
|
1911
3093
|
* EXISTING combinedAuth interaction route (WI-CHAN-3) — idempotent + retried.
|
|
1912
|
-
* `POST .../interactive-event {type, data}`. The server
|
|
1913
|
-
* interaction and posts a link to the proxied opencode-web
|
|
3094
|
+
* `POST .../interactive-event {type, data, source_message_id?}`. The server
|
|
3095
|
+
* persists the interaction and posts a link to the proxied opencode-web
|
|
3096
|
+
* conversation, @mentioning the user who triggered THIS message's turn.
|
|
3097
|
+
*
|
|
3098
|
+
* WI-3 / WI-4 contract: `source_message_id` is the PAUSED message's own Slack
|
|
3099
|
+
* ts (`message.source_message_id`). The server resolves the @mention from that
|
|
3100
|
+
* message's user FIRST (falling back to the old "newest processing" precedence
|
|
3101
|
+
* only when absent), so the correct person is mentioned under concurrency. It
|
|
3102
|
+
* is OPTIONAL for back-compat with older clients / legacy rows.
|
|
1914
3103
|
*/
|
|
1915
|
-
async reportInteraction(conversationId, type, data) {
|
|
3104
|
+
async reportInteraction(conversationId, type, data, sourceMessageId) {
|
|
1916
3105
|
try {
|
|
1917
3106
|
await this.callWithRetry(
|
|
1918
3107
|
"reporting interactive event",
|
|
@@ -1921,7 +3110,9 @@ var ChannelDriver = class {
|
|
|
1921
3110
|
{
|
|
1922
3111
|
method: "POST",
|
|
1923
3112
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
1924
|
-
body: JSON.stringify(
|
|
3113
|
+
body: JSON.stringify(
|
|
3114
|
+
sourceMessageId ? { type, data, source_message_id: sourceMessageId } : { type, data }
|
|
3115
|
+
)
|
|
1925
3116
|
}
|
|
1926
3117
|
)
|
|
1927
3118
|
);
|
|
@@ -1930,6 +3121,7 @@ var ChannelDriver = class {
|
|
|
1930
3121
|
message: `${type} surfaced to channel (id: ${data.id.slice(0, 8)})`,
|
|
1931
3122
|
conversation_id: conversationId
|
|
1932
3123
|
});
|
|
3124
|
+
return true;
|
|
1933
3125
|
} catch (err) {
|
|
1934
3126
|
if (err instanceof ChannelAuthError) throw err;
|
|
1935
3127
|
this.log({
|
|
@@ -1937,6 +3129,7 @@ var ChannelDriver = class {
|
|
|
1937
3129
|
message: `Failed to surface ${type}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1938
3130
|
conversation_id: conversationId
|
|
1939
3131
|
});
|
|
3132
|
+
return false;
|
|
1940
3133
|
}
|
|
1941
3134
|
}
|
|
1942
3135
|
// -------------------------------------------------------------------------
|
|
@@ -1975,8 +3168,9 @@ var ChannelDriver = class {
|
|
|
1975
3168
|
await this.sleep(backoffDelay(attempt, this.retry));
|
|
1976
3169
|
continue;
|
|
1977
3170
|
}
|
|
3171
|
+
break;
|
|
1978
3172
|
}
|
|
1979
|
-
throw new
|
|
3173
|
+
throw new ChannelTerminalError(`${context}: HTTP ${res.status}`, res.status);
|
|
1980
3174
|
}
|
|
1981
3175
|
throw lastError instanceof Error ? lastError : new Error(`${context}: exhausted retries`);
|
|
1982
3176
|
}
|
|
@@ -2199,7 +3393,9 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2199
3393
|
// src/commands/run.ts
|
|
2200
3394
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2201
3395
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2202
|
-
|
|
3396
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3397
|
+
var CHANNEL_SETTLE_MS = Number(process.env.EVIDENT_SETTLE_MS) || void 0;
|
|
3398
|
+
function log2(state, message, isError = false) {
|
|
2203
3399
|
if (state.json) {
|
|
2204
3400
|
console.log(
|
|
2205
3401
|
JSON.stringify({
|
|
@@ -2224,9 +3420,9 @@ function logActivity(state, entry) {
|
|
|
2224
3420
|
}
|
|
2225
3421
|
if (!state.interactive) {
|
|
2226
3422
|
if (entry.type === "error") {
|
|
2227
|
-
|
|
3423
|
+
log2(state, entry.error ?? "Unknown error", true);
|
|
2228
3424
|
} else if (entry.type === "info" && entry.message) {
|
|
2229
|
-
|
|
3425
|
+
log2(state, entry.message);
|
|
2230
3426
|
}
|
|
2231
3427
|
}
|
|
2232
3428
|
}
|
|
@@ -2312,6 +3508,7 @@ async function handleAuthError(state, error2) {
|
|
|
2312
3508
|
}
|
|
2313
3509
|
async function driveChannels(state, driver) {
|
|
2314
3510
|
let idlePolls = 0;
|
|
3511
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2315
3512
|
while (state.running) {
|
|
2316
3513
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2317
3514
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2321,9 +3518,11 @@ async function driveChannels(state, driver) {
|
|
|
2321
3518
|
try {
|
|
2322
3519
|
const processed = await driver.drainPending();
|
|
2323
3520
|
state.messageCount += processed;
|
|
2324
|
-
|
|
3521
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
3522
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
3523
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2325
3524
|
idlePolls = 0;
|
|
2326
|
-
if (state.interactive) displayStatus(state);
|
|
3525
|
+
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2327
3526
|
} else if (state.idleTimeout !== null) {
|
|
2328
3527
|
idlePolls++;
|
|
2329
3528
|
if (idlePolls === 1) {
|
|
@@ -2373,7 +3572,7 @@ async function cleanup(state) {
|
|
|
2373
3572
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2374
3573
|
displayStatus(state);
|
|
2375
3574
|
} else {
|
|
2376
|
-
|
|
3575
|
+
log2(state, "Stopped OpenCode process");
|
|
2377
3576
|
}
|
|
2378
3577
|
state.opencodeProcess = null;
|
|
2379
3578
|
}
|
|
@@ -2396,10 +3595,11 @@ async function run(options) {
|
|
|
2396
3595
|
running: true,
|
|
2397
3596
|
activityLog: [],
|
|
2398
3597
|
messageCount: 0,
|
|
3598
|
+
lastProxiedActivityAt: null,
|
|
2399
3599
|
authHeader: ""
|
|
2400
3600
|
};
|
|
2401
3601
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2402
|
-
|
|
3602
|
+
log2(
|
|
2403
3603
|
state,
|
|
2404
3604
|
"Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
2405
3605
|
false
|
|
@@ -2410,7 +3610,7 @@ async function run(options) {
|
|
|
2410
3610
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2411
3611
|
displayStatus(state);
|
|
2412
3612
|
} else {
|
|
2413
|
-
|
|
3613
|
+
log2(state, "Shutting down...");
|
|
2414
3614
|
}
|
|
2415
3615
|
await cleanup(state);
|
|
2416
3616
|
await shutdownTelemetry();
|
|
@@ -2443,7 +3643,7 @@ async function run(options) {
|
|
|
2443
3643
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2444
3644
|
if (resolved.agent_id) {
|
|
2445
3645
|
state.agentId = resolved.agent_id;
|
|
2446
|
-
|
|
3646
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2447
3647
|
if (state.interactive && !state.json) {
|
|
2448
3648
|
logActivity(state, {
|
|
2449
3649
|
type: "info",
|
|
@@ -2506,14 +3706,21 @@ async function run(options) {
|
|
|
2506
3706
|
port: state.port,
|
|
2507
3707
|
interactive: state.interactive,
|
|
2508
3708
|
agentId: state.agentId,
|
|
2509
|
-
log: (message) =>
|
|
3709
|
+
log: (message) => log2(state, message)
|
|
2510
3710
|
});
|
|
2511
3711
|
state.port = oc.port;
|
|
2512
3712
|
state.opencodeProcess = oc.process;
|
|
2513
3713
|
state.opencodeVersion = oc.version;
|
|
2514
3714
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2515
|
-
const
|
|
2516
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
3715
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
3716
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
3717
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
3718
|
+
if (versionWarning) {
|
|
3719
|
+
log2(state, versionWarning, false);
|
|
3720
|
+
if (state.interactive && !state.json) {
|
|
3721
|
+
logActivity(state, { type: "info", message: versionWarning });
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
2517
3724
|
} catch (error2) {
|
|
2518
3725
|
ocSpinner?.fail(error2.message);
|
|
2519
3726
|
throw error2;
|
|
@@ -2525,6 +3732,8 @@ async function run(options) {
|
|
|
2525
3732
|
apiUrl: getApiUrlConfig(),
|
|
2526
3733
|
getAuthHeader: () => state.authHeader,
|
|
2527
3734
|
conversationFilter: state.conversationFilter,
|
|
3735
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3736
|
+
settleMs: CHANNEL_SETTLE_MS,
|
|
2528
3737
|
log: (entry) => logActivity(state, {
|
|
2529
3738
|
type: entry.level === "error" ? "error" : "info",
|
|
2530
3739
|
message: entry.message,
|
|
@@ -2544,7 +3753,11 @@ async function run(options) {
|
|
|
2544
3753
|
type: "info",
|
|
2545
3754
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2546
3755
|
});
|
|
2547
|
-
emitAgentConnected(state.agentId, {
|
|
3756
|
+
emitAgentConnected(state.agentId, {
|
|
3757
|
+
port: state.port,
|
|
3758
|
+
cli_version: getCliVersion(),
|
|
3759
|
+
opencode_version: state.opencodeVersion
|
|
3760
|
+
});
|
|
2548
3761
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2549
3762
|
if (state.interactive) displayStatus(state);
|
|
2550
3763
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2578,9 +3791,14 @@ async function run(options) {
|
|
|
2578
3791
|
logActivity(state, { type: "error", error: error2 });
|
|
2579
3792
|
if (state.interactive) displayStatus(state);
|
|
2580
3793
|
},
|
|
2581
|
-
// Web traffic is proxied transparently;
|
|
3794
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
3795
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
3796
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
3797
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
3798
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2582
3799
|
onResponse: () => {
|
|
2583
3800
|
state.opencodeConnected = true;
|
|
3801
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2584
3802
|
},
|
|
2585
3803
|
// A channel message was queued and the api-worker pinged us over the
|
|
2586
3804
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2619,7 +3837,7 @@ async function run(options) {
|
|
|
2619
3837
|
throw error2;
|
|
2620
3838
|
}
|
|
2621
3839
|
if (!interactive || state.json) {
|
|
2622
|
-
|
|
3840
|
+
log2(state, "Driving channel messages...");
|
|
2623
3841
|
}
|
|
2624
3842
|
await driveChannels(state, channelDriver);
|
|
2625
3843
|
await cleanup(state);
|
|
@@ -2631,7 +3849,7 @@ async function run(options) {
|
|
|
2631
3849
|
})
|
|
2632
3850
|
);
|
|
2633
3851
|
} else if (!interactive) {
|
|
2634
|
-
|
|
3852
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2635
3853
|
}
|
|
2636
3854
|
await shutdownTelemetry();
|
|
2637
3855
|
process.exit(0);
|
|
@@ -2653,8 +3871,9 @@ async function run(options) {
|
|
|
2653
3871
|
}
|
|
2654
3872
|
|
|
2655
3873
|
// src/index.ts
|
|
3874
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2656
3875
|
var program = new Command();
|
|
2657
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
3876
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2658
3877
|
"--endpoint <url>",
|
|
2659
3878
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2660
3879
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|