@evident-ai/cli 3.0.1-dev.0590aa5 → 3.0.1-dev.07ee447
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +834 -73
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -484,8 +485,34 @@ var TelemetryEventTypes = {
|
|
|
484
485
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
486
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
487
|
|
|
488
|
+
// ../../packages/types/src/logging/index.ts
|
|
489
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
|
+
function log(level, event, fields) {
|
|
491
|
+
const method = level === "debug" ? "log" : level;
|
|
492
|
+
try {
|
|
493
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
494
|
+
} catch (err) {
|
|
495
|
+
console.error(
|
|
496
|
+
"[evident] log_serialize_failed",
|
|
497
|
+
event,
|
|
498
|
+
err instanceof Error ? err.message : String(err)
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
function stripQuery(url) {
|
|
503
|
+
try {
|
|
504
|
+
return new URL(url).pathname;
|
|
505
|
+
} catch {
|
|
506
|
+
const q = url.indexOf("?");
|
|
507
|
+
return q === -1 ? url : url.slice(0, q);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
487
511
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
489
516
|
var eventBuffer = [];
|
|
490
517
|
var flushTimeout = null;
|
|
491
518
|
var isShuttingDown = false;
|
|
@@ -685,14 +712,14 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
685
712
|
}
|
|
686
713
|
|
|
687
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
688
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
689
|
-
function isQueueValidatedVersion(
|
|
690
|
-
if (!
|
|
691
|
-
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(
|
|
692
|
-
}
|
|
693
|
-
function buildOpenCodeVersionWarning(
|
|
694
|
-
if (isQueueValidatedVersion(
|
|
695
|
-
const detected =
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
716
|
+
function isQueueValidatedVersion(version2) {
|
|
717
|
+
if (!version2) return false;
|
|
718
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
719
|
+
}
|
|
720
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
721
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
722
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
696
723
|
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
697
724
|
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
698
725
|
}
|
|
@@ -1009,7 +1036,11 @@ function roleOf(m) {
|
|
|
1009
1036
|
}
|
|
1010
1037
|
function completedOf(m) {
|
|
1011
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1012
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1013
1044
|
}
|
|
1014
1045
|
function idOf(m) {
|
|
1015
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1029,6 +1060,24 @@ function finishOf(m) {
|
|
|
1029
1060
|
const infoFinish = m.info?.finish;
|
|
1030
1061
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1031
1062
|
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1070
|
+
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1032
1081
|
async function createOpenCodeSession(port, directory) {
|
|
1033
1082
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1034
1083
|
if (directory && directory.trim()) {
|
|
@@ -1046,9 +1095,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1046
1095
|
const data = await response.json();
|
|
1047
1096
|
return data.id;
|
|
1048
1097
|
}
|
|
1049
|
-
|
|
1098
|
+
function messageText(m) {
|
|
1099
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1100
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1101
|
+
}
|
|
1102
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1103
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1104
|
+
const knownUserIds = new Set(
|
|
1105
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1106
|
+
);
|
|
1050
1107
|
const body = {
|
|
1051
|
-
messageID: messageId,
|
|
1052
1108
|
parts: [{ type: "text", text: content }]
|
|
1053
1109
|
};
|
|
1054
1110
|
if (options?.agent) {
|
|
@@ -1072,6 +1128,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1072
1128
|
const text = await res.text().catch(() => "");
|
|
1073
1129
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1074
1130
|
}
|
|
1131
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1132
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1133
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1134
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1135
|
+
if (after) {
|
|
1136
|
+
let best = null;
|
|
1137
|
+
for (const m of after) {
|
|
1138
|
+
if (roleOf(m) !== "user") continue;
|
|
1139
|
+
const id = idOf(m);
|
|
1140
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1141
|
+
if (messageText(m) !== content) continue;
|
|
1142
|
+
const created = createdOf(m) ?? 0;
|
|
1143
|
+
if (best === null || created > best.created) {
|
|
1144
|
+
best = { id, created };
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
if (best) return best.id;
|
|
1148
|
+
}
|
|
1149
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1150
|
+
await new Promise((resolve) => setTimeout(resolve, READ_BACK_DELAY_MS));
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
return null;
|
|
1075
1154
|
}
|
|
1076
1155
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1077
1156
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1088,19 +1167,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1088
1167
|
}
|
|
1089
1168
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1090
1169
|
if (!messages || messages.length === 0) return null;
|
|
1170
|
+
let lastCorrelated = null;
|
|
1171
|
+
let lastNonErrored = null;
|
|
1091
1172
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1092
1173
|
const m = messages[i];
|
|
1093
|
-
if (roleOf(m)
|
|
1174
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1175
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1176
|
+
if (errorOf(m) == null) {
|
|
1177
|
+
lastNonErrored = m;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1094
1180
|
}
|
|
1181
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1095
1182
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1096
1183
|
if (userIndex === -1) return null;
|
|
1097
1184
|
let last = null;
|
|
1185
|
+
let lastOk = null;
|
|
1098
1186
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1099
1187
|
const role = roleOf(messages[i]);
|
|
1100
1188
|
if (role === "user") break;
|
|
1101
|
-
if (role === "assistant")
|
|
1189
|
+
if (role === "assistant") {
|
|
1190
|
+
last = messages[i];
|
|
1191
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1192
|
+
}
|
|
1102
1193
|
}
|
|
1103
|
-
return last;
|
|
1194
|
+
return lastOk ?? last;
|
|
1104
1195
|
}
|
|
1105
1196
|
function messageRunState(messages, userMessageId) {
|
|
1106
1197
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1110,13 +1201,27 @@ function messageRunState(messages, userMessageId) {
|
|
|
1110
1201
|
if (!reply) return "unknown";
|
|
1111
1202
|
}
|
|
1112
1203
|
if (!reply) return "queued";
|
|
1113
|
-
if (
|
|
1114
|
-
|
|
1115
|
-
return "done";
|
|
1204
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1205
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1116
1206
|
}
|
|
1117
|
-
function
|
|
1118
|
-
const
|
|
1119
|
-
|
|
1207
|
+
function messageError(messages, userMessageId) {
|
|
1208
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1209
|
+
const error2 = errorOf(reply);
|
|
1210
|
+
if (error2 == null) return null;
|
|
1211
|
+
if (typeof error2 === "string") return error2;
|
|
1212
|
+
if (typeof error2 === "object") {
|
|
1213
|
+
const e = error2;
|
|
1214
|
+
const dataMessage = e.data?.message;
|
|
1215
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1216
|
+
if (typeof e.message === "string") return e.message;
|
|
1217
|
+
}
|
|
1218
|
+
return "The agent run failed.";
|
|
1219
|
+
}
|
|
1220
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1221
|
+
if (!messages || messages.length === 0) return false;
|
|
1222
|
+
return messages.some(
|
|
1223
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1224
|
+
);
|
|
1120
1225
|
}
|
|
1121
1226
|
|
|
1122
1227
|
// src/lib/tunnel/connection.ts
|
|
@@ -1187,12 +1292,20 @@ var StreamForwarder = class {
|
|
|
1187
1292
|
}
|
|
1188
1293
|
async handleOpen(frame) {
|
|
1189
1294
|
const { sid, method, path, headers, has_body } = frame;
|
|
1295
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1296
|
+
const startedAt = Date.now();
|
|
1190
1297
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1191
1298
|
this.callbacks.onDrainPing?.();
|
|
1192
1299
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1193
1300
|
this.send({ type: "res_end", sid });
|
|
1194
1301
|
return;
|
|
1195
1302
|
}
|
|
1303
|
+
log("info", "agent_request", {
|
|
1304
|
+
correlation_id: correlationId,
|
|
1305
|
+
sid,
|
|
1306
|
+
method,
|
|
1307
|
+
path: stripQuery(path)
|
|
1308
|
+
});
|
|
1196
1309
|
const ac = new AbortController();
|
|
1197
1310
|
let bodyPromise;
|
|
1198
1311
|
let pushBody;
|
|
@@ -1239,6 +1352,12 @@ var StreamForwarder = class {
|
|
|
1239
1352
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1240
1353
|
});
|
|
1241
1354
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1355
|
+
log("info", "agent_response", {
|
|
1356
|
+
correlation_id: correlationId,
|
|
1357
|
+
sid,
|
|
1358
|
+
status: upstream.status,
|
|
1359
|
+
duration_ms: Date.now() - startedAt
|
|
1360
|
+
});
|
|
1242
1361
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1243
1362
|
try {
|
|
1244
1363
|
if (upstream.body) {
|
|
@@ -1508,7 +1627,7 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1508
1627
|
};
|
|
1509
1628
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1510
1629
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1511
|
-
var
|
|
1630
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1512
1631
|
var ChannelAuthError = class extends Error {
|
|
1513
1632
|
constructor(message) {
|
|
1514
1633
|
super(message);
|
|
@@ -1543,10 +1662,18 @@ var ChannelDriver = class {
|
|
|
1543
1662
|
sleep;
|
|
1544
1663
|
pausedPollIntervalMs;
|
|
1545
1664
|
pausedMaxWaitMs;
|
|
1546
|
-
|
|
1665
|
+
stuckQueuedMs;
|
|
1547
1666
|
now;
|
|
1548
1667
|
/** Cache of conversationId → opencode sessionId. */
|
|
1549
1668
|
sessions = /* @__PURE__ */ new Map();
|
|
1669
|
+
/**
|
|
1670
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1671
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1672
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1673
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1674
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1675
|
+
*/
|
|
1676
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1550
1677
|
/**
|
|
1551
1678
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1552
1679
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1563,6 +1690,54 @@ var ChannelDriver = class {
|
|
|
1563
1690
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1564
1691
|
*/
|
|
1565
1692
|
dispatched = /* @__PURE__ */ new Set();
|
|
1693
|
+
/**
|
|
1694
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1695
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1696
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1697
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1698
|
+
* the processing list.
|
|
1699
|
+
*/
|
|
1700
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1701
|
+
/**
|
|
1702
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1703
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1704
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1705
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1706
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1707
|
+
*
|
|
1708
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1709
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1710
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1711
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1712
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1713
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1714
|
+
*/
|
|
1715
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1716
|
+
/**
|
|
1717
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1718
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1719
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1720
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1721
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1722
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1723
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1724
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1725
|
+
*/
|
|
1726
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1727
|
+
/**
|
|
1728
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1729
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1730
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1731
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1732
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1733
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1734
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1735
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1736
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1737
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1738
|
+
* so the NEXT tick may retry exactly once more).
|
|
1739
|
+
*/
|
|
1740
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1566
1741
|
/**
|
|
1567
1742
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1568
1743
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1585,7 +1760,7 @@ var ChannelDriver = class {
|
|
|
1585
1760
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1586
1761
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1587
1762
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1588
|
-
this.
|
|
1763
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1589
1764
|
this.now = config2.now ?? (() => Date.now());
|
|
1590
1765
|
}
|
|
1591
1766
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1618,6 +1793,7 @@ var ChannelDriver = class {
|
|
|
1618
1793
|
for (const conv of conversations) {
|
|
1619
1794
|
dispatched += await this.processConversation(conv);
|
|
1620
1795
|
}
|
|
1796
|
+
await this.readoptProcessing();
|
|
1621
1797
|
} finally {
|
|
1622
1798
|
this.draining = false;
|
|
1623
1799
|
}
|
|
@@ -1669,15 +1845,17 @@ var ChannelDriver = class {
|
|
|
1669
1845
|
const sessionId = await this.ensureSession(conv);
|
|
1670
1846
|
const messages = await this.getPendingMessages(conv.id);
|
|
1671
1847
|
let dispatched = 0;
|
|
1848
|
+
let skippedAlreadyDispatched = 0;
|
|
1672
1849
|
for (const message of messages) {
|
|
1673
1850
|
if (this.dispatched.has(message.id)) {
|
|
1851
|
+
skippedAlreadyDispatched += 1;
|
|
1674
1852
|
continue;
|
|
1675
1853
|
}
|
|
1676
|
-
const opencodeMessageId = opencodeMessageIdFor(message.id);
|
|
1677
1854
|
const options = {
|
|
1678
1855
|
agent: message.opencode_agent ?? void 0,
|
|
1679
1856
|
model: message.opencode_model ?? void 0
|
|
1680
1857
|
};
|
|
1858
|
+
let opencodeMessageId;
|
|
1681
1859
|
try {
|
|
1682
1860
|
this.log({
|
|
1683
1861
|
level: "info",
|
|
@@ -1685,7 +1863,10 @@ var ChannelDriver = class {
|
|
|
1685
1863
|
conversation_id: conv.id,
|
|
1686
1864
|
message_id: message.id
|
|
1687
1865
|
});
|
|
1688
|
-
await
|
|
1866
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
1867
|
+
sessionId,
|
|
1868
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
1869
|
+
);
|
|
1689
1870
|
} catch (err) {
|
|
1690
1871
|
if (err instanceof ChannelAuthError) throw err;
|
|
1691
1872
|
this.dispatched.delete(message.id);
|
|
@@ -1699,9 +1880,26 @@ var ChannelDriver = class {
|
|
|
1699
1880
|
});
|
|
1700
1881
|
continue;
|
|
1701
1882
|
}
|
|
1883
|
+
if (opencodeMessageId === null) {
|
|
1884
|
+
this.log({
|
|
1885
|
+
level: "error",
|
|
1886
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
1887
|
+
conversation_id: conv.id,
|
|
1888
|
+
message_id: message.id
|
|
1889
|
+
});
|
|
1890
|
+
continue;
|
|
1891
|
+
}
|
|
1702
1892
|
this.dispatched.add(message.id);
|
|
1703
1893
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1704
1894
|
dispatched += 1;
|
|
1895
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1896
|
+
}
|
|
1897
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1898
|
+
this.log({
|
|
1899
|
+
level: "error",
|
|
1900
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
1901
|
+
conversation_id: conv.id
|
|
1902
|
+
});
|
|
1705
1903
|
}
|
|
1706
1904
|
this.ensureWatcherRunning(sessionId);
|
|
1707
1905
|
return dispatched;
|
|
@@ -1739,6 +1937,25 @@ var ChannelDriver = class {
|
|
|
1739
1937
|
// -------------------------------------------------------------------------
|
|
1740
1938
|
// Per-session watcher (WI-3)
|
|
1741
1939
|
// -------------------------------------------------------------------------
|
|
1940
|
+
/**
|
|
1941
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
1942
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
1943
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
1944
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
1945
|
+
* (each dispatch reports its own outcome to its caller).
|
|
1946
|
+
*/
|
|
1947
|
+
dispatchLocked(sessionId, fn) {
|
|
1948
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
1949
|
+
const run2 = prior.then(fn, fn);
|
|
1950
|
+
this.sessionDispatchLocks.set(
|
|
1951
|
+
sessionId,
|
|
1952
|
+
run2.then(
|
|
1953
|
+
() => void 0,
|
|
1954
|
+
() => void 0
|
|
1955
|
+
)
|
|
1956
|
+
);
|
|
1957
|
+
return run2;
|
|
1958
|
+
}
|
|
1742
1959
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1743
1960
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1744
1961
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1760,7 +1977,54 @@ var ChannelDriver = class {
|
|
|
1760
1977
|
dispatchedAt: now,
|
|
1761
1978
|
deadline: now + this.pausedMaxWaitMs,
|
|
1762
1979
|
started: false,
|
|
1763
|
-
done: false
|
|
1980
|
+
done: false,
|
|
1981
|
+
stuckReported: false
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
/**
|
|
1985
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1986
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1987
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1988
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1989
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1990
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1991
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1992
|
+
*
|
|
1993
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1994
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1995
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1996
|
+
*
|
|
1997
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1998
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1999
|
+
* fire from the watcher's normal branches.
|
|
2000
|
+
*/
|
|
2001
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2002
|
+
let watcher = this.watchers.get(sessionId);
|
|
2003
|
+
if (!watcher) {
|
|
2004
|
+
watcher = {
|
|
2005
|
+
conv,
|
|
2006
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2007
|
+
loop: null,
|
|
2008
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2009
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2010
|
+
};
|
|
2011
|
+
this.watchers.set(sessionId, watcher);
|
|
2012
|
+
}
|
|
2013
|
+
watcher.inFlight.set(message.id, {
|
|
2014
|
+
evidentMessageId: message.id,
|
|
2015
|
+
opencodeMessageId,
|
|
2016
|
+
message,
|
|
2017
|
+
dispatchedAt: this.now(),
|
|
2018
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2019
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2020
|
+
started: true,
|
|
2021
|
+
done: false,
|
|
2022
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2023
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2024
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2025
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2026
|
+
// (#210/#220 observability).
|
|
2027
|
+
stuckReported: false
|
|
1764
2028
|
});
|
|
1765
2029
|
}
|
|
1766
2030
|
/**
|
|
@@ -1826,6 +2090,7 @@ var ChannelDriver = class {
|
|
|
1826
2090
|
conversation_id: watcher.conv.id
|
|
1827
2091
|
});
|
|
1828
2092
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2093
|
+
this.readopted.delete(evidentMessageId);
|
|
1829
2094
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1830
2095
|
}
|
|
1831
2096
|
return;
|
|
@@ -1846,10 +2111,15 @@ var ChannelDriver = class {
|
|
|
1846
2111
|
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
1847
2112
|
const conv = watcher.conv;
|
|
1848
2113
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1849
|
-
if ((state === "running" || state === "done") && !inFlight.started) {
|
|
2114
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1850
2115
|
let claimed;
|
|
1851
2116
|
try {
|
|
1852
|
-
claimed = await this.markProcessing(
|
|
2117
|
+
claimed = await this.markProcessing(
|
|
2118
|
+
conv.id,
|
|
2119
|
+
inFlight.evidentMessageId,
|
|
2120
|
+
sessionId,
|
|
2121
|
+
inFlight.opencodeMessageId
|
|
2122
|
+
);
|
|
1853
2123
|
} catch (err) {
|
|
1854
2124
|
if (err instanceof ChannelAuthError) throw err;
|
|
1855
2125
|
this.log({
|
|
@@ -1879,7 +2149,12 @@ var ChannelDriver = class {
|
|
|
1879
2149
|
message_id: inFlight.evidentMessageId
|
|
1880
2150
|
});
|
|
1881
2151
|
try {
|
|
1882
|
-
await this.markDone(
|
|
2152
|
+
await this.markDone(
|
|
2153
|
+
conv.id,
|
|
2154
|
+
inFlight.evidentMessageId,
|
|
2155
|
+
sessionId,
|
|
2156
|
+
inFlight.opencodeMessageId
|
|
2157
|
+
);
|
|
1883
2158
|
} catch (err) {
|
|
1884
2159
|
if (err instanceof ChannelAuthError) throw err;
|
|
1885
2160
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -1915,10 +2190,59 @@ var ChannelDriver = class {
|
|
|
1915
2190
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1916
2191
|
return;
|
|
1917
2192
|
}
|
|
1918
|
-
if (state === "
|
|
1919
|
-
if (
|
|
1920
|
-
|
|
2193
|
+
if (state === "failed") {
|
|
2194
|
+
if (!inFlight.done) {
|
|
2195
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2196
|
+
this.log({
|
|
2197
|
+
level: "error",
|
|
2198
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2199
|
+
conversation_id: conv.id,
|
|
2200
|
+
message_id: inFlight.evidentMessageId
|
|
2201
|
+
});
|
|
2202
|
+
try {
|
|
2203
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2204
|
+
} catch (err) {
|
|
2205
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2206
|
+
if (err instanceof ChannelTerminalError) {
|
|
2207
|
+
this.log({
|
|
2208
|
+
level: "error",
|
|
2209
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2210
|
+
conversation_id: conv.id,
|
|
2211
|
+
message_id: inFlight.evidentMessageId
|
|
2212
|
+
});
|
|
2213
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2214
|
+
return;
|
|
2215
|
+
}
|
|
2216
|
+
if (this.now() >= inFlight.deadline) {
|
|
2217
|
+
this.log({
|
|
2218
|
+
level: "error",
|
|
2219
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2220
|
+
conversation_id: conv.id,
|
|
2221
|
+
message_id: inFlight.evidentMessageId
|
|
2222
|
+
});
|
|
2223
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2224
|
+
return;
|
|
2225
|
+
}
|
|
2226
|
+
this.log({
|
|
2227
|
+
level: "error",
|
|
2228
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2229
|
+
conversation_id: conv.id,
|
|
2230
|
+
message_id: inFlight.evidentMessageId
|
|
2231
|
+
});
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
inFlight.done = true;
|
|
1921
2235
|
}
|
|
2236
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2240
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2241
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2242
|
+
inFlight.stuckReported = true;
|
|
2243
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2244
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2245
|
+
});
|
|
1922
2246
|
}
|
|
1923
2247
|
if (this.now() >= inFlight.deadline) {
|
|
1924
2248
|
this.log({
|
|
@@ -1927,48 +2251,386 @@ var ChannelDriver = class {
|
|
|
1927
2251
|
conversation_id: conv.id,
|
|
1928
2252
|
message_id: inFlight.evidentMessageId
|
|
1929
2253
|
});
|
|
2254
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2255
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2256
|
+
});
|
|
1930
2257
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1931
2258
|
}
|
|
1932
2259
|
}
|
|
2260
|
+
// -------------------------------------------------------------------------
|
|
2261
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2262
|
+
// -------------------------------------------------------------------------
|
|
2263
|
+
/**
|
|
2264
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2265
|
+
*
|
|
2266
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2267
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2268
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2269
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2270
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2271
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2272
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2273
|
+
*
|
|
2274
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2275
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2276
|
+
*/
|
|
2277
|
+
async readoptProcessing() {
|
|
2278
|
+
const rows = await this.getProcessingMessages();
|
|
2279
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2280
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2281
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2282
|
+
if (!stillProcessing.has(id)) {
|
|
2283
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2284
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2285
|
+
if (cleared || clearedUndeliverable) {
|
|
2286
|
+
this.log({
|
|
2287
|
+
level: "info",
|
|
2288
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2289
|
+
message_id: id
|
|
2290
|
+
});
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
if (rows.length === 0) return;
|
|
2296
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2297
|
+
for (const row of rows) {
|
|
2298
|
+
if (!row.opencode_session_id) {
|
|
2299
|
+
this.log({
|
|
2300
|
+
level: "error",
|
|
2301
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2302
|
+
conversation_id: row.conversation_id,
|
|
2303
|
+
message_id: row.id
|
|
2304
|
+
});
|
|
2305
|
+
continue;
|
|
2306
|
+
}
|
|
2307
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2308
|
+
list.push(row);
|
|
2309
|
+
bySession.set(row.opencode_session_id, list);
|
|
2310
|
+
}
|
|
2311
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2312
|
+
let messages;
|
|
2313
|
+
try {
|
|
2314
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2315
|
+
if (!res.ok) {
|
|
2316
|
+
this.log({
|
|
2317
|
+
level: "error",
|
|
2318
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2319
|
+
});
|
|
2320
|
+
continue;
|
|
2321
|
+
}
|
|
2322
|
+
const body = await res.json();
|
|
2323
|
+
if (!Array.isArray(body)) {
|
|
2324
|
+
this.log({
|
|
2325
|
+
level: "error",
|
|
2326
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2327
|
+
});
|
|
2328
|
+
continue;
|
|
2329
|
+
}
|
|
2330
|
+
messages = body;
|
|
2331
|
+
} catch (err) {
|
|
2332
|
+
this.log({
|
|
2333
|
+
level: "error",
|
|
2334
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2335
|
+
});
|
|
2336
|
+
continue;
|
|
2337
|
+
}
|
|
2338
|
+
for (const row of sessionRows) {
|
|
2339
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
/**
|
|
2344
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2345
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2346
|
+
*
|
|
2347
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2348
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2349
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2350
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2351
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2352
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2353
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2354
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2355
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2356
|
+
* tracking the stored id so the reply correlates by it;
|
|
2357
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2358
|
+
*
|
|
2359
|
+
* Only `ChannelAuthError` propagates.
|
|
2360
|
+
*/
|
|
2361
|
+
async readoptOne(sessionId, row, messages) {
|
|
2362
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2363
|
+
this.log({
|
|
2364
|
+
level: "info",
|
|
2365
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2366
|
+
conversation_id: row.conversation_id,
|
|
2367
|
+
message_id: row.id
|
|
2368
|
+
});
|
|
2369
|
+
return;
|
|
2370
|
+
}
|
|
2371
|
+
const ocId = row.opencode_message_id;
|
|
2372
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2373
|
+
if (state === "done") {
|
|
2374
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2375
|
+
this.log({
|
|
2376
|
+
level: "info",
|
|
2377
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2378
|
+
conversation_id: row.conversation_id,
|
|
2379
|
+
message_id: row.id
|
|
2380
|
+
});
|
|
2381
|
+
return;
|
|
2382
|
+
}
|
|
2383
|
+
this.log({
|
|
2384
|
+
level: "info",
|
|
2385
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2386
|
+
conversation_id: row.conversation_id,
|
|
2387
|
+
message_id: row.id
|
|
2388
|
+
});
|
|
2389
|
+
try {
|
|
2390
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2391
|
+
} catch (err) {
|
|
2392
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2393
|
+
if (err instanceof ChannelTerminalError) {
|
|
2394
|
+
this.doneUndeliverable.add(row.id);
|
|
2395
|
+
this.log({
|
|
2396
|
+
level: "error",
|
|
2397
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2398
|
+
conversation_id: row.conversation_id,
|
|
2399
|
+
message_id: row.id
|
|
2400
|
+
});
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
this.log({
|
|
2404
|
+
level: "error",
|
|
2405
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2406
|
+
conversation_id: row.conversation_id,
|
|
2407
|
+
message_id: row.id
|
|
2408
|
+
});
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
this.dontRedispatch.delete(row.id);
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
if (state === "failed") {
|
|
2415
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2416
|
+
this.log({
|
|
2417
|
+
level: "error",
|
|
2418
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2419
|
+
conversation_id: row.conversation_id,
|
|
2420
|
+
message_id: row.id
|
|
2421
|
+
});
|
|
2422
|
+
try {
|
|
2423
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2424
|
+
} catch (err) {
|
|
2425
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2426
|
+
if (err instanceof ChannelTerminalError) {
|
|
2427
|
+
this.doneUndeliverable.add(row.id);
|
|
2428
|
+
this.log({
|
|
2429
|
+
level: "error",
|
|
2430
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2431
|
+
conversation_id: row.conversation_id,
|
|
2432
|
+
message_id: row.id
|
|
2433
|
+
});
|
|
2434
|
+
return;
|
|
2435
|
+
}
|
|
2436
|
+
this.log({
|
|
2437
|
+
level: "error",
|
|
2438
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2439
|
+
conversation_id: row.conversation_id,
|
|
2440
|
+
message_id: row.id
|
|
2441
|
+
});
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2444
|
+
this.dontRedispatch.delete(row.id);
|
|
2445
|
+
return;
|
|
2446
|
+
}
|
|
2447
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2448
|
+
this.log({
|
|
2449
|
+
level: "info",
|
|
2450
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2451
|
+
conversation_id: row.conversation_id,
|
|
2452
|
+
message_id: row.id
|
|
2453
|
+
});
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2457
|
+
const conv = this.convForRow(sessionId, row);
|
|
2458
|
+
const message = this.queuedMessageForRow(row);
|
|
2459
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2460
|
+
this.dispatched.add(row.id);
|
|
2461
|
+
this.readopted.add(row.id);
|
|
2462
|
+
this.ensureWatcherRunning(sessionId);
|
|
2463
|
+
this.log({
|
|
2464
|
+
level: "info",
|
|
2465
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2466
|
+
conversation_id: row.conversation_id,
|
|
2467
|
+
message_id: row.id
|
|
2468
|
+
});
|
|
2469
|
+
return;
|
|
2470
|
+
}
|
|
2471
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2472
|
+
}
|
|
1933
2473
|
/**
|
|
1934
|
-
* Re-dispatch
|
|
1935
|
-
*
|
|
1936
|
-
*
|
|
1937
|
-
*
|
|
2474
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2475
|
+
*
|
|
2476
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
2477
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
2478
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
2479
|
+
* correlates server-side.
|
|
2480
|
+
*
|
|
2481
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
2482
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
2483
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
2484
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
2485
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
2486
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
2487
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
2488
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
2489
|
+
* may retry exactly once more).
|
|
2490
|
+
*
|
|
2491
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2492
|
+
* `processed_at` (Invariant 1).
|
|
1938
2493
|
*/
|
|
1939
|
-
async
|
|
2494
|
+
async forceReadoptRun(sessionId, row) {
|
|
2495
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
2496
|
+
this.log({
|
|
2497
|
+
level: "info",
|
|
2498
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
2499
|
+
conversation_id: row.conversation_id,
|
|
2500
|
+
message_id: row.id
|
|
2501
|
+
});
|
|
2502
|
+
return;
|
|
2503
|
+
}
|
|
2504
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2505
|
+
this.dontRedispatch.add(row.id);
|
|
2506
|
+
this.log({
|
|
2507
|
+
level: "info",
|
|
2508
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2509
|
+
conversation_id: row.conversation_id,
|
|
2510
|
+
message_id: row.id
|
|
2511
|
+
});
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
1940
2514
|
const options = {
|
|
1941
|
-
agent:
|
|
1942
|
-
model:
|
|
2515
|
+
agent: row.opencode_agent ?? void 0,
|
|
2516
|
+
model: row.opencode_model ?? void 0
|
|
1943
2517
|
};
|
|
1944
2518
|
this.log({
|
|
1945
2519
|
level: "info",
|
|
1946
|
-
message: `
|
|
1947
|
-
|
|
2520
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2521
|
+
conversation_id: row.conversation_id,
|
|
2522
|
+
message_id: row.id
|
|
1948
2523
|
});
|
|
2524
|
+
this.awaitingReadopt.add(row.id);
|
|
2525
|
+
let ocId;
|
|
1949
2526
|
try {
|
|
1950
|
-
await
|
|
1951
|
-
this.port,
|
|
2527
|
+
ocId = await this.dispatchLocked(
|
|
1952
2528
|
sessionId,
|
|
1953
|
-
|
|
1954
|
-
options,
|
|
1955
|
-
inFlight.opencodeMessageId
|
|
2529
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
1956
2530
|
);
|
|
1957
2531
|
} catch (err) {
|
|
2532
|
+
this.awaitingReadopt.delete(row.id);
|
|
2533
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
1958
2534
|
this.log({
|
|
1959
2535
|
level: "error",
|
|
1960
|
-
message: `Re-dispatch failed for message ${
|
|
1961
|
-
|
|
2536
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2537
|
+
conversation_id: row.conversation_id,
|
|
2538
|
+
message_id: row.id
|
|
2539
|
+
});
|
|
2540
|
+
return;
|
|
2541
|
+
}
|
|
2542
|
+
if (ocId === null) {
|
|
2543
|
+
this.awaitingReadopt.delete(row.id);
|
|
2544
|
+
this.log({
|
|
2545
|
+
level: "error",
|
|
2546
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
2547
|
+
conversation_id: row.conversation_id,
|
|
2548
|
+
message_id: row.id
|
|
1962
2549
|
});
|
|
2550
|
+
return;
|
|
1963
2551
|
}
|
|
1964
|
-
|
|
2552
|
+
const conv = this.convForRow(sessionId, row);
|
|
2553
|
+
const message = this.queuedMessageForRow(row);
|
|
2554
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2555
|
+
this.dispatched.add(row.id);
|
|
2556
|
+
this.readopted.add(row.id);
|
|
2557
|
+
this.awaitingReadopt.delete(row.id);
|
|
2558
|
+
this.ensureWatcherRunning(sessionId);
|
|
2559
|
+
}
|
|
2560
|
+
/**
|
|
2561
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2562
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2563
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2564
|
+
*/
|
|
2565
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2566
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2567
|
+
const watcher = this.watchers.get(sessionId);
|
|
2568
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2572
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2573
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2574
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2575
|
+
* intended, which is worth surfacing.
|
|
2576
|
+
*/
|
|
2577
|
+
processedAtMs(row) {
|
|
2578
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2579
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2580
|
+
this.log({
|
|
2581
|
+
level: "error",
|
|
2582
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2583
|
+
conversation_id: row.conversation_id,
|
|
2584
|
+
message_id: row.id
|
|
2585
|
+
});
|
|
2586
|
+
return this.now();
|
|
2587
|
+
}
|
|
2588
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2589
|
+
convForRow(sessionId, row) {
|
|
2590
|
+
return {
|
|
2591
|
+
id: row.conversation_id,
|
|
2592
|
+
agent_id: this.agentId,
|
|
2593
|
+
opencode_session_id: sessionId,
|
|
2594
|
+
pending_message_count: 0,
|
|
2595
|
+
oldest_pending_at: row.processed_at
|
|
2596
|
+
};
|
|
2597
|
+
}
|
|
2598
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2599
|
+
queuedMessageForRow(row) {
|
|
2600
|
+
return {
|
|
2601
|
+
id: row.id,
|
|
2602
|
+
content: row.content,
|
|
2603
|
+
status: "processing",
|
|
2604
|
+
opencode_agent: row.opencode_agent,
|
|
2605
|
+
opencode_model: row.opencode_model,
|
|
2606
|
+
source_message_id: row.source_message_id,
|
|
2607
|
+
slack_user_id: row.slack_user_id
|
|
2608
|
+
};
|
|
1965
2609
|
}
|
|
1966
2610
|
/**
|
|
1967
2611
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
1968
2612
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
1969
2613
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2614
|
+
*
|
|
2615
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2616
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2617
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2618
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2619
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2620
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2621
|
+
* the done branch still delivers it (Bugbot #202).
|
|
1970
2622
|
*/
|
|
1971
2623
|
removeInFlight(watcher, evidentMessageId) {
|
|
2624
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2625
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2626
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2627
|
+
this.log({
|
|
2628
|
+
level: "info",
|
|
2629
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2630
|
+
conversation_id: watcher.conv.id,
|
|
2631
|
+
message_id: evidentMessageId
|
|
2632
|
+
});
|
|
2633
|
+
}
|
|
1972
2634
|
watcher.inFlight.delete(evidentMessageId);
|
|
1973
2635
|
this.dispatched.delete(evidentMessageId);
|
|
1974
2636
|
}
|
|
@@ -2112,6 +2774,35 @@ var ChannelDriver = class {
|
|
|
2112
2774
|
}
|
|
2113
2775
|
return await res.json();
|
|
2114
2776
|
}
|
|
2777
|
+
/**
|
|
2778
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2779
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2780
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2781
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2782
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2783
|
+
* `opencode_session_id`, routing).
|
|
2784
|
+
*
|
|
2785
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2786
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2787
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2788
|
+
* next tick retries.
|
|
2789
|
+
*/
|
|
2790
|
+
async getProcessingMessages() {
|
|
2791
|
+
const res = await this.fetchImpl(
|
|
2792
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2793
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2794
|
+
);
|
|
2795
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2796
|
+
if (!res.ok) {
|
|
2797
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2798
|
+
}
|
|
2799
|
+
const data = await res.json();
|
|
2800
|
+
let messages = data.messages ?? [];
|
|
2801
|
+
if (this.conversationFilter) {
|
|
2802
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2803
|
+
}
|
|
2804
|
+
return messages;
|
|
2805
|
+
}
|
|
2115
2806
|
/**
|
|
2116
2807
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2117
2808
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2133,13 +2824,17 @@ var ChannelDriver = class {
|
|
|
2133
2824
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2134
2825
|
* retry vehicle for the swap-to-running.
|
|
2135
2826
|
*/
|
|
2136
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
2827
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2137
2828
|
const res = await this.fetchImpl(
|
|
2138
2829
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2139
2830
|
{
|
|
2140
2831
|
method: "PATCH",
|
|
2141
2832
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2142
|
-
body: JSON.stringify({
|
|
2833
|
+
body: JSON.stringify({
|
|
2834
|
+
status: "processing",
|
|
2835
|
+
opencode_session_id: sessionId,
|
|
2836
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2837
|
+
})
|
|
2143
2838
|
}
|
|
2144
2839
|
);
|
|
2145
2840
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2177,13 +2872,17 @@ var ChannelDriver = class {
|
|
|
2177
2872
|
* watcher retries next tick within the
|
|
2178
2873
|
* deadline, Finding 4).
|
|
2179
2874
|
*/
|
|
2180
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
2875
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2181
2876
|
const res = await this.fetchImpl(
|
|
2182
2877
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2183
2878
|
{
|
|
2184
2879
|
method: "PATCH",
|
|
2185
2880
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2186
|
-
body: JSON.stringify({
|
|
2881
|
+
body: JSON.stringify({
|
|
2882
|
+
status: "done",
|
|
2883
|
+
opencode_session_id: sessionId,
|
|
2884
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2885
|
+
})
|
|
2187
2886
|
}
|
|
2188
2887
|
);
|
|
2189
2888
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2193,7 +2892,17 @@ var ChannelDriver = class {
|
|
|
2193
2892
|
}
|
|
2194
2893
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2195
2894
|
}
|
|
2196
|
-
|
|
2895
|
+
/**
|
|
2896
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
2897
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
2898
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
2899
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
2900
|
+
* failure reason reaches the channel.
|
|
2901
|
+
*/
|
|
2902
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
2903
|
+
const body = { status: "failed" };
|
|
2904
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
2905
|
+
if (error2 !== void 0) body.error = error2;
|
|
2197
2906
|
await this.callWithRetry(
|
|
2198
2907
|
"marking message as failed",
|
|
2199
2908
|
() => this.fetchImpl(
|
|
@@ -2201,11 +2910,47 @@ var ChannelDriver = class {
|
|
|
2201
2910
|
{
|
|
2202
2911
|
method: "PATCH",
|
|
2203
2912
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2204
|
-
body: JSON.stringify(
|
|
2913
|
+
body: JSON.stringify(body)
|
|
2205
2914
|
}
|
|
2206
2915
|
)
|
|
2207
2916
|
);
|
|
2208
2917
|
}
|
|
2918
|
+
/**
|
|
2919
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
2920
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
2921
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
2922
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
2923
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2924
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2925
|
+
* context (no silent catch, per development-workflow).
|
|
2926
|
+
*/
|
|
2927
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
2928
|
+
try {
|
|
2929
|
+
const res = await this.fetchImpl(
|
|
2930
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
2931
|
+
{
|
|
2932
|
+
method: "POST",
|
|
2933
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2934
|
+
body: JSON.stringify({ signal, ...extra })
|
|
2935
|
+
}
|
|
2936
|
+
);
|
|
2937
|
+
if (!res.ok) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "error",
|
|
2940
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
2941
|
+
conversation_id: conversationId,
|
|
2942
|
+
message_id: messageId
|
|
2943
|
+
});
|
|
2944
|
+
}
|
|
2945
|
+
} catch (err) {
|
|
2946
|
+
this.log({
|
|
2947
|
+
level: "error",
|
|
2948
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
2949
|
+
conversation_id: conversationId,
|
|
2950
|
+
message_id: messageId
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2209
2954
|
async persistSession(conversationId, sessionId) {
|
|
2210
2955
|
const res = await this.fetchImpl(
|
|
2211
2956
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2521,7 +3266,8 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2521
3266
|
// src/commands/run.ts
|
|
2522
3267
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2523
3268
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2524
|
-
|
|
3269
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3270
|
+
function log2(state, message, isError = false) {
|
|
2525
3271
|
if (state.json) {
|
|
2526
3272
|
console.log(
|
|
2527
3273
|
JSON.stringify({
|
|
@@ -2546,9 +3292,9 @@ function logActivity(state, entry) {
|
|
|
2546
3292
|
}
|
|
2547
3293
|
if (!state.interactive) {
|
|
2548
3294
|
if (entry.type === "error") {
|
|
2549
|
-
|
|
3295
|
+
log2(state, entry.error ?? "Unknown error", true);
|
|
2550
3296
|
} else if (entry.type === "info" && entry.message) {
|
|
2551
|
-
|
|
3297
|
+
log2(state, entry.message);
|
|
2552
3298
|
}
|
|
2553
3299
|
}
|
|
2554
3300
|
}
|
|
@@ -2634,6 +3380,7 @@ async function handleAuthError(state, error2) {
|
|
|
2634
3380
|
}
|
|
2635
3381
|
async function driveChannels(state, driver) {
|
|
2636
3382
|
let idlePolls = 0;
|
|
3383
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2637
3384
|
while (state.running) {
|
|
2638
3385
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2639
3386
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2643,7 +3390,9 @@ async function driveChannels(state, driver) {
|
|
|
2643
3390
|
try {
|
|
2644
3391
|
const processed = await driver.drainPending();
|
|
2645
3392
|
state.messageCount += processed;
|
|
2646
|
-
|
|
3393
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
3394
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
3395
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2647
3396
|
idlePolls = 0;
|
|
2648
3397
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2649
3398
|
} else if (state.idleTimeout !== null) {
|
|
@@ -2695,7 +3444,7 @@ async function cleanup(state) {
|
|
|
2695
3444
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2696
3445
|
displayStatus(state);
|
|
2697
3446
|
} else {
|
|
2698
|
-
|
|
3447
|
+
log2(state, "Stopped OpenCode process");
|
|
2699
3448
|
}
|
|
2700
3449
|
state.opencodeProcess = null;
|
|
2701
3450
|
}
|
|
@@ -2718,10 +3467,11 @@ async function run(options) {
|
|
|
2718
3467
|
running: true,
|
|
2719
3468
|
activityLog: [],
|
|
2720
3469
|
messageCount: 0,
|
|
3470
|
+
lastProxiedActivityAt: null,
|
|
2721
3471
|
authHeader: ""
|
|
2722
3472
|
};
|
|
2723
3473
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2724
|
-
|
|
3474
|
+
log2(
|
|
2725
3475
|
state,
|
|
2726
3476
|
"Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
2727
3477
|
false
|
|
@@ -2732,7 +3482,7 @@ async function run(options) {
|
|
|
2732
3482
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2733
3483
|
displayStatus(state);
|
|
2734
3484
|
} else {
|
|
2735
|
-
|
|
3485
|
+
log2(state, "Shutting down...");
|
|
2736
3486
|
}
|
|
2737
3487
|
await cleanup(state);
|
|
2738
3488
|
await shutdownTelemetry();
|
|
@@ -2765,7 +3515,7 @@ async function run(options) {
|
|
|
2765
3515
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2766
3516
|
if (resolved.agent_id) {
|
|
2767
3517
|
state.agentId = resolved.agent_id;
|
|
2768
|
-
|
|
3518
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2769
3519
|
if (state.interactive && !state.json) {
|
|
2770
3520
|
logActivity(state, {
|
|
2771
3521
|
type: "info",
|
|
@@ -2828,17 +3578,17 @@ async function run(options) {
|
|
|
2828
3578
|
port: state.port,
|
|
2829
3579
|
interactive: state.interactive,
|
|
2830
3580
|
agentId: state.agentId,
|
|
2831
|
-
log: (message) =>
|
|
3581
|
+
log: (message) => log2(state, message)
|
|
2832
3582
|
});
|
|
2833
3583
|
state.port = oc.port;
|
|
2834
3584
|
state.opencodeProcess = oc.process;
|
|
2835
3585
|
state.opencodeVersion = oc.version;
|
|
2836
3586
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2837
|
-
const
|
|
2838
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
3587
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
3588
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
2839
3589
|
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
2840
3590
|
if (versionWarning) {
|
|
2841
|
-
|
|
3591
|
+
log2(state, versionWarning, false);
|
|
2842
3592
|
if (state.interactive && !state.json) {
|
|
2843
3593
|
logActivity(state, { type: "info", message: versionWarning });
|
|
2844
3594
|
}
|
|
@@ -2854,6 +3604,7 @@ async function run(options) {
|
|
|
2854
3604
|
apiUrl: getApiUrlConfig(),
|
|
2855
3605
|
getAuthHeader: () => state.authHeader,
|
|
2856
3606
|
conversationFilter: state.conversationFilter,
|
|
3607
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
2857
3608
|
log: (entry) => logActivity(state, {
|
|
2858
3609
|
type: entry.level === "error" ? "error" : "info",
|
|
2859
3610
|
message: entry.message,
|
|
@@ -2873,7 +3624,11 @@ async function run(options) {
|
|
|
2873
3624
|
type: "info",
|
|
2874
3625
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2875
3626
|
});
|
|
2876
|
-
emitAgentConnected(state.agentId, {
|
|
3627
|
+
emitAgentConnected(state.agentId, {
|
|
3628
|
+
port: state.port,
|
|
3629
|
+
cli_version: getCliVersion(),
|
|
3630
|
+
opencode_version: state.opencodeVersion
|
|
3631
|
+
});
|
|
2877
3632
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2878
3633
|
if (state.interactive) displayStatus(state);
|
|
2879
3634
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2907,9 +3662,14 @@ async function run(options) {
|
|
|
2907
3662
|
logActivity(state, { type: "error", error: error2 });
|
|
2908
3663
|
if (state.interactive) displayStatus(state);
|
|
2909
3664
|
},
|
|
2910
|
-
// Web traffic is proxied transparently;
|
|
3665
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
3666
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
3667
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
3668
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
3669
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2911
3670
|
onResponse: () => {
|
|
2912
3671
|
state.opencodeConnected = true;
|
|
3672
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2913
3673
|
},
|
|
2914
3674
|
// A channel message was queued and the api-worker pinged us over the
|
|
2915
3675
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2948,7 +3708,7 @@ async function run(options) {
|
|
|
2948
3708
|
throw error2;
|
|
2949
3709
|
}
|
|
2950
3710
|
if (!interactive || state.json) {
|
|
2951
|
-
|
|
3711
|
+
log2(state, "Driving channel messages...");
|
|
2952
3712
|
}
|
|
2953
3713
|
await driveChannels(state, channelDriver);
|
|
2954
3714
|
await cleanup(state);
|
|
@@ -2960,7 +3720,7 @@ async function run(options) {
|
|
|
2960
3720
|
})
|
|
2961
3721
|
);
|
|
2962
3722
|
} else if (!interactive) {
|
|
2963
|
-
|
|
3723
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2964
3724
|
}
|
|
2965
3725
|
await shutdownTelemetry();
|
|
2966
3726
|
process.exit(0);
|
|
@@ -2982,8 +3742,9 @@ async function run(options) {
|
|
|
2982
3742
|
}
|
|
2983
3743
|
|
|
2984
3744
|
// src/index.ts
|
|
3745
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2985
3746
|
var program = new Command();
|
|
2986
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
3747
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2987
3748
|
"--endpoint <url>",
|
|
2988
3749
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2989
3750
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|