@evident-ai/cli 3.0.1-dev.ced5d3d → 3.0.1-dev.d0ec1bd
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +754 -55
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -471,12 +471,6 @@ import chalk6 from "chalk";
|
|
|
471
471
|
import ora3 from "ora";
|
|
472
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
473
473
|
|
|
474
|
-
// ../../packages/types/src/opencode/index.ts
|
|
475
|
-
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
-
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
-
return `msg_${sanitized}`;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
474
|
// ../../packages/types/src/telemetry/index.ts
|
|
481
475
|
var TelemetryEventTypes = {
|
|
482
476
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -515,7 +509,10 @@ function stripQuery(url) {
|
|
|
515
509
|
}
|
|
516
510
|
|
|
517
511
|
// src/lib/telemetry.ts
|
|
518
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
512
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
+
function getCliVersion() {
|
|
514
|
+
return CLI_VERSION;
|
|
515
|
+
}
|
|
519
516
|
var eventBuffer = [];
|
|
520
517
|
var flushTimeout = null;
|
|
521
518
|
var isShuttingDown = false;
|
|
@@ -715,7 +712,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
715
712
|
}
|
|
716
713
|
|
|
717
714
|
// src/lib/opencode/opencode-version-gate.ts
|
|
718
|
-
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
715
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
719
716
|
function isQueueValidatedVersion(version2) {
|
|
720
717
|
if (!version2) return false;
|
|
721
718
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
@@ -1039,7 +1036,11 @@ function roleOf(m) {
|
|
|
1039
1036
|
}
|
|
1040
1037
|
function completedOf(m) {
|
|
1041
1038
|
if (!m || typeof m !== "object") return void 0;
|
|
1042
|
-
return m.info?.time?.completed;
|
|
1039
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1040
|
+
}
|
|
1041
|
+
function createdOf(m) {
|
|
1042
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1043
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1043
1044
|
}
|
|
1044
1045
|
function idOf(m) {
|
|
1045
1046
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1059,6 +1060,24 @@ function finishOf(m) {
|
|
|
1059
1060
|
const infoFinish = m.info?.finish;
|
|
1060
1061
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1061
1062
|
}
|
|
1063
|
+
function errorOf(m) {
|
|
1064
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1065
|
+
return m.info?.error ?? m.error;
|
|
1066
|
+
}
|
|
1067
|
+
function isAssistantInFlight(m) {
|
|
1068
|
+
if (completedOf(m) == null) return true;
|
|
1069
|
+
return finishOf(m) === "tool-calls";
|
|
1070
|
+
}
|
|
1071
|
+
async function getSessionMessages(port, sessionId) {
|
|
1072
|
+
try {
|
|
1073
|
+
const res = await fetch(`${opencodeBase(port)}/session/${sessionId}/message`);
|
|
1074
|
+
if (!res.ok) return null;
|
|
1075
|
+
const body = await res.json();
|
|
1076
|
+
return Array.isArray(body) ? body : null;
|
|
1077
|
+
} catch {
|
|
1078
|
+
return null;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1062
1081
|
async function createOpenCodeSession(port, directory) {
|
|
1063
1082
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1064
1083
|
if (directory && directory.trim()) {
|
|
@@ -1076,9 +1095,16 @@ async function createOpenCodeSession(port, directory) {
|
|
|
1076
1095
|
const data = await response.json();
|
|
1077
1096
|
return data.id;
|
|
1078
1097
|
}
|
|
1079
|
-
|
|
1098
|
+
function messageText(m) {
|
|
1099
|
+
if (!m || !Array.isArray(m.parts)) return "";
|
|
1100
|
+
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
1101
|
+
}
|
|
1102
|
+
async function sendPromptAsync(port, sessionId, content, options) {
|
|
1103
|
+
const before = await getSessionMessages(port, sessionId);
|
|
1104
|
+
const knownUserIds = new Set(
|
|
1105
|
+
(before ?? []).filter((m) => roleOf(m) === "user").map((m) => idOf(m)).filter((id) => typeof id === "string")
|
|
1106
|
+
);
|
|
1080
1107
|
const body = {
|
|
1081
|
-
messageID: messageId,
|
|
1082
1108
|
parts: [{ type: "text", text: content }]
|
|
1083
1109
|
};
|
|
1084
1110
|
if (options?.agent) {
|
|
@@ -1102,6 +1128,29 @@ async function sendPromptAsync(port, sessionId, content, options, messageId) {
|
|
|
1102
1128
|
const text = await res.text().catch(() => "");
|
|
1103
1129
|
throw new Error(`OpenCode prompt_async failed: HTTP ${res.status}${text ? `: ${text}` : ""}`);
|
|
1104
1130
|
}
|
|
1131
|
+
const READ_BACK_ATTEMPTS = 5;
|
|
1132
|
+
const READ_BACK_DELAY_MS = 150;
|
|
1133
|
+
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
1134
|
+
const after = await getSessionMessages(port, sessionId);
|
|
1135
|
+
if (after) {
|
|
1136
|
+
let best = null;
|
|
1137
|
+
for (const m of after) {
|
|
1138
|
+
if (roleOf(m) !== "user") continue;
|
|
1139
|
+
const id = idOf(m);
|
|
1140
|
+
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
1141
|
+
if (messageText(m) !== content) continue;
|
|
1142
|
+
const created = createdOf(m) ?? 0;
|
|
1143
|
+
if (best === null || created > best.created) {
|
|
1144
|
+
best = { id, created };
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
if (best) return best.id;
|
|
1148
|
+
}
|
|
1149
|
+
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1150
|
+
await new Promise((resolve) => setTimeout(resolve, READ_BACK_DELAY_MS));
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
return null;
|
|
1105
1154
|
}
|
|
1106
1155
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
1107
1156
|
if (!messages || messages.length === 0) return null;
|
|
@@ -1118,19 +1167,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1118
1167
|
}
|
|
1119
1168
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1120
1169
|
if (!messages || messages.length === 0) return null;
|
|
1170
|
+
let lastCorrelated = null;
|
|
1171
|
+
let lastNonErrored = null;
|
|
1121
1172
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1122
1173
|
const m = messages[i];
|
|
1123
|
-
if (roleOf(m)
|
|
1174
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1175
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1176
|
+
if (errorOf(m) == null) {
|
|
1177
|
+
lastNonErrored = m;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1124
1180
|
}
|
|
1181
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1125
1182
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1126
1183
|
if (userIndex === -1) return null;
|
|
1127
1184
|
let last = null;
|
|
1185
|
+
let lastOk = null;
|
|
1128
1186
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1129
1187
|
const role = roleOf(messages[i]);
|
|
1130
1188
|
if (role === "user") break;
|
|
1131
|
-
if (role === "assistant")
|
|
1189
|
+
if (role === "assistant") {
|
|
1190
|
+
last = messages[i];
|
|
1191
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1192
|
+
}
|
|
1132
1193
|
}
|
|
1133
|
-
return last;
|
|
1194
|
+
return lastOk ?? last;
|
|
1134
1195
|
}
|
|
1135
1196
|
function messageRunState(messages, userMessageId) {
|
|
1136
1197
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1140,12 +1201,27 @@ function messageRunState(messages, userMessageId) {
|
|
|
1140
1201
|
if (!reply) return "unknown";
|
|
1141
1202
|
}
|
|
1142
1203
|
if (!reply) return "queued";
|
|
1143
|
-
if (
|
|
1144
|
-
|
|
1145
|
-
return "done";
|
|
1204
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1205
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1146
1206
|
}
|
|
1147
|
-
function
|
|
1148
|
-
|
|
1207
|
+
function messageError(messages, userMessageId) {
|
|
1208
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1209
|
+
const error2 = errorOf(reply);
|
|
1210
|
+
if (error2 == null) return null;
|
|
1211
|
+
if (typeof error2 === "string") return error2;
|
|
1212
|
+
if (typeof error2 === "object") {
|
|
1213
|
+
const e = error2;
|
|
1214
|
+
const dataMessage = e.data?.message;
|
|
1215
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1216
|
+
if (typeof e.message === "string") return e.message;
|
|
1217
|
+
}
|
|
1218
|
+
return "The agent run failed.";
|
|
1219
|
+
}
|
|
1220
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1221
|
+
if (!messages || messages.length === 0) return false;
|
|
1222
|
+
return messages.some(
|
|
1223
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1224
|
+
);
|
|
1149
1225
|
}
|
|
1150
1226
|
|
|
1151
1227
|
// src/lib/tunnel/connection.ts
|
|
@@ -1551,7 +1627,7 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1551
1627
|
};
|
|
1552
1628
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1553
1629
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1554
|
-
var
|
|
1630
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1555
1631
|
var ChannelAuthError = class extends Error {
|
|
1556
1632
|
constructor(message) {
|
|
1557
1633
|
super(message);
|
|
@@ -1586,10 +1662,18 @@ var ChannelDriver = class {
|
|
|
1586
1662
|
sleep;
|
|
1587
1663
|
pausedPollIntervalMs;
|
|
1588
1664
|
pausedMaxWaitMs;
|
|
1589
|
-
|
|
1665
|
+
stuckQueuedMs;
|
|
1590
1666
|
now;
|
|
1591
1667
|
/** Cache of conversationId → opencode sessionId. */
|
|
1592
1668
|
sessions = /* @__PURE__ */ new Map();
|
|
1669
|
+
/**
|
|
1670
|
+
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
1671
|
+
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
1672
|
+
* "the one new user row" — which is only unambiguous if no OTHER dispatch into
|
|
1673
|
+
* the SAME session interleaves its snapshot→POST→read-back. This map chains each
|
|
1674
|
+
* session's dispatches so they run serially; distinct sessions stay concurrent.
|
|
1675
|
+
*/
|
|
1676
|
+
sessionDispatchLocks = /* @__PURE__ */ new Map();
|
|
1593
1677
|
/**
|
|
1594
1678
|
* Per-SESSION watchers (WI-3), keyed by opencode sessionId. Single-flight per
|
|
1595
1679
|
* session: one polling loop services all of that session's in-flight messages.
|
|
@@ -1606,6 +1690,54 @@ var ChannelDriver = class {
|
|
|
1606
1690
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1607
1691
|
*/
|
|
1608
1692
|
dispatched = /* @__PURE__ */ new Set();
|
|
1693
|
+
/**
|
|
1694
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1695
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1696
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1697
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1698
|
+
* the processing list.
|
|
1699
|
+
*/
|
|
1700
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1701
|
+
/**
|
|
1702
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1703
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1704
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1705
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1706
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1707
|
+
*
|
|
1708
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1709
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1710
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1711
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1712
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1713
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1714
|
+
*/
|
|
1715
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1716
|
+
/**
|
|
1717
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1718
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1719
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1720
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1721
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1722
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1723
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1724
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1725
|
+
*/
|
|
1726
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1727
|
+
/**
|
|
1728
|
+
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1729
|
+
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
1730
|
+
* is NOT idempotent: if `forceReadoptRun` dispatches on tick N but the read-back +
|
|
1731
|
+
* persist hasn't landed before tick N+1 re-reads the still-null
|
|
1732
|
+
* `row.opencode_message_id`, tick N+1 would dispatch AGAIN → duplicate user turns.
|
|
1733
|
+
* A row is added here right before its `sendPromptAsync` and `forceReadoptRun`
|
|
1734
|
+
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
1735
|
+
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
1736
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
1737
|
+
* re-entry) OR on a failed/unresolved dispatch (the message is genuinely un-sent,
|
|
1738
|
+
* so the NEXT tick may retry exactly once more).
|
|
1739
|
+
*/
|
|
1740
|
+
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
1609
1741
|
/**
|
|
1610
1742
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1611
1743
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1628,7 +1760,7 @@ var ChannelDriver = class {
|
|
|
1628
1760
|
this.sleep = config2.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1629
1761
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1630
1762
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1631
|
-
this.
|
|
1763
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1632
1764
|
this.now = config2.now ?? (() => Date.now());
|
|
1633
1765
|
}
|
|
1634
1766
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1661,6 +1793,7 @@ var ChannelDriver = class {
|
|
|
1661
1793
|
for (const conv of conversations) {
|
|
1662
1794
|
dispatched += await this.processConversation(conv);
|
|
1663
1795
|
}
|
|
1796
|
+
await this.readoptProcessing();
|
|
1664
1797
|
} finally {
|
|
1665
1798
|
this.draining = false;
|
|
1666
1799
|
}
|
|
@@ -1718,11 +1851,11 @@ var ChannelDriver = class {
|
|
|
1718
1851
|
skippedAlreadyDispatched += 1;
|
|
1719
1852
|
continue;
|
|
1720
1853
|
}
|
|
1721
|
-
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1722
1854
|
const options = {
|
|
1723
1855
|
agent: message.opencode_agent ?? void 0,
|
|
1724
1856
|
model: message.opencode_model ?? void 0
|
|
1725
1857
|
};
|
|
1858
|
+
let opencodeMessageId;
|
|
1726
1859
|
try {
|
|
1727
1860
|
this.log({
|
|
1728
1861
|
level: "info",
|
|
@@ -1730,7 +1863,10 @@ var ChannelDriver = class {
|
|
|
1730
1863
|
conversation_id: conv.id,
|
|
1731
1864
|
message_id: message.id
|
|
1732
1865
|
});
|
|
1733
|
-
await
|
|
1866
|
+
opencodeMessageId = await this.dispatchLocked(
|
|
1867
|
+
sessionId,
|
|
1868
|
+
() => sendPromptAsync(this.port, sessionId, message.content, options)
|
|
1869
|
+
);
|
|
1734
1870
|
} catch (err) {
|
|
1735
1871
|
if (err instanceof ChannelAuthError) throw err;
|
|
1736
1872
|
this.dispatched.delete(message.id);
|
|
@@ -1744,9 +1880,19 @@ var ChannelDriver = class {
|
|
|
1744
1880
|
});
|
|
1745
1881
|
continue;
|
|
1746
1882
|
}
|
|
1883
|
+
if (opencodeMessageId === null) {
|
|
1884
|
+
this.log({
|
|
1885
|
+
level: "error",
|
|
1886
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
1887
|
+
conversation_id: conv.id,
|
|
1888
|
+
message_id: message.id
|
|
1889
|
+
});
|
|
1890
|
+
continue;
|
|
1891
|
+
}
|
|
1747
1892
|
this.dispatched.add(message.id);
|
|
1748
1893
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1749
1894
|
dispatched += 1;
|
|
1895
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1750
1896
|
}
|
|
1751
1897
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1752
1898
|
this.log({
|
|
@@ -1791,6 +1937,25 @@ var ChannelDriver = class {
|
|
|
1791
1937
|
// -------------------------------------------------------------------------
|
|
1792
1938
|
// Per-session watcher (WI-3)
|
|
1793
1939
|
// -------------------------------------------------------------------------
|
|
1940
|
+
/**
|
|
1941
|
+
* Run one dispatch (`sendPromptAsync` snapshot→POST→read-back) serialized per
|
|
1942
|
+
* opencode session (Task 2.1a), so two dispatches into the SAME session can
|
|
1943
|
+
* never interleave and mis-correlate their read-backs. Distinct sessions run
|
|
1944
|
+
* concurrently. The chained tail intentionally ignores the prior result/error
|
|
1945
|
+
* (each dispatch reports its own outcome to its caller).
|
|
1946
|
+
*/
|
|
1947
|
+
dispatchLocked(sessionId, fn) {
|
|
1948
|
+
const prior = this.sessionDispatchLocks.get(sessionId) ?? Promise.resolve();
|
|
1949
|
+
const run2 = prior.then(fn, fn);
|
|
1950
|
+
this.sessionDispatchLocks.set(
|
|
1951
|
+
sessionId,
|
|
1952
|
+
run2.then(
|
|
1953
|
+
() => void 0,
|
|
1954
|
+
() => void 0
|
|
1955
|
+
)
|
|
1956
|
+
);
|
|
1957
|
+
return run2;
|
|
1958
|
+
}
|
|
1794
1959
|
/** Register a freshly-dispatched message with its session's watcher state. */
|
|
1795
1960
|
registerInFlight(conv, sessionId, message, opencodeMessageId) {
|
|
1796
1961
|
let watcher = this.watchers.get(sessionId);
|
|
@@ -1812,7 +1977,54 @@ var ChannelDriver = class {
|
|
|
1812
1977
|
dispatchedAt: now,
|
|
1813
1978
|
deadline: now + this.pausedMaxWaitMs,
|
|
1814
1979
|
started: false,
|
|
1815
|
-
done: false
|
|
1980
|
+
done: false,
|
|
1981
|
+
stuckReported: false
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
/**
|
|
1985
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1986
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1987
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1988
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1989
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1990
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1991
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1992
|
+
*
|
|
1993
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1994
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1995
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1996
|
+
*
|
|
1997
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1998
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1999
|
+
* fire from the watcher's normal branches.
|
|
2000
|
+
*/
|
|
2001
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
2002
|
+
let watcher = this.watchers.get(sessionId);
|
|
2003
|
+
if (!watcher) {
|
|
2004
|
+
watcher = {
|
|
2005
|
+
conv,
|
|
2006
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
2007
|
+
loop: null,
|
|
2008
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2009
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2010
|
+
};
|
|
2011
|
+
this.watchers.set(sessionId, watcher);
|
|
2012
|
+
}
|
|
2013
|
+
watcher.inFlight.set(message.id, {
|
|
2014
|
+
evidentMessageId: message.id,
|
|
2015
|
+
opencodeMessageId,
|
|
2016
|
+
message,
|
|
2017
|
+
dispatchedAt: this.now(),
|
|
2018
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2019
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2020
|
+
started: true,
|
|
2021
|
+
done: false,
|
|
2022
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
2023
|
+
// AND the stuck-queued observer INCLUDES re-adopted queued wedges: it gates
|
|
2024
|
+
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2025
|
+
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2026
|
+
// (#210/#220 observability).
|
|
2027
|
+
stuckReported: false
|
|
1816
2028
|
});
|
|
1817
2029
|
}
|
|
1818
2030
|
/**
|
|
@@ -1878,6 +2090,7 @@ var ChannelDriver = class {
|
|
|
1878
2090
|
conversation_id: watcher.conv.id
|
|
1879
2091
|
});
|
|
1880
2092
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2093
|
+
this.readopted.delete(evidentMessageId);
|
|
1881
2094
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1882
2095
|
}
|
|
1883
2096
|
return;
|
|
@@ -1898,10 +2111,15 @@ var ChannelDriver = class {
|
|
|
1898
2111
|
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
1899
2112
|
const conv = watcher.conv;
|
|
1900
2113
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1901
|
-
if ((state === "running" || state === "done") && !inFlight.started) {
|
|
2114
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1902
2115
|
let claimed;
|
|
1903
2116
|
try {
|
|
1904
|
-
claimed = await this.markProcessing(
|
|
2117
|
+
claimed = await this.markProcessing(
|
|
2118
|
+
conv.id,
|
|
2119
|
+
inFlight.evidentMessageId,
|
|
2120
|
+
sessionId,
|
|
2121
|
+
inFlight.opencodeMessageId
|
|
2122
|
+
);
|
|
1905
2123
|
} catch (err) {
|
|
1906
2124
|
if (err instanceof ChannelAuthError) throw err;
|
|
1907
2125
|
this.log({
|
|
@@ -1931,7 +2149,12 @@ var ChannelDriver = class {
|
|
|
1931
2149
|
message_id: inFlight.evidentMessageId
|
|
1932
2150
|
});
|
|
1933
2151
|
try {
|
|
1934
|
-
await this.markDone(
|
|
2152
|
+
await this.markDone(
|
|
2153
|
+
conv.id,
|
|
2154
|
+
inFlight.evidentMessageId,
|
|
2155
|
+
sessionId,
|
|
2156
|
+
inFlight.opencodeMessageId
|
|
2157
|
+
);
|
|
1935
2158
|
} catch (err) {
|
|
1936
2159
|
if (err instanceof ChannelAuthError) throw err;
|
|
1937
2160
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -1967,10 +2190,59 @@ var ChannelDriver = class {
|
|
|
1967
2190
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1968
2191
|
return;
|
|
1969
2192
|
}
|
|
1970
|
-
if (state === "
|
|
1971
|
-
if (
|
|
1972
|
-
|
|
2193
|
+
if (state === "failed") {
|
|
2194
|
+
if (!inFlight.done) {
|
|
2195
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2196
|
+
this.log({
|
|
2197
|
+
level: "error",
|
|
2198
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2199
|
+
conversation_id: conv.id,
|
|
2200
|
+
message_id: inFlight.evidentMessageId
|
|
2201
|
+
});
|
|
2202
|
+
try {
|
|
2203
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2204
|
+
} catch (err) {
|
|
2205
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2206
|
+
if (err instanceof ChannelTerminalError) {
|
|
2207
|
+
this.log({
|
|
2208
|
+
level: "error",
|
|
2209
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2210
|
+
conversation_id: conv.id,
|
|
2211
|
+
message_id: inFlight.evidentMessageId
|
|
2212
|
+
});
|
|
2213
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2214
|
+
return;
|
|
2215
|
+
}
|
|
2216
|
+
if (this.now() >= inFlight.deadline) {
|
|
2217
|
+
this.log({
|
|
2218
|
+
level: "error",
|
|
2219
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2220
|
+
conversation_id: conv.id,
|
|
2221
|
+
message_id: inFlight.evidentMessageId
|
|
2222
|
+
});
|
|
2223
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2224
|
+
return;
|
|
2225
|
+
}
|
|
2226
|
+
this.log({
|
|
2227
|
+
level: "error",
|
|
2228
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2229
|
+
conversation_id: conv.id,
|
|
2230
|
+
message_id: inFlight.evidentMessageId
|
|
2231
|
+
});
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
inFlight.done = true;
|
|
1973
2235
|
}
|
|
2236
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2240
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2241
|
+
if (state === "queued" && pastStuckBound && sessionIdle && !inFlight.stuckReported) {
|
|
2242
|
+
inFlight.stuckReported = true;
|
|
2243
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2244
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2245
|
+
});
|
|
1974
2246
|
}
|
|
1975
2247
|
if (this.now() >= inFlight.deadline) {
|
|
1976
2248
|
this.log({
|
|
@@ -1979,48 +2251,386 @@ var ChannelDriver = class {
|
|
|
1979
2251
|
conversation_id: conv.id,
|
|
1980
2252
|
message_id: inFlight.evidentMessageId
|
|
1981
2253
|
});
|
|
2254
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2255
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2256
|
+
});
|
|
1982
2257
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1983
2258
|
}
|
|
1984
2259
|
}
|
|
2260
|
+
// -------------------------------------------------------------------------
|
|
2261
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2262
|
+
// -------------------------------------------------------------------------
|
|
2263
|
+
/**
|
|
2264
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2265
|
+
*
|
|
2266
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2267
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2268
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2269
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2270
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2271
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2272
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2273
|
+
*
|
|
2274
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2275
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2276
|
+
*/
|
|
2277
|
+
async readoptProcessing() {
|
|
2278
|
+
const rows = await this.getProcessingMessages();
|
|
2279
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2280
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2281
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2282
|
+
if (!stillProcessing.has(id)) {
|
|
2283
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2284
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2285
|
+
if (cleared || clearedUndeliverable) {
|
|
2286
|
+
this.log({
|
|
2287
|
+
level: "info",
|
|
2288
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2289
|
+
message_id: id
|
|
2290
|
+
});
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
if (rows.length === 0) return;
|
|
2296
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2297
|
+
for (const row of rows) {
|
|
2298
|
+
if (!row.opencode_session_id) {
|
|
2299
|
+
this.log({
|
|
2300
|
+
level: "error",
|
|
2301
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2302
|
+
conversation_id: row.conversation_id,
|
|
2303
|
+
message_id: row.id
|
|
2304
|
+
});
|
|
2305
|
+
continue;
|
|
2306
|
+
}
|
|
2307
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2308
|
+
list.push(row);
|
|
2309
|
+
bySession.set(row.opencode_session_id, list);
|
|
2310
|
+
}
|
|
2311
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2312
|
+
let messages;
|
|
2313
|
+
try {
|
|
2314
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2315
|
+
if (!res.ok) {
|
|
2316
|
+
this.log({
|
|
2317
|
+
level: "error",
|
|
2318
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2319
|
+
});
|
|
2320
|
+
continue;
|
|
2321
|
+
}
|
|
2322
|
+
const body = await res.json();
|
|
2323
|
+
if (!Array.isArray(body)) {
|
|
2324
|
+
this.log({
|
|
2325
|
+
level: "error",
|
|
2326
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2327
|
+
});
|
|
2328
|
+
continue;
|
|
2329
|
+
}
|
|
2330
|
+
messages = body;
|
|
2331
|
+
} catch (err) {
|
|
2332
|
+
this.log({
|
|
2333
|
+
level: "error",
|
|
2334
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2335
|
+
});
|
|
2336
|
+
continue;
|
|
2337
|
+
}
|
|
2338
|
+
for (const row of sessionRows) {
|
|
2339
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
1985
2343
|
/**
|
|
1986
|
-
* Re-
|
|
1987
|
-
*
|
|
1988
|
-
*
|
|
1989
|
-
*
|
|
2344
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2345
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2346
|
+
*
|
|
2347
|
+
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
2348
|
+
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
2349
|
+
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
2350
|
+
* before the restart) has no id to correlate → treated as an orphan and
|
|
2351
|
+
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
2352
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2353
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2354
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2355
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2356
|
+
* tracking the stored id so the reply correlates by it;
|
|
2357
|
+
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
2358
|
+
*
|
|
2359
|
+
* Only `ChannelAuthError` propagates.
|
|
1990
2360
|
*/
|
|
1991
|
-
async
|
|
2361
|
+
async readoptOne(sessionId, row, messages) {
|
|
2362
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2363
|
+
this.log({
|
|
2364
|
+
level: "info",
|
|
2365
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2366
|
+
conversation_id: row.conversation_id,
|
|
2367
|
+
message_id: row.id
|
|
2368
|
+
});
|
|
2369
|
+
return;
|
|
2370
|
+
}
|
|
2371
|
+
const ocId = row.opencode_message_id;
|
|
2372
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
2373
|
+
if (state === "done") {
|
|
2374
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2375
|
+
this.log({
|
|
2376
|
+
level: "info",
|
|
2377
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2378
|
+
conversation_id: row.conversation_id,
|
|
2379
|
+
message_id: row.id
|
|
2380
|
+
});
|
|
2381
|
+
return;
|
|
2382
|
+
}
|
|
2383
|
+
this.log({
|
|
2384
|
+
level: "info",
|
|
2385
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2386
|
+
conversation_id: row.conversation_id,
|
|
2387
|
+
message_id: row.id
|
|
2388
|
+
});
|
|
2389
|
+
try {
|
|
2390
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId);
|
|
2391
|
+
} catch (err) {
|
|
2392
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2393
|
+
if (err instanceof ChannelTerminalError) {
|
|
2394
|
+
this.doneUndeliverable.add(row.id);
|
|
2395
|
+
this.log({
|
|
2396
|
+
level: "error",
|
|
2397
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2398
|
+
conversation_id: row.conversation_id,
|
|
2399
|
+
message_id: row.id
|
|
2400
|
+
});
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
this.log({
|
|
2404
|
+
level: "error",
|
|
2405
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2406
|
+
conversation_id: row.conversation_id,
|
|
2407
|
+
message_id: row.id
|
|
2408
|
+
});
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
this.dontRedispatch.delete(row.id);
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
if (state === "failed") {
|
|
2415
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
2416
|
+
this.log({
|
|
2417
|
+
level: "error",
|
|
2418
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2419
|
+
conversation_id: row.conversation_id,
|
|
2420
|
+
message_id: row.id
|
|
2421
|
+
});
|
|
2422
|
+
try {
|
|
2423
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2424
|
+
} catch (err) {
|
|
2425
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2426
|
+
if (err instanceof ChannelTerminalError) {
|
|
2427
|
+
this.doneUndeliverable.add(row.id);
|
|
2428
|
+
this.log({
|
|
2429
|
+
level: "error",
|
|
2430
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2431
|
+
conversation_id: row.conversation_id,
|
|
2432
|
+
message_id: row.id
|
|
2433
|
+
});
|
|
2434
|
+
return;
|
|
2435
|
+
}
|
|
2436
|
+
this.log({
|
|
2437
|
+
level: "error",
|
|
2438
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2439
|
+
conversation_id: row.conversation_id,
|
|
2440
|
+
message_id: row.id
|
|
2441
|
+
});
|
|
2442
|
+
return;
|
|
2443
|
+
}
|
|
2444
|
+
this.dontRedispatch.delete(row.id);
|
|
2445
|
+
return;
|
|
2446
|
+
}
|
|
2447
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2448
|
+
this.log({
|
|
2449
|
+
level: "info",
|
|
2450
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2451
|
+
conversation_id: row.conversation_id,
|
|
2452
|
+
message_id: row.id
|
|
2453
|
+
});
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
if ((state === "running" || state === "queued") && ocId) {
|
|
2457
|
+
const conv = this.convForRow(sessionId, row);
|
|
2458
|
+
const message = this.queuedMessageForRow(row);
|
|
2459
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2460
|
+
this.dispatched.add(row.id);
|
|
2461
|
+
this.readopted.add(row.id);
|
|
2462
|
+
this.ensureWatcherRunning(sessionId);
|
|
2463
|
+
this.log({
|
|
2464
|
+
level: "info",
|
|
2465
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stored id, no re-dispatch)`,
|
|
2466
|
+
conversation_id: row.conversation_id,
|
|
2467
|
+
message_id: row.id
|
|
2468
|
+
});
|
|
2469
|
+
return;
|
|
2470
|
+
}
|
|
2471
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2472
|
+
}
|
|
2473
|
+
/**
|
|
2474
|
+
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
2475
|
+
*
|
|
2476
|
+
* #218/WI-5: the row's user message is absent (never kept, or a null stored id),
|
|
2477
|
+
* so we re-`prompt_async` WITHOUT a caller id (opencode assigns a monotonic one),
|
|
2478
|
+
* read it back, and register the watcher under the assigned id so the reply
|
|
2479
|
+
* correlates server-side.
|
|
2480
|
+
*
|
|
2481
|
+
* ⚠️ AT-MOST-ONCE (High-2): dispatch is no longer idempotent (no caller-supplied
|
|
2482
|
+
* id). Without a guard, if this dispatches on tick N but the read-back+persist
|
|
2483
|
+
* hasn't landed before tick N+1 re-reads the still-null `opencode_message_id`,
|
|
2484
|
+
* tick N+1 would dispatch AGAIN → duplicate user turns. The `awaitingReadopt`
|
|
2485
|
+
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
2486
|
+
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
2487
|
+
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
2488
|
+
* re-entry) OR on a failed/unresolved dispatch (genuinely un-sent → the next tick
|
|
2489
|
+
* may retry exactly once more).
|
|
2490
|
+
*
|
|
2491
|
+
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
2492
|
+
* `processed_at` (Invariant 1).
|
|
2493
|
+
*/
|
|
2494
|
+
async forceReadoptRun(sessionId, row) {
|
|
2495
|
+
if (this.awaitingReadopt.has(row.id)) {
|
|
2496
|
+
this.log({
|
|
2497
|
+
level: "info",
|
|
2498
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already has a re-dispatch awaiting read-back \u2014 skipping (at most once)`,
|
|
2499
|
+
conversation_id: row.conversation_id,
|
|
2500
|
+
message_id: row.id
|
|
2501
|
+
});
|
|
2502
|
+
return;
|
|
2503
|
+
}
|
|
2504
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2505
|
+
this.dontRedispatch.add(row.id);
|
|
2506
|
+
this.log({
|
|
2507
|
+
level: "info",
|
|
2508
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2509
|
+
conversation_id: row.conversation_id,
|
|
2510
|
+
message_id: row.id
|
|
2511
|
+
});
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
1992
2514
|
const options = {
|
|
1993
|
-
agent:
|
|
1994
|
-
model:
|
|
2515
|
+
agent: row.opencode_agent ?? void 0,
|
|
2516
|
+
model: row.opencode_model ?? void 0
|
|
1995
2517
|
};
|
|
1996
2518
|
this.log({
|
|
1997
2519
|
level: "info",
|
|
1998
|
-
message: `
|
|
1999
|
-
|
|
2520
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching (opencode assigns a fresh id)`,
|
|
2521
|
+
conversation_id: row.conversation_id,
|
|
2522
|
+
message_id: row.id
|
|
2000
2523
|
});
|
|
2524
|
+
this.awaitingReadopt.add(row.id);
|
|
2525
|
+
let ocId;
|
|
2001
2526
|
try {
|
|
2002
|
-
await
|
|
2003
|
-
this.port,
|
|
2527
|
+
ocId = await this.dispatchLocked(
|
|
2004
2528
|
sessionId,
|
|
2005
|
-
|
|
2006
|
-
options,
|
|
2007
|
-
inFlight.opencodeMessageId
|
|
2529
|
+
() => sendPromptAsync(this.port, sessionId, row.content, options)
|
|
2008
2530
|
);
|
|
2009
2531
|
} catch (err) {
|
|
2532
|
+
this.awaitingReadopt.delete(row.id);
|
|
2533
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2010
2534
|
this.log({
|
|
2011
2535
|
level: "error",
|
|
2012
|
-
message: `Re-dispatch failed for message ${
|
|
2013
|
-
|
|
2536
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2537
|
+
conversation_id: row.conversation_id,
|
|
2538
|
+
message_id: row.id
|
|
2014
2539
|
});
|
|
2540
|
+
return;
|
|
2015
2541
|
}
|
|
2016
|
-
|
|
2542
|
+
if (ocId === null) {
|
|
2543
|
+
this.awaitingReadopt.delete(row.id);
|
|
2544
|
+
this.log({
|
|
2545
|
+
level: "error",
|
|
2546
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
2547
|
+
conversation_id: row.conversation_id,
|
|
2548
|
+
message_id: row.id
|
|
2549
|
+
});
|
|
2550
|
+
return;
|
|
2551
|
+
}
|
|
2552
|
+
const conv = this.convForRow(sessionId, row);
|
|
2553
|
+
const message = this.queuedMessageForRow(row);
|
|
2554
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2555
|
+
this.dispatched.add(row.id);
|
|
2556
|
+
this.readopted.add(row.id);
|
|
2557
|
+
this.awaitingReadopt.delete(row.id);
|
|
2558
|
+
this.ensureWatcherRunning(sessionId);
|
|
2559
|
+
}
|
|
2560
|
+
/**
|
|
2561
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2562
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2563
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2564
|
+
*/
|
|
2565
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2566
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2567
|
+
const watcher = this.watchers.get(sessionId);
|
|
2568
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2572
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2573
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2574
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2575
|
+
* intended, which is worth surfacing.
|
|
2576
|
+
*/
|
|
2577
|
+
processedAtMs(row) {
|
|
2578
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2579
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2580
|
+
this.log({
|
|
2581
|
+
level: "error",
|
|
2582
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2583
|
+
conversation_id: row.conversation_id,
|
|
2584
|
+
message_id: row.id
|
|
2585
|
+
});
|
|
2586
|
+
return this.now();
|
|
2587
|
+
}
|
|
2588
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2589
|
+
convForRow(sessionId, row) {
|
|
2590
|
+
return {
|
|
2591
|
+
id: row.conversation_id,
|
|
2592
|
+
agent_id: this.agentId,
|
|
2593
|
+
opencode_session_id: sessionId,
|
|
2594
|
+
pending_message_count: 0,
|
|
2595
|
+
oldest_pending_at: row.processed_at
|
|
2596
|
+
};
|
|
2597
|
+
}
|
|
2598
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2599
|
+
queuedMessageForRow(row) {
|
|
2600
|
+
return {
|
|
2601
|
+
id: row.id,
|
|
2602
|
+
content: row.content,
|
|
2603
|
+
status: "processing",
|
|
2604
|
+
opencode_agent: row.opencode_agent,
|
|
2605
|
+
opencode_model: row.opencode_model,
|
|
2606
|
+
source_message_id: row.source_message_id,
|
|
2607
|
+
slack_user_id: row.slack_user_id
|
|
2608
|
+
};
|
|
2017
2609
|
}
|
|
2018
2610
|
/**
|
|
2019
2611
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2020
2612
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2021
2613
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2614
|
+
*
|
|
2615
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2616
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2617
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2618
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2619
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2620
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2621
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2022
2622
|
*/
|
|
2023
2623
|
removeInFlight(watcher, evidentMessageId) {
|
|
2624
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2625
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2626
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2627
|
+
this.log({
|
|
2628
|
+
level: "info",
|
|
2629
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2630
|
+
conversation_id: watcher.conv.id,
|
|
2631
|
+
message_id: evidentMessageId
|
|
2632
|
+
});
|
|
2633
|
+
}
|
|
2024
2634
|
watcher.inFlight.delete(evidentMessageId);
|
|
2025
2635
|
this.dispatched.delete(evidentMessageId);
|
|
2026
2636
|
}
|
|
@@ -2164,6 +2774,35 @@ var ChannelDriver = class {
|
|
|
2164
2774
|
}
|
|
2165
2775
|
return await res.json();
|
|
2166
2776
|
}
|
|
2777
|
+
/**
|
|
2778
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2779
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2780
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2781
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2782
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2783
|
+
* `opencode_session_id`, routing).
|
|
2784
|
+
*
|
|
2785
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2786
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2787
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2788
|
+
* next tick retries.
|
|
2789
|
+
*/
|
|
2790
|
+
async getProcessingMessages() {
|
|
2791
|
+
const res = await this.fetchImpl(
|
|
2792
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2793
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2794
|
+
);
|
|
2795
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2796
|
+
if (!res.ok) {
|
|
2797
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2798
|
+
}
|
|
2799
|
+
const data = await res.json();
|
|
2800
|
+
let messages = data.messages ?? [];
|
|
2801
|
+
if (this.conversationFilter) {
|
|
2802
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2803
|
+
}
|
|
2804
|
+
return messages;
|
|
2805
|
+
}
|
|
2167
2806
|
/**
|
|
2168
2807
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2169
2808
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2185,13 +2824,17 @@ var ChannelDriver = class {
|
|
|
2185
2824
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2186
2825
|
* retry vehicle for the swap-to-running.
|
|
2187
2826
|
*/
|
|
2188
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
2827
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2189
2828
|
const res = await this.fetchImpl(
|
|
2190
2829
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2191
2830
|
{
|
|
2192
2831
|
method: "PATCH",
|
|
2193
2832
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2194
|
-
body: JSON.stringify({
|
|
2833
|
+
body: JSON.stringify({
|
|
2834
|
+
status: "processing",
|
|
2835
|
+
opencode_session_id: sessionId,
|
|
2836
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2837
|
+
})
|
|
2195
2838
|
}
|
|
2196
2839
|
);
|
|
2197
2840
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2229,13 +2872,17 @@ var ChannelDriver = class {
|
|
|
2229
2872
|
* watcher retries next tick within the
|
|
2230
2873
|
* deadline, Finding 4).
|
|
2231
2874
|
*/
|
|
2232
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
2875
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2233
2876
|
const res = await this.fetchImpl(
|
|
2234
2877
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2235
2878
|
{
|
|
2236
2879
|
method: "PATCH",
|
|
2237
2880
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2238
|
-
body: JSON.stringify({
|
|
2881
|
+
body: JSON.stringify({
|
|
2882
|
+
status: "done",
|
|
2883
|
+
opencode_session_id: sessionId,
|
|
2884
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2885
|
+
})
|
|
2239
2886
|
}
|
|
2240
2887
|
);
|
|
2241
2888
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2245,7 +2892,17 @@ var ChannelDriver = class {
|
|
|
2245
2892
|
}
|
|
2246
2893
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2247
2894
|
}
|
|
2248
|
-
|
|
2895
|
+
/**
|
|
2896
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
2897
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
2898
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
2899
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
2900
|
+
* failure reason reaches the channel.
|
|
2901
|
+
*/
|
|
2902
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
2903
|
+
const body = { status: "failed" };
|
|
2904
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
2905
|
+
if (error2 !== void 0) body.error = error2;
|
|
2249
2906
|
await this.callWithRetry(
|
|
2250
2907
|
"marking message as failed",
|
|
2251
2908
|
() => this.fetchImpl(
|
|
@@ -2253,11 +2910,47 @@ var ChannelDriver = class {
|
|
|
2253
2910
|
{
|
|
2254
2911
|
method: "PATCH",
|
|
2255
2912
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2256
|
-
body: JSON.stringify(
|
|
2913
|
+
body: JSON.stringify(body)
|
|
2257
2914
|
}
|
|
2258
2915
|
)
|
|
2259
2916
|
);
|
|
2260
2917
|
}
|
|
2918
|
+
/**
|
|
2919
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
2920
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
2921
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
2922
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
2923
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2924
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2925
|
+
* context (no silent catch, per development-workflow).
|
|
2926
|
+
*/
|
|
2927
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
2928
|
+
try {
|
|
2929
|
+
const res = await this.fetchImpl(
|
|
2930
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
2931
|
+
{
|
|
2932
|
+
method: "POST",
|
|
2933
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2934
|
+
body: JSON.stringify({ signal, ...extra })
|
|
2935
|
+
}
|
|
2936
|
+
);
|
|
2937
|
+
if (!res.ok) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "error",
|
|
2940
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
2941
|
+
conversation_id: conversationId,
|
|
2942
|
+
message_id: messageId
|
|
2943
|
+
});
|
|
2944
|
+
}
|
|
2945
|
+
} catch (err) {
|
|
2946
|
+
this.log({
|
|
2947
|
+
level: "error",
|
|
2948
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
2949
|
+
conversation_id: conversationId,
|
|
2950
|
+
message_id: messageId
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2261
2954
|
async persistSession(conversationId, sessionId) {
|
|
2262
2955
|
const res = await this.fetchImpl(
|
|
2263
2956
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2573,6 +3266,7 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2573
3266
|
// src/commands/run.ts
|
|
2574
3267
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2575
3268
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
3269
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
2576
3270
|
function log2(state, message, isError = false) {
|
|
2577
3271
|
if (state.json) {
|
|
2578
3272
|
console.log(
|
|
@@ -2910,6 +3604,7 @@ async function run(options) {
|
|
|
2910
3604
|
apiUrl: getApiUrlConfig(),
|
|
2911
3605
|
getAuthHeader: () => state.authHeader,
|
|
2912
3606
|
conversationFilter: state.conversationFilter,
|
|
3607
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
2913
3608
|
log: (entry) => logActivity(state, {
|
|
2914
3609
|
type: entry.level === "error" ? "error" : "info",
|
|
2915
3610
|
message: entry.message,
|
|
@@ -2929,7 +3624,11 @@ async function run(options) {
|
|
|
2929
3624
|
type: "info",
|
|
2930
3625
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2931
3626
|
});
|
|
2932
|
-
emitAgentConnected(state.agentId, {
|
|
3627
|
+
emitAgentConnected(state.agentId, {
|
|
3628
|
+
port: state.port,
|
|
3629
|
+
cli_version: getCliVersion(),
|
|
3630
|
+
opencode_version: state.opencodeVersion
|
|
3631
|
+
});
|
|
2933
3632
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2934
3633
|
if (state.interactive) displayStatus(state);
|
|
2935
3634
|
channelDriver.drainPending().then((processed) => {
|