@evident-ai/cli 3.0.1-dev.bf495ea → 3.0.1-dev.c152b49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +580 -7
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1059,6 +1059,14 @@ function finishOf(m) {
|
|
|
1059
1059
|
const infoFinish = m.info?.finish;
|
|
1060
1060
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1061
1061
|
}
|
|
1062
|
+
function errorOf(m) {
|
|
1063
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1064
|
+
return m.info?.error ?? m.error;
|
|
1065
|
+
}
|
|
1066
|
+
function isAssistantInFlight(m) {
|
|
1067
|
+
if (completedOf(m) == null) return true;
|
|
1068
|
+
return finishOf(m) === "tool-calls";
|
|
1069
|
+
}
|
|
1062
1070
|
async function createOpenCodeSession(port, directory) {
|
|
1063
1071
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1064
1072
|
if (directory && directory.trim()) {
|
|
@@ -1140,9 +1148,27 @@ function messageRunState(messages, userMessageId) {
|
|
|
1140
1148
|
if (!reply) return "unknown";
|
|
1141
1149
|
}
|
|
1142
1150
|
if (!reply) return "queued";
|
|
1143
|
-
if (
|
|
1144
|
-
|
|
1145
|
-
|
|
1151
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1152
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1153
|
+
}
|
|
1154
|
+
function messageError(messages, userMessageId) {
|
|
1155
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1156
|
+
const error2 = errorOf(reply);
|
|
1157
|
+
if (error2 == null) return null;
|
|
1158
|
+
if (typeof error2 === "string") return error2;
|
|
1159
|
+
if (typeof error2 === "object") {
|
|
1160
|
+
const e = error2;
|
|
1161
|
+
const dataMessage = e.data?.message;
|
|
1162
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1163
|
+
if (typeof e.message === "string") return e.message;
|
|
1164
|
+
}
|
|
1165
|
+
return "The agent run failed.";
|
|
1166
|
+
}
|
|
1167
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1168
|
+
if (!messages || messages.length === 0) return false;
|
|
1169
|
+
return messages.some(
|
|
1170
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1171
|
+
);
|
|
1146
1172
|
}
|
|
1147
1173
|
function opencodeMessageIdFor2(queuedMessageId) {
|
|
1148
1174
|
return opencodeMessageIdFor(queuedMessageId);
|
|
@@ -1552,6 +1578,7 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1552
1578
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1553
1579
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1554
1580
|
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1581
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1555
1582
|
var ChannelAuthError = class extends Error {
|
|
1556
1583
|
constructor(message) {
|
|
1557
1584
|
super(message);
|
|
@@ -1587,6 +1614,7 @@ var ChannelDriver = class {
|
|
|
1587
1614
|
pausedPollIntervalMs;
|
|
1588
1615
|
pausedMaxWaitMs;
|
|
1589
1616
|
dispatchConfirmMs;
|
|
1617
|
+
stuckQueuedMs;
|
|
1590
1618
|
now;
|
|
1591
1619
|
/** Cache of conversationId → opencode sessionId. */
|
|
1592
1620
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -1606,6 +1634,40 @@ var ChannelDriver = class {
|
|
|
1606
1634
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1607
1635
|
*/
|
|
1608
1636
|
dispatched = /* @__PURE__ */ new Set();
|
|
1637
|
+
/**
|
|
1638
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1639
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1640
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1641
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1642
|
+
* the processing list.
|
|
1643
|
+
*/
|
|
1644
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1645
|
+
/**
|
|
1646
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1647
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1648
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1649
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1650
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1651
|
+
*
|
|
1652
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1653
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1654
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1655
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1656
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1657
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1658
|
+
*/
|
|
1659
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1660
|
+
/**
|
|
1661
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1662
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1663
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1664
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1665
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1666
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1667
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1668
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1669
|
+
*/
|
|
1670
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1609
1671
|
/**
|
|
1610
1672
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1611
1673
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1629,6 +1691,7 @@ var ChannelDriver = class {
|
|
|
1629
1691
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1630
1692
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1631
1693
|
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1694
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1632
1695
|
this.now = config2.now ?? (() => Date.now());
|
|
1633
1696
|
}
|
|
1634
1697
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1661,6 +1724,7 @@ var ChannelDriver = class {
|
|
|
1661
1724
|
for (const conv of conversations) {
|
|
1662
1725
|
dispatched += await this.processConversation(conv);
|
|
1663
1726
|
}
|
|
1727
|
+
await this.readoptProcessing();
|
|
1664
1728
|
} finally {
|
|
1665
1729
|
this.draining = false;
|
|
1666
1730
|
}
|
|
@@ -1747,6 +1811,7 @@ var ChannelDriver = class {
|
|
|
1747
1811
|
this.dispatched.add(message.id);
|
|
1748
1812
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1749
1813
|
dispatched += 1;
|
|
1814
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1750
1815
|
}
|
|
1751
1816
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1752
1817
|
this.log({
|
|
@@ -1812,7 +1877,54 @@ var ChannelDriver = class {
|
|
|
1812
1877
|
dispatchedAt: now,
|
|
1813
1878
|
deadline: now + this.pausedMaxWaitMs,
|
|
1814
1879
|
started: false,
|
|
1815
|
-
done: false
|
|
1880
|
+
done: false,
|
|
1881
|
+
stuckReported: false
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
/**
|
|
1885
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1886
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1887
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1888
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1889
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1890
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1891
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1892
|
+
*
|
|
1893
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1894
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1895
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1896
|
+
*
|
|
1897
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1898
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1899
|
+
* fire from the watcher's normal branches.
|
|
1900
|
+
*/
|
|
1901
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
1902
|
+
let watcher = this.watchers.get(sessionId);
|
|
1903
|
+
if (!watcher) {
|
|
1904
|
+
watcher = {
|
|
1905
|
+
conv,
|
|
1906
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1907
|
+
loop: null,
|
|
1908
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1909
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
1910
|
+
};
|
|
1911
|
+
this.watchers.set(sessionId, watcher);
|
|
1912
|
+
}
|
|
1913
|
+
watcher.inFlight.set(message.id, {
|
|
1914
|
+
evidentMessageId: message.id,
|
|
1915
|
+
opencodeMessageId,
|
|
1916
|
+
message,
|
|
1917
|
+
dispatchedAt: this.now(),
|
|
1918
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1919
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1920
|
+
started: true,
|
|
1921
|
+
done: false,
|
|
1922
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1923
|
+
// AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
|
|
1924
|
+
// gates on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
1925
|
+
// so a re-adopted row left wedged in `queued` still emits the signal once
|
|
1926
|
+
// (queued-followup-redrive, #210).
|
|
1927
|
+
stuckReported: false
|
|
1816
1928
|
});
|
|
1817
1929
|
}
|
|
1818
1930
|
/**
|
|
@@ -1878,6 +1990,7 @@ var ChannelDriver = class {
|
|
|
1878
1990
|
conversation_id: watcher.conv.id
|
|
1879
1991
|
});
|
|
1880
1992
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
1993
|
+
this.readopted.delete(evidentMessageId);
|
|
1881
1994
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1882
1995
|
}
|
|
1883
1996
|
return;
|
|
@@ -1898,7 +2011,7 @@ var ChannelDriver = class {
|
|
|
1898
2011
|
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
1899
2012
|
const conv = watcher.conv;
|
|
1900
2013
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1901
|
-
if ((state === "running" || state === "done") && !inFlight.started) {
|
|
2014
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1902
2015
|
let claimed;
|
|
1903
2016
|
try {
|
|
1904
2017
|
claimed = await this.markProcessing(conv.id, inFlight.evidentMessageId, sessionId);
|
|
@@ -1967,11 +2080,63 @@ var ChannelDriver = class {
|
|
|
1967
2080
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1968
2081
|
return;
|
|
1969
2082
|
}
|
|
2083
|
+
if (state === "failed") {
|
|
2084
|
+
if (!inFlight.done) {
|
|
2085
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2086
|
+
this.log({
|
|
2087
|
+
level: "error",
|
|
2088
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2089
|
+
conversation_id: conv.id,
|
|
2090
|
+
message_id: inFlight.evidentMessageId
|
|
2091
|
+
});
|
|
2092
|
+
try {
|
|
2093
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2094
|
+
} catch (err) {
|
|
2095
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2096
|
+
if (err instanceof ChannelTerminalError) {
|
|
2097
|
+
this.log({
|
|
2098
|
+
level: "error",
|
|
2099
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2100
|
+
conversation_id: conv.id,
|
|
2101
|
+
message_id: inFlight.evidentMessageId
|
|
2102
|
+
});
|
|
2103
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2104
|
+
return;
|
|
2105
|
+
}
|
|
2106
|
+
if (this.now() >= inFlight.deadline) {
|
|
2107
|
+
this.log({
|
|
2108
|
+
level: "error",
|
|
2109
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2110
|
+
conversation_id: conv.id,
|
|
2111
|
+
message_id: inFlight.evidentMessageId
|
|
2112
|
+
});
|
|
2113
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2114
|
+
return;
|
|
2115
|
+
}
|
|
2116
|
+
this.log({
|
|
2117
|
+
level: "error",
|
|
2118
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2119
|
+
conversation_id: conv.id,
|
|
2120
|
+
message_id: inFlight.evidentMessageId
|
|
2121
|
+
});
|
|
2122
|
+
return;
|
|
2123
|
+
}
|
|
2124
|
+
inFlight.done = true;
|
|
2125
|
+
}
|
|
2126
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2127
|
+
return;
|
|
2128
|
+
}
|
|
1970
2129
|
if (state === "unknown") {
|
|
1971
2130
|
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
1972
2131
|
await this.redispatchInFlight(sessionId, inFlight);
|
|
1973
2132
|
}
|
|
1974
2133
|
}
|
|
2134
|
+
if (state === "queued" && !inFlight.stuckReported && this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId)) {
|
|
2135
|
+
inFlight.stuckReported = true;
|
|
2136
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2137
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2138
|
+
});
|
|
2139
|
+
}
|
|
1975
2140
|
if (this.now() >= inFlight.deadline) {
|
|
1976
2141
|
this.log({
|
|
1977
2142
|
level: "info",
|
|
@@ -2015,12 +2180,345 @@ var ChannelDriver = class {
|
|
|
2015
2180
|
}
|
|
2016
2181
|
inFlight.dispatchedAt = this.now();
|
|
2017
2182
|
}
|
|
2183
|
+
// -------------------------------------------------------------------------
|
|
2184
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2185
|
+
// -------------------------------------------------------------------------
|
|
2186
|
+
/**
|
|
2187
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2188
|
+
*
|
|
2189
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2190
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2191
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2192
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2193
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2194
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2195
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2196
|
+
*
|
|
2197
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2198
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2199
|
+
*/
|
|
2200
|
+
async readoptProcessing() {
|
|
2201
|
+
const rows = await this.getProcessingMessages();
|
|
2202
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2203
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2204
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2205
|
+
if (!stillProcessing.has(id)) {
|
|
2206
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2207
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2208
|
+
if (cleared || clearedUndeliverable) {
|
|
2209
|
+
this.log({
|
|
2210
|
+
level: "info",
|
|
2211
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2212
|
+
message_id: id
|
|
2213
|
+
});
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
if (rows.length === 0) return;
|
|
2219
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2220
|
+
for (const row of rows) {
|
|
2221
|
+
if (!row.opencode_session_id) {
|
|
2222
|
+
this.log({
|
|
2223
|
+
level: "error",
|
|
2224
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2225
|
+
conversation_id: row.conversation_id,
|
|
2226
|
+
message_id: row.id
|
|
2227
|
+
});
|
|
2228
|
+
continue;
|
|
2229
|
+
}
|
|
2230
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2231
|
+
list.push(row);
|
|
2232
|
+
bySession.set(row.opencode_session_id, list);
|
|
2233
|
+
}
|
|
2234
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2235
|
+
let messages;
|
|
2236
|
+
try {
|
|
2237
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2238
|
+
if (!res.ok) {
|
|
2239
|
+
this.log({
|
|
2240
|
+
level: "error",
|
|
2241
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2242
|
+
});
|
|
2243
|
+
continue;
|
|
2244
|
+
}
|
|
2245
|
+
const body = await res.json();
|
|
2246
|
+
if (!Array.isArray(body)) {
|
|
2247
|
+
this.log({
|
|
2248
|
+
level: "error",
|
|
2249
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2250
|
+
});
|
|
2251
|
+
continue;
|
|
2252
|
+
}
|
|
2253
|
+
messages = body;
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
this.log({
|
|
2256
|
+
level: "error",
|
|
2257
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2258
|
+
});
|
|
2259
|
+
continue;
|
|
2260
|
+
}
|
|
2261
|
+
for (const row of sessionRows) {
|
|
2262
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
/**
|
|
2267
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2268
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2269
|
+
*
|
|
2270
|
+
* Branches on `messageRunState(messages, opencodeMessageIdFor(row.id))`:
|
|
2271
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2272
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2273
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2274
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch);
|
|
2275
|
+
* - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
|
|
2276
|
+
*
|
|
2277
|
+
* Only `ChannelAuthError` propagates.
|
|
2278
|
+
*/
|
|
2279
|
+
async readoptOne(sessionId, row, messages) {
|
|
2280
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2281
|
+
this.log({
|
|
2282
|
+
level: "info",
|
|
2283
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2284
|
+
conversation_id: row.conversation_id,
|
|
2285
|
+
message_id: row.id
|
|
2286
|
+
});
|
|
2287
|
+
return;
|
|
2288
|
+
}
|
|
2289
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2290
|
+
const state = messageRunState(messages, ocId);
|
|
2291
|
+
if (state === "done") {
|
|
2292
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2293
|
+
this.log({
|
|
2294
|
+
level: "info",
|
|
2295
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2296
|
+
conversation_id: row.conversation_id,
|
|
2297
|
+
message_id: row.id
|
|
2298
|
+
});
|
|
2299
|
+
return;
|
|
2300
|
+
}
|
|
2301
|
+
this.log({
|
|
2302
|
+
level: "info",
|
|
2303
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2304
|
+
conversation_id: row.conversation_id,
|
|
2305
|
+
message_id: row.id
|
|
2306
|
+
});
|
|
2307
|
+
try {
|
|
2308
|
+
await this.markDone(row.conversation_id, row.id, sessionId);
|
|
2309
|
+
} catch (err) {
|
|
2310
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2311
|
+
if (err instanceof ChannelTerminalError) {
|
|
2312
|
+
this.doneUndeliverable.add(row.id);
|
|
2313
|
+
this.log({
|
|
2314
|
+
level: "error",
|
|
2315
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2316
|
+
conversation_id: row.conversation_id,
|
|
2317
|
+
message_id: row.id
|
|
2318
|
+
});
|
|
2319
|
+
return;
|
|
2320
|
+
}
|
|
2321
|
+
this.log({
|
|
2322
|
+
level: "error",
|
|
2323
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2324
|
+
conversation_id: row.conversation_id,
|
|
2325
|
+
message_id: row.id
|
|
2326
|
+
});
|
|
2327
|
+
return;
|
|
2328
|
+
}
|
|
2329
|
+
this.dontRedispatch.delete(row.id);
|
|
2330
|
+
return;
|
|
2331
|
+
}
|
|
2332
|
+
if (state === "failed") {
|
|
2333
|
+
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2334
|
+
this.log({
|
|
2335
|
+
level: "error",
|
|
2336
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2337
|
+
conversation_id: row.conversation_id,
|
|
2338
|
+
message_id: row.id
|
|
2339
|
+
});
|
|
2340
|
+
try {
|
|
2341
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2342
|
+
} catch (err) {
|
|
2343
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2344
|
+
if (err instanceof ChannelTerminalError) {
|
|
2345
|
+
this.doneUndeliverable.add(row.id);
|
|
2346
|
+
this.log({
|
|
2347
|
+
level: "error",
|
|
2348
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2349
|
+
conversation_id: row.conversation_id,
|
|
2350
|
+
message_id: row.id
|
|
2351
|
+
});
|
|
2352
|
+
return;
|
|
2353
|
+
}
|
|
2354
|
+
this.log({
|
|
2355
|
+
level: "error",
|
|
2356
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2357
|
+
conversation_id: row.conversation_id,
|
|
2358
|
+
message_id: row.id
|
|
2359
|
+
});
|
|
2360
|
+
return;
|
|
2361
|
+
}
|
|
2362
|
+
this.dontRedispatch.delete(row.id);
|
|
2363
|
+
return;
|
|
2364
|
+
}
|
|
2365
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2366
|
+
this.log({
|
|
2367
|
+
level: "info",
|
|
2368
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2369
|
+
conversation_id: row.conversation_id,
|
|
2370
|
+
message_id: row.id
|
|
2371
|
+
});
|
|
2372
|
+
return;
|
|
2373
|
+
}
|
|
2374
|
+
if (state === "running" || state === "queued") {
|
|
2375
|
+
const conv = this.convForRow(sessionId, row);
|
|
2376
|
+
const message = this.queuedMessageForRow(row);
|
|
2377
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2378
|
+
this.dispatched.add(row.id);
|
|
2379
|
+
this.readopted.add(row.id);
|
|
2380
|
+
this.ensureWatcherRunning(sessionId);
|
|
2381
|
+
this.log({
|
|
2382
|
+
level: "info",
|
|
2383
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stable id, no re-dispatch)`,
|
|
2384
|
+
conversation_id: row.conversation_id,
|
|
2385
|
+
message_id: row.id
|
|
2386
|
+
});
|
|
2387
|
+
return;
|
|
2388
|
+
}
|
|
2389
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
|
|
2393
|
+
*
|
|
2394
|
+
* The stable-id user message is absent from the session, so we (re-)dispatch with
|
|
2395
|
+
* the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
|
|
2396
|
+
* This is what keeps the reply correlatable: the server's completion
|
|
2397
|
+
* notification looks for the reply under the stable id, so the fresh turn's reply
|
|
2398
|
+
* (which hangs off the stable id) is found and delivered. The residual
|
|
2399
|
+
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2400
|
+
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2401
|
+
* to swallow the duplicate.
|
|
2402
|
+
*
|
|
2403
|
+
* `evidentMessageId = row.id` addresses the SERVER row; the stable
|
|
2404
|
+
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
2405
|
+
* `processed_at` (Invariant 1).
|
|
2406
|
+
*/
|
|
2407
|
+
async forceReadoptRun(sessionId, row) {
|
|
2408
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2409
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2410
|
+
this.dontRedispatch.add(row.id);
|
|
2411
|
+
this.log({
|
|
2412
|
+
level: "info",
|
|
2413
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2414
|
+
conversation_id: row.conversation_id,
|
|
2415
|
+
message_id: row.id
|
|
2416
|
+
});
|
|
2417
|
+
return;
|
|
2418
|
+
}
|
|
2419
|
+
const options = {
|
|
2420
|
+
agent: row.opencode_agent ?? void 0,
|
|
2421
|
+
model: row.opencode_model ?? void 0
|
|
2422
|
+
};
|
|
2423
|
+
this.log({
|
|
2424
|
+
level: "info",
|
|
2425
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
|
|
2426
|
+
conversation_id: row.conversation_id,
|
|
2427
|
+
message_id: row.id
|
|
2428
|
+
});
|
|
2429
|
+
try {
|
|
2430
|
+
await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
|
|
2431
|
+
} catch (err) {
|
|
2432
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2433
|
+
this.log({
|
|
2434
|
+
level: "error",
|
|
2435
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2436
|
+
conversation_id: row.conversation_id,
|
|
2437
|
+
message_id: row.id
|
|
2438
|
+
});
|
|
2439
|
+
return;
|
|
2440
|
+
}
|
|
2441
|
+
const conv = this.convForRow(sessionId, row);
|
|
2442
|
+
const message = this.queuedMessageForRow(row);
|
|
2443
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2444
|
+
this.dispatched.add(row.id);
|
|
2445
|
+
this.readopted.add(row.id);
|
|
2446
|
+
this.ensureWatcherRunning(sessionId);
|
|
2447
|
+
}
|
|
2448
|
+
/**
|
|
2449
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2450
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2451
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2452
|
+
*/
|
|
2453
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2454
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2455
|
+
const watcher = this.watchers.get(sessionId);
|
|
2456
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2457
|
+
}
|
|
2458
|
+
/**
|
|
2459
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2460
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2461
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2462
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2463
|
+
* intended, which is worth surfacing.
|
|
2464
|
+
*/
|
|
2465
|
+
processedAtMs(row) {
|
|
2466
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2467
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2468
|
+
this.log({
|
|
2469
|
+
level: "error",
|
|
2470
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2471
|
+
conversation_id: row.conversation_id,
|
|
2472
|
+
message_id: row.id
|
|
2473
|
+
});
|
|
2474
|
+
return this.now();
|
|
2475
|
+
}
|
|
2476
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2477
|
+
convForRow(sessionId, row) {
|
|
2478
|
+
return {
|
|
2479
|
+
id: row.conversation_id,
|
|
2480
|
+
agent_id: this.agentId,
|
|
2481
|
+
opencode_session_id: sessionId,
|
|
2482
|
+
pending_message_count: 0,
|
|
2483
|
+
oldest_pending_at: row.processed_at
|
|
2484
|
+
};
|
|
2485
|
+
}
|
|
2486
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2487
|
+
queuedMessageForRow(row) {
|
|
2488
|
+
return {
|
|
2489
|
+
id: row.id,
|
|
2490
|
+
content: row.content,
|
|
2491
|
+
status: "processing",
|
|
2492
|
+
opencode_agent: row.opencode_agent,
|
|
2493
|
+
opencode_model: row.opencode_model,
|
|
2494
|
+
source_message_id: row.source_message_id,
|
|
2495
|
+
slack_user_id: row.slack_user_id
|
|
2496
|
+
};
|
|
2497
|
+
}
|
|
2018
2498
|
/**
|
|
2019
2499
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2020
2500
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2021
2501
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2502
|
+
*
|
|
2503
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2504
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2505
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2506
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2507
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2508
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2509
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2022
2510
|
*/
|
|
2023
2511
|
removeInFlight(watcher, evidentMessageId) {
|
|
2512
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2513
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2514
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2515
|
+
this.log({
|
|
2516
|
+
level: "info",
|
|
2517
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2518
|
+
conversation_id: watcher.conv.id,
|
|
2519
|
+
message_id: evidentMessageId
|
|
2520
|
+
});
|
|
2521
|
+
}
|
|
2024
2522
|
watcher.inFlight.delete(evidentMessageId);
|
|
2025
2523
|
this.dispatched.delete(evidentMessageId);
|
|
2026
2524
|
}
|
|
@@ -2164,6 +2662,35 @@ var ChannelDriver = class {
|
|
|
2164
2662
|
}
|
|
2165
2663
|
return await res.json();
|
|
2166
2664
|
}
|
|
2665
|
+
/**
|
|
2666
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2667
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2668
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2669
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2670
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2671
|
+
* `opencode_session_id`, routing).
|
|
2672
|
+
*
|
|
2673
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2674
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2675
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2676
|
+
* next tick retries.
|
|
2677
|
+
*/
|
|
2678
|
+
async getProcessingMessages() {
|
|
2679
|
+
const res = await this.fetchImpl(
|
|
2680
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2681
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2682
|
+
);
|
|
2683
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2684
|
+
if (!res.ok) {
|
|
2685
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2686
|
+
}
|
|
2687
|
+
const data = await res.json();
|
|
2688
|
+
let messages = data.messages ?? [];
|
|
2689
|
+
if (this.conversationFilter) {
|
|
2690
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2691
|
+
}
|
|
2692
|
+
return messages;
|
|
2693
|
+
}
|
|
2167
2694
|
/**
|
|
2168
2695
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2169
2696
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2245,7 +2772,17 @@ var ChannelDriver = class {
|
|
|
2245
2772
|
}
|
|
2246
2773
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2247
2774
|
}
|
|
2248
|
-
|
|
2775
|
+
/**
|
|
2776
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
2777
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
2778
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
2779
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
2780
|
+
* failure reason reaches the channel.
|
|
2781
|
+
*/
|
|
2782
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
2783
|
+
const body = { status: "failed" };
|
|
2784
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
2785
|
+
if (error2 !== void 0) body.error = error2;
|
|
2249
2786
|
await this.callWithRetry(
|
|
2250
2787
|
"marking message as failed",
|
|
2251
2788
|
() => this.fetchImpl(
|
|
@@ -2253,11 +2790,47 @@ var ChannelDriver = class {
|
|
|
2253
2790
|
{
|
|
2254
2791
|
method: "PATCH",
|
|
2255
2792
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2256
|
-
body: JSON.stringify(
|
|
2793
|
+
body: JSON.stringify(body)
|
|
2257
2794
|
}
|
|
2258
2795
|
)
|
|
2259
2796
|
);
|
|
2260
2797
|
}
|
|
2798
|
+
/**
|
|
2799
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
2800
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
2801
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
2802
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
2803
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2804
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2805
|
+
* context (no silent catch, per development-workflow).
|
|
2806
|
+
*/
|
|
2807
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
2808
|
+
try {
|
|
2809
|
+
const res = await this.fetchImpl(
|
|
2810
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
2811
|
+
{
|
|
2812
|
+
method: "POST",
|
|
2813
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2814
|
+
body: JSON.stringify({ signal, ...extra })
|
|
2815
|
+
}
|
|
2816
|
+
);
|
|
2817
|
+
if (!res.ok) {
|
|
2818
|
+
this.log({
|
|
2819
|
+
level: "error",
|
|
2820
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
2821
|
+
conversation_id: conversationId,
|
|
2822
|
+
message_id: messageId
|
|
2823
|
+
});
|
|
2824
|
+
}
|
|
2825
|
+
} catch (err) {
|
|
2826
|
+
this.log({
|
|
2827
|
+
level: "error",
|
|
2828
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
2829
|
+
conversation_id: conversationId,
|
|
2830
|
+
message_id: messageId
|
|
2831
|
+
});
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2261
2834
|
async persistSession(conversationId, sessionId) {
|
|
2262
2835
|
const res = await this.fetchImpl(
|
|
2263
2836
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|