@evident-ai/cli 3.0.1-dev.66ce6db → 3.0.1-dev.6a683e7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +468 -4
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1059,6 +1059,10 @@ function finishOf(m) {
|
|
|
1059
1059
|
const infoFinish = m.info?.finish;
|
|
1060
1060
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1061
1061
|
}
|
|
1062
|
+
function isAssistantInFlight(m) {
|
|
1063
|
+
if (completedOf(m) == null) return true;
|
|
1064
|
+
return finishOf(m) === "tool-calls";
|
|
1065
|
+
}
|
|
1062
1066
|
async function createOpenCodeSession(port, directory) {
|
|
1063
1067
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1064
1068
|
if (directory && directory.trim()) {
|
|
@@ -1140,9 +1144,13 @@ function messageRunState(messages, userMessageId) {
|
|
|
1140
1144
|
if (!reply) return "unknown";
|
|
1141
1145
|
}
|
|
1142
1146
|
if (!reply) return "queued";
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1147
|
+
return isAssistantInFlight(reply) ? "running" : "done";
|
|
1148
|
+
}
|
|
1149
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1150
|
+
if (!messages || messages.length === 0) return false;
|
|
1151
|
+
return messages.some(
|
|
1152
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1153
|
+
);
|
|
1146
1154
|
}
|
|
1147
1155
|
function opencodeMessageIdFor2(queuedMessageId) {
|
|
1148
1156
|
return opencodeMessageIdFor(queuedMessageId);
|
|
@@ -1552,6 +1560,7 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1552
1560
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1553
1561
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1554
1562
|
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1563
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1555
1564
|
var ChannelAuthError = class extends Error {
|
|
1556
1565
|
constructor(message) {
|
|
1557
1566
|
super(message);
|
|
@@ -1587,6 +1596,7 @@ var ChannelDriver = class {
|
|
|
1587
1596
|
pausedPollIntervalMs;
|
|
1588
1597
|
pausedMaxWaitMs;
|
|
1589
1598
|
dispatchConfirmMs;
|
|
1599
|
+
stuckQueuedMs;
|
|
1590
1600
|
now;
|
|
1591
1601
|
/** Cache of conversationId → opencode sessionId. */
|
|
1592
1602
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -1606,6 +1616,40 @@ var ChannelDriver = class {
|
|
|
1606
1616
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1607
1617
|
*/
|
|
1608
1618
|
dispatched = /* @__PURE__ */ new Set();
|
|
1619
|
+
/**
|
|
1620
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1621
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1622
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1623
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1624
|
+
* the processing list.
|
|
1625
|
+
*/
|
|
1626
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1627
|
+
/**
|
|
1628
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1629
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1630
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1631
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1632
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1633
|
+
*
|
|
1634
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1635
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1636
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1637
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1638
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1639
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1640
|
+
*/
|
|
1641
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1642
|
+
/**
|
|
1643
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1644
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1645
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1646
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1647
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1648
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1649
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1650
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1651
|
+
*/
|
|
1652
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1609
1653
|
/**
|
|
1610
1654
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1611
1655
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1629,6 +1673,7 @@ var ChannelDriver = class {
|
|
|
1629
1673
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1630
1674
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1631
1675
|
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1676
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1632
1677
|
this.now = config2.now ?? (() => Date.now());
|
|
1633
1678
|
}
|
|
1634
1679
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1661,6 +1706,7 @@ var ChannelDriver = class {
|
|
|
1661
1706
|
for (const conv of conversations) {
|
|
1662
1707
|
dispatched += await this.processConversation(conv);
|
|
1663
1708
|
}
|
|
1709
|
+
await this.readoptProcessing();
|
|
1664
1710
|
} finally {
|
|
1665
1711
|
this.draining = false;
|
|
1666
1712
|
}
|
|
@@ -1747,6 +1793,7 @@ var ChannelDriver = class {
|
|
|
1747
1793
|
this.dispatched.add(message.id);
|
|
1748
1794
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1749
1795
|
dispatched += 1;
|
|
1796
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1750
1797
|
}
|
|
1751
1798
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1752
1799
|
this.log({
|
|
@@ -1812,7 +1859,54 @@ var ChannelDriver = class {
|
|
|
1812
1859
|
dispatchedAt: now,
|
|
1813
1860
|
deadline: now + this.pausedMaxWaitMs,
|
|
1814
1861
|
started: false,
|
|
1815
|
-
done: false
|
|
1862
|
+
done: false,
|
|
1863
|
+
stuckReported: false
|
|
1864
|
+
});
|
|
1865
|
+
}
|
|
1866
|
+
/**
|
|
1867
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1868
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1869
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1870
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1871
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1872
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1873
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1874
|
+
*
|
|
1875
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1876
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1877
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1878
|
+
*
|
|
1879
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1880
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1881
|
+
* fire from the watcher's normal branches.
|
|
1882
|
+
*/
|
|
1883
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs) {
|
|
1884
|
+
let watcher = this.watchers.get(sessionId);
|
|
1885
|
+
if (!watcher) {
|
|
1886
|
+
watcher = {
|
|
1887
|
+
conv,
|
|
1888
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1889
|
+
loop: null,
|
|
1890
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1891
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
1892
|
+
};
|
|
1893
|
+
this.watchers.set(sessionId, watcher);
|
|
1894
|
+
}
|
|
1895
|
+
watcher.inFlight.set(message.id, {
|
|
1896
|
+
evidentMessageId: message.id,
|
|
1897
|
+
opencodeMessageId,
|
|
1898
|
+
message,
|
|
1899
|
+
dispatchedAt: this.now(),
|
|
1900
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1901
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1902
|
+
started: true,
|
|
1903
|
+
done: false,
|
|
1904
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1905
|
+
// AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
|
|
1906
|
+
// gates on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
1907
|
+
// so a re-adopted row left wedged in `queued` still emits the signal once
|
|
1908
|
+
// (queued-followup-redrive, #210).
|
|
1909
|
+
stuckReported: false
|
|
1816
1910
|
});
|
|
1817
1911
|
}
|
|
1818
1912
|
/**
|
|
@@ -1878,6 +1972,7 @@ var ChannelDriver = class {
|
|
|
1878
1972
|
conversation_id: watcher.conv.id
|
|
1879
1973
|
});
|
|
1880
1974
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
1975
|
+
this.readopted.delete(evidentMessageId);
|
|
1881
1976
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1882
1977
|
}
|
|
1883
1978
|
return;
|
|
@@ -1972,6 +2067,12 @@ var ChannelDriver = class {
|
|
|
1972
2067
|
await this.redispatchInFlight(sessionId, inFlight);
|
|
1973
2068
|
}
|
|
1974
2069
|
}
|
|
2070
|
+
if (state === "queued" && !inFlight.stuckReported && this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId)) {
|
|
2071
|
+
inFlight.stuckReported = true;
|
|
2072
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2073
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2074
|
+
});
|
|
2075
|
+
}
|
|
1975
2076
|
if (this.now() >= inFlight.deadline) {
|
|
1976
2077
|
this.log({
|
|
1977
2078
|
level: "info",
|
|
@@ -2015,12 +2116,310 @@ var ChannelDriver = class {
|
|
|
2015
2116
|
}
|
|
2016
2117
|
inFlight.dispatchedAt = this.now();
|
|
2017
2118
|
}
|
|
2119
|
+
// -------------------------------------------------------------------------
|
|
2120
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2121
|
+
// -------------------------------------------------------------------------
|
|
2122
|
+
/**
|
|
2123
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2124
|
+
*
|
|
2125
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2126
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2127
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2128
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2129
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2130
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2131
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2132
|
+
*
|
|
2133
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2134
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2135
|
+
*/
|
|
2136
|
+
async readoptProcessing() {
|
|
2137
|
+
const rows = await this.getProcessingMessages();
|
|
2138
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2139
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2140
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2141
|
+
if (!stillProcessing.has(id)) {
|
|
2142
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2143
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2144
|
+
if (cleared || clearedUndeliverable) {
|
|
2145
|
+
this.log({
|
|
2146
|
+
level: "info",
|
|
2147
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2148
|
+
message_id: id
|
|
2149
|
+
});
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
if (rows.length === 0) return;
|
|
2155
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2156
|
+
for (const row of rows) {
|
|
2157
|
+
if (!row.opencode_session_id) {
|
|
2158
|
+
this.log({
|
|
2159
|
+
level: "error",
|
|
2160
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2161
|
+
conversation_id: row.conversation_id,
|
|
2162
|
+
message_id: row.id
|
|
2163
|
+
});
|
|
2164
|
+
continue;
|
|
2165
|
+
}
|
|
2166
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2167
|
+
list.push(row);
|
|
2168
|
+
bySession.set(row.opencode_session_id, list);
|
|
2169
|
+
}
|
|
2170
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2171
|
+
let messages;
|
|
2172
|
+
try {
|
|
2173
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2174
|
+
if (!res.ok) {
|
|
2175
|
+
this.log({
|
|
2176
|
+
level: "error",
|
|
2177
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2178
|
+
});
|
|
2179
|
+
continue;
|
|
2180
|
+
}
|
|
2181
|
+
const body = await res.json();
|
|
2182
|
+
if (!Array.isArray(body)) {
|
|
2183
|
+
this.log({
|
|
2184
|
+
level: "error",
|
|
2185
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2186
|
+
});
|
|
2187
|
+
continue;
|
|
2188
|
+
}
|
|
2189
|
+
messages = body;
|
|
2190
|
+
} catch (err) {
|
|
2191
|
+
this.log({
|
|
2192
|
+
level: "error",
|
|
2193
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2194
|
+
});
|
|
2195
|
+
continue;
|
|
2196
|
+
}
|
|
2197
|
+
for (const row of sessionRows) {
|
|
2198
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
/**
|
|
2203
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2204
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2205
|
+
*
|
|
2206
|
+
* Branches on `messageRunState(messages, opencodeMessageIdFor(row.id))`:
|
|
2207
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2208
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch);
|
|
2209
|
+
* - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
|
|
2210
|
+
*
|
|
2211
|
+
* Only `ChannelAuthError` propagates.
|
|
2212
|
+
*/
|
|
2213
|
+
async readoptOne(sessionId, row, messages) {
|
|
2214
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2215
|
+
this.log({
|
|
2216
|
+
level: "info",
|
|
2217
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2218
|
+
conversation_id: row.conversation_id,
|
|
2219
|
+
message_id: row.id
|
|
2220
|
+
});
|
|
2221
|
+
return;
|
|
2222
|
+
}
|
|
2223
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2224
|
+
const state = messageRunState(messages, ocId);
|
|
2225
|
+
if (state === "done") {
|
|
2226
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2227
|
+
this.log({
|
|
2228
|
+
level: "info",
|
|
2229
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2230
|
+
conversation_id: row.conversation_id,
|
|
2231
|
+
message_id: row.id
|
|
2232
|
+
});
|
|
2233
|
+
return;
|
|
2234
|
+
}
|
|
2235
|
+
this.log({
|
|
2236
|
+
level: "info",
|
|
2237
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2238
|
+
conversation_id: row.conversation_id,
|
|
2239
|
+
message_id: row.id
|
|
2240
|
+
});
|
|
2241
|
+
try {
|
|
2242
|
+
await this.markDone(row.conversation_id, row.id, sessionId);
|
|
2243
|
+
} catch (err) {
|
|
2244
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2245
|
+
if (err instanceof ChannelTerminalError) {
|
|
2246
|
+
this.doneUndeliverable.add(row.id);
|
|
2247
|
+
this.log({
|
|
2248
|
+
level: "error",
|
|
2249
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2250
|
+
conversation_id: row.conversation_id,
|
|
2251
|
+
message_id: row.id
|
|
2252
|
+
});
|
|
2253
|
+
return;
|
|
2254
|
+
}
|
|
2255
|
+
this.log({
|
|
2256
|
+
level: "error",
|
|
2257
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2258
|
+
conversation_id: row.conversation_id,
|
|
2259
|
+
message_id: row.id
|
|
2260
|
+
});
|
|
2261
|
+
return;
|
|
2262
|
+
}
|
|
2263
|
+
this.dontRedispatch.delete(row.id);
|
|
2264
|
+
return;
|
|
2265
|
+
}
|
|
2266
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2267
|
+
this.log({
|
|
2268
|
+
level: "info",
|
|
2269
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2270
|
+
conversation_id: row.conversation_id,
|
|
2271
|
+
message_id: row.id
|
|
2272
|
+
});
|
|
2273
|
+
return;
|
|
2274
|
+
}
|
|
2275
|
+
if (state === "running" || state === "queued") {
|
|
2276
|
+
const conv = this.convForRow(sessionId, row);
|
|
2277
|
+
const message = this.queuedMessageForRow(row);
|
|
2278
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2279
|
+
this.dispatched.add(row.id);
|
|
2280
|
+
this.readopted.add(row.id);
|
|
2281
|
+
this.ensureWatcherRunning(sessionId);
|
|
2282
|
+
this.log({
|
|
2283
|
+
level: "info",
|
|
2284
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (stable id, no re-dispatch)`,
|
|
2285
|
+
conversation_id: row.conversation_id,
|
|
2286
|
+
message_id: row.id
|
|
2287
|
+
});
|
|
2288
|
+
return;
|
|
2289
|
+
}
|
|
2290
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2291
|
+
}
|
|
2292
|
+
/**
|
|
2293
|
+
* Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
|
|
2294
|
+
*
|
|
2295
|
+
* The stable-id user message is absent from the session, so we (re-)dispatch with
|
|
2296
|
+
* the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
|
|
2297
|
+
* This is what keeps the reply correlatable: the server's completion
|
|
2298
|
+
* notification looks for the reply under the stable id, so the fresh turn's reply
|
|
2299
|
+
* (which hangs off the stable id) is found and delivered. The residual
|
|
2300
|
+
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2301
|
+
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2302
|
+
* to swallow the duplicate.
|
|
2303
|
+
*
|
|
2304
|
+
* `evidentMessageId = row.id` addresses the SERVER row; the stable
|
|
2305
|
+
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
2306
|
+
* `processed_at` (Invariant 1).
|
|
2307
|
+
*/
|
|
2308
|
+
async forceReadoptRun(sessionId, row) {
|
|
2309
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2310
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2311
|
+
this.dontRedispatch.add(row.id);
|
|
2312
|
+
this.log({
|
|
2313
|
+
level: "info",
|
|
2314
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2315
|
+
conversation_id: row.conversation_id,
|
|
2316
|
+
message_id: row.id
|
|
2317
|
+
});
|
|
2318
|
+
return;
|
|
2319
|
+
}
|
|
2320
|
+
const options = {
|
|
2321
|
+
agent: row.opencode_agent ?? void 0,
|
|
2322
|
+
model: row.opencode_model ?? void 0
|
|
2323
|
+
};
|
|
2324
|
+
this.log({
|
|
2325
|
+
level: "info",
|
|
2326
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
|
|
2327
|
+
conversation_id: row.conversation_id,
|
|
2328
|
+
message_id: row.id
|
|
2329
|
+
});
|
|
2330
|
+
try {
|
|
2331
|
+
await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
|
|
2332
|
+
} catch (err) {
|
|
2333
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2334
|
+
this.log({
|
|
2335
|
+
level: "error",
|
|
2336
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2337
|
+
conversation_id: row.conversation_id,
|
|
2338
|
+
message_id: row.id
|
|
2339
|
+
});
|
|
2340
|
+
return;
|
|
2341
|
+
}
|
|
2342
|
+
const conv = this.convForRow(sessionId, row);
|
|
2343
|
+
const message = this.queuedMessageForRow(row);
|
|
2344
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2345
|
+
this.dispatched.add(row.id);
|
|
2346
|
+
this.readopted.add(row.id);
|
|
2347
|
+
this.ensureWatcherRunning(sessionId);
|
|
2348
|
+
}
|
|
2349
|
+
/**
|
|
2350
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2351
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2352
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2353
|
+
*/
|
|
2354
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2355
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2356
|
+
const watcher = this.watchers.get(sessionId);
|
|
2357
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2358
|
+
}
|
|
2359
|
+
/**
|
|
2360
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2361
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2362
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2363
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2364
|
+
* intended, which is worth surfacing.
|
|
2365
|
+
*/
|
|
2366
|
+
processedAtMs(row) {
|
|
2367
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2368
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2369
|
+
this.log({
|
|
2370
|
+
level: "error",
|
|
2371
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2372
|
+
conversation_id: row.conversation_id,
|
|
2373
|
+
message_id: row.id
|
|
2374
|
+
});
|
|
2375
|
+
return this.now();
|
|
2376
|
+
}
|
|
2377
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2378
|
+
convForRow(sessionId, row) {
|
|
2379
|
+
return {
|
|
2380
|
+
id: row.conversation_id,
|
|
2381
|
+
agent_id: this.agentId,
|
|
2382
|
+
opencode_session_id: sessionId,
|
|
2383
|
+
pending_message_count: 0,
|
|
2384
|
+
oldest_pending_at: row.processed_at
|
|
2385
|
+
};
|
|
2386
|
+
}
|
|
2387
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2388
|
+
queuedMessageForRow(row) {
|
|
2389
|
+
return {
|
|
2390
|
+
id: row.id,
|
|
2391
|
+
content: row.content,
|
|
2392
|
+
status: "processing",
|
|
2393
|
+
opencode_agent: row.opencode_agent,
|
|
2394
|
+
opencode_model: row.opencode_model,
|
|
2395
|
+
source_message_id: row.source_message_id,
|
|
2396
|
+
slack_user_id: row.slack_user_id
|
|
2397
|
+
};
|
|
2398
|
+
}
|
|
2018
2399
|
/**
|
|
2019
2400
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
2020
2401
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
2021
2402
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2403
|
+
*
|
|
2404
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2405
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2406
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2407
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2408
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2409
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2410
|
+
* the done branch still delivers it (Bugbot #202).
|
|
2022
2411
|
*/
|
|
2023
2412
|
removeInFlight(watcher, evidentMessageId) {
|
|
2413
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2414
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2415
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2416
|
+
this.log({
|
|
2417
|
+
level: "info",
|
|
2418
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2419
|
+
conversation_id: watcher.conv.id,
|
|
2420
|
+
message_id: evidentMessageId
|
|
2421
|
+
});
|
|
2422
|
+
}
|
|
2024
2423
|
watcher.inFlight.delete(evidentMessageId);
|
|
2025
2424
|
this.dispatched.delete(evidentMessageId);
|
|
2026
2425
|
}
|
|
@@ -2164,6 +2563,35 @@ var ChannelDriver = class {
|
|
|
2164
2563
|
}
|
|
2165
2564
|
return await res.json();
|
|
2166
2565
|
}
|
|
2566
|
+
/**
|
|
2567
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2568
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2569
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2570
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2571
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2572
|
+
* `opencode_session_id`, routing).
|
|
2573
|
+
*
|
|
2574
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2575
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2576
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2577
|
+
* next tick retries.
|
|
2578
|
+
*/
|
|
2579
|
+
async getProcessingMessages() {
|
|
2580
|
+
const res = await this.fetchImpl(
|
|
2581
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2582
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2583
|
+
);
|
|
2584
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2585
|
+
if (!res.ok) {
|
|
2586
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2587
|
+
}
|
|
2588
|
+
const data = await res.json();
|
|
2589
|
+
let messages = data.messages ?? [];
|
|
2590
|
+
if (this.conversationFilter) {
|
|
2591
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2592
|
+
}
|
|
2593
|
+
return messages;
|
|
2594
|
+
}
|
|
2167
2595
|
/**
|
|
2168
2596
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2169
2597
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2258,6 +2686,42 @@ var ChannelDriver = class {
|
|
|
2258
2686
|
)
|
|
2259
2687
|
);
|
|
2260
2688
|
}
|
|
2689
|
+
/**
|
|
2690
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
2691
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
2692
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
2693
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
2694
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
2695
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
2696
|
+
* context (no silent catch, per development-workflow).
|
|
2697
|
+
*/
|
|
2698
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
2699
|
+
try {
|
|
2700
|
+
const res = await this.fetchImpl(
|
|
2701
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
2702
|
+
{
|
|
2703
|
+
method: "POST",
|
|
2704
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2705
|
+
body: JSON.stringify({ signal, ...extra })
|
|
2706
|
+
}
|
|
2707
|
+
);
|
|
2708
|
+
if (!res.ok) {
|
|
2709
|
+
this.log({
|
|
2710
|
+
level: "error",
|
|
2711
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
2712
|
+
conversation_id: conversationId,
|
|
2713
|
+
message_id: messageId
|
|
2714
|
+
});
|
|
2715
|
+
}
|
|
2716
|
+
} catch (err) {
|
|
2717
|
+
this.log({
|
|
2718
|
+
level: "error",
|
|
2719
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
2720
|
+
conversation_id: conversationId,
|
|
2721
|
+
message_id: messageId
|
|
2722
|
+
});
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2261
2725
|
async persistSession(conversationId, sessionId) {
|
|
2262
2726
|
const res = await this.fetchImpl(
|
|
2263
2727
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|