@evident-ai/cli 3.0.1-dev.3007897 → 3.0.1-dev.379acd9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  // src/index.ts
4
+ import { createRequire } from "module";
4
5
  import { Command } from "commander";
5
6
 
6
7
  // src/commands/login.ts
@@ -470,6 +471,12 @@ import chalk6 from "chalk";
470
471
  import ora3 from "ora";
471
472
  import { select as select3 } from "@inquirer/prompts";
472
473
 
474
+ // ../../packages/types/src/opencode/index.ts
475
+ function opencodeMessageIdFor(queuedMessageId) {
476
+ const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
477
+ return `msg_${sanitized}`;
478
+ }
479
+
473
480
  // ../../packages/types/src/telemetry/index.ts
474
481
  var TelemetryEventTypes = {
475
482
  // Agent activity events (shown in web UI activity log)
@@ -484,8 +491,34 @@ var TelemetryEventTypes = {
484
491
  var MAX_FRAME_BYTES = 256 * 1024;
485
492
  var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
486
493
 
494
+ // ../../packages/types/src/logging/index.ts
495
+ var CORRELATION_ID_HEADER = "x-evident-correlation-id";
496
+ function log(level, event, fields) {
497
+ const method = level === "debug" ? "log" : level;
498
+ try {
499
+ console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
500
+ } catch (err) {
501
+ console.error(
502
+ "[evident] log_serialize_failed",
503
+ event,
504
+ err instanceof Error ? err.message : String(err)
505
+ );
506
+ }
507
+ }
508
+ function stripQuery(url) {
509
+ try {
510
+ return new URL(url).pathname;
511
+ } catch {
512
+ const q = url.indexOf("?");
513
+ return q === -1 ? url : url.slice(0, q);
514
+ }
515
+ }
516
+
487
517
  // src/lib/telemetry.ts
488
- var CLI_VERSION = process.env.npm_package_version || "unknown";
518
+ var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
519
+ function getCliVersion() {
520
+ return CLI_VERSION;
521
+ }
489
522
  var eventBuffer = [];
490
523
  var flushTimeout = null;
491
524
  var isShuttingDown = false;
@@ -686,13 +719,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
686
719
 
687
720
  // src/lib/opencode/opencode-version-gate.ts
688
721
  var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
689
- function isQueueValidatedVersion(version) {
690
- if (!version) return false;
691
- return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version);
722
+ function isQueueValidatedVersion(version2) {
723
+ if (!version2) return false;
724
+ return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
692
725
  }
693
- function buildOpenCodeVersionWarning(version) {
694
- if (isQueueValidatedVersion(version)) return null;
695
- const detected = version ? `v${version}` : "unknown";
726
+ function buildOpenCodeVersionWarning(version2) {
727
+ if (isQueueValidatedVersion(version2)) return null;
728
+ const detected = version2 ? `v${version2}` : "unknown";
696
729
  const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
697
730
  return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
698
731
  }
@@ -1009,7 +1042,11 @@ function roleOf(m) {
1009
1042
  }
1010
1043
  function completedOf(m) {
1011
1044
  if (!m || typeof m !== "object") return void 0;
1012
- return m.info?.time?.completed;
1045
+ return m.info?.time?.completed ?? m.time?.completed;
1046
+ }
1047
+ function createdOf(m) {
1048
+ if (!m || typeof m !== "object") return void 0;
1049
+ return m.info?.time?.created ?? m.time?.created;
1013
1050
  }
1014
1051
  function idOf(m) {
1015
1052
  if (!m || typeof m !== "object") return void 0;
@@ -1029,6 +1066,14 @@ function finishOf(m) {
1029
1066
  const infoFinish = m.info?.finish;
1030
1067
  return typeof infoFinish === "string" ? infoFinish : void 0;
1031
1068
  }
1069
+ function errorOf(m) {
1070
+ if (!m || typeof m !== "object") return void 0;
1071
+ return m.info?.error ?? m.error;
1072
+ }
1073
+ function isAssistantInFlight(m) {
1074
+ if (completedOf(m) == null) return true;
1075
+ return finishOf(m) === "tool-calls";
1076
+ }
1032
1077
  async function createOpenCodeSession(port, directory) {
1033
1078
  const url = new URL(`${opencodeBase(port)}/session`);
1034
1079
  if (directory && directory.trim()) {
@@ -1088,19 +1133,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
1088
1133
  }
1089
1134
  function findLastAssistantReplyFor(messages, userMessageId) {
1090
1135
  if (!messages || messages.length === 0) return null;
1136
+ let lastCorrelated = null;
1137
+ let lastNonErrored = null;
1091
1138
  for (let i = messages.length - 1; i >= 0; i--) {
1092
1139
  const m = messages[i];
1093
- if (roleOf(m) === "assistant" && parentIdOf(m) === userMessageId) return m;
1140
+ if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
1141
+ if (lastCorrelated === null) lastCorrelated = m;
1142
+ if (errorOf(m) == null) {
1143
+ lastNonErrored = m;
1144
+ break;
1145
+ }
1094
1146
  }
1147
+ if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
1095
1148
  const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
1096
1149
  if (userIndex === -1) return null;
1097
1150
  let last = null;
1151
+ let lastOk = null;
1098
1152
  for (let i = userIndex + 1; i < messages.length; i++) {
1099
1153
  const role = roleOf(messages[i]);
1100
1154
  if (role === "user") break;
1101
- if (role === "assistant") last = messages[i];
1155
+ if (role === "assistant") {
1156
+ last = messages[i];
1157
+ if (errorOf(messages[i]) == null) lastOk = messages[i];
1158
+ }
1102
1159
  }
1103
- return last;
1160
+ return lastOk ?? last;
1104
1161
  }
1105
1162
  function messageRunState(messages, userMessageId) {
1106
1163
  if (!messages || messages.length === 0) return "unknown";
@@ -1110,13 +1167,52 @@ function messageRunState(messages, userMessageId) {
1110
1167
  if (!reply) return "unknown";
1111
1168
  }
1112
1169
  if (!reply) return "queued";
1113
- if (completedOf(reply) == null) return "running";
1114
- if (finishOf(reply) === "tool-calls") return "running";
1115
- return "done";
1170
+ if (isAssistantInFlight(reply)) return "running";
1171
+ return errorOf(reply) != null ? "failed" : "done";
1116
1172
  }
1117
- function opencodeMessageIdFor(queuedMessageId) {
1118
- const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
1119
- return `msg_${sanitized}`;
1173
+ function messageError(messages, userMessageId) {
1174
+ const reply = findLastAssistantReplyFor(messages, userMessageId);
1175
+ const error2 = errorOf(reply);
1176
+ if (error2 == null) return null;
1177
+ if (typeof error2 === "string") return error2;
1178
+ if (typeof error2 === "object") {
1179
+ const e = error2;
1180
+ const dataMessage = e.data?.message;
1181
+ if (typeof dataMessage === "string") return dataMessage;
1182
+ if (typeof e.message === "string") return e.message;
1183
+ }
1184
+ return "The agent run failed.";
1185
+ }
1186
+ function hasRunningAssistantExcept(messages, exceptUserMessageId) {
1187
+ if (!messages || messages.length === 0) return false;
1188
+ return messages.some(
1189
+ (m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
1190
+ );
1191
+ }
1192
+ function isSessionSettled(messages, now, settleMs) {
1193
+ if (!messages || messages.length === 0) return true;
1194
+ let newestAssistantCreated = null;
1195
+ for (const m of messages) {
1196
+ if (roleOf(m) !== "assistant") continue;
1197
+ if (isAssistantInFlight(m)) return false;
1198
+ const created = createdOf(m);
1199
+ if (typeof created === "number" && (newestAssistantCreated === null || created > newestAssistantCreated)) {
1200
+ newestAssistantCreated = created;
1201
+ }
1202
+ }
1203
+ if (newestAssistantCreated === null) return true;
1204
+ return now - newestAssistantCreated >= settleMs;
1205
+ }
1206
+ function opencodeMessageIdFor2(queuedMessageId) {
1207
+ return opencodeMessageIdFor(queuedMessageId);
1208
+ }
1209
+ var NATIVE_ID_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
1210
+ function nativeOpencodeMessageId() {
1211
+ let id = "msg_";
1212
+ for (let i = 0; i < 24; i++) {
1213
+ id += NATIVE_ID_ALPHABET[Math.floor(Math.random() * NATIVE_ID_ALPHABET.length)];
1214
+ }
1215
+ return id;
1120
1216
  }
1121
1217
 
1122
1218
  // src/lib/tunnel/connection.ts
@@ -1187,12 +1283,20 @@ var StreamForwarder = class {
1187
1283
  }
1188
1284
  async handleOpen(frame) {
1189
1285
  const { sid, method, path, headers, has_body } = frame;
1286
+ const correlationId = headers?.[CORRELATION_ID_HEADER];
1287
+ const startedAt = Date.now();
1190
1288
  if (path === TUNNEL_DRAIN_PING_PATH) {
1191
1289
  this.callbacks.onDrainPing?.();
1192
1290
  this.send({ type: "head", sid, status: 204, headers: {} });
1193
1291
  this.send({ type: "res_end", sid });
1194
1292
  return;
1195
1293
  }
1294
+ log("info", "agent_request", {
1295
+ correlation_id: correlationId,
1296
+ sid,
1297
+ method,
1298
+ path: stripQuery(path)
1299
+ });
1196
1300
  const ac = new AbortController();
1197
1301
  let bodyPromise;
1198
1302
  let pushBody;
@@ -1239,6 +1343,12 @@ var StreamForwarder = class {
1239
1343
  if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
1240
1344
  });
1241
1345
  this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
1346
+ log("info", "agent_response", {
1347
+ correlation_id: correlationId,
1348
+ sid,
1349
+ status: upstream.status,
1350
+ duration_ms: Date.now() - startedAt
1351
+ });
1242
1352
  this.callbacks.onHead?.(sid, upstream.status);
1243
1353
  try {
1244
1354
  if (upstream.body) {
@@ -1509,6 +1619,9 @@ var DEFAULT_RETRY_POLICY = {
1509
1619
  var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
1510
1620
  var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
1511
1621
  var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
1622
+ var DEFAULT_STUCK_QUEUED_MS = 6e4;
1623
+ var DEFAULT_STUCK_QUEUED_REDRIVE_MAX = 3;
1624
+ var DEFAULT_SETTLE_MS = 3500;
1512
1625
  var ChannelAuthError = class extends Error {
1513
1626
  constructor(message) {
1514
1627
  super(message);
@@ -1544,6 +1657,9 @@ var ChannelDriver = class {
1544
1657
  pausedPollIntervalMs;
1545
1658
  pausedMaxWaitMs;
1546
1659
  dispatchConfirmMs;
1660
+ stuckQueuedMs;
1661
+ stuckQueuedRedriveMax;
1662
+ settleMs;
1547
1663
  now;
1548
1664
  /** Cache of conversationId → opencode sessionId. */
1549
1665
  sessions = /* @__PURE__ */ new Map();
@@ -1563,6 +1679,40 @@ var ChannelDriver = class {
1563
1679
  * a steady-state-poll re-dispatch will not double-run the message.
1564
1680
  */
1565
1681
  dispatched = /* @__PURE__ */ new Set();
1682
+ /**
1683
+ * Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
1684
+ * Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
1685
+ * so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
1686
+ * it is re-adopted and removed when its watcher settles or it is observed off
1687
+ * the processing list.
1688
+ */
1689
+ readopted = /* @__PURE__ */ new Set();
1690
+ /**
1691
+ * "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
1692
+ * re-adopted running/orphan row's watcher hit its `processed_at`-anchored
1693
+ * deadline (or an orphan whose window already elapsed): the still-`processing`
1694
+ * server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
1695
+ * drain until the 15-min cron resets it — spamming new turns.
1696
+ *
1697
+ * CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
1698
+ * does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
1699
+ * in opencode must still be delivered via `markDone` on the next drain — so
1700
+ * `readoptOne` computes `state` FIRST and this set is checked only on the
1701
+ * non-done path. It is cleared once the row leaves the processing list (cron
1702
+ * reset → it drains normally as `pending`), so it can never leak.
1703
+ */
1704
+ dontRedispatch = /* @__PURE__ */ new Set();
1705
+ /**
1706
+ * "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
1707
+ * re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
1708
+ * never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
1709
+ * that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
1710
+ * markDone failure must NOT land here (it must still retry next drain). Separate
1711
+ * from `dontRedispatch` because the two concerns are independent: a row can need
1712
+ * "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
1713
+ * the row leaves the processing list, exactly like `dontRedispatch`.
1714
+ */
1715
+ doneUndeliverable = /* @__PURE__ */ new Set();
1566
1716
  /**
1567
1717
  * Cache of the opencode root directory (from `GET /path`). Resolved lazily on
1568
1718
  * first session creation so drain-created sessions are rooted at the project
@@ -1586,6 +1736,9 @@ var ChannelDriver = class {
1586
1736
  this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
1587
1737
  this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
1588
1738
  this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
1739
+ this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
1740
+ this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
1741
+ this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
1589
1742
  this.now = config2.now ?? (() => Date.now());
1590
1743
  }
1591
1744
  /** The IPv4-loopback base URL for the local `opencode serve`. */
@@ -1618,6 +1771,7 @@ var ChannelDriver = class {
1618
1771
  for (const conv of conversations) {
1619
1772
  dispatched += await this.processConversation(conv);
1620
1773
  }
1774
+ await this.readoptProcessing();
1621
1775
  } finally {
1622
1776
  this.draining = false;
1623
1777
  }
@@ -1669,11 +1823,13 @@ var ChannelDriver = class {
1669
1823
  const sessionId = await this.ensureSession(conv);
1670
1824
  const messages = await this.getPendingMessages(conv.id);
1671
1825
  let dispatched = 0;
1826
+ let skippedAlreadyDispatched = 0;
1672
1827
  for (const message of messages) {
1673
1828
  if (this.dispatched.has(message.id)) {
1829
+ skippedAlreadyDispatched += 1;
1674
1830
  continue;
1675
1831
  }
1676
- const opencodeMessageId = opencodeMessageIdFor(message.id);
1832
+ const opencodeMessageId = opencodeMessageIdFor2(message.id);
1677
1833
  const options = {
1678
1834
  agent: message.opencode_agent ?? void 0,
1679
1835
  model: message.opencode_model ?? void 0
@@ -1702,6 +1858,14 @@ var ChannelDriver = class {
1702
1858
  this.dispatched.add(message.id);
1703
1859
  this.registerInFlight(conv, sessionId, message, opencodeMessageId);
1704
1860
  dispatched += 1;
1861
+ void this.postSignal(conv.id, message.id, "dispatched");
1862
+ }
1863
+ if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
1864
+ this.log({
1865
+ level: "error",
1866
+ message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
1867
+ conversation_id: conv.id
1868
+ });
1705
1869
  }
1706
1870
  this.ensureWatcherRunning(sessionId);
1707
1871
  return dispatched;
@@ -1760,7 +1924,64 @@ var ChannelDriver = class {
1760
1924
  dispatchedAt: now,
1761
1925
  deadline: now + this.pausedMaxWaitMs,
1762
1926
  started: false,
1763
- done: false
1927
+ done: false,
1928
+ stuckReported: false,
1929
+ redriveAttempts: 0,
1930
+ lastRedriveAt: null,
1931
+ redriveOpencodeMessageId: null,
1932
+ attemptedOpencodeMessageIds: [opencodeMessageId]
1933
+ });
1934
+ }
1935
+ /**
1936
+ * Register a RE-ADOPTED `processing` message with its session watcher
1937
+ * (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
1938
+ * `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
1939
+ * `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
1940
+ * (10 min after `processed_at`), not 10 min from now — otherwise its deadline
1941
+ * lands ~15 min after `processed_at`, coinciding with the cron reset →
1942
+ * double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
1943
+ *
1944
+ * `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
1945
+ * `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
1946
+ * fresh-run path these differ (a fresh opencode id under the same server row).
1947
+ *
1948
+ * `started` is set true so the watcher does NOT re-`markProcessing` a row the
1949
+ * server already flipped to `processing`; the running/done transitions still
1950
+ * fire from the watcher's normal branches.
1951
+ */
1952
+ registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs, redriveOpencodeMessageId = null) {
1953
+ let watcher = this.watchers.get(sessionId);
1954
+ if (!watcher) {
1955
+ watcher = {
1956
+ conv,
1957
+ inFlight: /* @__PURE__ */ new Map(),
1958
+ loop: null,
1959
+ reportedQuestions: /* @__PURE__ */ new Set(),
1960
+ reportedPermissions: /* @__PURE__ */ new Set()
1961
+ };
1962
+ this.watchers.set(sessionId, watcher);
1963
+ }
1964
+ watcher.inFlight.set(message.id, {
1965
+ evidentMessageId: message.id,
1966
+ opencodeMessageId,
1967
+ message,
1968
+ dispatchedAt: this.now(),
1969
+ deadline: processedAtMs + this.pausedMaxWaitMs,
1970
+ // The server row is ALREADY `processing`; do not re-fire markProcessing.
1971
+ started: true,
1972
+ done: false,
1973
+ // Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
1974
+ // AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
1975
+ // gates on `state === 'queued'` (turn produced no reply), not on `started`,
1976
+ // so a re-adopted row left wedged in `queued` still emits the signal once
1977
+ // (queued-followup-redrive, #210). A re-adopted `queued` wedge is ALSO
1978
+ // re-driven by the same path (ADR-0046 §c deferred the live-session redrive
1979
+ // to here) — `redriveAttempts` starts fresh so it gets the full budget.
1980
+ stuckReported: false,
1981
+ redriveAttempts: 0,
1982
+ lastRedriveAt: null,
1983
+ redriveOpencodeMessageId,
1984
+ attemptedOpencodeMessageIds: [opencodeMessageId]
1764
1985
  });
1765
1986
  }
1766
1987
  /**
@@ -1826,6 +2047,7 @@ var ChannelDriver = class {
1826
2047
  conversation_id: watcher.conv.id
1827
2048
  });
1828
2049
  for (const evidentMessageId of [...watcher.inFlight.keys()]) {
2050
+ this.readopted.delete(evidentMessageId);
1829
2051
  this.removeInFlight(watcher, evidentMessageId);
1830
2052
  }
1831
2053
  return;
@@ -1846,10 +2068,15 @@ var ChannelDriver = class {
1846
2068
  async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
1847
2069
  const conv = watcher.conv;
1848
2070
  const state = messageRunState(messages, inFlight.opencodeMessageId);
1849
- if ((state === "running" || state === "done") && !inFlight.started) {
2071
+ if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
1850
2072
  let claimed;
1851
2073
  try {
1852
- claimed = await this.markProcessing(conv.id, inFlight.evidentMessageId, sessionId);
2074
+ claimed = await this.markProcessing(
2075
+ conv.id,
2076
+ inFlight.evidentMessageId,
2077
+ sessionId,
2078
+ inFlight.redriveOpencodeMessageId
2079
+ );
1853
2080
  } catch (err) {
1854
2081
  if (err instanceof ChannelAuthError) throw err;
1855
2082
  this.log({
@@ -1879,7 +2106,12 @@ var ChannelDriver = class {
1879
2106
  message_id: inFlight.evidentMessageId
1880
2107
  });
1881
2108
  try {
1882
- await this.markDone(conv.id, inFlight.evidentMessageId, sessionId);
2109
+ await this.markDone(
2110
+ conv.id,
2111
+ inFlight.evidentMessageId,
2112
+ sessionId,
2113
+ inFlight.redriveOpencodeMessageId
2114
+ );
1883
2115
  } catch (err) {
1884
2116
  if (err instanceof ChannelAuthError) throw err;
1885
2117
  if (err instanceof ChannelTerminalError) {
@@ -1915,9 +2147,70 @@ var ChannelDriver = class {
1915
2147
  this.removeInFlight(watcher, inFlight.evidentMessageId);
1916
2148
  return;
1917
2149
  }
2150
+ if (state === "failed") {
2151
+ if (!inFlight.done) {
2152
+ const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
2153
+ this.log({
2154
+ level: "error",
2155
+ message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
2156
+ conversation_id: conv.id,
2157
+ message_id: inFlight.evidentMessageId
2158
+ });
2159
+ try {
2160
+ await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
2161
+ } catch (err) {
2162
+ if (err instanceof ChannelAuthError) throw err;
2163
+ if (err instanceof ChannelTerminalError) {
2164
+ this.log({
2165
+ level: "error",
2166
+ message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
2167
+ conversation_id: conv.id,
2168
+ message_id: inFlight.evidentMessageId
2169
+ });
2170
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2171
+ return;
2172
+ }
2173
+ if (this.now() >= inFlight.deadline) {
2174
+ this.log({
2175
+ level: "error",
2176
+ message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
2177
+ conversation_id: conv.id,
2178
+ message_id: inFlight.evidentMessageId
2179
+ });
2180
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2181
+ return;
2182
+ }
2183
+ this.log({
2184
+ level: "error",
2185
+ message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
2186
+ conversation_id: conv.id,
2187
+ message_id: inFlight.evidentMessageId
2188
+ });
2189
+ return;
2190
+ }
2191
+ inFlight.done = true;
2192
+ }
2193
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2194
+ return;
2195
+ }
1918
2196
  if (state === "unknown") {
1919
2197
  if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
1920
- await this.redispatchInFlight(sessionId, inFlight);
2198
+ await this.redispatchInFlight(conv.id, sessionId, inFlight);
2199
+ }
2200
+ }
2201
+ const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
2202
+ const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
2203
+ if (state === "queued" && pastStuckBound && sessionIdle) {
2204
+ if (!inFlight.stuckReported) {
2205
+ inFlight.stuckReported = true;
2206
+ void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
2207
+ stuck_for_ms: this.now() - inFlight.dispatchedAt
2208
+ });
2209
+ }
2210
+ const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
2211
+ if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
2212
+ const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
2213
+ if (removed) return;
1921
2214
  }
1922
2215
  }
1923
2216
  if (this.now() >= inFlight.deadline) {
@@ -1927,6 +2220,9 @@ var ChannelDriver = class {
1927
2220
  conversation_id: conv.id,
1928
2221
  message_id: inFlight.evidentMessageId
1929
2222
  });
2223
+ void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
2224
+ watched_for_ms: this.now() - inFlight.dispatchedAt
2225
+ });
1930
2226
  this.removeInFlight(watcher, inFlight.evidentMessageId);
1931
2227
  }
1932
2228
  }
@@ -1936,7 +2232,7 @@ var ChannelDriver = class {
1936
2232
  * fact 9) — one user message + one reply even if the original DID land. Resets
1937
2233
  * the dispatch timestamp so the guard doesn't immediately fire again.
1938
2234
  */
1939
- async redispatchInFlight(sessionId, inFlight) {
2235
+ async redispatchInFlight(conversationId, sessionId, inFlight) {
1940
2236
  const options = {
1941
2237
  agent: inFlight.message.opencode_agent ?? void 0,
1942
2238
  model: inFlight.message.opencode_model ?? void 0
@@ -1946,6 +2242,7 @@ var ChannelDriver = class {
1946
2242
  message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
1947
2243
  message_id: inFlight.evidentMessageId
1948
2244
  });
2245
+ void this.postSignal(conversationId, inFlight.evidentMessageId, "redispatched");
1949
2246
  try {
1950
2247
  await sendPromptAsync(
1951
2248
  this.port,
@@ -1963,12 +2260,504 @@ var ChannelDriver = class {
1963
2260
  }
1964
2261
  inFlight.dispatchedAt = this.now();
1965
2262
  }
2263
+ /**
2264
+ * RE-DRIVE a stuck-`queued` follow-up so opencode actually runs it
2265
+ * (queued-followup-redrive; the fix for the 2026-07-18 dev incident).
2266
+ *
2267
+ * The wedge: a follow-up `prompt_async`'d mid-turn / in the post-turn settling
2268
+ * window is orphaned — its user message persists but the turn never runs
2269
+ * (`messageRunState === 'queued'`). Proven against real opencode 1.18.3 by an
2270
+ * isolated, interleaved experiment (`.harness/followup-send-mechanism-finding.md`):
2271
+ * the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
2272
+ * runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
2273
+ * turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
2274
+ * (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
2275
+ * dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
2276
+ *
2277
+ * So we re-`prompt_async` into the SAME opencode session (opencode-web does
2278
+ * exactly this and it works — UPDATE 2 in the investigation doc), preserving the
2279
+ * conversation's history/continuity: we do NOT create a fresh session and we do
2280
+ * NOT overwrite the conversation's `opencode_session_id`. Only the opencode
2281
+ * user-message id changes — to a fresh native id per attempt (opencode's
2282
+ * caller-supplied id dedup is global+permanent, so each attempt needs a never-
2283
+ * seen id).
2284
+ *
2285
+ * Reply correlation is PRESERVED: the native id is random and NOT re-derivable
2286
+ * from the row id, so the runner carries it to the server (on markProcessing/
2287
+ * markDone via `redriveOpencodeMessageId`), which persists it on the row and
2288
+ * correlates the reply by THAT id (`conversation-notification.ts`). A normal,
2289
+ * never-re-driven message still correlates by the derived stable id — unchanged.
2290
+ *
2291
+ * Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
2292
+ * FAILED (existing channel failure affordance) so the user is TOLD it could not
2293
+ * be answered rather than left silent, and it is removed from the in-flight set.
2294
+ *
2295
+ * @returns true if the message LEFT this watcher's in-flight set (gave up +
2296
+ * marked failed) — the caller then stops servicing it this tick. Returns false
2297
+ * when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
2298
+ * send failed and the next window retries).
2299
+ */
2300
+ async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
2301
+ if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
2302
+ return true;
2303
+ }
2304
+ if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
2305
+ this.log({
2306
+ level: "error",
2307
+ message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
2308
+ conversation_id: watcher.conv.id,
2309
+ message_id: inFlight.evidentMessageId
2310
+ });
2311
+ try {
2312
+ await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
2313
+ } catch (err) {
2314
+ if (err instanceof ChannelAuthError) throw err;
2315
+ this.log({
2316
+ level: "error",
2317
+ message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
2318
+ conversation_id: watcher.conv.id,
2319
+ message_id: inFlight.evidentMessageId
2320
+ });
2321
+ }
2322
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2323
+ return true;
2324
+ }
2325
+ inFlight.redriveAttempts += 1;
2326
+ inFlight.lastRedriveAt = this.now();
2327
+ this.log({
2328
+ level: "info",
2329
+ message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
2330
+ conversation_id: watcher.conv.id,
2331
+ message_id: inFlight.evidentMessageId
2332
+ });
2333
+ void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
2334
+ redrive_attempt: inFlight.redriveAttempts
2335
+ });
2336
+ const options = {
2337
+ agent: inFlight.message.opencode_agent ?? void 0,
2338
+ model: inFlight.message.opencode_model ?? void 0
2339
+ };
2340
+ const nativeId = nativeOpencodeMessageId();
2341
+ try {
2342
+ await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
2343
+ } catch (err) {
2344
+ if (err instanceof ChannelAuthError) throw err;
2345
+ inFlight.redriveAttempts -= 1;
2346
+ this.log({
2347
+ level: "error",
2348
+ message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
2349
+ conversation_id: watcher.conv.id,
2350
+ message_id: inFlight.evidentMessageId
2351
+ });
2352
+ return false;
2353
+ }
2354
+ inFlight.opencodeMessageId = nativeId;
2355
+ inFlight.redriveOpencodeMessageId = nativeId;
2356
+ inFlight.attemptedOpencodeMessageIds.push(nativeId);
2357
+ inFlight.started = false;
2358
+ return false;
2359
+ }
2360
+ /**
2361
+ * Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
2362
+ * driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
2363
+ * in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
2364
+ * earlier re-drive attempt eventually produced is delivered) instead of
2365
+ * re-driving again or marking failed. Idempotent + guarded like the watcher's
2366
+ * done branch. `@returns` true when the message left the in-flight set (delivered
2367
+ * or terminally-undeliverable) — the caller then stops servicing it this tick.
2368
+ *
2369
+ * The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
2370
+ * branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
2371
+ * polls. `messageRunState === 'done'` means a completed, non-errored correlated
2372
+ * reply exists for that id.
2373
+ */
2374
+ async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
2375
+ const completedId = inFlight.attemptedOpencodeMessageIds.find(
2376
+ (id) => messageRunState(messages, id) === "done"
2377
+ );
2378
+ if (!completedId) return false;
2379
+ if (inFlight.done) {
2380
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2381
+ return true;
2382
+ }
2383
+ this.log({
2384
+ level: "info",
2385
+ message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
2386
+ conversation_id: watcher.conv.id,
2387
+ message_id: inFlight.evidentMessageId
2388
+ });
2389
+ try {
2390
+ await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
2391
+ } catch (err) {
2392
+ if (err instanceof ChannelAuthError) throw err;
2393
+ this.log({
2394
+ level: "error",
2395
+ message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done off an earlier re-drive attempt (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
2396
+ conversation_id: watcher.conv.id,
2397
+ message_id: inFlight.evidentMessageId
2398
+ });
2399
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2400
+ return true;
2401
+ }
2402
+ inFlight.done = true;
2403
+ this.removeInFlight(watcher, inFlight.evidentMessageId);
2404
+ return true;
2405
+ }
2406
+ // -------------------------------------------------------------------------
2407
+ // Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
2408
+ // -------------------------------------------------------------------------
2409
+ /**
2410
+ * Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
2411
+ *
2412
+ * The pending drain only re-drives `pending` rows; a message already flipped to
2413
+ * `processing` before the runner died is watched by nobody until the 15-min
2414
+ * cron resets it. Here we fetch those rows, and per row resolve its correlated
2415
+ * reply against opencode's OWN session store — completing, re-attaching, or
2416
+ * (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
2417
+ * is idempotent per message (Invariant 2): a row a watcher already tracks is
2418
+ * skipped in `readoptOne` — one driver, no double-drive.
2419
+ *
2420
+ * Only `ChannelAuthError` propagates (to `drainPending`, like the pending
2421
+ * path); every other early return LOGS a reason with context — no silent drop.
2422
+ */
2423
+ async readoptProcessing() {
2424
+ const rows = await this.getProcessingMessages();
2425
+ if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
2426
+ const stillProcessing = new Set(rows.map((r) => r.id));
2427
+ for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
2428
+ if (!stillProcessing.has(id)) {
2429
+ const cleared = this.dontRedispatch.delete(id);
2430
+ const clearedUndeliverable = this.doneUndeliverable.delete(id);
2431
+ if (cleared || clearedUndeliverable) {
2432
+ this.log({
2433
+ level: "info",
2434
+ message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
2435
+ message_id: id
2436
+ });
2437
+ }
2438
+ }
2439
+ }
2440
+ }
2441
+ if (rows.length === 0) return;
2442
+ const bySession = /* @__PURE__ */ new Map();
2443
+ for (const row of rows) {
2444
+ if (!row.opencode_session_id) {
2445
+ this.log({
2446
+ level: "error",
2447
+ message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
2448
+ conversation_id: row.conversation_id,
2449
+ message_id: row.id
2450
+ });
2451
+ continue;
2452
+ }
2453
+ const list = bySession.get(row.opencode_session_id) ?? [];
2454
+ list.push(row);
2455
+ bySession.set(row.opencode_session_id, list);
2456
+ }
2457
+ for (const [sessionId, sessionRows] of bySession) {
2458
+ let messages;
2459
+ try {
2460
+ const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
2461
+ if (!res.ok) {
2462
+ this.log({
2463
+ level: "error",
2464
+ message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
2465
+ });
2466
+ continue;
2467
+ }
2468
+ const body = await res.json();
2469
+ if (!Array.isArray(body)) {
2470
+ this.log({
2471
+ level: "error",
2472
+ message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
2473
+ });
2474
+ continue;
2475
+ }
2476
+ messages = body;
2477
+ } catch (err) {
2478
+ this.log({
2479
+ level: "error",
2480
+ message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
2481
+ });
2482
+ continue;
2483
+ }
2484
+ for (const row of sessionRows) {
2485
+ await this.readoptOne(sessionId, row, messages);
2486
+ }
2487
+ }
2488
+ }
2489
+ /**
2490
+ * Re-adopt ONE `processing` row against the tick's session message snapshot
2491
+ * (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
2492
+ *
2493
+ * Branches on `messageRunState(messages, effectiveId)`, where `effectiveId` is
2494
+ * the NATIVE id a prior lifetime's re-drive ran this row under
2495
+ * (`row.opencode_message_id`) if present, else the STABLE derived id
2496
+ * (`opencodeMessageIdFor(row.id)`). Consulting the native id is what makes a
2497
+ * re-driven row that ALREADY ran/completed resolve correctly on restart instead
2498
+ * of looking `queued` under the (never-run) stable id and being re-driven AGAIN
2499
+ * (Bugbot #217 Finding 3 — the duplicate-turn bug):
2500
+ * - `done` → `markDone` now (guarded like the watcher's done branch);
2501
+ * - `failed` → `markFailed` with the surfaced error (issue #182), so an
2502
+ * errored turn is reported failed on restart, NOT re-dispatched;
2503
+ * - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
2504
+ * tracking the effective id so the reply correlates by it;
2505
+ * - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
2506
+ *
2507
+ * Only `ChannelAuthError` propagates.
2508
+ */
2509
+ async readoptOne(sessionId, row, messages) {
2510
+ if (this.isTracked(sessionId, row.id)) {
2511
+ this.log({
2512
+ level: "info",
2513
+ message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
2514
+ conversation_id: row.conversation_id,
2515
+ message_id: row.id
2516
+ });
2517
+ return;
2518
+ }
2519
+ const stableId = opencodeMessageIdFor2(row.id);
2520
+ const nativeRedriveId = row.opencode_message_id;
2521
+ const ocId = nativeRedriveId ?? stableId;
2522
+ const state = messageRunState(messages, ocId);
2523
+ if (state === "done") {
2524
+ if (this.doneUndeliverable.has(row.id)) {
2525
+ this.log({
2526
+ level: "info",
2527
+ message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
2528
+ conversation_id: row.conversation_id,
2529
+ message_id: row.id
2530
+ });
2531
+ return;
2532
+ }
2533
+ this.log({
2534
+ level: "info",
2535
+ message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
2536
+ conversation_id: row.conversation_id,
2537
+ message_id: row.id
2538
+ });
2539
+ try {
2540
+ await this.markDone(row.conversation_id, row.id, sessionId, nativeRedriveId);
2541
+ } catch (err) {
2542
+ if (err instanceof ChannelAuthError) throw err;
2543
+ if (err instanceof ChannelTerminalError) {
2544
+ this.doneUndeliverable.add(row.id);
2545
+ this.log({
2546
+ level: "error",
2547
+ message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
2548
+ conversation_id: row.conversation_id,
2549
+ message_id: row.id
2550
+ });
2551
+ return;
2552
+ }
2553
+ this.log({
2554
+ level: "error",
2555
+ message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
2556
+ conversation_id: row.conversation_id,
2557
+ message_id: row.id
2558
+ });
2559
+ return;
2560
+ }
2561
+ this.dontRedispatch.delete(row.id);
2562
+ return;
2563
+ }
2564
+ if (state === "failed") {
2565
+ const error2 = messageError(messages, ocId) ?? void 0;
2566
+ this.log({
2567
+ level: "error",
2568
+ message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
2569
+ conversation_id: row.conversation_id,
2570
+ message_id: row.id
2571
+ });
2572
+ try {
2573
+ await this.markFailed(row.conversation_id, row.id, sessionId, error2);
2574
+ } catch (err) {
2575
+ if (err instanceof ChannelAuthError) throw err;
2576
+ if (err instanceof ChannelTerminalError) {
2577
+ this.doneUndeliverable.add(row.id);
2578
+ this.log({
2579
+ level: "error",
2580
+ message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
2581
+ conversation_id: row.conversation_id,
2582
+ message_id: row.id
2583
+ });
2584
+ return;
2585
+ }
2586
+ this.log({
2587
+ level: "error",
2588
+ message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
2589
+ conversation_id: row.conversation_id,
2590
+ message_id: row.id
2591
+ });
2592
+ return;
2593
+ }
2594
+ this.dontRedispatch.delete(row.id);
2595
+ return;
2596
+ }
2597
+ if (this.dontRedispatch.has(row.id)) {
2598
+ this.log({
2599
+ level: "info",
2600
+ message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
2601
+ conversation_id: row.conversation_id,
2602
+ message_id: row.id
2603
+ });
2604
+ return;
2605
+ }
2606
+ if (state === "running" || state === "queued") {
2607
+ const conv = this.convForRow(sessionId, row);
2608
+ const message = this.queuedMessageForRow(row);
2609
+ this.registerReadopted(
2610
+ conv,
2611
+ sessionId,
2612
+ message,
2613
+ ocId,
2614
+ this.processedAtMs(row),
2615
+ nativeRedriveId
2616
+ );
2617
+ this.dispatched.add(row.id);
2618
+ this.readopted.add(row.id);
2619
+ this.ensureWatcherRunning(sessionId);
2620
+ this.log({
2621
+ level: "info",
2622
+ message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (${nativeRedriveId ? "native re-drive id" : "stable id"}, no re-dispatch)`,
2623
+ conversation_id: row.conversation_id,
2624
+ message_id: row.id
2625
+ });
2626
+ return;
2627
+ }
2628
+ await this.forceReadoptRun(sessionId, row);
2629
+ }
2630
+ /**
2631
+ * Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
2632
+ *
2633
+ * The stable-id user message is absent from the session, so we (re-)dispatch with
2634
+ * the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
2635
+ * This is what keeps the reply correlatable: the server's completion
2636
+ * notification looks for the reply under the stable id, so the fresh turn's reply
2637
+ * (which hangs off the stable id) is found and delivered. The residual
2638
+ * duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
2639
+ * are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
2640
+ * to swallow the duplicate.
2641
+ *
2642
+ * `evidentMessageId = row.id` addresses the SERVER row; the stable
2643
+ * `opencodeMessageId` is what the watcher polls. Deadline anchored to
2644
+ * `processed_at` (Invariant 1).
2645
+ */
2646
+ async forceReadoptRun(sessionId, row) {
2647
+ const ocId = opencodeMessageIdFor2(row.id);
2648
+ if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
2649
+ this.dontRedispatch.add(row.id);
2650
+ this.log({
2651
+ level: "info",
2652
+ message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
2653
+ conversation_id: row.conversation_id,
2654
+ message_id: row.id
2655
+ });
2656
+ return;
2657
+ }
2658
+ const options = {
2659
+ agent: row.opencode_agent ?? void 0,
2660
+ model: row.opencode_model ?? void 0
2661
+ };
2662
+ this.log({
2663
+ level: "info",
2664
+ message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
2665
+ conversation_id: row.conversation_id,
2666
+ message_id: row.id
2667
+ });
2668
+ try {
2669
+ await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
2670
+ } catch (err) {
2671
+ if (err instanceof ChannelAuthError) throw err;
2672
+ this.log({
2673
+ level: "error",
2674
+ message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
2675
+ conversation_id: row.conversation_id,
2676
+ message_id: row.id
2677
+ });
2678
+ return;
2679
+ }
2680
+ const conv = this.convForRow(sessionId, row);
2681
+ const message = this.queuedMessageForRow(row);
2682
+ this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
2683
+ this.dispatched.add(row.id);
2684
+ this.readopted.add(row.id);
2685
+ this.ensureWatcherRunning(sessionId);
2686
+ }
2687
+ /**
2688
+ * True if `evidentMessageId` is already being driven — either in the
2689
+ * authoritative `dispatched` set or a live watcher's in-flight set for this
2690
+ * session (Invariant 2, WI-5). Either signal means a watcher owns the row.
2691
+ */
2692
+ isTracked(sessionId, evidentMessageId) {
2693
+ if (this.dispatched.has(evidentMessageId)) return true;
2694
+ const watcher = this.watchers.get(sessionId);
2695
+ return watcher?.inFlight.has(evidentMessageId) ?? false;
2696
+ }
2697
+ /**
2698
+ * Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
2699
+ * deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
2700
+ * for `processing` rows, but if it is somehow null/unparseable fall back to
2701
+ * `now` (defensive) AND log — a fallback means the anchor is weaker than
2702
+ * intended, which is worth surfacing.
2703
+ */
2704
+ processedAtMs(row) {
2705
+ const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
2706
+ if (!Number.isNaN(parsed)) return parsed;
2707
+ this.log({
2708
+ level: "error",
2709
+ message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
2710
+ conversation_id: row.conversation_id,
2711
+ message_id: row.id
2712
+ });
2713
+ return this.now();
2714
+ }
2715
+ /** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
2716
+ convForRow(sessionId, row) {
2717
+ return {
2718
+ id: row.conversation_id,
2719
+ agent_id: this.agentId,
2720
+ opencode_session_id: sessionId,
2721
+ pending_message_count: 0,
2722
+ oldest_pending_at: row.processed_at
2723
+ };
2724
+ }
2725
+ /** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
2726
+ queuedMessageForRow(row) {
2727
+ return {
2728
+ id: row.id,
2729
+ content: row.content,
2730
+ status: "processing",
2731
+ opencode_agent: row.opencode_agent,
2732
+ opencode_model: row.opencode_model,
2733
+ source_message_id: row.source_message_id,
2734
+ slack_user_id: row.slack_user_id
2735
+ };
2736
+ }
1966
2737
  /**
1967
2738
  * Remove a message from the in-flight set AND the authoritative dispatched
1968
2739
  * set. Once the in-flight set empties, the watcher loop's `while` guard exits
1969
2740
  * and its `.finally` removes the session entry from `this.watchers`.
2741
+ *
2742
+ * Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
2743
+ * (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
2744
+ * cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
2745
+ * re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
2746
+ * re-adopted message that completed (`done`) needs no marker — it's leaving
2747
+ * `processing`. This suppresses only re-dispatch: if its reply later completes,
2748
+ * the done branch still delivers it (Bugbot #202).
1970
2749
  */
1971
2750
  removeInFlight(watcher, evidentMessageId) {
2751
+ const inFlight = watcher.inFlight.get(evidentMessageId);
2752
+ if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
2753
+ this.dontRedispatch.add(evidentMessageId);
2754
+ this.log({
2755
+ level: "info",
2756
+ message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
2757
+ conversation_id: watcher.conv.id,
2758
+ message_id: evidentMessageId
2759
+ });
2760
+ }
1972
2761
  watcher.inFlight.delete(evidentMessageId);
1973
2762
  this.dispatched.delete(evidentMessageId);
1974
2763
  }
@@ -2112,6 +2901,35 @@ var ChannelDriver = class {
2112
2901
  }
2113
2902
  return await res.json();
2114
2903
  }
2904
+ /**
2905
+ * Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
2906
+ * The pending path (`getPendingConversations`/`getPendingMessages`) only
2907
+ * surfaces `pending` rows, so a message already `processing` when the runner
2908
+ * died is invisible to it — this dedicated endpoint returns exactly those rows
2909
+ * with the fields the re-adopt path needs (`processed_at`,
2910
+ * `opencode_session_id`, routing).
2911
+ *
2912
+ * Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
2913
+ * array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
2914
+ * other non-ok so `drainPending`'s try/finally leaves `draining` false and the
2915
+ * next tick retries.
2916
+ */
2917
+ async getProcessingMessages() {
2918
+ const res = await this.fetchImpl(
2919
+ `${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
2920
+ { headers: { Authorization: this.getAuthHeader() } }
2921
+ );
2922
+ this.assertAuth(res, "fetching processing messages");
2923
+ if (!res.ok) {
2924
+ throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
2925
+ }
2926
+ const data = await res.json();
2927
+ let messages = data.messages ?? [];
2928
+ if (this.conversationFilter) {
2929
+ messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
2930
+ }
2931
+ return messages;
2932
+ }
2115
2933
  /**
2116
2934
  * EXISTING combinedAuth route — now fired by the watcher on queued→running
2117
2935
  * (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
@@ -2133,13 +2951,17 @@ var ChannelDriver = class {
2133
2951
  * A single attempt (no internal retry): the watcher's per-tick loop is the
2134
2952
  * retry vehicle for the swap-to-running.
2135
2953
  */
2136
- async markProcessing(conversationId, messageId, sessionId) {
2954
+ async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
2137
2955
  const res = await this.fetchImpl(
2138
2956
  `${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
2139
2957
  {
2140
2958
  method: "PATCH",
2141
2959
  headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
2142
- body: JSON.stringify({ status: "processing", opencode_session_id: sessionId })
2960
+ body: JSON.stringify({
2961
+ status: "processing",
2962
+ opencode_session_id: sessionId,
2963
+ ...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
2964
+ })
2143
2965
  }
2144
2966
  );
2145
2967
  this.assertAuth(res, "marking message as processing");
@@ -2177,13 +2999,17 @@ var ChannelDriver = class {
2177
2999
  * watcher retries next tick within the
2178
3000
  * deadline, Finding 4).
2179
3001
  */
2180
- async markDone(conversationId, messageId, sessionId) {
3002
+ async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
2181
3003
  const res = await this.fetchImpl(
2182
3004
  `${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
2183
3005
  {
2184
3006
  method: "PATCH",
2185
3007
  headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
2186
- body: JSON.stringify({ status: "done", opencode_session_id: sessionId })
3008
+ body: JSON.stringify({
3009
+ status: "done",
3010
+ opencode_session_id: sessionId,
3011
+ ...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
3012
+ })
2187
3013
  }
2188
3014
  );
2189
3015
  this.assertAuth(res, "marking message as done");
@@ -2193,7 +3019,17 @@ var ChannelDriver = class {
2193
3019
  }
2194
3020
  throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
2195
3021
  }
2196
- async markFailed(conversationId, messageId) {
3022
+ /**
3023
+ * Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
3024
+ * when provided (issue #182): a bare `markFailed(conv, msg)` sends
3025
+ * `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
3026
+ * OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
3027
+ * failure reason reaches the channel.
3028
+ */
3029
+ async markFailed(conversationId, messageId, sessionId, error2) {
3030
+ const body = { status: "failed" };
3031
+ if (sessionId !== void 0) body.opencode_session_id = sessionId;
3032
+ if (error2 !== void 0) body.error = error2;
2197
3033
  await this.callWithRetry(
2198
3034
  "marking message as failed",
2199
3035
  () => this.fetchImpl(
@@ -2201,11 +3037,47 @@ var ChannelDriver = class {
2201
3037
  {
2202
3038
  method: "PATCH",
2203
3039
  headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
2204
- body: JSON.stringify({ status: "failed" })
3040
+ body: JSON.stringify(body)
2205
3041
  }
2206
3042
  )
2207
3043
  );
2208
3044
  }
3045
+ /**
3046
+ * Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
3047
+ * (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
3048
+ * — the server records it via `log()` (no DB write, no notification). This is
3049
+ * fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
3050
+ * MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
3051
+ * watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
3052
+ * context (no silent catch, per development-workflow).
3053
+ */
3054
+ async postSignal(conversationId, messageId, signal, extra) {
3055
+ try {
3056
+ const res = await this.fetchImpl(
3057
+ `${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
3058
+ {
3059
+ method: "POST",
3060
+ headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
3061
+ body: JSON.stringify({ signal, ...extra })
3062
+ }
3063
+ );
3064
+ if (!res.ok) {
3065
+ this.log({
3066
+ level: "error",
3067
+ message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
3068
+ conversation_id: conversationId,
3069
+ message_id: messageId
3070
+ });
3071
+ }
3072
+ } catch (err) {
3073
+ this.log({
3074
+ level: "error",
3075
+ message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
3076
+ conversation_id: conversationId,
3077
+ message_id: messageId
3078
+ });
3079
+ }
3080
+ }
2209
3081
  async persistSession(conversationId, sessionId) {
2210
3082
  const res = await this.fetchImpl(
2211
3083
  `${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
@@ -2521,7 +3393,9 @@ async function getAgentInfo(agentId, authHeader) {
2521
3393
  // src/commands/run.ts
2522
3394
  var MAX_ACTIVITY_LOG_ENTRIES = 10;
2523
3395
  var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
2524
- function log(state, message, isError = false) {
3396
+ var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
3397
+ var CHANNEL_SETTLE_MS = Number(process.env.EVIDENT_SETTLE_MS) || void 0;
3398
+ function log2(state, message, isError = false) {
2525
3399
  if (state.json) {
2526
3400
  console.log(
2527
3401
  JSON.stringify({
@@ -2546,9 +3420,9 @@ function logActivity(state, entry) {
2546
3420
  }
2547
3421
  if (!state.interactive) {
2548
3422
  if (entry.type === "error") {
2549
- log(state, entry.error ?? "Unknown error", true);
3423
+ log2(state, entry.error ?? "Unknown error", true);
2550
3424
  } else if (entry.type === "info" && entry.message) {
2551
- log(state, entry.message);
3425
+ log2(state, entry.message);
2552
3426
  }
2553
3427
  }
2554
3428
  }
@@ -2634,6 +3508,7 @@ async function handleAuthError(state, error2) {
2634
3508
  }
2635
3509
  async function driveChannels(state, driver) {
2636
3510
  let idlePolls = 0;
3511
+ let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
2637
3512
  while (state.running) {
2638
3513
  if (state.connection?.reconnecting && state.connection.reconnectPromise) {
2639
3514
  logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
@@ -2643,7 +3518,9 @@ async function driveChannels(state, driver) {
2643
3518
  try {
2644
3519
  const processed = await driver.drainPending();
2645
3520
  state.messageCount += processed;
2646
- if (processed > 0 || driver.hasInFlightWatchers()) {
3521
+ const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
3522
+ lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
3523
+ if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
2647
3524
  idlePolls = 0;
2648
3525
  if (processed > 0 && state.interactive) displayStatus(state);
2649
3526
  } else if (state.idleTimeout !== null) {
@@ -2695,7 +3572,7 @@ async function cleanup(state) {
2695
3572
  logActivity(state, { type: "info", message: "Stopped OpenCode process" });
2696
3573
  displayStatus(state);
2697
3574
  } else {
2698
- log(state, "Stopped OpenCode process");
3575
+ log2(state, "Stopped OpenCode process");
2699
3576
  }
2700
3577
  state.opencodeProcess = null;
2701
3578
  }
@@ -2718,10 +3595,11 @@ async function run(options) {
2718
3595
  running: true,
2719
3596
  activityLog: [],
2720
3597
  messageCount: 0,
3598
+ lastProxiedActivityAt: null,
2721
3599
  authHeader: ""
2722
3600
  };
2723
3601
  if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
2724
- log(
3602
+ log2(
2725
3603
  state,
2726
3604
  "Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
2727
3605
  false
@@ -2732,7 +3610,7 @@ async function run(options) {
2732
3610
  logActivity(state, { type: "info", message: "Shutting down..." });
2733
3611
  displayStatus(state);
2734
3612
  } else {
2735
- log(state, "Shutting down...");
3613
+ log2(state, "Shutting down...");
2736
3614
  }
2737
3615
  await cleanup(state);
2738
3616
  await shutdownTelemetry();
@@ -2765,7 +3643,7 @@ async function run(options) {
2765
3643
  const resolved = await resolveAgentIdFromKey(state.authHeader);
2766
3644
  if (resolved.agent_id) {
2767
3645
  state.agentId = resolved.agent_id;
2768
- log(state, `Resolved agent ID from key: ${state.agentId}`);
3646
+ log2(state, `Resolved agent ID from key: ${state.agentId}`);
2769
3647
  if (state.interactive && !state.json) {
2770
3648
  logActivity(state, {
2771
3649
  type: "info",
@@ -2828,17 +3706,17 @@ async function run(options) {
2828
3706
  port: state.port,
2829
3707
  interactive: state.interactive,
2830
3708
  agentId: state.agentId,
2831
- log: (message) => log(state, message)
3709
+ log: (message) => log2(state, message)
2832
3710
  });
2833
3711
  state.port = oc.port;
2834
3712
  state.opencodeProcess = oc.process;
2835
3713
  state.opencodeVersion = oc.version;
2836
3714
  state.opencodeConnected = oc.process !== null || oc.version !== null;
2837
- const version = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
2838
- ocSpinner?.succeed(`OpenCode running on port ${state.port}${version}`);
3715
+ const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
3716
+ ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
2839
3717
  const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
2840
3718
  if (versionWarning) {
2841
- log(state, versionWarning, false);
3719
+ log2(state, versionWarning, false);
2842
3720
  if (state.interactive && !state.json) {
2843
3721
  logActivity(state, { type: "info", message: versionWarning });
2844
3722
  }
@@ -2854,6 +3732,8 @@ async function run(options) {
2854
3732
  apiUrl: getApiUrlConfig(),
2855
3733
  getAuthHeader: () => state.authHeader,
2856
3734
  conversationFilter: state.conversationFilter,
3735
+ stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
3736
+ settleMs: CHANNEL_SETTLE_MS,
2857
3737
  log: (entry) => logActivity(state, {
2858
3738
  type: entry.level === "error" ? "error" : "info",
2859
3739
  message: entry.message,
@@ -2873,7 +3753,11 @@ async function run(options) {
2873
3753
  type: "info",
2874
3754
  message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
2875
3755
  });
2876
- emitAgentConnected(state.agentId, { port: state.port });
3756
+ emitAgentConnected(state.agentId, {
3757
+ port: state.port,
3758
+ cli_version: getCliVersion(),
3759
+ opencode_version: state.opencodeVersion
3760
+ });
2877
3761
  if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
2878
3762
  if (state.interactive) displayStatus(state);
2879
3763
  channelDriver.drainPending().then((processed) => {
@@ -2907,9 +3791,14 @@ async function run(options) {
2907
3791
  logActivity(state, { type: "error", error: error2 });
2908
3792
  if (state.interactive) displayStatus(state);
2909
3793
  },
2910
- // Web traffic is proxied transparently; only note opencode is live.
3794
+ // Web traffic is proxied transparently; note opencode is live and stamp
3795
+ // proxied activity so the idle loop treats interactive proxy use as work.
3796
+ // Fires per forwarded response head (incl. every SSE open) and excludes
3797
+ // the internal drain-ping, so an actively-used proxy keeps the timer
3798
+ // fresh while a lone idle SSE with no follow-up requests still ages out.
2911
3799
  onResponse: () => {
2912
3800
  state.opencodeConnected = true;
3801
+ state.lastProxiedActivityAt = Date.now();
2913
3802
  },
2914
3803
  // A channel message was queued and the api-worker pinged us over the
2915
3804
  // tunnel to drain immediately instead of waiting for the next poll tick.
@@ -2948,7 +3837,7 @@ async function run(options) {
2948
3837
  throw error2;
2949
3838
  }
2950
3839
  if (!interactive || state.json) {
2951
- log(state, "Driving channel messages...");
3840
+ log2(state, "Driving channel messages...");
2952
3841
  }
2953
3842
  await driveChannels(state, channelDriver);
2954
3843
  await cleanup(state);
@@ -2960,7 +3849,7 @@ async function run(options) {
2960
3849
  })
2961
3850
  );
2962
3851
  } else if (!interactive) {
2963
- log(state, `Completed. Processed ${state.messageCount} message(s).`);
3852
+ log2(state, `Completed. Processed ${state.messageCount} message(s).`);
2964
3853
  }
2965
3854
  await shutdownTelemetry();
2966
3855
  process.exit(0);
@@ -2982,8 +3871,9 @@ async function run(options) {
2982
3871
  }
2983
3872
 
2984
3873
  // src/index.ts
3874
+ var { version } = createRequire(import.meta.url)("../package.json");
2985
3875
  var program = new Command();
2986
- program.name("evident").description("Run OpenCode locally and connect it to Evident").version("0.1.0").option(
3876
+ program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
2987
3877
  "--endpoint <url>",
2988
3878
  "Evident API base URL (default: production; e.g. http://localhost:3001)"
2989
3879
  ).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {