@evident-ai/cli 3.0.1-dev.3007897 → 3.0.1-dev.379acd9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +937 -47
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
+
import { createRequire } from "module";
|
|
4
5
|
import { Command } from "commander";
|
|
5
6
|
|
|
6
7
|
// src/commands/login.ts
|
|
@@ -470,6 +471,12 @@ import chalk6 from "chalk";
|
|
|
470
471
|
import ora3 from "ora";
|
|
471
472
|
import { select as select3 } from "@inquirer/prompts";
|
|
472
473
|
|
|
474
|
+
// ../../packages/types/src/opencode/index.ts
|
|
475
|
+
function opencodeMessageIdFor(queuedMessageId) {
|
|
476
|
+
const sanitized = queuedMessageId.replace(/[^a-zA-Z0-9]/g, "_");
|
|
477
|
+
return `msg_${sanitized}`;
|
|
478
|
+
}
|
|
479
|
+
|
|
473
480
|
// ../../packages/types/src/telemetry/index.ts
|
|
474
481
|
var TelemetryEventTypes = {
|
|
475
482
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -484,8 +491,34 @@ var TelemetryEventTypes = {
|
|
|
484
491
|
var MAX_FRAME_BYTES = 256 * 1024;
|
|
485
492
|
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
486
493
|
|
|
494
|
+
// ../../packages/types/src/logging/index.ts
|
|
495
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
496
|
+
function log(level, event, fields) {
|
|
497
|
+
const method = level === "debug" ? "log" : level;
|
|
498
|
+
try {
|
|
499
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
500
|
+
} catch (err) {
|
|
501
|
+
console.error(
|
|
502
|
+
"[evident] log_serialize_failed",
|
|
503
|
+
event,
|
|
504
|
+
err instanceof Error ? err.message : String(err)
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
function stripQuery(url) {
|
|
509
|
+
try {
|
|
510
|
+
return new URL(url).pathname;
|
|
511
|
+
} catch {
|
|
512
|
+
const q = url.indexOf("?");
|
|
513
|
+
return q === -1 ? url : url.slice(0, q);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
487
517
|
// src/lib/telemetry.ts
|
|
488
|
-
var CLI_VERSION = process.env.npm_package_version
|
|
518
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
519
|
+
function getCliVersion() {
|
|
520
|
+
return CLI_VERSION;
|
|
521
|
+
}
|
|
489
522
|
var eventBuffer = [];
|
|
490
523
|
var flushTimeout = null;
|
|
491
524
|
var isShuttingDown = false;
|
|
@@ -686,13 +719,13 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
686
719
|
|
|
687
720
|
// src/lib/opencode/opencode-version-gate.ts
|
|
688
721
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11"];
|
|
689
|
-
function isQueueValidatedVersion(
|
|
690
|
-
if (!
|
|
691
|
-
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(
|
|
722
|
+
function isQueueValidatedVersion(version2) {
|
|
723
|
+
if (!version2) return false;
|
|
724
|
+
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
692
725
|
}
|
|
693
|
-
function buildOpenCodeVersionWarning(
|
|
694
|
-
if (isQueueValidatedVersion(
|
|
695
|
-
const detected =
|
|
726
|
+
function buildOpenCodeVersionWarning(version2) {
|
|
727
|
+
if (isQueueValidatedVersion(version2)) return null;
|
|
728
|
+
const detected = version2 ? `v${version2}` : "unknown";
|
|
696
729
|
const validated = QUEUE_VALIDATED_OPENCODE_VERSIONS.map((v) => `v${v}`).join(", ");
|
|
697
730
|
return `Warning: opencode ${detected} is not a queue-validated version (validated: ${validated}). Native message queuing \u2014 which channel (Slack/WhatsApp) message handling relies on \u2014 is unverified on this version; queued/follow-up messages may behave unexpectedly. Continuing anyway. Bumping the validated set requires re-running the queue validation.`;
|
|
698
731
|
}
|
|
@@ -1009,7 +1042,11 @@ function roleOf(m) {
|
|
|
1009
1042
|
}
|
|
1010
1043
|
function completedOf(m) {
|
|
1011
1044
|
if (!m || typeof m !== "object") return void 0;
|
|
1012
|
-
return m.info?.time?.completed;
|
|
1045
|
+
return m.info?.time?.completed ?? m.time?.completed;
|
|
1046
|
+
}
|
|
1047
|
+
function createdOf(m) {
|
|
1048
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1049
|
+
return m.info?.time?.created ?? m.time?.created;
|
|
1013
1050
|
}
|
|
1014
1051
|
function idOf(m) {
|
|
1015
1052
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -1029,6 +1066,14 @@ function finishOf(m) {
|
|
|
1029
1066
|
const infoFinish = m.info?.finish;
|
|
1030
1067
|
return typeof infoFinish === "string" ? infoFinish : void 0;
|
|
1031
1068
|
}
|
|
1069
|
+
function errorOf(m) {
|
|
1070
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1071
|
+
return m.info?.error ?? m.error;
|
|
1072
|
+
}
|
|
1073
|
+
function isAssistantInFlight(m) {
|
|
1074
|
+
if (completedOf(m) == null) return true;
|
|
1075
|
+
return finishOf(m) === "tool-calls";
|
|
1076
|
+
}
|
|
1032
1077
|
async function createOpenCodeSession(port, directory) {
|
|
1033
1078
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1034
1079
|
if (directory && directory.trim()) {
|
|
@@ -1088,19 +1133,31 @@ function findAssistantReplyAfter(messages, userMessageId) {
|
|
|
1088
1133
|
}
|
|
1089
1134
|
function findLastAssistantReplyFor(messages, userMessageId) {
|
|
1090
1135
|
if (!messages || messages.length === 0) return null;
|
|
1136
|
+
let lastCorrelated = null;
|
|
1137
|
+
let lastNonErrored = null;
|
|
1091
1138
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1092
1139
|
const m = messages[i];
|
|
1093
|
-
if (roleOf(m)
|
|
1140
|
+
if (roleOf(m) !== "assistant" || parentIdOf(m) !== userMessageId) continue;
|
|
1141
|
+
if (lastCorrelated === null) lastCorrelated = m;
|
|
1142
|
+
if (errorOf(m) == null) {
|
|
1143
|
+
lastNonErrored = m;
|
|
1144
|
+
break;
|
|
1145
|
+
}
|
|
1094
1146
|
}
|
|
1147
|
+
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
1095
1148
|
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1096
1149
|
if (userIndex === -1) return null;
|
|
1097
1150
|
let last = null;
|
|
1151
|
+
let lastOk = null;
|
|
1098
1152
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1099
1153
|
const role = roleOf(messages[i]);
|
|
1100
1154
|
if (role === "user") break;
|
|
1101
|
-
if (role === "assistant")
|
|
1155
|
+
if (role === "assistant") {
|
|
1156
|
+
last = messages[i];
|
|
1157
|
+
if (errorOf(messages[i]) == null) lastOk = messages[i];
|
|
1158
|
+
}
|
|
1102
1159
|
}
|
|
1103
|
-
return last;
|
|
1160
|
+
return lastOk ?? last;
|
|
1104
1161
|
}
|
|
1105
1162
|
function messageRunState(messages, userMessageId) {
|
|
1106
1163
|
if (!messages || messages.length === 0) return "unknown";
|
|
@@ -1110,13 +1167,52 @@ function messageRunState(messages, userMessageId) {
|
|
|
1110
1167
|
if (!reply) return "unknown";
|
|
1111
1168
|
}
|
|
1112
1169
|
if (!reply) return "queued";
|
|
1113
|
-
if (
|
|
1114
|
-
|
|
1115
|
-
return "done";
|
|
1170
|
+
if (isAssistantInFlight(reply)) return "running";
|
|
1171
|
+
return errorOf(reply) != null ? "failed" : "done";
|
|
1116
1172
|
}
|
|
1117
|
-
function
|
|
1118
|
-
const
|
|
1119
|
-
|
|
1173
|
+
function messageError(messages, userMessageId) {
|
|
1174
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1175
|
+
const error2 = errorOf(reply);
|
|
1176
|
+
if (error2 == null) return null;
|
|
1177
|
+
if (typeof error2 === "string") return error2;
|
|
1178
|
+
if (typeof error2 === "object") {
|
|
1179
|
+
const e = error2;
|
|
1180
|
+
const dataMessage = e.data?.message;
|
|
1181
|
+
if (typeof dataMessage === "string") return dataMessage;
|
|
1182
|
+
if (typeof e.message === "string") return e.message;
|
|
1183
|
+
}
|
|
1184
|
+
return "The agent run failed.";
|
|
1185
|
+
}
|
|
1186
|
+
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1187
|
+
if (!messages || messages.length === 0) return false;
|
|
1188
|
+
return messages.some(
|
|
1189
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
function isSessionSettled(messages, now, settleMs) {
|
|
1193
|
+
if (!messages || messages.length === 0) return true;
|
|
1194
|
+
let newestAssistantCreated = null;
|
|
1195
|
+
for (const m of messages) {
|
|
1196
|
+
if (roleOf(m) !== "assistant") continue;
|
|
1197
|
+
if (isAssistantInFlight(m)) return false;
|
|
1198
|
+
const created = createdOf(m);
|
|
1199
|
+
if (typeof created === "number" && (newestAssistantCreated === null || created > newestAssistantCreated)) {
|
|
1200
|
+
newestAssistantCreated = created;
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
if (newestAssistantCreated === null) return true;
|
|
1204
|
+
return now - newestAssistantCreated >= settleMs;
|
|
1205
|
+
}
|
|
1206
|
+
function opencodeMessageIdFor2(queuedMessageId) {
|
|
1207
|
+
return opencodeMessageIdFor(queuedMessageId);
|
|
1208
|
+
}
|
|
1209
|
+
var NATIVE_ID_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
1210
|
+
function nativeOpencodeMessageId() {
|
|
1211
|
+
let id = "msg_";
|
|
1212
|
+
for (let i = 0; i < 24; i++) {
|
|
1213
|
+
id += NATIVE_ID_ALPHABET[Math.floor(Math.random() * NATIVE_ID_ALPHABET.length)];
|
|
1214
|
+
}
|
|
1215
|
+
return id;
|
|
1120
1216
|
}
|
|
1121
1217
|
|
|
1122
1218
|
// src/lib/tunnel/connection.ts
|
|
@@ -1187,12 +1283,20 @@ var StreamForwarder = class {
|
|
|
1187
1283
|
}
|
|
1188
1284
|
async handleOpen(frame) {
|
|
1189
1285
|
const { sid, method, path, headers, has_body } = frame;
|
|
1286
|
+
const correlationId = headers?.[CORRELATION_ID_HEADER];
|
|
1287
|
+
const startedAt = Date.now();
|
|
1190
1288
|
if (path === TUNNEL_DRAIN_PING_PATH) {
|
|
1191
1289
|
this.callbacks.onDrainPing?.();
|
|
1192
1290
|
this.send({ type: "head", sid, status: 204, headers: {} });
|
|
1193
1291
|
this.send({ type: "res_end", sid });
|
|
1194
1292
|
return;
|
|
1195
1293
|
}
|
|
1294
|
+
log("info", "agent_request", {
|
|
1295
|
+
correlation_id: correlationId,
|
|
1296
|
+
sid,
|
|
1297
|
+
method,
|
|
1298
|
+
path: stripQuery(path)
|
|
1299
|
+
});
|
|
1196
1300
|
const ac = new AbortController();
|
|
1197
1301
|
let bodyPromise;
|
|
1198
1302
|
let pushBody;
|
|
@@ -1239,6 +1343,12 @@ var StreamForwarder = class {
|
|
|
1239
1343
|
if (!STRIP_RES.has(key.toLowerCase())) resHeaders[key] = value;
|
|
1240
1344
|
});
|
|
1241
1345
|
this.send({ type: "head", sid, status: upstream.status, headers: resHeaders });
|
|
1346
|
+
log("info", "agent_response", {
|
|
1347
|
+
correlation_id: correlationId,
|
|
1348
|
+
sid,
|
|
1349
|
+
status: upstream.status,
|
|
1350
|
+
duration_ms: Date.now() - startedAt
|
|
1351
|
+
});
|
|
1242
1352
|
this.callbacks.onHead?.(sid, upstream.status);
|
|
1243
1353
|
try {
|
|
1244
1354
|
if (upstream.body) {
|
|
@@ -1509,6 +1619,9 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1509
1619
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1510
1620
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1511
1621
|
var DEFAULT_DISPATCH_CONFIRM_MS = 6e3;
|
|
1622
|
+
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1623
|
+
var DEFAULT_STUCK_QUEUED_REDRIVE_MAX = 3;
|
|
1624
|
+
var DEFAULT_SETTLE_MS = 3500;
|
|
1512
1625
|
var ChannelAuthError = class extends Error {
|
|
1513
1626
|
constructor(message) {
|
|
1514
1627
|
super(message);
|
|
@@ -1544,6 +1657,9 @@ var ChannelDriver = class {
|
|
|
1544
1657
|
pausedPollIntervalMs;
|
|
1545
1658
|
pausedMaxWaitMs;
|
|
1546
1659
|
dispatchConfirmMs;
|
|
1660
|
+
stuckQueuedMs;
|
|
1661
|
+
stuckQueuedRedriveMax;
|
|
1662
|
+
settleMs;
|
|
1547
1663
|
now;
|
|
1548
1664
|
/** Cache of conversationId → opencode sessionId. */
|
|
1549
1665
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -1563,6 +1679,40 @@ var ChannelDriver = class {
|
|
|
1563
1679
|
* a steady-state-poll re-dispatch will not double-run the message.
|
|
1564
1680
|
*/
|
|
1565
1681
|
dispatched = /* @__PURE__ */ new Set();
|
|
1682
|
+
/**
|
|
1683
|
+
* Re-adopted (ADR-0046) Evident message ids currently tracked by a watcher.
|
|
1684
|
+
* Used only to distinguish a RE-ADOPTED give-up from a normal-dispatch give-up
|
|
1685
|
+
* so the former can be parked in `dontRedispatch` (Bug 2). A row is added when
|
|
1686
|
+
* it is re-adopted and removed when its watcher settles or it is observed off
|
|
1687
|
+
* the processing list.
|
|
1688
|
+
*/
|
|
1689
|
+
readopted = /* @__PURE__ */ new Set();
|
|
1690
|
+
/**
|
|
1691
|
+
* "Don't re-DISPATCH / re-attach this orphan again" (Bug 2/5). Set when a
|
|
1692
|
+
* re-adopted running/orphan row's watcher hit its `processed_at`-anchored
|
|
1693
|
+
* deadline (or an orphan whose window already elapsed): the still-`processing`
|
|
1694
|
+
* server row would otherwise be re-adopted (and re-dispatched) on EVERY ~2s
|
|
1695
|
+
* drain until the 15-min cron resets it — spamming new turns.
|
|
1696
|
+
*
|
|
1697
|
+
* CRITICAL (Bugbot #202): this suppresses ONLY the dispatch/re-attach paths, it
|
|
1698
|
+
* does NOT suppress DONE delivery. A row parked here whose reply later COMPLETES
|
|
1699
|
+
* in opencode must still be delivered via `markDone` on the next drain — so
|
|
1700
|
+
* `readoptOne` computes `state` FIRST and this set is checked only on the
|
|
1701
|
+
* non-done path. It is cleared once the row leaves the processing list (cron
|
|
1702
|
+
* reset → it drains normally as `pending`), so it can never leak.
|
|
1703
|
+
*/
|
|
1704
|
+
dontRedispatch = /* @__PURE__ */ new Set();
|
|
1705
|
+
/**
|
|
1706
|
+
* "markDone for this row is TERMINALLY undeliverable" (Bug 4). Set ONLY when a
|
|
1707
|
+
* re-adopted DONE row's `markDone` returned a terminal 4xx (a status that will
|
|
1708
|
+
* never succeed). Checked at the TOP of the `done` branch so we do NOT re-attempt
|
|
1709
|
+
* that markDone every ~2s drain while the row stays `processing`. A TRANSIENT
|
|
1710
|
+
* markDone failure must NOT land here (it must still retry next drain). Separate
|
|
1711
|
+
* from `dontRedispatch` because the two concerns are independent: a row can need
|
|
1712
|
+
* "stop re-dispatching" without "stop delivering", and vice versa. Cleared once
|
|
1713
|
+
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1714
|
+
*/
|
|
1715
|
+
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1566
1716
|
/**
|
|
1567
1717
|
* Cache of the opencode root directory (from `GET /path`). Resolved lazily on
|
|
1568
1718
|
* first session creation so drain-created sessions are rooted at the project
|
|
@@ -1586,6 +1736,9 @@ var ChannelDriver = class {
|
|
|
1586
1736
|
this.pausedPollIntervalMs = config2.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
1587
1737
|
this.pausedMaxWaitMs = config2.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
1588
1738
|
this.dispatchConfirmMs = config2.dispatchConfirmMs ?? DEFAULT_DISPATCH_CONFIRM_MS;
|
|
1739
|
+
this.stuckQueuedMs = config2.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
1740
|
+
this.stuckQueuedRedriveMax = config2.stuckQueuedRedriveMax ?? DEFAULT_STUCK_QUEUED_REDRIVE_MAX;
|
|
1741
|
+
this.settleMs = config2.settleMs ?? DEFAULT_SETTLE_MS;
|
|
1589
1742
|
this.now = config2.now ?? (() => Date.now());
|
|
1590
1743
|
}
|
|
1591
1744
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
@@ -1618,6 +1771,7 @@ var ChannelDriver = class {
|
|
|
1618
1771
|
for (const conv of conversations) {
|
|
1619
1772
|
dispatched += await this.processConversation(conv);
|
|
1620
1773
|
}
|
|
1774
|
+
await this.readoptProcessing();
|
|
1621
1775
|
} finally {
|
|
1622
1776
|
this.draining = false;
|
|
1623
1777
|
}
|
|
@@ -1669,11 +1823,13 @@ var ChannelDriver = class {
|
|
|
1669
1823
|
const sessionId = await this.ensureSession(conv);
|
|
1670
1824
|
const messages = await this.getPendingMessages(conv.id);
|
|
1671
1825
|
let dispatched = 0;
|
|
1826
|
+
let skippedAlreadyDispatched = 0;
|
|
1672
1827
|
for (const message of messages) {
|
|
1673
1828
|
if (this.dispatched.has(message.id)) {
|
|
1829
|
+
skippedAlreadyDispatched += 1;
|
|
1674
1830
|
continue;
|
|
1675
1831
|
}
|
|
1676
|
-
const opencodeMessageId =
|
|
1832
|
+
const opencodeMessageId = opencodeMessageIdFor2(message.id);
|
|
1677
1833
|
const options = {
|
|
1678
1834
|
agent: message.opencode_agent ?? void 0,
|
|
1679
1835
|
model: message.opencode_model ?? void 0
|
|
@@ -1702,6 +1858,14 @@ var ChannelDriver = class {
|
|
|
1702
1858
|
this.dispatched.add(message.id);
|
|
1703
1859
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
1704
1860
|
dispatched += 1;
|
|
1861
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
1862
|
+
}
|
|
1863
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
1864
|
+
this.log({
|
|
1865
|
+
level: "error",
|
|
1866
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
1867
|
+
conversation_id: conv.id
|
|
1868
|
+
});
|
|
1705
1869
|
}
|
|
1706
1870
|
this.ensureWatcherRunning(sessionId);
|
|
1707
1871
|
return dispatched;
|
|
@@ -1760,7 +1924,64 @@ var ChannelDriver = class {
|
|
|
1760
1924
|
dispatchedAt: now,
|
|
1761
1925
|
deadline: now + this.pausedMaxWaitMs,
|
|
1762
1926
|
started: false,
|
|
1763
|
-
done: false
|
|
1927
|
+
done: false,
|
|
1928
|
+
stuckReported: false,
|
|
1929
|
+
redriveAttempts: 0,
|
|
1930
|
+
lastRedriveAt: null,
|
|
1931
|
+
redriveOpencodeMessageId: null,
|
|
1932
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1935
|
+
/**
|
|
1936
|
+
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
1937
|
+
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
1938
|
+
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
1939
|
+
* `now`: a row already `processing` for e.g. 5 min must give up ~5 min from now
|
|
1940
|
+
* (10 min after `processed_at`), not 10 min from now — otherwise its deadline
|
|
1941
|
+
* lands ~15 min after `processed_at`, coinciding with the cron reset →
|
|
1942
|
+
* double-drive race. `dispatchedAt` stays `now` (only the appear-guard uses it).
|
|
1943
|
+
*
|
|
1944
|
+
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
1945
|
+
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
1946
|
+
* fresh-run path these differ (a fresh opencode id under the same server row).
|
|
1947
|
+
*
|
|
1948
|
+
* `started` is set true so the watcher does NOT re-`markProcessing` a row the
|
|
1949
|
+
* server already flipped to `processing`; the running/done transitions still
|
|
1950
|
+
* fire from the watcher's normal branches.
|
|
1951
|
+
*/
|
|
1952
|
+
registerReadopted(conv, sessionId, message, opencodeMessageId, processedAtMs, redriveOpencodeMessageId = null) {
|
|
1953
|
+
let watcher = this.watchers.get(sessionId);
|
|
1954
|
+
if (!watcher) {
|
|
1955
|
+
watcher = {
|
|
1956
|
+
conv,
|
|
1957
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1958
|
+
loop: null,
|
|
1959
|
+
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
1960
|
+
reportedPermissions: /* @__PURE__ */ new Set()
|
|
1961
|
+
};
|
|
1962
|
+
this.watchers.set(sessionId, watcher);
|
|
1963
|
+
}
|
|
1964
|
+
watcher.inFlight.set(message.id, {
|
|
1965
|
+
evidentMessageId: message.id,
|
|
1966
|
+
opencodeMessageId,
|
|
1967
|
+
message,
|
|
1968
|
+
dispatchedAt: this.now(),
|
|
1969
|
+
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
1970
|
+
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
1971
|
+
started: true,
|
|
1972
|
+
done: false,
|
|
1973
|
+
// Not yet reported stuck-queued. The once-guard (`stuckReported`) applies,
|
|
1974
|
+
// AND the stuck-queued observer now INCLUDES re-adopted queued wedges: it
|
|
1975
|
+
// gates on `state === 'queued'` (turn produced no reply), not on `started`,
|
|
1976
|
+
// so a re-adopted row left wedged in `queued` still emits the signal once
|
|
1977
|
+
// (queued-followup-redrive, #210). A re-adopted `queued` wedge is ALSO
|
|
1978
|
+
// re-driven by the same path (ADR-0046 §c deferred the live-session redrive
|
|
1979
|
+
// to here) — `redriveAttempts` starts fresh so it gets the full budget.
|
|
1980
|
+
stuckReported: false,
|
|
1981
|
+
redriveAttempts: 0,
|
|
1982
|
+
lastRedriveAt: null,
|
|
1983
|
+
redriveOpencodeMessageId,
|
|
1984
|
+
attemptedOpencodeMessageIds: [opencodeMessageId]
|
|
1764
1985
|
});
|
|
1765
1986
|
}
|
|
1766
1987
|
/**
|
|
@@ -1826,6 +2047,7 @@ var ChannelDriver = class {
|
|
|
1826
2047
|
conversation_id: watcher.conv.id
|
|
1827
2048
|
});
|
|
1828
2049
|
for (const evidentMessageId of [...watcher.inFlight.keys()]) {
|
|
2050
|
+
this.readopted.delete(evidentMessageId);
|
|
1829
2051
|
this.removeInFlight(watcher, evidentMessageId);
|
|
1830
2052
|
}
|
|
1831
2053
|
return;
|
|
@@ -1846,10 +2068,15 @@ var ChannelDriver = class {
|
|
|
1846
2068
|
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
1847
2069
|
const conv = watcher.conv;
|
|
1848
2070
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
1849
|
-
if ((state === "running" || state === "done") && !inFlight.started) {
|
|
2071
|
+
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
1850
2072
|
let claimed;
|
|
1851
2073
|
try {
|
|
1852
|
-
claimed = await this.markProcessing(
|
|
2074
|
+
claimed = await this.markProcessing(
|
|
2075
|
+
conv.id,
|
|
2076
|
+
inFlight.evidentMessageId,
|
|
2077
|
+
sessionId,
|
|
2078
|
+
inFlight.redriveOpencodeMessageId
|
|
2079
|
+
);
|
|
1853
2080
|
} catch (err) {
|
|
1854
2081
|
if (err instanceof ChannelAuthError) throw err;
|
|
1855
2082
|
this.log({
|
|
@@ -1879,7 +2106,12 @@ var ChannelDriver = class {
|
|
|
1879
2106
|
message_id: inFlight.evidentMessageId
|
|
1880
2107
|
});
|
|
1881
2108
|
try {
|
|
1882
|
-
await this.markDone(
|
|
2109
|
+
await this.markDone(
|
|
2110
|
+
conv.id,
|
|
2111
|
+
inFlight.evidentMessageId,
|
|
2112
|
+
sessionId,
|
|
2113
|
+
inFlight.redriveOpencodeMessageId
|
|
2114
|
+
);
|
|
1883
2115
|
} catch (err) {
|
|
1884
2116
|
if (err instanceof ChannelAuthError) throw err;
|
|
1885
2117
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -1915,9 +2147,70 @@ var ChannelDriver = class {
|
|
|
1915
2147
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1916
2148
|
return;
|
|
1917
2149
|
}
|
|
2150
|
+
if (state === "failed") {
|
|
2151
|
+
if (!inFlight.done) {
|
|
2152
|
+
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2153
|
+
this.log({
|
|
2154
|
+
level: "error",
|
|
2155
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2156
|
+
conversation_id: conv.id,
|
|
2157
|
+
message_id: inFlight.evidentMessageId
|
|
2158
|
+
});
|
|
2159
|
+
try {
|
|
2160
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
2161
|
+
} catch (err) {
|
|
2162
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2163
|
+
if (err instanceof ChannelTerminalError) {
|
|
2164
|
+
this.log({
|
|
2165
|
+
level: "error",
|
|
2166
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2167
|
+
conversation_id: conv.id,
|
|
2168
|
+
message_id: inFlight.evidentMessageId
|
|
2169
|
+
});
|
|
2170
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
if (this.now() >= inFlight.deadline) {
|
|
2174
|
+
this.log({
|
|
2175
|
+
level: "error",
|
|
2176
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2177
|
+
conversation_id: conv.id,
|
|
2178
|
+
message_id: inFlight.evidentMessageId
|
|
2179
|
+
});
|
|
2180
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2181
|
+
return;
|
|
2182
|
+
}
|
|
2183
|
+
this.log({
|
|
2184
|
+
level: "error",
|
|
2185
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2186
|
+
conversation_id: conv.id,
|
|
2187
|
+
message_id: inFlight.evidentMessageId
|
|
2188
|
+
});
|
|
2189
|
+
return;
|
|
2190
|
+
}
|
|
2191
|
+
inFlight.done = true;
|
|
2192
|
+
}
|
|
2193
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2194
|
+
return;
|
|
2195
|
+
}
|
|
1918
2196
|
if (state === "unknown") {
|
|
1919
2197
|
if (this.now() - inFlight.dispatchedAt >= this.dispatchConfirmMs) {
|
|
1920
|
-
await this.redispatchInFlight(sessionId, inFlight);
|
|
2198
|
+
await this.redispatchInFlight(conv.id, sessionId, inFlight);
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
const pastStuckBound = this.now() - inFlight.dispatchedAt >= this.stuckQueuedMs;
|
|
2202
|
+
const sessionIdle = state === "queued" && !hasRunningAssistantExcept(messages, inFlight.opencodeMessageId);
|
|
2203
|
+
if (state === "queued" && pastStuckBound && sessionIdle) {
|
|
2204
|
+
if (!inFlight.stuckReported) {
|
|
2205
|
+
inFlight.stuckReported = true;
|
|
2206
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "stuck_queued", {
|
|
2207
|
+
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
const dueForRedrive = inFlight.lastRedriveAt == null || this.now() - inFlight.lastRedriveAt >= this.stuckQueuedMs;
|
|
2211
|
+
if (dueForRedrive && isSessionSettled(messages, this.now(), this.settleMs)) {
|
|
2212
|
+
const removed = await this.redriveStuckQueued(sessionId, watcher, inFlight, messages);
|
|
2213
|
+
if (removed) return;
|
|
1921
2214
|
}
|
|
1922
2215
|
}
|
|
1923
2216
|
if (this.now() >= inFlight.deadline) {
|
|
@@ -1927,6 +2220,9 @@ var ChannelDriver = class {
|
|
|
1927
2220
|
conversation_id: conv.id,
|
|
1928
2221
|
message_id: inFlight.evidentMessageId
|
|
1929
2222
|
});
|
|
2223
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2224
|
+
watched_for_ms: this.now() - inFlight.dispatchedAt
|
|
2225
|
+
});
|
|
1930
2226
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
1931
2227
|
}
|
|
1932
2228
|
}
|
|
@@ -1936,7 +2232,7 @@ var ChannelDriver = class {
|
|
|
1936
2232
|
* fact 9) — one user message + one reply even if the original DID land. Resets
|
|
1937
2233
|
* the dispatch timestamp so the guard doesn't immediately fire again.
|
|
1938
2234
|
*/
|
|
1939
|
-
async redispatchInFlight(sessionId, inFlight) {
|
|
2235
|
+
async redispatchInFlight(conversationId, sessionId, inFlight) {
|
|
1940
2236
|
const options = {
|
|
1941
2237
|
agent: inFlight.message.opencode_agent ?? void 0,
|
|
1942
2238
|
model: inFlight.message.opencode_model ?? void 0
|
|
@@ -1946,6 +2242,7 @@ var ChannelDriver = class {
|
|
|
1946
2242
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} not observed after dispatch \u2014 re-dispatching (idle-path guard)`,
|
|
1947
2243
|
message_id: inFlight.evidentMessageId
|
|
1948
2244
|
});
|
|
2245
|
+
void this.postSignal(conversationId, inFlight.evidentMessageId, "redispatched");
|
|
1949
2246
|
try {
|
|
1950
2247
|
await sendPromptAsync(
|
|
1951
2248
|
this.port,
|
|
@@ -1963,12 +2260,504 @@ var ChannelDriver = class {
|
|
|
1963
2260
|
}
|
|
1964
2261
|
inFlight.dispatchedAt = this.now();
|
|
1965
2262
|
}
|
|
2263
|
+
/**
|
|
2264
|
+
* RE-DRIVE a stuck-`queued` follow-up so opencode actually runs it
|
|
2265
|
+
* (queued-followup-redrive; the fix for the 2026-07-18 dev incident).
|
|
2266
|
+
*
|
|
2267
|
+
* The wedge: a follow-up `prompt_async`'d mid-turn / in the post-turn settling
|
|
2268
|
+
* window is orphaned — its user message persists but the turn never runs
|
|
2269
|
+
* (`messageRunState === 'queued'`). Proven against real opencode 1.18.3 by an
|
|
2270
|
+
* isolated, interleaved experiment (`.harness/followup-send-mechanism-finding.md`):
|
|
2271
|
+
* the un-sticking variable is the `messageID` FORMAT. A follow-up carrying the
|
|
2272
|
+
* runner's CUSTOM underscore id (`msg_<sanitized-uuid>`) is NEVER picked up mid-
|
|
2273
|
+
* turn (0/24 trials); the SAME follow-up carrying a NATIVE-format id
|
|
2274
|
+
* (`msg_`+24 base62) runs once the prior turn settles (~50% locally, reliably on
|
|
2275
|
+
* dev). `parts[].id` and `agent`/`model` were proven IRRELEVANT.
|
|
2276
|
+
*
|
|
2277
|
+
* So we re-`prompt_async` into the SAME opencode session (opencode-web does
|
|
2278
|
+
* exactly this and it works — UPDATE 2 in the investigation doc), preserving the
|
|
2279
|
+
* conversation's history/continuity: we do NOT create a fresh session and we do
|
|
2280
|
+
* NOT overwrite the conversation's `opencode_session_id`. Only the opencode
|
|
2281
|
+
* user-message id changes — to a fresh native id per attempt (opencode's
|
|
2282
|
+
* caller-supplied id dedup is global+permanent, so each attempt needs a never-
|
|
2283
|
+
* seen id).
|
|
2284
|
+
*
|
|
2285
|
+
* Reply correlation is PRESERVED: the native id is random and NOT re-derivable
|
|
2286
|
+
* from the row id, so the runner carries it to the server (on markProcessing/
|
|
2287
|
+
* markDone via `redriveOpencodeMessageId`), which persists it on the row and
|
|
2288
|
+
* correlates the reply by THAT id (`conversation-notification.ts`). A normal,
|
|
2289
|
+
* never-re-driven message still correlates by the derived stable id — unchanged.
|
|
2290
|
+
*
|
|
2291
|
+
* Bounded to `stuckQueuedRedriveMax` attempts. On exhaustion the row is marked
|
|
2292
|
+
* FAILED (existing channel failure affordance) so the user is TOLD it could not
|
|
2293
|
+
* be answered rather than left silent, and it is removed from the in-flight set.
|
|
2294
|
+
*
|
|
2295
|
+
* @returns true if the message LEFT this watcher's in-flight set (gave up +
|
|
2296
|
+
* marked failed) — the caller then stops servicing it this tick. Returns false
|
|
2297
|
+
* when the follow-up stays in THIS watcher (re-driven in place, or the re-drive
|
|
2298
|
+
* send failed and the next window retries).
|
|
2299
|
+
*/
|
|
2300
|
+
async redriveStuckQueued(sessionId, watcher, inFlight, messages) {
|
|
2301
|
+
if (await this.deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages)) {
|
|
2302
|
+
return true;
|
|
2303
|
+
}
|
|
2304
|
+
if (inFlight.redriveAttempts >= this.stuckQueuedRedriveMax) {
|
|
2305
|
+
this.log({
|
|
2306
|
+
level: "error",
|
|
2307
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued after ${inFlight.redriveAttempts} re-drive attempt(s) \u2014 marking failed`,
|
|
2308
|
+
conversation_id: watcher.conv.id,
|
|
2309
|
+
message_id: inFlight.evidentMessageId
|
|
2310
|
+
});
|
|
2311
|
+
try {
|
|
2312
|
+
await this.markFailed(watcher.conv.id, inFlight.evidentMessageId);
|
|
2313
|
+
} catch (err) {
|
|
2314
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2315
|
+
this.log({
|
|
2316
|
+
level: "error",
|
|
2317
|
+
message: `Failed to mark stuck message ${inFlight.evidentMessageId.slice(0, 8)} failed (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2318
|
+
conversation_id: watcher.conv.id,
|
|
2319
|
+
message_id: inFlight.evidentMessageId
|
|
2320
|
+
});
|
|
2321
|
+
}
|
|
2322
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2323
|
+
return true;
|
|
2324
|
+
}
|
|
2325
|
+
inFlight.redriveAttempts += 1;
|
|
2326
|
+
inFlight.lastRedriveAt = this.now();
|
|
2327
|
+
this.log({
|
|
2328
|
+
level: "info",
|
|
2329
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} stuck queued on an idle session \u2014 re-driving into the SAME session with a native id (attempt ${inFlight.redriveAttempts}/${this.stuckQueuedRedriveMax})`,
|
|
2330
|
+
conversation_id: watcher.conv.id,
|
|
2331
|
+
message_id: inFlight.evidentMessageId
|
|
2332
|
+
});
|
|
2333
|
+
void this.postSignal(watcher.conv.id, inFlight.evidentMessageId, "redriven", {
|
|
2334
|
+
redrive_attempt: inFlight.redriveAttempts
|
|
2335
|
+
});
|
|
2336
|
+
const options = {
|
|
2337
|
+
agent: inFlight.message.opencode_agent ?? void 0,
|
|
2338
|
+
model: inFlight.message.opencode_model ?? void 0
|
|
2339
|
+
};
|
|
2340
|
+
const nativeId = nativeOpencodeMessageId();
|
|
2341
|
+
try {
|
|
2342
|
+
await sendPromptAsync(this.port, sessionId, inFlight.message.content, options, nativeId);
|
|
2343
|
+
} catch (err) {
|
|
2344
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2345
|
+
inFlight.redriveAttempts -= 1;
|
|
2346
|
+
this.log({
|
|
2347
|
+
level: "error",
|
|
2348
|
+
message: `Re-drive failed for stuck message ${inFlight.evidentMessageId.slice(0, 8)} (will retry next window): ${err instanceof Error ? err.message : String(err)}`,
|
|
2349
|
+
conversation_id: watcher.conv.id,
|
|
2350
|
+
message_id: inFlight.evidentMessageId
|
|
2351
|
+
});
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
inFlight.opencodeMessageId = nativeId;
|
|
2355
|
+
inFlight.redriveOpencodeMessageId = nativeId;
|
|
2356
|
+
inFlight.attemptedOpencodeMessageIds.push(nativeId);
|
|
2357
|
+
inFlight.started = false;
|
|
2358
|
+
return false;
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* Finding 2 (Bugbot #217): scan EVERY opencode id this stuck message has been
|
|
2362
|
+
* driven under (`attemptedOpencodeMessageIds`) for a COMPLETED correlated reply
|
|
2363
|
+
* in the tick's snapshot; if one is found, markDone off THAT id (so the reply an
|
|
2364
|
+
* earlier re-drive attempt eventually produced is delivered) instead of
|
|
2365
|
+
* re-driving again or marking failed. Idempotent + guarded like the watcher's
|
|
2366
|
+
* done branch. `@returns` true when the message left the in-flight set (delivered
|
|
2367
|
+
* or terminally-undeliverable) — the caller then stops servicing it this tick.
|
|
2368
|
+
*
|
|
2369
|
+
* The LATEST id is normally handled by `serviceInFlightMessage`'s own `done`
|
|
2370
|
+
* branch; this covers the ids a re-drive OVERWROTE, which that branch no longer
|
|
2371
|
+
* polls. `messageRunState === 'done'` means a completed, non-errored correlated
|
|
2372
|
+
* reply exists for that id.
|
|
2373
|
+
*/
|
|
2374
|
+
async deliverIfAnyAttemptCompleted(sessionId, watcher, inFlight, messages) {
|
|
2375
|
+
const completedId = inFlight.attemptedOpencodeMessageIds.find(
|
|
2376
|
+
(id) => messageRunState(messages, id) === "done"
|
|
2377
|
+
);
|
|
2378
|
+
if (!completedId) return false;
|
|
2379
|
+
if (inFlight.done) {
|
|
2380
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2381
|
+
return true;
|
|
2382
|
+
}
|
|
2383
|
+
this.log({
|
|
2384
|
+
level: "info",
|
|
2385
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed under an earlier re-drive attempt's id \u2014 marking done (not re-driving/failing)`,
|
|
2386
|
+
conversation_id: watcher.conv.id,
|
|
2387
|
+
message_id: inFlight.evidentMessageId
|
|
2388
|
+
});
|
|
2389
|
+
try {
|
|
2390
|
+
await this.markDone(watcher.conv.id, inFlight.evidentMessageId, sessionId, completedId);
|
|
2391
|
+
} catch (err) {
|
|
2392
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2393
|
+
this.log({
|
|
2394
|
+
level: "error",
|
|
2395
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done off an earlier re-drive attempt (leaving for the cron safety net): ${err instanceof Error ? err.message : String(err)}`,
|
|
2396
|
+
conversation_id: watcher.conv.id,
|
|
2397
|
+
message_id: inFlight.evidentMessageId
|
|
2398
|
+
});
|
|
2399
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2400
|
+
return true;
|
|
2401
|
+
}
|
|
2402
|
+
inFlight.done = true;
|
|
2403
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2404
|
+
return true;
|
|
2405
|
+
}
|
|
2406
|
+
// -------------------------------------------------------------------------
|
|
2407
|
+
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
2408
|
+
// -------------------------------------------------------------------------
|
|
2409
|
+
/**
|
|
2410
|
+
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
2411
|
+
*
|
|
2412
|
+
* The pending drain only re-drives `pending` rows; a message already flipped to
|
|
2413
|
+
* `processing` before the runner died is watched by nobody until the 15-min
|
|
2414
|
+
* cron resets it. Here we fetch those rows, and per row resolve its correlated
|
|
2415
|
+
* reply against opencode's OWN session store — completing, re-attaching, or
|
|
2416
|
+
* (for an orphan) forcing a genuine fresh run. Runs on EVERY drain tick, so it
|
|
2417
|
+
* is idempotent per message (Invariant 2): a row a watcher already tracks is
|
|
2418
|
+
* skipped in `readoptOne` — one driver, no double-drive.
|
|
2419
|
+
*
|
|
2420
|
+
* Only `ChannelAuthError` propagates (to `drainPending`, like the pending
|
|
2421
|
+
* path); every other early return LOGS a reason with context — no silent drop.
|
|
2422
|
+
*/
|
|
2423
|
+
async readoptProcessing() {
|
|
2424
|
+
const rows = await this.getProcessingMessages();
|
|
2425
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2426
|
+
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2427
|
+
for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
|
|
2428
|
+
if (!stillProcessing.has(id)) {
|
|
2429
|
+
const cleared = this.dontRedispatch.delete(id);
|
|
2430
|
+
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2431
|
+
if (cleared || clearedUndeliverable) {
|
|
2432
|
+
this.log({
|
|
2433
|
+
level: "info",
|
|
2434
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left the processing list (cron reset) \u2014 cleared gave-up marker`,
|
|
2435
|
+
message_id: id
|
|
2436
|
+
});
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
if (rows.length === 0) return;
|
|
2442
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
2443
|
+
for (const row of rows) {
|
|
2444
|
+
if (!row.opencode_session_id) {
|
|
2445
|
+
this.log({
|
|
2446
|
+
level: "error",
|
|
2447
|
+
message: `Cannot re-adopt processing message ${row.id.slice(0, 8)} \u2014 no opencode session id; leaving for the cron safety net`,
|
|
2448
|
+
conversation_id: row.conversation_id,
|
|
2449
|
+
message_id: row.id
|
|
2450
|
+
});
|
|
2451
|
+
continue;
|
|
2452
|
+
}
|
|
2453
|
+
const list = bySession.get(row.opencode_session_id) ?? [];
|
|
2454
|
+
list.push(row);
|
|
2455
|
+
bySession.set(row.opencode_session_id, list);
|
|
2456
|
+
}
|
|
2457
|
+
for (const [sessionId, sessionRows] of bySession) {
|
|
2458
|
+
let messages;
|
|
2459
|
+
try {
|
|
2460
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
2461
|
+
if (!res.ok) {
|
|
2462
|
+
this.log({
|
|
2463
|
+
level: "error",
|
|
2464
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned HTTP ${res.status} \u2014 skipping this session this tick`
|
|
2465
|
+
});
|
|
2466
|
+
continue;
|
|
2467
|
+
}
|
|
2468
|
+
const body = await res.json();
|
|
2469
|
+
if (!Array.isArray(body)) {
|
|
2470
|
+
this.log({
|
|
2471
|
+
level: "error",
|
|
2472
|
+
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned a non-array message body \u2014 skipping this session this tick`
|
|
2473
|
+
});
|
|
2474
|
+
continue;
|
|
2475
|
+
}
|
|
2476
|
+
messages = body;
|
|
2477
|
+
} catch (err) {
|
|
2478
|
+
this.log({
|
|
2479
|
+
level: "error",
|
|
2480
|
+
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
2481
|
+
});
|
|
2482
|
+
continue;
|
|
2483
|
+
}
|
|
2484
|
+
for (const row of sessionRows) {
|
|
2485
|
+
await this.readoptOne(sessionId, row, messages);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
/**
|
|
2490
|
+
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
2491
|
+
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
2492
|
+
*
|
|
2493
|
+
* Branches on `messageRunState(messages, effectiveId)`, where `effectiveId` is
|
|
2494
|
+
* the NATIVE id a prior lifetime's re-drive ran this row under
|
|
2495
|
+
* (`row.opencode_message_id`) if present, else the STABLE derived id
|
|
2496
|
+
* (`opencodeMessageIdFor(row.id)`). Consulting the native id is what makes a
|
|
2497
|
+
* re-driven row that ALREADY ran/completed resolve correctly on restart instead
|
|
2498
|
+
* of looking `queued` under the (never-run) stable id and being re-driven AGAIN
|
|
2499
|
+
* (Bugbot #217 Finding 3 — the duplicate-turn bug):
|
|
2500
|
+
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
2501
|
+
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
2502
|
+
* errored turn is reported failed on restart, NOT re-dispatched;
|
|
2503
|
+
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
2504
|
+
* tracking the effective id so the reply correlates by it;
|
|
2505
|
+
* - `unknown` → re-dispatch the STABLE id + attach a watcher (orphan).
|
|
2506
|
+
*
|
|
2507
|
+
* Only `ChannelAuthError` propagates.
|
|
2508
|
+
*/
|
|
2509
|
+
async readoptOne(sessionId, row, messages) {
|
|
2510
|
+
if (this.isTracked(sessionId, row.id)) {
|
|
2511
|
+
this.log({
|
|
2512
|
+
level: "info",
|
|
2513
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already tracked in-flight \u2014 skipping (owned by the watcher loop)`,
|
|
2514
|
+
conversation_id: row.conversation_id,
|
|
2515
|
+
message_id: row.id
|
|
2516
|
+
});
|
|
2517
|
+
return;
|
|
2518
|
+
}
|
|
2519
|
+
const stableId = opencodeMessageIdFor2(row.id);
|
|
2520
|
+
const nativeRedriveId = row.opencode_message_id;
|
|
2521
|
+
const ocId = nativeRedriveId ?? stableId;
|
|
2522
|
+
const state = messageRunState(messages, ocId);
|
|
2523
|
+
if (state === "done") {
|
|
2524
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
2525
|
+
this.log({
|
|
2526
|
+
level: "info",
|
|
2527
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
2528
|
+
conversation_id: row.conversation_id,
|
|
2529
|
+
message_id: row.id
|
|
2530
|
+
});
|
|
2531
|
+
return;
|
|
2532
|
+
}
|
|
2533
|
+
this.log({
|
|
2534
|
+
level: "info",
|
|
2535
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
2536
|
+
conversation_id: row.conversation_id,
|
|
2537
|
+
message_id: row.id
|
|
2538
|
+
});
|
|
2539
|
+
try {
|
|
2540
|
+
await this.markDone(row.conversation_id, row.id, sessionId, nativeRedriveId);
|
|
2541
|
+
} catch (err) {
|
|
2542
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2543
|
+
if (err instanceof ChannelTerminalError) {
|
|
2544
|
+
this.doneUndeliverable.add(row.id);
|
|
2545
|
+
this.log({
|
|
2546
|
+
level: "error",
|
|
2547
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2548
|
+
conversation_id: row.conversation_id,
|
|
2549
|
+
message_id: row.id
|
|
2550
|
+
});
|
|
2551
|
+
return;
|
|
2552
|
+
}
|
|
2553
|
+
this.log({
|
|
2554
|
+
level: "error",
|
|
2555
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2556
|
+
conversation_id: row.conversation_id,
|
|
2557
|
+
message_id: row.id
|
|
2558
|
+
});
|
|
2559
|
+
return;
|
|
2560
|
+
}
|
|
2561
|
+
this.dontRedispatch.delete(row.id);
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
if (state === "failed") {
|
|
2565
|
+
const error2 = messageError(messages, ocId) ?? void 0;
|
|
2566
|
+
this.log({
|
|
2567
|
+
level: "error",
|
|
2568
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
2569
|
+
conversation_id: row.conversation_id,
|
|
2570
|
+
message_id: row.id
|
|
2571
|
+
});
|
|
2572
|
+
try {
|
|
2573
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2);
|
|
2574
|
+
} catch (err) {
|
|
2575
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2576
|
+
if (err instanceof ChannelTerminalError) {
|
|
2577
|
+
this.doneUndeliverable.add(row.id);
|
|
2578
|
+
this.log({
|
|
2579
|
+
level: "error",
|
|
2580
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
2581
|
+
conversation_id: row.conversation_id,
|
|
2582
|
+
message_id: row.id
|
|
2583
|
+
});
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
this.log({
|
|
2587
|
+
level: "error",
|
|
2588
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2589
|
+
conversation_id: row.conversation_id,
|
|
2590
|
+
message_id: row.id
|
|
2591
|
+
});
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
this.dontRedispatch.delete(row.id);
|
|
2595
|
+
return;
|
|
2596
|
+
}
|
|
2597
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
2598
|
+
this.log({
|
|
2599
|
+
level: "info",
|
|
2600
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
2601
|
+
conversation_id: row.conversation_id,
|
|
2602
|
+
message_id: row.id
|
|
2603
|
+
});
|
|
2604
|
+
return;
|
|
2605
|
+
}
|
|
2606
|
+
if (state === "running" || state === "queued") {
|
|
2607
|
+
const conv = this.convForRow(sessionId, row);
|
|
2608
|
+
const message = this.queuedMessageForRow(row);
|
|
2609
|
+
this.registerReadopted(
|
|
2610
|
+
conv,
|
|
2611
|
+
sessionId,
|
|
2612
|
+
message,
|
|
2613
|
+
ocId,
|
|
2614
|
+
this.processedAtMs(row),
|
|
2615
|
+
nativeRedriveId
|
|
2616
|
+
);
|
|
2617
|
+
this.dispatched.add(row.id);
|
|
2618
|
+
this.readopted.add(row.id);
|
|
2619
|
+
this.ensureWatcherRunning(sessionId);
|
|
2620
|
+
this.log({
|
|
2621
|
+
level: "info",
|
|
2622
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} ${state} \u2014 re-attached watcher (${nativeRedriveId ? "native re-drive id" : "stable id"}, no re-dispatch)`,
|
|
2623
|
+
conversation_id: row.conversation_id,
|
|
2624
|
+
message_id: row.id
|
|
2625
|
+
});
|
|
2626
|
+
return;
|
|
2627
|
+
}
|
|
2628
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2629
|
+
}
|
|
2630
|
+
/**
|
|
2631
|
+
* Re-dispatch an orphaned (`unknown`) `processing` row (ADR-0046 Decision §2).
|
|
2632
|
+
*
|
|
2633
|
+
* The stable-id user message is absent from the session, so we (re-)dispatch with
|
|
2634
|
+
* the STABLE id (`opencodeMessageIdFor(row.id)`) — NOT a divergent per-attempt id.
|
|
2635
|
+
* This is what keeps the reply correlatable: the server's completion
|
|
2636
|
+
* notification looks for the reply under the stable id, so the fresh turn's reply
|
|
2637
|
+
* (which hangs off the stable id) is found and delivered. The residual
|
|
2638
|
+
* duplicate-incomplete-turn semantics (ADR §2, `.harness/restart-recovery-orphan-finding.md`)
|
|
2639
|
+
* are unchanged and, for an ABSENT id, cannot bite — there is no existing turn
|
|
2640
|
+
* to swallow the duplicate.
|
|
2641
|
+
*
|
|
2642
|
+
* `evidentMessageId = row.id` addresses the SERVER row; the stable
|
|
2643
|
+
* `opencodeMessageId` is what the watcher polls. Deadline anchored to
|
|
2644
|
+
* `processed_at` (Invariant 1).
|
|
2645
|
+
*/
|
|
2646
|
+
async forceReadoptRun(sessionId, row) {
|
|
2647
|
+
const ocId = opencodeMessageIdFor2(row.id);
|
|
2648
|
+
if (this.processedAtMs(row) + this.pausedMaxWaitMs <= this.now()) {
|
|
2649
|
+
this.dontRedispatch.add(row.id);
|
|
2650
|
+
this.log({
|
|
2651
|
+
level: "info",
|
|
2652
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned but its re-adopt window has already elapsed \u2014 not dispatching an unwatchable turn; parking until it leaves processing (cron will reset it)`,
|
|
2653
|
+
conversation_id: row.conversation_id,
|
|
2654
|
+
message_id: row.id
|
|
2655
|
+
});
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
const options = {
|
|
2659
|
+
agent: row.opencode_agent ?? void 0,
|
|
2660
|
+
model: row.opencode_model ?? void 0
|
|
2661
|
+
};
|
|
2662
|
+
this.log({
|
|
2663
|
+
level: "info",
|
|
2664
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} orphaned (user message absent) \u2014 re-dispatching with the stable id`,
|
|
2665
|
+
conversation_id: row.conversation_id,
|
|
2666
|
+
message_id: row.id
|
|
2667
|
+
});
|
|
2668
|
+
try {
|
|
2669
|
+
await sendPromptAsync(this.port, sessionId, row.content, options, ocId);
|
|
2670
|
+
} catch (err) {
|
|
2671
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
2672
|
+
this.log({
|
|
2673
|
+
level: "error",
|
|
2674
|
+
message: `Re-adopt: re-dispatch failed for message ${row.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
2675
|
+
conversation_id: row.conversation_id,
|
|
2676
|
+
message_id: row.id
|
|
2677
|
+
});
|
|
2678
|
+
return;
|
|
2679
|
+
}
|
|
2680
|
+
const conv = this.convForRow(sessionId, row);
|
|
2681
|
+
const message = this.queuedMessageForRow(row);
|
|
2682
|
+
this.registerReadopted(conv, sessionId, message, ocId, this.processedAtMs(row));
|
|
2683
|
+
this.dispatched.add(row.id);
|
|
2684
|
+
this.readopted.add(row.id);
|
|
2685
|
+
this.ensureWatcherRunning(sessionId);
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* True if `evidentMessageId` is already being driven — either in the
|
|
2689
|
+
* authoritative `dispatched` set or a live watcher's in-flight set for this
|
|
2690
|
+
* session (Invariant 2, WI-5). Either signal means a watcher owns the row.
|
|
2691
|
+
*/
|
|
2692
|
+
isTracked(sessionId, evidentMessageId) {
|
|
2693
|
+
if (this.dispatched.has(evidentMessageId)) return true;
|
|
2694
|
+
const watcher = this.watchers.get(sessionId);
|
|
2695
|
+
return watcher?.inFlight.has(evidentMessageId) ?? false;
|
|
2696
|
+
}
|
|
2697
|
+
/**
|
|
2698
|
+
* Parse a re-adopt row's `processed_at` (ISO string) to epoch ms for the
|
|
2699
|
+
* deadline anchor (Invariant 1). The endpoint guarantees `processed_at` is set
|
|
2700
|
+
* for `processing` rows, but if it is somehow null/unparseable fall back to
|
|
2701
|
+
* `now` (defensive) AND log — a fallback means the anchor is weaker than
|
|
2702
|
+
* intended, which is worth surfacing.
|
|
2703
|
+
*/
|
|
2704
|
+
processedAtMs(row) {
|
|
2705
|
+
const parsed = row.processed_at ? Date.parse(row.processed_at) : NaN;
|
|
2706
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
2707
|
+
this.log({
|
|
2708
|
+
level: "error",
|
|
2709
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has null/unparseable processed_at (${String(row.processed_at)}) \u2014 anchoring deadline to now (defensive)`,
|
|
2710
|
+
conversation_id: row.conversation_id,
|
|
2711
|
+
message_id: row.id
|
|
2712
|
+
});
|
|
2713
|
+
return this.now();
|
|
2714
|
+
}
|
|
2715
|
+
/** Build the `PendingConversation` shape the watcher needs from a re-adopt row. */
|
|
2716
|
+
convForRow(sessionId, row) {
|
|
2717
|
+
return {
|
|
2718
|
+
id: row.conversation_id,
|
|
2719
|
+
agent_id: this.agentId,
|
|
2720
|
+
opencode_session_id: sessionId,
|
|
2721
|
+
pending_message_count: 0,
|
|
2722
|
+
oldest_pending_at: row.processed_at
|
|
2723
|
+
};
|
|
2724
|
+
}
|
|
2725
|
+
/** Build the `QueuedMessage` shape the watcher/re-dispatch needs from a re-adopt row. */
|
|
2726
|
+
queuedMessageForRow(row) {
|
|
2727
|
+
return {
|
|
2728
|
+
id: row.id,
|
|
2729
|
+
content: row.content,
|
|
2730
|
+
status: "processing",
|
|
2731
|
+
opencode_agent: row.opencode_agent,
|
|
2732
|
+
opencode_model: row.opencode_model,
|
|
2733
|
+
source_message_id: row.source_message_id,
|
|
2734
|
+
slack_user_id: row.slack_user_id
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
1966
2737
|
/**
|
|
1967
2738
|
* Remove a message from the in-flight set AND the authoritative dispatched
|
|
1968
2739
|
* set. Once the in-flight set empties, the watcher loop's `while` guard exits
|
|
1969
2740
|
* and its `.finally` removes the session entry from `this.watchers`.
|
|
2741
|
+
*
|
|
2742
|
+
* Bug 2: if a RE-ADOPTED message is removed WITHOUT having completed
|
|
2743
|
+
* (`!inFlight.done` — i.e. a give-up: deadline reached, or markDone left to the
|
|
2744
|
+
* cron), park it in `dontRedispatch` so the next drain does NOT re-adopt (and
|
|
2745
|
+
* re-dispatch) the still-`processing` row every ~2s until the 15-min cron. A
|
|
2746
|
+
* re-adopted message that completed (`done`) needs no marker — it's leaving
|
|
2747
|
+
* `processing`. This suppresses only re-dispatch: if its reply later completes,
|
|
2748
|
+
* the done branch still delivers it (Bugbot #202).
|
|
1970
2749
|
*/
|
|
1971
2750
|
removeInFlight(watcher, evidentMessageId) {
|
|
2751
|
+
const inFlight = watcher.inFlight.get(evidentMessageId);
|
|
2752
|
+
if (this.readopted.delete(evidentMessageId) && inFlight && !inFlight.done) {
|
|
2753
|
+
this.dontRedispatch.add(evidentMessageId);
|
|
2754
|
+
this.log({
|
|
2755
|
+
level: "info",
|
|
2756
|
+
message: `Re-adopt: message ${evidentMessageId.slice(0, 8)} gave up \u2014 parking until it leaves the processing list (cron reset)`,
|
|
2757
|
+
conversation_id: watcher.conv.id,
|
|
2758
|
+
message_id: evidentMessageId
|
|
2759
|
+
});
|
|
2760
|
+
}
|
|
1972
2761
|
watcher.inFlight.delete(evidentMessageId);
|
|
1973
2762
|
this.dispatched.delete(evidentMessageId);
|
|
1974
2763
|
}
|
|
@@ -2112,6 +2901,35 @@ var ChannelDriver = class {
|
|
|
2112
2901
|
}
|
|
2113
2902
|
return await res.json();
|
|
2114
2903
|
}
|
|
2904
|
+
/**
|
|
2905
|
+
* Fetch this agent's `processing` messages for re-adoption (ADR-0046, WI-1).
|
|
2906
|
+
* The pending path (`getPendingConversations`/`getPendingMessages`) only
|
|
2907
|
+
* surfaces `pending` rows, so a message already `processing` when the runner
|
|
2908
|
+
* died is invisible to it — this dedicated endpoint returns exactly those rows
|
|
2909
|
+
* with the fields the re-adopt path needs (`processed_at`,
|
|
2910
|
+
* `opencode_session_id`, routing).
|
|
2911
|
+
*
|
|
2912
|
+
* Response is an OBJECT WRAPPER `{ messages: [...] }` (snake_case) — NOT a bare
|
|
2913
|
+
* array. Propagates `ChannelAuthError` on 401/403; throws a plain `Error` on
|
|
2914
|
+
* other non-ok so `drainPending`'s try/finally leaves `draining` false and the
|
|
2915
|
+
* next tick retries.
|
|
2916
|
+
*/
|
|
2917
|
+
async getProcessingMessages() {
|
|
2918
|
+
const res = await this.fetchImpl(
|
|
2919
|
+
`${this.apiUrl}/agents/${this.agentId}/conversations/processing`,
|
|
2920
|
+
{ headers: { Authorization: this.getAuthHeader() } }
|
|
2921
|
+
);
|
|
2922
|
+
this.assertAuth(res, "fetching processing messages");
|
|
2923
|
+
if (!res.ok) {
|
|
2924
|
+
throw new Error(`Failed to get processing messages: HTTP ${res.status}`);
|
|
2925
|
+
}
|
|
2926
|
+
const data = await res.json();
|
|
2927
|
+
let messages = data.messages ?? [];
|
|
2928
|
+
if (this.conversationFilter) {
|
|
2929
|
+
messages = messages.filter((m) => m.conversation_id === this.conversationFilter);
|
|
2930
|
+
}
|
|
2931
|
+
return messages;
|
|
2932
|
+
}
|
|
2115
2933
|
/**
|
|
2116
2934
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
2117
2935
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
@@ -2133,13 +2951,17 @@ var ChannelDriver = class {
|
|
|
2133
2951
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
2134
2952
|
* retry vehicle for the swap-to-running.
|
|
2135
2953
|
*/
|
|
2136
|
-
async markProcessing(conversationId, messageId, sessionId) {
|
|
2954
|
+
async markProcessing(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2137
2955
|
const res = await this.fetchImpl(
|
|
2138
2956
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2139
2957
|
{
|
|
2140
2958
|
method: "PATCH",
|
|
2141
2959
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2142
|
-
body: JSON.stringify({
|
|
2960
|
+
body: JSON.stringify({
|
|
2961
|
+
status: "processing",
|
|
2962
|
+
opencode_session_id: sessionId,
|
|
2963
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
2964
|
+
})
|
|
2143
2965
|
}
|
|
2144
2966
|
);
|
|
2145
2967
|
this.assertAuth(res, "marking message as processing");
|
|
@@ -2177,13 +2999,17 @@ var ChannelDriver = class {
|
|
|
2177
2999
|
* watcher retries next tick within the
|
|
2178
3000
|
* deadline, Finding 4).
|
|
2179
3001
|
*/
|
|
2180
|
-
async markDone(conversationId, messageId, sessionId) {
|
|
3002
|
+
async markDone(conversationId, messageId, sessionId, opencodeMessageId) {
|
|
2181
3003
|
const res = await this.fetchImpl(
|
|
2182
3004
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}`,
|
|
2183
3005
|
{
|
|
2184
3006
|
method: "PATCH",
|
|
2185
3007
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2186
|
-
body: JSON.stringify({
|
|
3008
|
+
body: JSON.stringify({
|
|
3009
|
+
status: "done",
|
|
3010
|
+
opencode_session_id: sessionId,
|
|
3011
|
+
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {}
|
|
3012
|
+
})
|
|
2187
3013
|
}
|
|
2188
3014
|
);
|
|
2189
3015
|
this.assertAuth(res, "marking message as done");
|
|
@@ -2193,7 +3019,17 @@ var ChannelDriver = class {
|
|
|
2193
3019
|
}
|
|
2194
3020
|
throw new ChannelTerminalError(`marking message as done: HTTP ${res.status}`, res.status);
|
|
2195
3021
|
}
|
|
2196
|
-
|
|
3022
|
+
/**
|
|
3023
|
+
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
3024
|
+
* when provided (issue #182): a bare `markFailed(conv, msg)` sends
|
|
3025
|
+
* `{status:'failed'}` unchanged (the dispatch-failure path), while an errored
|
|
3026
|
+
* OpenCode turn sends `{status:'failed', opencode_session_id, error}` so the
|
|
3027
|
+
* failure reason reaches the channel.
|
|
3028
|
+
*/
|
|
3029
|
+
async markFailed(conversationId, messageId, sessionId, error2) {
|
|
3030
|
+
const body = { status: "failed" };
|
|
3031
|
+
if (sessionId !== void 0) body.opencode_session_id = sessionId;
|
|
3032
|
+
if (error2 !== void 0) body.error = error2;
|
|
2197
3033
|
await this.callWithRetry(
|
|
2198
3034
|
"marking message as failed",
|
|
2199
3035
|
() => this.fetchImpl(
|
|
@@ -2201,11 +3037,47 @@ var ChannelDriver = class {
|
|
|
2201
3037
|
{
|
|
2202
3038
|
method: "PATCH",
|
|
2203
3039
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
2204
|
-
body: JSON.stringify(
|
|
3040
|
+
body: JSON.stringify(body)
|
|
2205
3041
|
}
|
|
2206
3042
|
)
|
|
2207
3043
|
);
|
|
2208
3044
|
}
|
|
3045
|
+
/**
|
|
3046
|
+
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
3047
|
+
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
3048
|
+
* — the server records it via `log()` (no DB write, no notification). This is
|
|
3049
|
+
* fire-and-forget: it MUST NEVER throw into the drain or the watcher tick, and
|
|
3050
|
+
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3051
|
+
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3052
|
+
* context (no silent catch, per development-workflow).
|
|
3053
|
+
*/
|
|
3054
|
+
async postSignal(conversationId, messageId, signal, extra) {
|
|
3055
|
+
try {
|
|
3056
|
+
const res = await this.fetchImpl(
|
|
3057
|
+
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}/messages/${messageId}/signal`,
|
|
3058
|
+
{
|
|
3059
|
+
method: "POST",
|
|
3060
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3061
|
+
body: JSON.stringify({ signal, ...extra })
|
|
3062
|
+
}
|
|
3063
|
+
);
|
|
3064
|
+
if (!res.ok) {
|
|
3065
|
+
this.log({
|
|
3066
|
+
level: "error",
|
|
3067
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} returned HTTP ${res.status} (telemetry-only, ignored)`,
|
|
3068
|
+
conversation_id: conversationId,
|
|
3069
|
+
message_id: messageId
|
|
3070
|
+
});
|
|
3071
|
+
}
|
|
3072
|
+
} catch (err) {
|
|
3073
|
+
this.log({
|
|
3074
|
+
level: "error",
|
|
3075
|
+
message: `Signal '${signal}' for message ${messageId.slice(0, 8)} failed (telemetry-only, ignored): ${err instanceof Error ? err.message : String(err)}`,
|
|
3076
|
+
conversation_id: conversationId,
|
|
3077
|
+
message_id: messageId
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
2209
3081
|
async persistSession(conversationId, sessionId) {
|
|
2210
3082
|
const res = await this.fetchImpl(
|
|
2211
3083
|
`${this.apiUrl}/agents/${this.agentId}/threads/${conversationId}`,
|
|
@@ -2521,7 +3393,9 @@ async function getAgentInfo(agentId, authHeader) {
|
|
|
2521
3393
|
// src/commands/run.ts
|
|
2522
3394
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
2523
3395
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
2524
|
-
|
|
3396
|
+
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
3397
|
+
var CHANNEL_SETTLE_MS = Number(process.env.EVIDENT_SETTLE_MS) || void 0;
|
|
3398
|
+
function log2(state, message, isError = false) {
|
|
2525
3399
|
if (state.json) {
|
|
2526
3400
|
console.log(
|
|
2527
3401
|
JSON.stringify({
|
|
@@ -2546,9 +3420,9 @@ function logActivity(state, entry) {
|
|
|
2546
3420
|
}
|
|
2547
3421
|
if (!state.interactive) {
|
|
2548
3422
|
if (entry.type === "error") {
|
|
2549
|
-
|
|
3423
|
+
log2(state, entry.error ?? "Unknown error", true);
|
|
2550
3424
|
} else if (entry.type === "info" && entry.message) {
|
|
2551
|
-
|
|
3425
|
+
log2(state, entry.message);
|
|
2552
3426
|
}
|
|
2553
3427
|
}
|
|
2554
3428
|
}
|
|
@@ -2634,6 +3508,7 @@ async function handleAuthError(state, error2) {
|
|
|
2634
3508
|
}
|
|
2635
3509
|
async function driveChannels(state, driver) {
|
|
2636
3510
|
let idlePolls = 0;
|
|
3511
|
+
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
2637
3512
|
while (state.running) {
|
|
2638
3513
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
2639
3514
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
@@ -2643,7 +3518,9 @@ async function driveChannels(state, driver) {
|
|
|
2643
3518
|
try {
|
|
2644
3519
|
const processed = await driver.drainPending();
|
|
2645
3520
|
state.messageCount += processed;
|
|
2646
|
-
|
|
3521
|
+
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
3522
|
+
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
3523
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity) {
|
|
2647
3524
|
idlePolls = 0;
|
|
2648
3525
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
2649
3526
|
} else if (state.idleTimeout !== null) {
|
|
@@ -2695,7 +3572,7 @@ async function cleanup(state) {
|
|
|
2695
3572
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
2696
3573
|
displayStatus(state);
|
|
2697
3574
|
} else {
|
|
2698
|
-
|
|
3575
|
+
log2(state, "Stopped OpenCode process");
|
|
2699
3576
|
}
|
|
2700
3577
|
state.opencodeProcess = null;
|
|
2701
3578
|
}
|
|
@@ -2718,10 +3595,11 @@ async function run(options) {
|
|
|
2718
3595
|
running: true,
|
|
2719
3596
|
activityLog: [],
|
|
2720
3597
|
messageCount: 0,
|
|
3598
|
+
lastProxiedActivityAt: null,
|
|
2721
3599
|
authHeader: ""
|
|
2722
3600
|
};
|
|
2723
3601
|
if (state.idleTimeout === null && (process.env.GITHUB_ACTIONS || process.env.CI)) {
|
|
2724
|
-
|
|
3602
|
+
log2(
|
|
2725
3603
|
state,
|
|
2726
3604
|
"Warning: No --idle-timeout set in CI environment. The runner will poll indefinitely until the job times out. Consider adding --idle-timeout 30 to avoid wasting runner minutes.",
|
|
2727
3605
|
false
|
|
@@ -2732,7 +3610,7 @@ async function run(options) {
|
|
|
2732
3610
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
2733
3611
|
displayStatus(state);
|
|
2734
3612
|
} else {
|
|
2735
|
-
|
|
3613
|
+
log2(state, "Shutting down...");
|
|
2736
3614
|
}
|
|
2737
3615
|
await cleanup(state);
|
|
2738
3616
|
await shutdownTelemetry();
|
|
@@ -2765,7 +3643,7 @@ async function run(options) {
|
|
|
2765
3643
|
const resolved = await resolveAgentIdFromKey(state.authHeader);
|
|
2766
3644
|
if (resolved.agent_id) {
|
|
2767
3645
|
state.agentId = resolved.agent_id;
|
|
2768
|
-
|
|
3646
|
+
log2(state, `Resolved agent ID from key: ${state.agentId}`);
|
|
2769
3647
|
if (state.interactive && !state.json) {
|
|
2770
3648
|
logActivity(state, {
|
|
2771
3649
|
type: "info",
|
|
@@ -2828,17 +3706,17 @@ async function run(options) {
|
|
|
2828
3706
|
port: state.port,
|
|
2829
3707
|
interactive: state.interactive,
|
|
2830
3708
|
agentId: state.agentId,
|
|
2831
|
-
log: (message) =>
|
|
3709
|
+
log: (message) => log2(state, message)
|
|
2832
3710
|
});
|
|
2833
3711
|
state.port = oc.port;
|
|
2834
3712
|
state.opencodeProcess = oc.process;
|
|
2835
3713
|
state.opencodeVersion = oc.version;
|
|
2836
3714
|
state.opencodeConnected = oc.process !== null || oc.version !== null;
|
|
2837
|
-
const
|
|
2838
|
-
ocSpinner?.succeed(`OpenCode running on port ${state.port}${
|
|
3715
|
+
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
3716
|
+
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
2839
3717
|
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
2840
3718
|
if (versionWarning) {
|
|
2841
|
-
|
|
3719
|
+
log2(state, versionWarning, false);
|
|
2842
3720
|
if (state.interactive && !state.json) {
|
|
2843
3721
|
logActivity(state, { type: "info", message: versionWarning });
|
|
2844
3722
|
}
|
|
@@ -2854,6 +3732,8 @@ async function run(options) {
|
|
|
2854
3732
|
apiUrl: getApiUrlConfig(),
|
|
2855
3733
|
getAuthHeader: () => state.authHeader,
|
|
2856
3734
|
conversationFilter: state.conversationFilter,
|
|
3735
|
+
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
3736
|
+
settleMs: CHANNEL_SETTLE_MS,
|
|
2857
3737
|
log: (entry) => logActivity(state, {
|
|
2858
3738
|
type: entry.level === "error" ? "error" : "info",
|
|
2859
3739
|
message: entry.message,
|
|
@@ -2873,7 +3753,11 @@ async function run(options) {
|
|
|
2873
3753
|
type: "info",
|
|
2874
3754
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (agent: ${agentId})`
|
|
2875
3755
|
});
|
|
2876
|
-
emitAgentConnected(state.agentId, {
|
|
3756
|
+
emitAgentConnected(state.agentId, {
|
|
3757
|
+
port: state.port,
|
|
3758
|
+
cli_version: getCliVersion(),
|
|
3759
|
+
opencode_version: state.opencodeVersion
|
|
3760
|
+
});
|
|
2877
3761
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
2878
3762
|
if (state.interactive) displayStatus(state);
|
|
2879
3763
|
channelDriver.drainPending().then((processed) => {
|
|
@@ -2907,9 +3791,14 @@ async function run(options) {
|
|
|
2907
3791
|
logActivity(state, { type: "error", error: error2 });
|
|
2908
3792
|
if (state.interactive) displayStatus(state);
|
|
2909
3793
|
},
|
|
2910
|
-
// Web traffic is proxied transparently;
|
|
3794
|
+
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
3795
|
+
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
3796
|
+
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
3797
|
+
// the internal drain-ping, so an actively-used proxy keeps the timer
|
|
3798
|
+
// fresh while a lone idle SSE with no follow-up requests still ages out.
|
|
2911
3799
|
onResponse: () => {
|
|
2912
3800
|
state.opencodeConnected = true;
|
|
3801
|
+
state.lastProxiedActivityAt = Date.now();
|
|
2913
3802
|
},
|
|
2914
3803
|
// A channel message was queued and the api-worker pinged us over the
|
|
2915
3804
|
// tunnel to drain immediately instead of waiting for the next poll tick.
|
|
@@ -2948,7 +3837,7 @@ async function run(options) {
|
|
|
2948
3837
|
throw error2;
|
|
2949
3838
|
}
|
|
2950
3839
|
if (!interactive || state.json) {
|
|
2951
|
-
|
|
3840
|
+
log2(state, "Driving channel messages...");
|
|
2952
3841
|
}
|
|
2953
3842
|
await driveChannels(state, channelDriver);
|
|
2954
3843
|
await cleanup(state);
|
|
@@ -2960,7 +3849,7 @@ async function run(options) {
|
|
|
2960
3849
|
})
|
|
2961
3850
|
);
|
|
2962
3851
|
} else if (!interactive) {
|
|
2963
|
-
|
|
3852
|
+
log2(state, `Completed. Processed ${state.messageCount} message(s).`);
|
|
2964
3853
|
}
|
|
2965
3854
|
await shutdownTelemetry();
|
|
2966
3855
|
process.exit(0);
|
|
@@ -2982,8 +3871,9 @@ async function run(options) {
|
|
|
2982
3871
|
}
|
|
2983
3872
|
|
|
2984
3873
|
// src/index.ts
|
|
3874
|
+
var { version } = createRequire(import.meta.url)("../package.json");
|
|
2985
3875
|
var program = new Command();
|
|
2986
|
-
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(
|
|
3876
|
+
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
2987
3877
|
"--endpoint <url>",
|
|
2988
3878
|
"Evident API base URL (default: production; e.g. http://localhost:3001)"
|
|
2989
3879
|
).option("--tunnel <url>", "Tunnel WebSocket URL (default: production; e.g. ws://localhost:8787)").hook("preAction", (thisCommand) => {
|