@evident-ai/cli 3.0.1-dev.0d0721c → 3.0.1-dev.10c527f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +397 -21
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1078,6 +1078,12 @@ async function getSessionMessages(port, sessionId) {
|
|
|
1078
1078
|
return null;
|
|
1079
1079
|
}
|
|
1080
1080
|
}
|
|
1081
|
+
function isSessionActivelyGenerating(messages) {
|
|
1082
|
+
if (!messages || messages.length === 0) return false;
|
|
1083
|
+
const last = messages[messages.length - 1];
|
|
1084
|
+
if (roleOf(last) !== "assistant") return false;
|
|
1085
|
+
return completedOf(last) == null;
|
|
1086
|
+
}
|
|
1081
1087
|
function sessionLastActivityMs(session) {
|
|
1082
1088
|
const candidates = [
|
|
1083
1089
|
session.time?.updated,
|
|
@@ -1120,6 +1126,36 @@ async function sessionExists(port, id) {
|
|
|
1120
1126
|
return null;
|
|
1121
1127
|
}
|
|
1122
1128
|
}
|
|
1129
|
+
async function getSessionStatuses(port) {
|
|
1130
|
+
try {
|
|
1131
|
+
const res = await fetch(`${opencodeBase(port)}/session/status`);
|
|
1132
|
+
if (!res.ok) {
|
|
1133
|
+
console.error(
|
|
1134
|
+
`[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
|
|
1135
|
+
);
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
const body = await res.json();
|
|
1139
|
+
if (body == null || typeof body !== "object" || Array.isArray(body)) {
|
|
1140
|
+
console.error(
|
|
1141
|
+
`[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
|
|
1142
|
+
);
|
|
1143
|
+
return null;
|
|
1144
|
+
}
|
|
1145
|
+
return body;
|
|
1146
|
+
} catch (err) {
|
|
1147
|
+
console.error(
|
|
1148
|
+
`[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
|
|
1149
|
+
);
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
async function isSessionOngoing(port, id) {
|
|
1154
|
+
const map = await getSessionStatuses(port);
|
|
1155
|
+
if (map == null) return null;
|
|
1156
|
+
const entry = map[id];
|
|
1157
|
+
return entry != null && entry.type !== "idle";
|
|
1158
|
+
}
|
|
1123
1159
|
async function createOpenCodeSession(port, directory) {
|
|
1124
1160
|
const url = new URL(`${opencodeBase(port)}/session`);
|
|
1125
1161
|
if (directory && directory.trim()) {
|
|
@@ -1246,6 +1282,11 @@ function messageRunState(messages, userMessageId) {
|
|
|
1246
1282
|
if (isAssistantInFlight(reply)) return "running";
|
|
1247
1283
|
return errorOf(reply) != null ? "failed" : "done";
|
|
1248
1284
|
}
|
|
1285
|
+
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
1286
|
+
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
1287
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1288
|
+
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1289
|
+
}
|
|
1249
1290
|
function messageError(messages, userMessageId) {
|
|
1250
1291
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1251
1292
|
const error2 = errorOf(reply);
|
|
@@ -1778,6 +1819,9 @@ var DEFAULT_RETRY_POLICY = {
|
|
|
1778
1819
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
1779
1820
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
1780
1821
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
1822
|
+
var HEARTBEAT_MS = 6e4;
|
|
1823
|
+
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
1824
|
+
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
1781
1825
|
var ChannelAuthError = class extends Error {
|
|
1782
1826
|
constructor(message) {
|
|
1783
1827
|
super(message);
|
|
@@ -1874,6 +1918,15 @@ var ChannelDriver = class {
|
|
|
1874
1918
|
* the row leaves the processing list, exactly like `dontRedispatch`.
|
|
1875
1919
|
*/
|
|
1876
1920
|
doneUndeliverable = /* @__PURE__ */ new Set();
|
|
1921
|
+
/**
|
|
1922
|
+
* "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
|
|
1923
|
+
* unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
|
|
1924
|
+
* every ~2s drain until the status map becomes readable — but the server-visible
|
|
1925
|
+
* signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
|
|
1926
|
+
* (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
|
|
1927
|
+
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
1928
|
+
*/
|
|
1929
|
+
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
1877
1930
|
/**
|
|
1878
1931
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
1879
1932
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -2249,7 +2302,9 @@ var ChannelDriver = class {
|
|
|
2249
2302
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2250
2303
|
loop: null,
|
|
2251
2304
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2252
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2305
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2306
|
+
lastGoodPollAt: this.now(),
|
|
2307
|
+
hadUsablePoll: false
|
|
2253
2308
|
};
|
|
2254
2309
|
this.watchers.set(sessionId, watcher);
|
|
2255
2310
|
}
|
|
@@ -2259,20 +2314,37 @@ var ChannelDriver = class {
|
|
|
2259
2314
|
opencodeMessageId,
|
|
2260
2315
|
message,
|
|
2261
2316
|
dispatchedAt: now,
|
|
2317
|
+
processingAnchorMs: now,
|
|
2262
2318
|
deadline: now + this.pausedMaxWaitMs,
|
|
2263
2319
|
started: false,
|
|
2264
2320
|
done: false,
|
|
2265
|
-
stuckReported: false
|
|
2321
|
+
stuckReported: false,
|
|
2322
|
+
lastAliveAt: 0,
|
|
2323
|
+
aliveInFlight: false,
|
|
2324
|
+
awaitingHumanLatched: false,
|
|
2325
|
+
pausedOnQuestion: false,
|
|
2326
|
+
pausedOnPermission: false,
|
|
2327
|
+
pausedClearConfirmed: false,
|
|
2328
|
+
pausedInFlight: false,
|
|
2329
|
+
deliveryDeadlineAnchored: false
|
|
2266
2330
|
});
|
|
2267
2331
|
}
|
|
2268
2332
|
/**
|
|
2269
2333
|
* Register a RE-ADOPTED `processing` message with its session watcher
|
|
2270
2334
|
* (ADR-0046, WI-4). Mirrors `registerInFlight` but anchors the give-up
|
|
2271
2335
|
* `deadline` to the row's SERVER-SIDE `processed_at` (Invariant 1), NEVER to
|
|
2272
|
-
* `now
|
|
2273
|
-
* (10 min after `processed_at
|
|
2274
|
-
*
|
|
2275
|
-
*
|
|
2336
|
+
* `now`, so the paused/queued/unreachable cases settle on the same wall-clock a
|
|
2337
|
+
* fresh dispatch would (10 min after `processed_at`, not 10 min from now).
|
|
2338
|
+
*
|
|
2339
|
+
* This re-attaches into the SAME watcher, so the ADR-0047 progressing-vs-paused
|
|
2340
|
+
* give-up (`serviceInFlightMessage`) applies unchanged: a re-adopted turn
|
|
2341
|
+
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2342
|
+
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2343
|
+
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2344
|
+
* handed to the cron. The old "the `deadline` must settle before the ~15-min
|
|
2345
|
+
* cron or they double-drive" reasoning is superseded: liveness now settles the
|
|
2346
|
+
* actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
|
|
2347
|
+
* (only the appear-guard uses it).
|
|
2276
2348
|
*
|
|
2277
2349
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
2278
2350
|
* `opencodeMessageId` is the id the watcher polls for a reply — for the orphan
|
|
@@ -2290,7 +2362,9 @@ var ChannelDriver = class {
|
|
|
2290
2362
|
inFlight: /* @__PURE__ */ new Map(),
|
|
2291
2363
|
loop: null,
|
|
2292
2364
|
reportedQuestions: /* @__PURE__ */ new Set(),
|
|
2293
|
-
reportedPermissions: /* @__PURE__ */ new Set()
|
|
2365
|
+
reportedPermissions: /* @__PURE__ */ new Set(),
|
|
2366
|
+
lastGoodPollAt: this.now(),
|
|
2367
|
+
hadUsablePoll: false
|
|
2294
2368
|
};
|
|
2295
2369
|
this.watchers.set(sessionId, watcher);
|
|
2296
2370
|
}
|
|
@@ -2299,6 +2373,10 @@ var ChannelDriver = class {
|
|
|
2299
2373
|
opencodeMessageId,
|
|
2300
2374
|
message,
|
|
2301
2375
|
dispatchedAt: this.now(),
|
|
2376
|
+
// Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
|
|
2377
|
+
// value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
|
|
2378
|
+
// reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
|
|
2379
|
+
processingAnchorMs: processedAtMs,
|
|
2302
2380
|
deadline: processedAtMs + this.pausedMaxWaitMs,
|
|
2303
2381
|
// The server row is ALREADY `processing`; do not re-fire markProcessing.
|
|
2304
2382
|
started: true,
|
|
@@ -2308,7 +2386,20 @@ var ChannelDriver = class {
|
|
|
2308
2386
|
// on `state === 'queued'` (turn produced no reply), not on `started`, so a
|
|
2309
2387
|
// re-adopted row left wedged in `queued` still emits the signal once
|
|
2310
2388
|
// (#210/#220 observability).
|
|
2311
|
-
stuckReported: false
|
|
2389
|
+
stuckReported: false,
|
|
2390
|
+
// Task 5.2: a re-adopted actively-running row re-attaches into the SAME
|
|
2391
|
+
// watcher and so hits the SAME actively-running heartbeat branch in
|
|
2392
|
+
// `serviceInFlightMessage` as a fresh dispatch — monitoring observes "runner
|
|
2393
|
+
// re-adopted and is confirming this row alive" via that `alive` heartbeat,
|
|
2394
|
+
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2395
|
+
lastAliveAt: 0,
|
|
2396
|
+
aliveInFlight: false,
|
|
2397
|
+
awaitingHumanLatched: false,
|
|
2398
|
+
pausedOnQuestion: false,
|
|
2399
|
+
pausedOnPermission: false,
|
|
2400
|
+
pausedClearConfirmed: false,
|
|
2401
|
+
pausedInFlight: false,
|
|
2402
|
+
deliveryDeadlineAnchored: false
|
|
2312
2403
|
});
|
|
2313
2404
|
}
|
|
2314
2405
|
/**
|
|
@@ -2359,12 +2450,30 @@ var ChannelDriver = class {
|
|
|
2359
2450
|
messages = Array.isArray(body) ? body : null;
|
|
2360
2451
|
}
|
|
2361
2452
|
} catch {
|
|
2362
|
-
continue;
|
|
2363
2453
|
}
|
|
2454
|
+
if (messages != null && messages.length > 0) {
|
|
2455
|
+
watcher.lastGoodPollAt = this.now();
|
|
2456
|
+
watcher.hadUsablePoll = true;
|
|
2457
|
+
} else {
|
|
2458
|
+
const emptyButReachable = messages != null;
|
|
2459
|
+
const graceApplies = !emptyButReachable || watcher.hadUsablePoll;
|
|
2460
|
+
if (graceApplies && this.now() - watcher.lastGoodPollAt < POLL_MISS_GRACE_MS) {
|
|
2461
|
+
continue;
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
const { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk } = await this.pollInteractions(sessionId, watcher, messages);
|
|
2364
2465
|
for (const inFlight of [...watcher.inFlight.values()]) {
|
|
2365
|
-
await this.serviceInFlightMessage(
|
|
2466
|
+
await this.serviceInFlightMessage(
|
|
2467
|
+
sessionId,
|
|
2468
|
+
watcher,
|
|
2469
|
+
inFlight,
|
|
2470
|
+
messages,
|
|
2471
|
+
openQuestions,
|
|
2472
|
+
openPermissions,
|
|
2473
|
+
questionsPolledOk,
|
|
2474
|
+
permissionsPolledOk
|
|
2475
|
+
);
|
|
2366
2476
|
}
|
|
2367
|
-
await this.pollInteractions(sessionId, watcher, messages);
|
|
2368
2477
|
}
|
|
2369
2478
|
} catch (err) {
|
|
2370
2479
|
if (err instanceof ChannelAuthError) {
|
|
@@ -2386,15 +2495,40 @@ var ChannelDriver = class {
|
|
|
2386
2495
|
});
|
|
2387
2496
|
}
|
|
2388
2497
|
}
|
|
2498
|
+
/**
|
|
2499
|
+
* On FIRST observing a terminal (done/failed) state, ensure the delivery
|
|
2500
|
+
* (markDone/markFailed) transient-retry path has a real window. A long
|
|
2501
|
+
* ACTIVELY-running turn is kept past its original `deadline`, so by completion
|
|
2502
|
+
* `now >= deadline` already holds and the retry bound below would fire on the
|
|
2503
|
+
* first transient PATCH failure — dropping the message before its reply lands
|
|
2504
|
+
* (Bugbot "Stale deadline aborts long-turn delivery"). Re-anchor once (latched)
|
|
2505
|
+
* to a fresh `pausedMaxWaitMs` window; only extend if the current deadline is at
|
|
2506
|
+
* or past now, so a still-ample window is left untouched.
|
|
2507
|
+
*/
|
|
2508
|
+
anchorDeliveryDeadline(inFlight) {
|
|
2509
|
+
if (inFlight.deliveryDeadlineAnchored) return;
|
|
2510
|
+
inFlight.deliveryDeadlineAnchored = true;
|
|
2511
|
+
if (this.now() >= inFlight.deadline) {
|
|
2512
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2389
2515
|
/**
|
|
2390
2516
|
* Drive ONE in-flight message's lifecycle from the tick's message snapshot.
|
|
2391
2517
|
* Fires markProcessing on queued→running and markDone on done (each once),
|
|
2392
2518
|
* applies the idle-path re-dispatch guard, and removes the message from the
|
|
2393
2519
|
* in-flight set on completion or timeout.
|
|
2394
2520
|
*/
|
|
2395
|
-
async serviceInFlightMessage(sessionId, watcher, inFlight, messages) {
|
|
2521
|
+
async serviceInFlightMessage(sessionId, watcher, inFlight, messages, openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk) {
|
|
2396
2522
|
const conv = watcher.conv;
|
|
2397
2523
|
const state = messageRunState(messages, inFlight.opencodeMessageId);
|
|
2524
|
+
const id = inFlight.evidentMessageId;
|
|
2525
|
+
if (openQuestions.has(id)) inFlight.pausedOnQuestion = true;
|
|
2526
|
+
else if (questionsPolledOk) inFlight.pausedOnQuestion = false;
|
|
2527
|
+
if (openPermissions.has(id)) inFlight.pausedOnPermission = true;
|
|
2528
|
+
else if (permissionsPolledOk) inFlight.pausedOnPermission = false;
|
|
2529
|
+
const observedOpen = openQuestions.has(id) || openPermissions.has(id);
|
|
2530
|
+
const latchedPaused = inFlight.pausedOnQuestion || inFlight.pausedOnPermission;
|
|
2531
|
+
const awaitingHuman = observedOpen || latchedPaused;
|
|
2398
2532
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2399
2533
|
let claimed;
|
|
2400
2534
|
try {
|
|
@@ -2425,6 +2559,7 @@ var ChannelDriver = class {
|
|
|
2425
2559
|
}
|
|
2426
2560
|
}
|
|
2427
2561
|
if (state === "done") {
|
|
2562
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2428
2563
|
if (!inFlight.done) {
|
|
2429
2564
|
this.log({
|
|
2430
2565
|
level: "info",
|
|
@@ -2475,6 +2610,7 @@ var ChannelDriver = class {
|
|
|
2475
2610
|
return;
|
|
2476
2611
|
}
|
|
2477
2612
|
if (state === "failed") {
|
|
2613
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
2478
2614
|
if (!inFlight.done) {
|
|
2479
2615
|
const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
|
|
2480
2616
|
this.log({
|
|
@@ -2528,7 +2664,51 @@ var ChannelDriver = class {
|
|
|
2528
2664
|
stuck_for_ms: this.now() - inFlight.dispatchedAt
|
|
2529
2665
|
});
|
|
2530
2666
|
}
|
|
2531
|
-
|
|
2667
|
+
const activelyRunning = state === "running" && !awaitingHuman;
|
|
2668
|
+
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
2669
|
+
this.log({
|
|
2670
|
+
level: "error",
|
|
2671
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
|
|
2672
|
+
conversation_id: conv.id,
|
|
2673
|
+
message_id: inFlight.evidentMessageId
|
|
2674
|
+
});
|
|
2675
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
|
|
2676
|
+
watched_for_ms: this.now() - inFlight.processingAnchorMs
|
|
2677
|
+
});
|
|
2678
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2679
|
+
return;
|
|
2680
|
+
}
|
|
2681
|
+
if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
|
|
2682
|
+
inFlight.aliveInFlight = true;
|
|
2683
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
|
|
2684
|
+
inFlight.aliveInFlight = false;
|
|
2685
|
+
if (ok) inFlight.lastAliveAt = this.now();
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
if (awaitingHuman) {
|
|
2689
|
+
if (!inFlight.awaitingHumanLatched) {
|
|
2690
|
+
inFlight.deadline = this.now() + this.pausedMaxWaitMs;
|
|
2691
|
+
inFlight.awaitingHumanLatched = true;
|
|
2692
|
+
}
|
|
2693
|
+
if (!inFlight.pausedClearConfirmed && !inFlight.pausedInFlight) {
|
|
2694
|
+
inFlight.pausedInFlight = true;
|
|
2695
|
+
void this.postSignal(conv.id, inFlight.evidentMessageId, "paused").then((ok) => {
|
|
2696
|
+
inFlight.pausedInFlight = false;
|
|
2697
|
+
if (ok && inFlight.awaitingHumanLatched) inFlight.pausedClearConfirmed = true;
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2700
|
+
} else if (inFlight.awaitingHumanLatched) {
|
|
2701
|
+
inFlight.awaitingHumanLatched = false;
|
|
2702
|
+
inFlight.pausedOnQuestion = false;
|
|
2703
|
+
inFlight.pausedOnPermission = false;
|
|
2704
|
+
inFlight.pausedClearConfirmed = false;
|
|
2705
|
+
}
|
|
2706
|
+
const siblingPaused = (sib) => openQuestions.has(sib.evidentMessageId) || openPermissions.has(sib.evidentMessageId) || sib.awaitingHumanLatched || sib.pausedOnQuestion || sib.pausedOnPermission;
|
|
2707
|
+
const hasActivelyRunningSibling = [...watcher.inFlight.values()].some(
|
|
2708
|
+
(sib) => sib.evidentMessageId !== inFlight.evidentMessageId && messageRunState(messages, sib.opencodeMessageId) === "running" && !siblingPaused(sib)
|
|
2709
|
+
);
|
|
2710
|
+
const queuedBehindRunningSibling = state === "queued" && hasActivelyRunningSibling;
|
|
2711
|
+
if (!activelyRunning && !queuedBehindRunningSibling && this.now() >= inFlight.deadline) {
|
|
2532
2712
|
this.log({
|
|
2533
2713
|
level: "info",
|
|
2534
2714
|
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} did not complete within the watch window \u2014 leaving for the cron safety net`,
|
|
@@ -2560,12 +2740,17 @@ var ChannelDriver = class {
|
|
|
2560
2740
|
*/
|
|
2561
2741
|
async readoptProcessing() {
|
|
2562
2742
|
const rows = await this.getProcessingMessages();
|
|
2563
|
-
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
|
|
2743
|
+
if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
|
|
2564
2744
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
2565
|
-
for (const id of [
|
|
2745
|
+
for (const id of [
|
|
2746
|
+
...this.dontRedispatch,
|
|
2747
|
+
...this.doneUndeliverable,
|
|
2748
|
+
...this.readoptPollUnresolvedSignalled
|
|
2749
|
+
]) {
|
|
2566
2750
|
if (!stillProcessing.has(id)) {
|
|
2567
2751
|
const cleared = this.dontRedispatch.delete(id);
|
|
2568
2752
|
const clearedUndeliverable = this.doneUndeliverable.delete(id);
|
|
2753
|
+
this.readoptPollUnresolvedSignalled.delete(id);
|
|
2569
2754
|
if (cleared || clearedUndeliverable) {
|
|
2570
2755
|
this.log({
|
|
2571
2756
|
level: "info",
|
|
@@ -2619,8 +2804,10 @@ var ChannelDriver = class {
|
|
|
2619
2804
|
});
|
|
2620
2805
|
continue;
|
|
2621
2806
|
}
|
|
2807
|
+
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
2808
|
+
const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
|
|
2622
2809
|
for (const row of sessionRows) {
|
|
2623
|
-
await this.readoptOne(sessionId, row, messages);
|
|
2810
|
+
await this.readoptOne(sessionId, row, messages, sessionOngoing);
|
|
2624
2811
|
}
|
|
2625
2812
|
}
|
|
2626
2813
|
}
|
|
@@ -2642,7 +2829,7 @@ var ChannelDriver = class {
|
|
|
2642
2829
|
*
|
|
2643
2830
|
* Only `ChannelAuthError` propagates.
|
|
2644
2831
|
*/
|
|
2645
|
-
async readoptOne(sessionId, row, messages) {
|
|
2832
|
+
async readoptOne(sessionId, row, messages, sessionOngoing) {
|
|
2646
2833
|
if (this.isTracked(sessionId, row.id)) {
|
|
2647
2834
|
this.log({
|
|
2648
2835
|
level: "info",
|
|
@@ -2682,6 +2869,7 @@ var ChannelDriver = class {
|
|
|
2682
2869
|
conversation_id: row.conversation_id,
|
|
2683
2870
|
message_id: row.id
|
|
2684
2871
|
});
|
|
2872
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2685
2873
|
return;
|
|
2686
2874
|
}
|
|
2687
2875
|
this.log({
|
|
@@ -2693,6 +2881,7 @@ var ChannelDriver = class {
|
|
|
2693
2881
|
return;
|
|
2694
2882
|
}
|
|
2695
2883
|
this.dontRedispatch.delete(row.id);
|
|
2884
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
2696
2885
|
return;
|
|
2697
2886
|
}
|
|
2698
2887
|
if (state === "failed") {
|
|
@@ -2715,6 +2904,7 @@ var ChannelDriver = class {
|
|
|
2715
2904
|
conversation_id: row.conversation_id,
|
|
2716
2905
|
message_id: row.id
|
|
2717
2906
|
});
|
|
2907
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
2718
2908
|
return;
|
|
2719
2909
|
}
|
|
2720
2910
|
this.log({
|
|
@@ -2726,6 +2916,7 @@ var ChannelDriver = class {
|
|
|
2726
2916
|
return;
|
|
2727
2917
|
}
|
|
2728
2918
|
this.dontRedispatch.delete(row.id);
|
|
2919
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
2729
2920
|
return;
|
|
2730
2921
|
}
|
|
2731
2922
|
if (this.dontRedispatch.has(row.id)) {
|
|
@@ -2737,6 +2928,71 @@ var ChannelDriver = class {
|
|
|
2737
2928
|
});
|
|
2738
2929
|
return;
|
|
2739
2930
|
}
|
|
2931
|
+
let statusReadableOngoing = null;
|
|
2932
|
+
if (state === "running" && ocId) {
|
|
2933
|
+
const reply = findLastAssistantReplyFor(messages, ocId);
|
|
2934
|
+
const shape = this.replyCompletionShape(reply);
|
|
2935
|
+
const ongoing = sessionOngoing;
|
|
2936
|
+
statusReadableOngoing = ongoing;
|
|
2937
|
+
if (ongoing === false) {
|
|
2938
|
+
this.log({
|
|
2939
|
+
level: "info",
|
|
2940
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
2941
|
+
conversation_id: row.conversation_id,
|
|
2942
|
+
message_id: row.id
|
|
2943
|
+
});
|
|
2944
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2945
|
+
return;
|
|
2946
|
+
}
|
|
2947
|
+
if (ongoing === true) {
|
|
2948
|
+
this.log({
|
|
2949
|
+
level: "info",
|
|
2950
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
|
|
2951
|
+
conversation_id: row.conversation_id,
|
|
2952
|
+
message_id: row.id
|
|
2953
|
+
});
|
|
2954
|
+
} else {
|
|
2955
|
+
if (shape === "b1") {
|
|
2956
|
+
this.log({
|
|
2957
|
+
level: "info",
|
|
2958
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
|
|
2959
|
+
conversation_id: row.conversation_id,
|
|
2960
|
+
message_id: row.id
|
|
2961
|
+
});
|
|
2962
|
+
if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
|
|
2963
|
+
this.readoptPollUnresolvedSignalled.add(row.id);
|
|
2964
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
|
|
2965
|
+
}
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
this.log({
|
|
2969
|
+
level: "info",
|
|
2970
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
|
|
2971
|
+
conversation_id: row.conversation_id,
|
|
2972
|
+
message_id: row.id
|
|
2973
|
+
});
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
|
|
2977
|
+
const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
|
|
2978
|
+
if (descendantAlive === true) {
|
|
2979
|
+
this.log({
|
|
2980
|
+
level: "info",
|
|
2981
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
|
|
2982
|
+
conversation_id: row.conversation_id,
|
|
2983
|
+
message_id: row.id
|
|
2984
|
+
});
|
|
2985
|
+
} else {
|
|
2986
|
+
this.log({
|
|
2987
|
+
level: "info",
|
|
2988
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
|
|
2989
|
+
conversation_id: row.conversation_id,
|
|
2990
|
+
message_id: row.id
|
|
2991
|
+
});
|
|
2992
|
+
await this.forceReadoptRun(sessionId, row);
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2740
2996
|
if ((state === "running" || state === "queued") && ocId) {
|
|
2741
2997
|
const conv = this.convForRow(sessionId, row);
|
|
2742
2998
|
const message = this.queuedMessageForRow(row);
|
|
@@ -2750,6 +3006,7 @@ var ChannelDriver = class {
|
|
|
2750
3006
|
conversation_id: row.conversation_id,
|
|
2751
3007
|
message_id: row.id
|
|
2752
3008
|
});
|
|
3009
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
|
|
2753
3010
|
return;
|
|
2754
3011
|
}
|
|
2755
3012
|
await this.forceReadoptRun(sessionId, row);
|
|
@@ -2802,6 +3059,7 @@ var ChannelDriver = class {
|
|
|
2802
3059
|
conversation_id: row.conversation_id,
|
|
2803
3060
|
message_id: row.id
|
|
2804
3061
|
});
|
|
3062
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
|
|
2805
3063
|
return;
|
|
2806
3064
|
}
|
|
2807
3065
|
const options = {
|
|
@@ -2830,6 +3088,7 @@ var ChannelDriver = class {
|
|
|
2830
3088
|
conversation_id: row.conversation_id,
|
|
2831
3089
|
message_id: row.id
|
|
2832
3090
|
});
|
|
3091
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2833
3092
|
return;
|
|
2834
3093
|
}
|
|
2835
3094
|
if (ocId === null) {
|
|
@@ -2840,6 +3099,7 @@ var ChannelDriver = class {
|
|
|
2840
3099
|
conversation_id: row.conversation_id,
|
|
2841
3100
|
message_id: row.id
|
|
2842
3101
|
});
|
|
3102
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
2843
3103
|
return;
|
|
2844
3104
|
}
|
|
2845
3105
|
const conv = this.convForRow(sessionId, row);
|
|
@@ -2849,6 +3109,7 @@ var ChannelDriver = class {
|
|
|
2849
3109
|
this.readopted.add(row.id);
|
|
2850
3110
|
this.awaitingReadopt.delete(row.id);
|
|
2851
3111
|
this.ensureWatcherRunning(sessionId);
|
|
3112
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
2852
3113
|
}
|
|
2853
3114
|
/**
|
|
2854
3115
|
* True if `evidentMessageId` is already being driven — either in the
|
|
@@ -2940,21 +3201,41 @@ var ChannelDriver = class {
|
|
|
2940
3201
|
* RUNNING (not done) is the one that paused. With one running message that is
|
|
2941
3202
|
* unambiguous; with several we prefer an explicit messageID match, else the
|
|
2942
3203
|
* oldest running message.
|
|
3204
|
+
*
|
|
3205
|
+
* Returns the set of in-flight Evident message ids that are paused awaiting a
|
|
3206
|
+
* human — an outstanding (still-open) question/permission is attributed to them.
|
|
3207
|
+
* `serviceInFlightMessage` uses this to keep an actively-running turn watched
|
|
3208
|
+
* forever (ADR-0047) while still bounding a turn merely blocked on a person who
|
|
3209
|
+
* may never answer. Attribution here covers ALL open interactions, not just
|
|
3210
|
+
* NEW (un-deduped) ones — a question stays "awaiting a human" until answered,
|
|
3211
|
+
* even after it was already surfaced to the channel.
|
|
2943
3212
|
*/
|
|
2944
3213
|
async pollInteractions(sessionId, watcher, messages) {
|
|
3214
|
+
const openQuestions = /* @__PURE__ */ new Set();
|
|
3215
|
+
const openPermissions = /* @__PURE__ */ new Set();
|
|
3216
|
+
let questionsPolledOk = true;
|
|
3217
|
+
let permissionsPolledOk = true;
|
|
2945
3218
|
let questions = [];
|
|
2946
3219
|
try {
|
|
2947
3220
|
const res = await this.fetchImpl(`${this.opencodeBase}/question`);
|
|
2948
3221
|
if (res.ok) {
|
|
2949
3222
|
const body = await res.json();
|
|
2950
|
-
|
|
3223
|
+
if (Array.isArray(body)) {
|
|
3224
|
+
questions = body;
|
|
3225
|
+
} else {
|
|
3226
|
+
questionsPolledOk = false;
|
|
3227
|
+
}
|
|
3228
|
+
} else {
|
|
3229
|
+
questionsPolledOk = false;
|
|
2951
3230
|
}
|
|
2952
3231
|
} catch {
|
|
3232
|
+
questionsPolledOk = false;
|
|
2953
3233
|
}
|
|
2954
3234
|
for (const q of questions) {
|
|
2955
|
-
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2956
3235
|
if (!await this.sessionBelongsTo(q.sessionID, sessionId)) continue;
|
|
2957
3236
|
const paused = this.attributeInteraction(watcher, q.tool?.messageID, messages);
|
|
3237
|
+
if (paused) openQuestions.add(paused.evidentMessageId);
|
|
3238
|
+
if (watcher.reportedQuestions.has(q.id)) continue;
|
|
2958
3239
|
const reported = await this.reportInteraction(
|
|
2959
3240
|
watcher.conv.id,
|
|
2960
3241
|
"question",
|
|
@@ -2968,14 +3249,22 @@ var ChannelDriver = class {
|
|
|
2968
3249
|
const res = await this.fetchImpl(`${this.opencodeBase}/permission`);
|
|
2969
3250
|
if (res.ok) {
|
|
2970
3251
|
const body = await res.json();
|
|
2971
|
-
|
|
3252
|
+
if (Array.isArray(body)) {
|
|
3253
|
+
permissions = body;
|
|
3254
|
+
} else {
|
|
3255
|
+
permissionsPolledOk = false;
|
|
3256
|
+
}
|
|
3257
|
+
} else {
|
|
3258
|
+
permissionsPolledOk = false;
|
|
2972
3259
|
}
|
|
2973
3260
|
} catch {
|
|
3261
|
+
permissionsPolledOk = false;
|
|
2974
3262
|
}
|
|
2975
3263
|
for (const p of permissions) {
|
|
2976
|
-
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2977
3264
|
if (!await this.sessionBelongsTo(p.sessionID, sessionId)) continue;
|
|
2978
3265
|
const paused = this.attributeInteraction(watcher, p.messageID, messages);
|
|
3266
|
+
if (paused) openPermissions.add(paused.evidentMessageId);
|
|
3267
|
+
if (watcher.reportedPermissions.has(p.id)) continue;
|
|
2979
3268
|
const reported = await this.reportInteraction(
|
|
2980
3269
|
watcher.conv.id,
|
|
2981
3270
|
"permission",
|
|
@@ -2984,6 +3273,7 @@ var ChannelDriver = class {
|
|
|
2984
3273
|
);
|
|
2985
3274
|
if (reported) watcher.reportedPermissions.add(p.id);
|
|
2986
3275
|
}
|
|
3276
|
+
return { openQuestions, openPermissions, questionsPolledOk, permissionsPolledOk };
|
|
2987
3277
|
}
|
|
2988
3278
|
/**
|
|
2989
3279
|
* True when `sessionId` is the `rootSessionId` itself OR a descendant of it —
|
|
@@ -3029,6 +3319,83 @@ var ChannelDriver = class {
|
|
|
3029
3319
|
if (parent !== void 0) this.sessionParents.set(sessionId, parent);
|
|
3030
3320
|
return parent;
|
|
3031
3321
|
}
|
|
3322
|
+
/**
|
|
3323
|
+
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3324
|
+
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
3325
|
+
*
|
|
3326
|
+
* The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
|
|
3327
|
+
* runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
|
|
3328
|
+
* completed `finish: "tool-calls"` root reply encountered during re-adoption is
|
|
3329
|
+
* idle by OpenCode's own definition and is re-dispatched. This method exists only
|
|
3330
|
+
* so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
|
|
3331
|
+
* provably in flight at the exact moment of recovery.
|
|
3332
|
+
*
|
|
3333
|
+
* "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
|
|
3334
|
+
* ACTIVELY generating — its LAST message is an assistant still mid-generation
|
|
3335
|
+
* (`completed == null`, via `isSessionActivelyGenerating`). An
|
|
3336
|
+
* INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
|
|
3337
|
+
* completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
|
|
3338
|
+
* is generating once the runner is gone), so it does NOT veto. (This is
|
|
3339
|
+
* deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
|
|
3340
|
+
* shapes and would falsely veto — re-hanging the very turn this path recovers.)
|
|
3341
|
+
*
|
|
3342
|
+
* Return contract (encoded so WI-3 need not re-derive it):
|
|
3343
|
+
* - `true` → a descendant is provably, actively generating (veto re-dispatch).
|
|
3344
|
+
* - `false` → descendants exist but none is actively generating (the restart
|
|
3345
|
+
* case), OR no descendant is found at all.
|
|
3346
|
+
* - `null` → liveness is INDETERMINATE (enumeration via `listSessions` failed).
|
|
3347
|
+
*
|
|
3348
|
+
* ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
|
|
3349
|
+
* as `false` and does NOT veto — a restart guarantees no live runner, so an
|
|
3350
|
+
* indeterminate cross-check almost always means "couldn't reach a child that no
|
|
3351
|
+
* longer exists". The inversion lives in the caller; this method just reports
|
|
3352
|
+
* true/false/null faithfully.
|
|
3353
|
+
*
|
|
3354
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
3355
|
+
* (already proven by the existing child-session interaction tests, via
|
|
3356
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
3357
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
3358
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
3359
|
+
* `SessionStatus` only.
|
|
3360
|
+
*/
|
|
3361
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
3362
|
+
const sessions = await listSessions(this.port);
|
|
3363
|
+
if (!sessions) {
|
|
3364
|
+
this.log({
|
|
3365
|
+
level: "error",
|
|
3366
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
3367
|
+
});
|
|
3368
|
+
return null;
|
|
3369
|
+
}
|
|
3370
|
+
for (const candidate of sessions) {
|
|
3371
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
3372
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
3373
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
3374
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
3375
|
+
return true;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
return false;
|
|
3379
|
+
}
|
|
3380
|
+
/**
|
|
3381
|
+
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3382
|
+
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
3383
|
+
* - `b1` — the reply itself is still in flight (`time.completed == null`) —
|
|
3384
|
+
* the aborted-in-flight production bug after a restart.
|
|
3385
|
+
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
3386
|
+
* (the sub-agent preamble — #253's shape).
|
|
3387
|
+
* - `other` — any other shape (defensive; a running row is normally b1 or b2).
|
|
3388
|
+
* Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
|
|
3389
|
+
* shape) directly rather than re-importing the module-private `completedOf`/
|
|
3390
|
+
* `finishOf` — this is a display label only, not a correctness predicate.
|
|
3391
|
+
*/
|
|
3392
|
+
replyCompletionShape(reply) {
|
|
3393
|
+
if (!reply) return "other";
|
|
3394
|
+
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
3395
|
+
if (completed == null) return "b1";
|
|
3396
|
+
const finish = reply.info?.finish ?? reply.finish;
|
|
3397
|
+
return finish === "tool-calls" ? "b2" : "other";
|
|
3398
|
+
}
|
|
3032
3399
|
/**
|
|
3033
3400
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
3034
3401
|
*
|
|
@@ -3260,6 +3627,12 @@ var ChannelDriver = class {
|
|
|
3260
3627
|
* MUST NOT use `callWithRetry` (a telemetry ping must not block the sequential
|
|
3261
3628
|
* watcher tick — one attempt is enough). A failure is SWALLOWED but LOGGED with
|
|
3262
3629
|
* context (no silent catch, per development-workflow).
|
|
3630
|
+
*
|
|
3631
|
+
* Returns whether the POST SUCCEEDED (2xx). Most callers ignore this (pure
|
|
3632
|
+
* telemetry), but the `paused` liveness-clear uses it to know whether to
|
|
3633
|
+
* RE-ASSERT on a later tick — a single dropped `paused` POST must not leave a
|
|
3634
|
+
* stale `last_seen_alive_at` on a still-paused row (Bugbot "Failed paused signal
|
|
3635
|
+
* leaves liveness").
|
|
3263
3636
|
*/
|
|
3264
3637
|
async postSignal(conversationId, messageId, signal, extra) {
|
|
3265
3638
|
try {
|
|
@@ -3278,7 +3651,9 @@ var ChannelDriver = class {
|
|
|
3278
3651
|
conversation_id: conversationId,
|
|
3279
3652
|
message_id: messageId
|
|
3280
3653
|
});
|
|
3654
|
+
return false;
|
|
3281
3655
|
}
|
|
3656
|
+
return true;
|
|
3282
3657
|
} catch (err) {
|
|
3283
3658
|
this.log({
|
|
3284
3659
|
level: "error",
|
|
@@ -3286,6 +3661,7 @@ var ChannelDriver = class {
|
|
|
3286
3661
|
conversation_id: conversationId,
|
|
3287
3662
|
message_id: messageId
|
|
3288
3663
|
});
|
|
3664
|
+
return false;
|
|
3289
3665
|
}
|
|
3290
3666
|
}
|
|
3291
3667
|
async persistSession(conversationId, sessionId) {
|