@evident-ai/cli 3.0.1-dev.39d9b5f → 3.0.1-dev.3e59bb5

package/dist/index.js

@@ -1078,6 +1078,12 @@ async function getSessionMessages(port, sessionId) {
     return null;
   }
 }
+function isSessionActivelyGenerating(messages) {
+  if (!messages || messages.length === 0) return false;
+  const last = messages[messages.length - 1];
+  if (roleOf(last) !== "assistant") return false;
+  return completedOf(last) == null;
+}
 function sessionLastActivityMs(session) {
   const candidates = [
     session.time?.updated,
@@ -1120,6 +1126,36 @@ async function sessionExists(port, id) {
     return null;
   }
 }
+async function getSessionStatuses(port) {
+  try {
+    const res = await fetch(`${opencodeBase(port)}/session/status`);
+    if (!res.ok) {
+      console.error(
+        `[getSessionStatuses] GET /session/status returned HTTP ${res.status} (port ${port})`
+      );
+      return null;
+    }
+    const body = await res.json();
+    if (body == null || typeof body !== "object" || Array.isArray(body)) {
+      console.error(
+        `[getSessionStatuses] GET /session/status body was not a plain object (port ${port})`
+      );
+      return null;
+    }
+    return body;
+  } catch (err) {
+    console.error(
+      `[getSessionStatuses] GET /session/status failed (port ${port}): ${err instanceof Error ? err.message : String(err)}`
+    );
+    return null;
+  }
+}
+async function isSessionOngoing(port, id) {
+  const map = await getSessionStatuses(port);
+  if (map == null) return null;
+  const entry = map[id];
+  return entry != null && entry.type !== "idle";
+}
 async function createOpenCodeSession(port, directory) {
   const url = new URL(`${opencodeBase(port)}/session`);
   if (directory && directory.trim()) {
@@ -1246,6 +1282,11 @@ function messageRunState(messages, userMessageId) {
   if (isAssistantInFlight(reply)) return "running";
   return errorOf(reply) != null ? "failed" : "done";
 }
+function isPreamblePinnedRunning(messages, userMessageId) {
+  if (messageRunState(messages, userMessageId) !== "running") return false;
+  const reply = findLastAssistantReplyFor(messages, userMessageId);
+  return completedOf(reply) != null && finishOf(reply) === "tool-calls";
+}
 function messageError(messages, userMessageId) {
   const reply = findLastAssistantReplyFor(messages, userMessageId);
   const error2 = errorOf(reply);
@@ -1779,6 +1820,7 @@ var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
 var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
 var DEFAULT_STUCK_QUEUED_MS = 6e4;
 var HEARTBEAT_MS = 6e4;
+var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
 var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
 var ChannelAuthError = class extends Error {
   constructor(message) {
@@ -1876,6 +1918,15 @@ var ChannelDriver = class {
    * the row leaves the processing list, exactly like `dontRedispatch`.
    */
   doneUndeliverable = /* @__PURE__ */ new Set();
+  /**
+   * "Already emitted `readopt_poll_unresolved` for this row" (#229). The b1 /
+   * unreadable-status re-evaluate leaf leaves the row UN-tracked so it is re-read
+   * every ~2s drain until the status map becomes readable — but the server-visible
+   * signal is an OUTCOME, so it must fire at most ONCE per row, not once per drain
+   * (Bugbot "Re-adopt signals flood every drain"). Cleared when the row leaves the
+   * processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
+   */
+  readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
   /**
    * "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
    * Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
@@ -2263,6 +2314,7 @@ var ChannelDriver = class {
       opencodeMessageId,
       message,
       dispatchedAt: now,
+      processingAnchorMs: now,
       deadline: now + this.pausedMaxWaitMs,
       started: false,
       done: false,
@@ -2321,6 +2373,10 @@ var ChannelDriver = class {
       opencodeMessageId,
       message,
       dispatchedAt: this.now(),
+      // Anchor the absolute-age ceiling to the SERVER-SIDE `processed_at` (the same
+      // value seeding `deadline`), NOT `dispatchedAt` — so a re-adopted zombie's age
+      // reflects the real turn duration and the ceiling fires on the ORIGINAL turn.
+      processingAnchorMs: processedAtMs,
       deadline: processedAtMs + this.pausedMaxWaitMs,
       // The server row is ALREADY `processing`; do not re-fire markProcessing.
       started: true,
@@ -2609,6 +2665,19 @@ var ChannelDriver = class {
       });
     }
     const activelyRunning = state === "running" && !awaitingHuman;
+    if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
+      this.log({
+        level: "error",
+        message: `Message ${inFlight.evidentMessageId.slice(0, 8)} exceeded the absolute processing ceiling (${Math.round((this.now() - inFlight.processingAnchorMs) / 6e4)}min, session ${sessionId}) while still actively running \u2014 releasing so the cron can reclaim it`,
+        conversation_id: conv.id,
+        message_id: inFlight.evidentMessageId
+      });
+      void this.postSignal(conv.id, inFlight.evidentMessageId, "gave_up", {
+        watched_for_ms: this.now() - inFlight.processingAnchorMs
+      });
+      this.removeInFlight(watcher, inFlight.evidentMessageId);
+      return;
+    }
     if (activelyRunning && !inFlight.awaitingHumanLatched && !inFlight.aliveInFlight && this.now() - inFlight.lastAliveAt >= HEARTBEAT_MS) {
       inFlight.aliveInFlight = true;
       void this.postSignal(conv.id, inFlight.evidentMessageId, "alive").then((ok) => {
@@ -2671,12 +2740,17 @@ var ChannelDriver = class {
    */
   async readoptProcessing() {
     const rows = await this.getProcessingMessages();
-    if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0) {
+    if (this.dontRedispatch.size > 0 || this.doneUndeliverable.size > 0 || this.readoptPollUnresolvedSignalled.size > 0) {
       const stillProcessing = new Set(rows.map((r) => r.id));
-      for (const id of [...this.dontRedispatch, ...this.doneUndeliverable]) {
+      for (const id of [
+        ...this.dontRedispatch,
+        ...this.doneUndeliverable,
+        ...this.readoptPollUnresolvedSignalled
+      ]) {
         if (!stillProcessing.has(id)) {
           const cleared = this.dontRedispatch.delete(id);
           const clearedUndeliverable = this.doneUndeliverable.delete(id);
+          this.readoptPollUnresolvedSignalled.delete(id);
           if (cleared || clearedUndeliverable) {
             this.log({
               level: "info",
@@ -2730,8 +2804,10 @@ var ChannelDriver = class {
         });
         continue;
       }
+      const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
+      const sessionOngoing = anyUntracked ? await isSessionOngoing(this.port, sessionId) : null;
       for (const row of sessionRows) {
-        await this.readoptOne(sessionId, row, messages);
+        await this.readoptOne(sessionId, row, messages, sessionOngoing);
       }
     }
   }
@@ -2753,7 +2829,7 @@ var ChannelDriver = class {
    *
    * Only `ChannelAuthError` propagates.
    */
-  async readoptOne(sessionId, row, messages) {
+  async readoptOne(sessionId, row, messages, sessionOngoing) {
     if (this.isTracked(sessionId, row.id)) {
       this.log({
         level: "info",
@@ -2793,6 +2869,7 @@ var ChannelDriver = class {
             conversation_id: row.conversation_id,
             message_id: row.id
           });
+          void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
           return;
         }
         this.log({
@@ -2804,6 +2881,7 @@ var ChannelDriver = class {
         return;
       }
       this.dontRedispatch.delete(row.id);
+      void this.postSignal(row.conversation_id, row.id, "readopt_done");
       return;
     }
     if (state === "failed") {
@@ -2826,6 +2904,7 @@ var ChannelDriver = class {
             conversation_id: row.conversation_id,
             message_id: row.id
           });
+          void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
           return;
         }
         this.log({
@@ -2837,6 +2916,7 @@ var ChannelDriver = class {
         return;
       }
       this.dontRedispatch.delete(row.id);
+      void this.postSignal(row.conversation_id, row.id, "readopt_failed");
       return;
     }
     if (this.dontRedispatch.has(row.id)) {
@@ -2848,6 +2928,71 @@ var ChannelDriver = class {
       });
       return;
     }
+    let statusReadableOngoing = null;
+    if (state === "running" && ocId) {
+      const reply = findLastAssistantReplyFor(messages, ocId);
+      const shape = this.replyCompletionShape(reply);
+      const ongoing = sessionOngoing;
+      statusReadableOngoing = ongoing;
+      if (ongoing === false) {
+        this.log({
+          level: "info",
+          message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+        await this.forceReadoptRun(sessionId, row);
+        return;
+      }
+      if (ongoing === true) {
+        this.log({
+          level: "info",
+          message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} and session ${sessionId.slice(0, 8)} is ongoing per GET /session/status (busy/retry) \u2014 re-attaching watcher (no re-dispatch)`,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+      } else {
+        if (shape === "b1") {
+          this.log({
+            level: "info",
+            message: `Re-adopt: message ${row.id.slice(0, 8)} running/b1 but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 NOT latching a b1 row on a transient status blip; leaving it un-tracked to re-evaluate on the next drain`,
+            conversation_id: row.conversation_id,
+            message_id: row.id
+          });
+          if (!this.readoptPollUnresolvedSignalled.has(row.id)) {
+            this.readoptPollUnresolvedSignalled.add(row.id);
+            void this.postSignal(row.conversation_id, row.id, "readopt_poll_unresolved");
+          }
+          return;
+        }
+        this.log({
+          level: "info",
+          message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but GET /session/status was unreadable (null) for session ${sessionId.slice(0, 8)} \u2014 falling back to the #253 preamble + descendant cross-check`,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+      }
+    }
+    if (statusReadableOngoing === null && state === "running" && ocId && isPreamblePinnedRunning(messages, ocId)) {
+      const descendantAlive = await this.isAnyDescendantSessionAlive(sessionId);
+      if (descendantAlive === true) {
+        this.log({
+          level: "info",
+          message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned running (root ${sessionId.slice(0, 8)}) but a live descendant sub-agent session was found \u2014 treating as still running, re-attaching watcher (no re-dispatch)`,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+      } else {
+        this.log({
+          level: "info",
+          message: `Re-adopt: message ${row.id.slice(0, 8)} preamble-pinned on recovery (root ${sessionId.slice(0, 8)}), no live descendant runner \u2014 re-dispatching from scratch${descendantAlive === null ? " (descendant liveness indeterminate; a restart guarantees no live runner, so this does NOT block the re-dispatch)" : ""}`,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+        await this.forceReadoptRun(sessionId, row);
+        return;
+      }
+    }
     if ((state === "running" || state === "queued") && ocId) {
       const conv = this.convForRow(sessionId, row);
       const message = this.queuedMessageForRow(row);
@@ -2861,6 +3006,7 @@ var ChannelDriver = class {
         conversation_id: row.conversation_id,
         message_id: row.id
       });
+      void this.postSignal(row.conversation_id, row.id, "readopt_reattached");
       return;
     }
     await this.forceReadoptRun(sessionId, row);
@@ -2913,6 +3059,7 @@ var ChannelDriver = class {
         conversation_id: row.conversation_id,
         message_id: row.id
       });
+      void this.postSignal(row.conversation_id, row.id, "readopt_window_elapsed");
       return;
     }
     const options = {
@@ -2941,6 +3088,7 @@ var ChannelDriver = class {
         conversation_id: row.conversation_id,
         message_id: row.id
       });
+      void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
       return;
     }
     if (ocId === null) {
@@ -2951,6 +3099,7 @@ var ChannelDriver = class {
         conversation_id: row.conversation_id,
         message_id: row.id
       });
+      void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
       return;
     }
     const conv = this.convForRow(sessionId, row);
@@ -2960,6 +3109,7 @@ var ChannelDriver = class {
     this.readopted.add(row.id);
     this.awaitingReadopt.delete(row.id);
     this.ensureWatcherRunning(sessionId);
+    void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
   }
   /**
    * True if `evidentMessageId` is already being driven — either in the
@@ -3169,6 +3319,83 @@ var ChannelDriver = class {
     if (parent !== void 0) this.sessionParents.set(sessionId, parent);
     return parent;
   }
+  /**
+   * DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
+   * (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
+   *
+   * The PRIMARY recovery trigger is "preamble-pinned on recovery ⇒ idle" — a
+   * runner restart wipes OpenCode's in-memory `SessionStatus`/`Runner`, so a
+   * completed `finish: "tool-calls"` root reply encountered during re-adoption is
+   * idle by OpenCode's own definition and is re-dispatched. This method exists only
+   * so the WI-3 caller can VETO that re-dispatch in the rare case a descendant is
+   * provably in flight at the exact moment of recovery.
+   *
+   * "Alive" criterion (TIGHTENED): a descendant is alive only when it is PROVABLY,
+   * ACTIVELY generating — its LAST message is an assistant still mid-generation
+   * (`completed == null`, via `isSessionActivelyGenerating`). An
+   * INCOMPLETE-BUT-NOT-GENERATING child — last message a user message, or a
+   * completed `finish: "tool-calls"` step — is NOT alive after a restart (nothing
+   * is generating once the runner is gone), so it does NOT veto. (This is
+   * deliberately NOT `!isTurnComplete`, which also matches those dead-but-non-terminal
+   * shapes and would falsely veto — re-hanging the very turn this path recovers.)
+   *
+   * Return contract (encoded so WI-3 need not re-derive it):
+   *  - `true`  → a descendant is provably, actively generating (veto re-dispatch).
+   *  - `false` → descendants exist but none is actively generating (the restart
+   *              case), OR no descendant is found at all.
+   *  - `null`  → liveness is INDETERMINATE (enumeration via `listSessions` failed).
+   *
+   * ⚠️ `null` (UNKNOWN) MUST NOT be treated as "alive": WI-3 treats `null` the same
+   * as `false` and does NOT veto — a restart guarantees no live runner, so an
+   * indeterminate cross-check almost always means "couldn't reach a child that no
+   * longer exists". The inversion lives in the caller; this method just reports
+   * true/false/null faithfully.
+   *
+   * VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
+   * (already proven by the existing child-session interaction tests, via
+   * `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
+   * terminal state. We do NOT depend on any session-level `busy`/`idle` field —
+   * there is none on `GET /session/:id`; OpenCode's busy state is in-memory
+   * `SessionStatus` only.
+   */
+  async isAnyDescendantSessionAlive(rootSessionId) {
+    const sessions = await listSessions(this.port);
+    if (!sessions) {
+      this.log({
+        level: "error",
+        message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
+      });
+      return null;
+    }
+    for (const candidate of sessions) {
+      if (!candidate?.id || candidate.id === rootSessionId) continue;
+      if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
+      const childMsgs = await getSessionMessages(this.port, candidate.id);
+      if (isSessionActivelyGenerating(childMsgs)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  /**
+   * Cheap decision-telemetry label for a running row's LAST correlated reply
+   * (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
+   *   - `b1`    — the reply itself is still in flight (`time.completed == null`) —
+   *               the aborted-in-flight production bug after a restart.
+   *   - `b2`    — a COMPLETED reply pinned running only by `finish === "tool-calls"`
+   *               (the sub-agent preamble — #253's shape).
+   *   - `other` — any other shape (defensive; a running row is normally b1 or b2).
+   * Reads `info.time.completed` / `info.finish` (tolerating the legacy top-level
+   * shape) directly rather than re-importing the module-private `completedOf`/
+   * `finishOf` — this is a display label only, not a correctness predicate.
+   */
+  replyCompletionShape(reply) {
+    if (!reply) return "other";
+    const completed = reply.info?.time?.completed ?? reply.time?.completed;
+    if (completed == null) return "b1";
+    const finish = reply.info?.finish ?? reply.finish;
+    return finish === "tool-calls" ? "b2" : "other";
+  }
   /**
    * Attribute a surfaced interaction to the in-flight message it paused on (M-1).
    *