@evident-ai/cli 3.1.1-dev.1c351d6 → 3.1.1-dev.1e270a5

package/dist/index.js

@@ -3141,6 +3141,7 @@ var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
 var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
 var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
 var MAX_SUPERSEDED_CONVERSATIONS = 256;
+var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
 var ChannelAuthError = class extends Error {
   constructor(message) {
     super(message);
@@ -3163,6 +3164,10 @@ function backoffDelay(attempt, policy) {
 function isRetryableStatus(status2) {
   return status2 === 429 || status2 >= 500 && status2 <= 599;
 }
+var VOLATILE_BODY_FIELD_PATTERN = /("(?:ref|requestId|request_id|traceId|trace_id)"\s*:\s*)"[^"]*"/gi;
+function normalizeRedrivePollFailureBody(body) {
+  return body.replace(VOLATILE_BODY_FIELD_PATTERN, '$1"<redacted>"').replace(/\s+/g, " ").trim().slice(0, 200);
+}
 var ChannelDriver = class _ChannelDriver {
   agentId;
   port;
@@ -3294,6 +3299,40 @@ var ChannelDriver = class _ChannelDriver {
    * ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
    */
   redriveUnresolvedSince = /* @__PURE__ */ new Map();
+  /**
+   * Consecutive-identical-poll-failure streak for the re-drive fence (#1348),
+   * keyed by Evident **message id** (not session) so `clearRedriveUnresolved`
+   * can drop it with the other two trackers and it cannot leak. `sessionId` is
+   * carried inside the entry, not the key: a session change is a different
+   * situation and resets the streak, which gives the `(sessionId, message.id)`
+   * pairing #1348 asks for without a composite map key.
+   */
+  redrivePollFailures = /* @__PURE__ */ new Map();
+  /**
+   * "Already emitted `redrive_outcome_unreported` for THIS (message, outcome)
+   * streak" (Class B, #1340: the runner DECIDED reattach/settle/fail_permanent
+   * but its own PATCH to record it failed — distinct from Class A's
+   * `redrive_poll_failed`, where opencode itself can't be observed). Keyed by
+   * message id, valued by the outcome currently failing to report, so a
+   * change of outcome starts a fresh signal. Cleared by
+   * `clearRedriveUnresolved` the instant either PATCH succeeds.
+   */
+  redriveOutcomeUnreportedSignalled = /* @__PURE__ */ new Map();
+  /**
+   * Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
+   * `opencode_message_id` yet — i.e. one that has never even reached the
+   * re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
+   * read-back retries can never confirm the assigned id when the session's
+   * message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
+   * SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
+   * dispatched instead of after). Unlike an already-dispatched row, THIS row has
+   * no other safety net at all: the lifecycle cron only reclaims `status =
+   * 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
+   * by message id, carrying `sessionId` so a session change (a fresh one bound
+   * after abandonment) starts a new streak rather than inheriting the old
+   * session's count — same shape as `redrivePollFailures` above.
+   */
+  unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
   /**
    * "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
    * Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
@@ -3618,7 +3657,7 @@ var ChannelDriver = class _ChannelDriver {
    * @returns the count of messages NEWLY dispatched (not already in-flight).
    */
   async processConversation(conv) {
-    const { sessionId, refusedSessionId } = await this.ensureSession(conv);
+    const { sessionId, refusedSessionId, created: sessionCreated } = await this.ensureSession(conv);
     const messages = await this.getPendingMessages(conv.id);
     let dispatched = 0;
     let skippedAlreadyDispatched = 0;
@@ -3634,7 +3673,7 @@ var ChannelDriver = class _ChannelDriver {
         continue;
       }
       if (message.opencode_message_id) {
-        const outcome = await this.resolveRedrive(conv, sessionId, message, refusedSessionId);
+        const outcome = await this.resolveRedrive(conv, sessionId, message, sessionCreated);
         if (outcome !== "dispatch") {
           break;
         }
@@ -3705,14 +3744,37 @@ var ChannelDriver = class _ChannelDriver {
         break;
       }
       if (opencodeMessageId === null) {
+        const streak = this.recordUnconfirmedDispatch(message.id, sessionId);
+        if (streak < MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
+          this.log({
+            level: "warn",
+            message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next tick`,
+            conversation_id: conv.id,
+            message_id: message.id
+          });
+          continue;
+        }
+        this.unconfirmedDispatchFailures.delete(message.id);
+        this.sessions.delete(conv.id);
+        this.supersede(conv.id, sessionId);
+        const errorMessage = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
         this.log({
-          level: "warn",
-          message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
+          level: "error",
+          message: errorMessage,
           conversation_id: conv.id,
           message_id: message.id
         });
-        continue;
+        await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
+          this.log({
+            level: "warn",
+            message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
+            conversation_id: conv.id,
+            message_id: message.id
+          });
+        });
+        break;
       }
+      this.unconfirmedDispatchFailures.delete(message.id);
       this.dispatched.add(message.id);
       this.registerInFlight(conv, sessionId, message, opencodeMessageId);
       dispatched += 1;
@@ -3733,21 +3795,29 @@ var ChannelDriver = class _ChannelDriver {
    * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
    * hits the global `fetch` and would bypass the same override every other
    * opencode poll in this file respects. Mirrors `readoptProcessing`'s own
-   * snapshot fetch (`:3081-3111`). `null` = unreadable (non-OK response,
-   * non-array body, or a network exception) — treated as "can't observe",
-   * never as "confirmed gone".
+   * snapshot fetch (`:3081-3111`).
+   *
+   * Returns `{ ok: true, messages }` on a readable snapshot, or
+   * `{ ok: false, signature }` on failure — `signature` is a string that
+   * repeats across attempts for the SAME underlying fault (used by the
+   * consecutive-identical-failure bound, #1348), or `null` for a thrown
+   * exception, which is NOT countable toward that bound (a network blip / an
+   * opencode restart also throws identically every tick, and must keep
+   * retrying unbounded rather than ever being treated as permanent).
    */
   async pollSessionMessagesForRedrive(conv, message, sessionId) {
     try {
       const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
       if (!res.ok) {
+        const rawBody = await res.text();
+        const normalized = normalizeRedrivePollFailureBody(rawBody);
         this.log({
           level: "warn",
-          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status} \u2014 treating as unreadable this tick`,
+          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status}${normalized ? `: ${normalized}` : ""} \u2014 treating as unreadable this tick`,
           conversation_id: conv.id,
           message_id: message.id
         });
-        return null;
+        return { ok: false, signature: `HTTP ${res.status}${normalized ? `: ${normalized}` : ""}` };
       }
       const body = await res.json();
       if (!Array.isArray(body)) {
@@ -3757,9 +3827,9 @@ var ChannelDriver = class _ChannelDriver {
           conversation_id: conv.id,
           message_id: message.id
         });
-        return null;
+        return { ok: false, signature: "non-array message body" };
       }
-      return body;
+      return { ok: true, messages: body };
     } catch (err) {
       this.log({
         level: "warn",
@@ -3767,7 +3837,7 @@ var ChannelDriver = class _ChannelDriver {
         conversation_id: conv.id,
         message_id: message.id
       });
-      return null;
+      return { ok: false, signature: null };
     }
   }
   /**
@@ -3779,21 +3849,34 @@ var ChannelDriver = class _ChannelDriver {
    * without this fence the drain loop would re-`prompt_async` the SAME turn a
    * second time against live GitHub state. Mirrors `readoptOne`'s job for the
    * `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
-   * needed here because `refusedSessionId` already handles the one case
-   * (#553 abandoned session) that path exists for.
+   * needed here because `sessionCreated` already handles the cases (a #553
+   * abandoned session, a #190 vanished one) that path exists for.
    *
-   * Only `ChannelAuthError` propagates; every other failure resolves to
-   * `unresolved` and is retried whole on the next ~2s drain tick.
+   * Only `ChannelAuthError` propagates. A poll that fails identically
+   * `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row reports the message
+   * failed instead of retrying it (#1348) — SEPARATE from, not a replacement
+   * for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
+   * other failure resolves to `unresolved` and is retried whole on the next
+   * ~2s drain tick.
    */
-  async resolveRedrive(conv, sessionId, message, refusedSessionId) {
+  async resolveRedrive(conv, sessionId, message, sessionCreated) {
     const ocId = message.opencode_message_id ?? null;
-    if (refusedSessionId) {
+    if (sessionCreated) {
       this.clearRedriveUnresolved(message.id);
       void this.postSignal(conv.id, message.id, "redrive_redispatched");
       return "dispatch";
     }
-    const messages = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
-    if (messages == null || messages.length === 0) {
+    const polled = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
+    if (!polled.ok) {
+      const streak = this.recordRedrivePollFailure(message.id, sessionId, polled.signature);
+      if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES && polled.signature !== null) {
+        return this.failRedrivePollPermanent(conv, sessionId, message, polled.signature, streak);
+      }
+      return this.resolveRedriveUnresolved(conv, message);
+    }
+    this.redrivePollFailures.delete(message.id);
+    const messages = polled.messages;
+    if (messages.length === 0) {
       return this.resolveRedriveUnresolved(conv, message);
     }
     const state = messageRunState(messages, ocId ?? "");
@@ -3860,6 +3943,7 @@ var ChannelDriver = class _ChannelDriver {
         conversation_id: conv.id,
         message_id: message.id
       });
+      this.signalRedriveOutcomeUnreported(conv, message, "reattach");
       return "unresolved";
     }
     this.clearRedriveUnresolved(message.id);
@@ -3920,12 +4004,53 @@ var ChannelDriver = class _ChannelDriver {
         conversation_id: conv.id,
         message_id: message.id
       });
+      this.signalRedriveOutcomeUnreported(conv, message, "settle");
       return "unresolved";
     }
     this.clearRedriveUnresolved(message.id);
     void this.postSignal(conv.id, message.id, "redrive_settled");
     return "settled";
   }
+  /**
+   * The permanent-failure outcome (#1348): the fence's own poll of this session
+   * failed with the SAME opencode-answered signature
+   * `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row — a transient blip
+   * would have varied or eventually cleared (see `pollSessionMessagesForRedrive`
+   * and `recordRedrivePollFailure`), so this is a durable fault (e.g. #1345's
+   * corrupted opencode session) rather than something worth retrying forever.
+   * Mirrors `settleRedrive`'s error discipline: no `usage`/`failure` args to
+   * `markFailed` (no opencode snapshot to extract them from — this poll never
+   * got a readable one).
+   */
+  async failRedrivePollPermanent(conv, sessionId, message, signature, streak) {
+    this.log({
+      level: "error",
+      message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) failed to poll with the identical signature "${signature}" ${streak} times in a row \u2014 reporting the message failed instead of retrying forever`,
+      conversation_id: conv.id,
+      message_id: message.id
+    });
+    try {
+      await this.markFailed(
+        conv.id,
+        message.id,
+        sessionId,
+        `The runner could not read this conversation's state from OpenCode (${signature}). The same failure repeated ${streak} times in a row, so the message was not retried further.`
+      );
+    } catch (err) {
+      if (err instanceof ChannelAuthError) throw err;
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to report message ${message.id.slice(0, 8)} permanently failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      this.signalRedriveOutcomeUnreported(conv, message, "fail_permanent");
+      return "unresolved";
+    }
+    this.clearRedriveUnresolved(message.id);
+    void this.postSignal(conv.id, message.id, "redrive_poll_failed");
+    return "settled";
+  }
   /**
    * The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
    * observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
@@ -3951,10 +4076,68 @@ var ChannelDriver = class _ChannelDriver {
     }
     return "unresolved";
   }
-  /** Clear both `unresolved`-bound trackers for a row (any non-`unresolved` outcome). */
+  /** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
   clearRedriveUnresolved(messageId) {
     this.redriveUnresolvedSince.delete(messageId);
     this.redriveUnresolvedSignalled.delete(messageId);
+    this.redrivePollFailures.delete(messageId);
+    this.redriveOutcomeUnreportedSignalled.delete(messageId);
+  }
+  /**
+   * Class B (#1340): the runner DECIDED an outcome (reattach/settle/fail_permanent)
+   * but its own PATCH to record it failed. Deliberately NOT bounded like
+   * `resolveRedriveUnresolved`, for two load-bearing reasons: (1) the only
+   * remedy is a PATCH to the server — the very call that is failing, so a bound
+   * would act by retrying it; (2) this class is self-clearing by construction
+   * (the next successful PATCH resolves it via `clearRedriveUnresolved`),
+   * unlike #1340's deterministic, permanent shape. Fires at most once per
+   * (message, outcome) streak.
+   */
+  signalRedriveOutcomeUnreported(conv, message, outcome) {
+    if (this.redriveOutcomeUnreportedSignalled.get(message.id) === outcome) return;
+    this.redriveOutcomeUnreportedSignalled.set(message.id, outcome);
+    void this.postSignal(conv.id, message.id, "redrive_outcome_unreported", {
+      attempted_outcome: outcome
+    });
+  }
+  /**
+   * Record one poll outcome toward the re-drive fence's consecutive-identical-
+   * failure streak (#1348) and return the resulting count. `signature === null`
+   * (a thrown exception, H1) always clears the streak and returns `0` — it is
+   * never countable. Otherwise the streak continues only when BOTH the session
+   * and the signature match the previous failure; anything else (a different
+   * session, or the same session failing a DIFFERENT way) starts a fresh streak
+   * at `1`.
+   */
+  recordRedrivePollFailure(messageId, sessionId, signature) {
+    if (signature === null) {
+      this.redrivePollFailures.delete(messageId);
+      return 0;
+    }
+    const existing = this.redrivePollFailures.get(messageId);
+    if (existing && existing.sessionId === sessionId && existing.signature === signature) {
+      existing.count += 1;
+      return existing.count;
+    }
+    this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
+    return 1;
+  }
+  /**
+   * Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
+   * `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
+   * bound in `processConversation`'s dispatch loop, and return the resulting
+   * count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
+   * change starts a fresh streak at `1` rather than inheriting the old one's
+   * count, since a new session is a genuinely different attempt.
+   */
+  recordUnconfirmedDispatch(messageId, sessionId) {
+    const existing = this.unconfirmedDispatchFailures.get(messageId);
+    if (existing && existing.sessionId === sessionId) {
+      existing.count += 1;
+      return existing.count;
+    }
+    this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
+    return 1;
   }
   /**
    * Record that `sessionId` is no longer a valid binding for `conversationId`
@@ -3980,6 +4163,13 @@ var ChannelDriver = class _ChannelDriver {
    * `refusedSessionId` is set when the #553 guard fired — i.e. the persisted
    * binding was an id this runner had abandoned, so a resurrection genuinely
    * happened and a fresh session was bound instead. The caller reports it.
+   *
+   * `created` says the returned session was made JUST NOW, so it provably holds
+   * no prior turn. The re-drive fence needs that as CONTRARY evidence ("nothing
+   * to reconcile against") — distinct from the ambiguous "I polled and saw an
+   * empty transcript", which stays a deferral. Keep it separate from
+   * `refusedSessionId`: only the latter means a #553 resurrection happened, and
+   * only it may drive the `session_superseded` signal.
    */
   async ensureSession(conv) {
     const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
@@ -3990,7 +4180,11 @@ var ChannelDriver = class _ChannelDriver {
         conversation_id: conv.id
       });
       this.sessions.delete(conv.id);
-      return { sessionId: await this.createAndBindSession(conv.id), refusedSessionId: bound };
+      return {
+        sessionId: await this.createAndBindSession(conv.id),
+        refusedSessionId: bound,
+        created: true
+      };
     }
     if (bound) {
       const exists = await sessionExists(this.port, bound);
@@ -4001,12 +4195,12 @@ var ChannelDriver = class _ChannelDriver {
           conversation_id: conv.id
         });
         this.sessions.delete(conv.id);
-        return { sessionId: await this.createAndBindSession(conv.id) };
+        return { sessionId: await this.createAndBindSession(conv.id), created: true };
       }
       this.sessions.set(conv.id, bound);
-      return { sessionId: bound };
+      return { sessionId: bound, created: false };
     }
-    return { sessionId: await this.createAndBindSession(conv.id) };
+    return { sessionId: await this.createAndBindSession(conv.id), created: true };
   }
   /**
    * Create a fresh OpenCode session for a conversation, cache the binding, and
@@ -5092,15 +5286,39 @@ var ChannelDriver = class _ChannelDriver {
     }
     if (ocId === null) {
       this.awaitingReadopt.delete(row.id);
+      const streak = this.recordUnconfirmedDispatch(row.id, sessionId);
+      if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
+        this.unconfirmedDispatchFailures.delete(row.id);
+        this.sessions.delete(readoptConv.id);
+        this.supersede(readoptConv.id, sessionId);
+        const errorMessage = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
+        this.log({
+          level: "error",
+          message: errorMessage,
+          conversation_id: row.conversation_id,
+          message_id: row.id
+        });
+        await this.markFailed(row.conversation_id, row.id, null, errorMessage).catch((markErr) => {
+          this.log({
+            level: "warn",
+            message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
+            conversation_id: row.conversation_id,
+            message_id: row.id
+          });
+        });
+        void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
+        return;
+      }
       this.log({
         level: "warn",
-        message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
+        message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next drain`,
         conversation_id: row.conversation_id,
         message_id: row.id
       });
       void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
       return;
     }
+    this.unconfirmedDispatchFailures.delete(row.id);
     this.registerReadopted(readoptConv, sessionId, readoptMessage, ocId, this.processedAtMs(row));
     this.dispatched.add(row.id);
     this.readopted.add(row.id);