@evident-ai/cli 3.1.1-dev.9efabd5 → 3.1.1-dev.9fe57bf

package/dist/index.js

@@ -997,6 +997,9 @@ import { homedir as homedir3 } from "os";
 import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
 import chalk6 from "chalk";
 
+// ../../packages/types/src/agents/index.ts
+var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
+
 // ../../packages/types/src/telemetry/index.ts
 var TelemetryEventTypes = {
   // Agent activity events (shown in web UI activity log)
@@ -2086,6 +2089,21 @@ function messageError(messages, userMessageId) {
   }
   return "The agent run failed.";
 }
+function isAbortedTerminalReply(messages, userMessageId) {
+  const reply = findLastAssistantReplyFor(messages, userMessageId);
+  const error2 = errorOf(reply);
+  if (error2 == null) return false;
+  if (typeof error2 === "string") return error2.trim() === "Aborted";
+  if (typeof error2 === "object") {
+    const e = error2;
+    if (e.name === "MessageAbortedError") return true;
+    if (e.name === "AbortError") return true;
+    const dataMessage = e.data?.message;
+    const rendered = typeof dataMessage === "string" ? dataMessage : typeof e.message === "string" ? e.message : null;
+    return rendered != null && rendered.trim() === "Aborted";
+  }
+  return false;
+}
 function messageFailure(messages, userMessageId) {
   const reply = findLastAssistantReplyFor(messages, userMessageId);
   const error2 = errorOf(reply);
@@ -3258,6 +3276,24 @@ var ChannelDriver = class _ChannelDriver {
    * processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
    */
   readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
+  /**
+   * "Already emitted `redrive_unresolved` for this row" (#965). Mirrors
+   * `readoptPollUnresolvedSignalled`: `resolveRedrive`'s `unresolved` leaf recurs
+   * every ~2s drain until opencode's status becomes readable, but the
+   * server-visible signal is an OUTCOME, so it fires at most once per row. Cleared
+   * on any non-`unresolved` outcome so the set cannot grow beyond the currently
+   * unresolvable rows.
+   */
+  redriveUnresolvedSignalled = /* @__PURE__ */ new Set();
+  /**
+   * First `now()` a `pending` row's re-drive was observed `unresolved` (#965). A
+   * `pending` row is invisible to every cron arm (all require `status =
+   * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
+   * nothing driving it. Once `now - since >= pausedMaxWaitMs`, `resolveRedrive`
+   * takes `dispatch` instead of `unresolved` (reusing the existing knob — see
+   * ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
+   */
+  redriveUnresolvedSince = /* @__PURE__ */ new Map();
   /**
    * "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
    * Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
@@ -3597,6 +3633,12 @@ var ChannelDriver = class _ChannelDriver {
         skippedAlreadyDispatched += 1;
         continue;
       }
+      if (message.opencode_message_id) {
+        const outcome = await this.resolveRedrive(conv, sessionId, message, refusedSessionId);
+        if (outcome !== "dispatch") {
+          break;
+        }
+      }
       const options = {
         agent: message.opencode_agent ?? void 0,
         model: message.opencode_model ?? void 0
@@ -3686,6 +3728,234 @@ var ChannelDriver = class _ChannelDriver {
     this.ensureWatcherRunning(sessionId);
     return dispatched;
   }
+  /**
+   * Poll a session's message list for the re-drive fence (#965), via the
+   * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
+   * hits the global `fetch` and would bypass the same override every other
+   * opencode poll in this file respects. Mirrors `readoptProcessing`'s own
+   * snapshot fetch (`:3081-3111`). `null` = unreadable (non-OK response,
+   * non-array body, or a network exception) — treated as "can't observe",
+   * never as "confirmed gone".
+   */
+  async pollSessionMessagesForRedrive(conv, message, sessionId) {
+    try {
+      const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
+      if (!res.ok) {
+        this.log({
+          level: "warn",
+          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status} \u2014 treating as unreadable this tick`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        return null;
+      }
+      const body = await res.json();
+      if (!Array.isArray(body)) {
+        this.log({
+          level: "warn",
+          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned a non-array message body \u2014 treating as unreadable this tick`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        return null;
+      }
+      return body;
+    } catch (err) {
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to poll session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return null;
+    }
+  }
+  /**
+   * The re-drive fence for a `pending` row that already carries a stored
+   * `opencode_message_id` (#965) — i.e. it has already been handed to opencode at
+   * least once (see the invariant at `QueuedMessage.opencode_message_id`'s doc).
+   * The lifecycle cron can falsely reclaim a `processing` row back to `pending`
+   * mid-turn (a 5-minute liveness-staleness check racing a still-running turn);
+   * without this fence the drain loop would re-`prompt_async` the SAME turn a
+   * second time against live GitHub state. Mirrors `readoptOne`'s job for the
+   * `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
+   * needed here because `refusedSessionId` already handles the one case
+   * (#553 abandoned session) that path exists for.
+   *
+   * Only `ChannelAuthError` propagates; every other failure resolves to
+   * `unresolved` and is retried whole on the next ~2s drain tick.
+   */
+  async resolveRedrive(conv, sessionId, message, refusedSessionId) {
+    const ocId = message.opencode_message_id ?? null;
+    if (refusedSessionId) {
+      this.clearRedriveUnresolved(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_redispatched");
+      return "dispatch";
+    }
+    const messages = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
+    if (messages == null || messages.length === 0) {
+      return this.resolveRedriveUnresolved(conv, message);
+    }
+    const state = messageRunState(messages, ocId ?? "");
+    if (state === "failed" && isAbortedTerminalReply(messages, ocId ?? "")) {
+      const ongoing = await isSessionOngoing(this.port, sessionId);
+      if (ongoing === false) {
+        this.log({
+          level: "info",
+          message: `Re-drive: message ${message.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        this.clearRedriveUnresolved(message.id);
+        void this.postSignal(conv.id, message.id, "redrive_redispatched");
+        return "dispatch";
+      }
+    }
+    if (state === "done" || state === "failed") {
+      return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
+    }
+    if (state === "running" || state === "queued") {
+      const ongoing = await isSessionOngoing(this.port, sessionId);
+      if (ongoing === true) {
+        return this.reattachRedrive(conv, sessionId, message, ocId);
+      }
+      if (ongoing === false) {
+        this.clearRedriveUnresolved(message.id);
+        void this.postSignal(conv.id, message.id, "redrive_redispatched");
+        return "dispatch";
+      }
+      return this.resolveRedriveUnresolved(conv, message);
+    }
+    this.clearRedriveUnresolved(message.id);
+    void this.postSignal(conv.id, message.id, "redrive_redispatched");
+    return "dispatch";
+  }
+  /**
+   * The `reattached` outcome (Task 3.3): the prior turn is STILL ONGOING per
+   * opencode's own status map — undo the false reclaim instead of starting a
+   * second turn.
+   */
+  async reattachRedrive(conv, sessionId, message, ocId) {
+    let anchorMs;
+    const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
+    if (!Number.isNaN(parsed)) {
+      anchorMs = parsed;
+    } else {
+      anchorMs = this.now();
+      this.log({
+        level: "error",
+        message: `Re-drive: message ${message.id.slice(0, 8)} has null/unparseable processing_started_at (${String(message.processing_started_at)}) \u2014 anchoring the watcher's absolute-age ceiling to now (defensive)`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+    }
+    const title = await this.resolveSessionTitle(sessionId, conv.id);
+    try {
+      await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
+    } catch (err) {
+      if (err instanceof ChannelAuthError) throw err;
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return "unresolved";
+    }
+    this.clearRedriveUnresolved(message.id);
+    this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
+    this.dispatched.add(message.id);
+    this.readopted.add(message.id);
+    this.ensureWatcherRunning(sessionId);
+    const watchedForMs = this.now() - anchorMs;
+    void this.postSignal(conv.id, message.id, "redrive_reattached", {
+      watched_for_ms: watchedForMs
+    });
+    this.log({
+      level: "warn",
+      message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) was wrongly reclaimed to pending while its turn was still running (watched ${watchedForMs}ms) \u2014 restored to processing instead of re-dispatching`,
+      conversation_id: conv.id,
+      message_id: message.id
+    });
+    return "reattached";
+  }
+  /**
+   * The `settled` outcome (Task 3.2): the prior turn already finished (or
+   * errored) while nobody was watching — deliver/report it instead of re-running.
+   * Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
+   * (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
+   * drain, same as any other non-auth failure). The restart-abort carve-out that
+   * keeps the two in step for `failed` lives in the caller (`resolveRedrive`, #1310),
+   * so a row reaching this `failed` branch is a GENUINE failure.
+   */
+  async settleRedrive(conv, sessionId, message, ocId, messages, state) {
+    try {
+      if (state === "done") {
+        const title = await this.resolveSessionTitle(sessionId, conv.id);
+        const usage = messageUsage(messages, ocId ?? "");
+        this.log({
+          level: "info",
+          message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        await this.markDone(conv.id, message.id, sessionId, ocId, title, usage);
+      } else {
+        const error2 = messageError(messages, ocId ?? "") ?? void 0;
+        const usage = messageUsage(messages, ocId ?? "");
+        const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
+        this.log({
+          level: "error",
+          message: `Re-drive: message ${message.id.slice(0, 8)} errored while its row was wrongly reclaimed to pending \u2014 marking failed instead of re-dispatching: ${error2 ?? "(no error text)"}`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
+      }
+    } catch (err) {
+      if (err instanceof ChannelAuthError) throw err;
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to report message ${message.id.slice(0, 8)} ${state} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return "unresolved";
+    }
+    this.clearRedriveUnresolved(message.id);
+    void this.postSignal(conv.id, message.id, "redrive_settled");
+    return "settled";
+  }
+  /**
+   * The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
+   * observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
+   * A `pending` row is invisible to every cron arm (all require `status =
+   * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
+   * nothing driving it — bound it to the existing `pausedMaxWaitMs` window
+   * (reusing the knob, not a new constant) and take `dispatch` once elapsed.
+   */
+  resolveRedriveUnresolved(conv, message) {
+    const now = this.now();
+    const since = this.redriveUnresolvedSince.get(message.id);
+    if (since !== void 0 && now - since >= this.pausedMaxWaitMs) {
+      this.clearRedriveUnresolved(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_redispatched");
+      return "dispatch";
+    }
+    if (since === void 0) {
+      this.redriveUnresolvedSince.set(message.id, now);
+    }
+    if (!this.redriveUnresolvedSignalled.has(message.id)) {
+      this.redriveUnresolvedSignalled.add(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_unresolved");
+    }
+    return "unresolved";
+  }
+  /** Clear both `unresolved`-bound trackers for a row (any non-`unresolved` outcome). */
+  clearRedriveUnresolved(messageId) {
+    this.redriveUnresolvedSince.delete(messageId);
+    this.redriveUnresolvedSignalled.delete(messageId);
+  }
   /**
    * Record that `sessionId` is no longer a valid binding for `conversationId`
    * (#553). Keyed by conversation and hard-capped, so it cannot grow with the
@@ -3969,9 +4239,10 @@ var ChannelDriver = class _ChannelDriver {
    * opencode reports ACTIVELY `running` is watched to completion (its liveness
    * heartbeat keeps the cron off its row), while a re-adopted turn that is paused
    * awaiting a human — or queued/unreachable — is still bounded by `deadline` and
-   * handed to the cron. The old "the `deadline` must settle before the ~15-min
-   * cron or they double-drive" reasoning is superseded: liveness now settles the
-   * actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
+   * handed to the cron. Real invariant (#965): the cron MAY reclaim a row this
+   * runner still holds; a reclaimed row that already ran is never re-dispatched
+   * while opencode reports its turn ongoing (readopt's own gate here, and the
+   * `pending`-row re-drive fence, `resolveRedrive`). `dispatchedAt` stays `now`
    * (only the appear-guard uses it).
    *
    * `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
@@ -4531,7 +4802,10 @@ var ChannelDriver = class _ChannelDriver {
    * re-dispatched (at most once, see `forceReadoptRun`):
    *   - `done`             → `markDone` now (guarded like the watcher's done branch);
    *   - `failed`           → `markFailed` with the surfaced error (issue #182), so an
-   *                          errored turn is reported failed on restart, NOT re-dispatched;
+   *                          errored turn is reported failed on restart, NOT re-dispatched —
+   *                          EXCEPT a restart-ABORTED turn under a not-ongoing session,
+   *                          which is a restart orphan wearing a terminal error and is
+   *                          re-dispatched instead (issue #1310, see the branch below);
    *   - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
    *                          tracking the stored id so the reply correlates by it;
    *   - `unknown`/null id  → re-dispatch (opencode assigns a fresh id) + attach a watcher.
@@ -4595,7 +4869,16 @@ var ChannelDriver = class _ChannelDriver {
       void this.postSignal(row.conversation_id, row.id, "readopt_done");
       return;
     }
-    if (state === "failed") {
+    const restartAborted = state === "failed" && sessionOngoing === false && isAbortedTerminalReply(messages, ocId ?? "");
+    if (restartAborted) {
+      this.log({
+        level: "info",
+        message: `Re-adopt: message ${row.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
+        conversation_id: row.conversation_id,
+        message_id: row.id
+      });
+    }
+    if (state === "failed" && !restartAborted) {
       const error2 = messageError(messages, ocId ?? "") ?? void 0;
       const usage = messageUsage(messages, ocId ?? "");
       const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
@@ -4873,7 +5156,8 @@ var ChannelDriver = class _ChannelDriver {
       opencode_model: row.opencode_model,
       source_message_id: row.source_message_id,
       slack_user_id: row.slack_user_id,
-      attachments: row.attachments ?? null
+      attachments: row.attachments ?? null,
+      opencode_message_id: row.opencode_message_id
     };
   }
   /**
@@ -6999,7 +7283,7 @@ program.command("run").description("Connect to Evident and process messages").op
   []
 ).option(
   "--tunnel-ready-file <path>",
-  "Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
+  "Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
 ).action(
   (options) => {
     run({