@evident-ai/cli 3.1.1-dev.2ab3b19 → 3.1.1-dev.2ea09d2

package/dist/index.js

@@ -997,6 +997,9 @@ import { homedir as homedir3 } from "os";
 import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
 import chalk6 from "chalk";
 
+// ../../packages/types/src/agents/index.ts
+var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
+
 // ../../packages/types/src/telemetry/index.ts
 var TelemetryEventTypes = {
   // Agent activity events (shown in web UI activity log)
@@ -2688,6 +2691,10 @@ function nextReportDelayMs(random = Math.random) {
   return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
 }
 var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
+var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
+function claudeUsageFailureLogLevel(consecutiveFailures) {
+  return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
+}
 
 // src/lib/channels/driver.ts
 import { homedir as homedir2 } from "os";
@@ -3254,6 +3261,24 @@ var ChannelDriver = class _ChannelDriver {
    * processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
    */
   readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
+  /**
+   * "Already emitted `redrive_unresolved` for this row" (#965). Mirrors
+   * `readoptPollUnresolvedSignalled`: `resolveRedrive`'s `unresolved` leaf recurs
+   * every ~2s drain until opencode's status becomes readable, but the
+   * server-visible signal is an OUTCOME, so it fires at most once per row. Cleared
+   * on any non-`unresolved` outcome so the set cannot grow beyond the currently
+   * unresolvable rows.
+   */
+  redriveUnresolvedSignalled = /* @__PURE__ */ new Set();
+  /**
+   * First `now()` a `pending` row's re-drive was observed `unresolved` (#965). A
+   * `pending` row is invisible to every cron arm (all require `status =
+   * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
+   * nothing driving it. Once `now - since >= pausedMaxWaitMs`, `resolveRedrive`
+   * takes `dispatch` instead of `unresolved` (reusing the existing knob — see
+   * ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
+   */
+  redriveUnresolvedSince = /* @__PURE__ */ new Map();
   /**
    * "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
    * Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
@@ -3593,6 +3618,12 @@ var ChannelDriver = class _ChannelDriver {
         skippedAlreadyDispatched += 1;
         continue;
       }
+      if (message.opencode_message_id) {
+        const outcome = await this.resolveRedrive(conv, sessionId, message, refusedSessionId);
+        if (outcome !== "dispatch") {
+          break;
+        }
+      }
       const options = {
         agent: message.opencode_agent ?? void 0,
         model: message.opencode_model ?? void 0
@@ -3682,6 +3713,218 @@ var ChannelDriver = class _ChannelDriver {
     this.ensureWatcherRunning(sessionId);
     return dispatched;
   }
+  /**
+   * Poll a session's message list for the re-drive fence (#965), via the
+   * INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
+   * hits the global `fetch` and would bypass the same override every other
+   * opencode poll in this file respects. Mirrors `readoptProcessing`'s own
+   * snapshot fetch (`:3081-3111`). `null` = unreadable (non-OK response,
+   * non-array body, or a network exception) — treated as "can't observe",
+   * never as "confirmed gone".
+   */
+  async pollSessionMessagesForRedrive(conv, message, sessionId) {
+    try {
+      const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
+      if (!res.ok) {
+        this.log({
+          level: "warn",
+          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status} \u2014 treating as unreadable this tick`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        return null;
+      }
+      const body = await res.json();
+      if (!Array.isArray(body)) {
+        this.log({
+          level: "warn",
+          message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned a non-array message body \u2014 treating as unreadable this tick`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        return null;
+      }
+      return body;
+    } catch (err) {
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to poll session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return null;
+    }
+  }
+  /**
+   * The re-drive fence for a `pending` row that already carries a stored
+   * `opencode_message_id` (#965) — i.e. it has already been handed to opencode at
+   * least once (see the invariant at `QueuedMessage.opencode_message_id`'s doc).
+   * The lifecycle cron can falsely reclaim a `processing` row back to `pending`
+   * mid-turn (a 5-minute liveness-staleness check racing a still-running turn);
+   * without this fence the drain loop would re-`prompt_async` the SAME turn a
+   * second time against live GitHub state. Mirrors `readoptOne`'s job for the
+   * `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
+   * needed here because `refusedSessionId` already handles the one case
+   * (#553 abandoned session) that path exists for.
+   *
+   * Only `ChannelAuthError` propagates; every other failure resolves to
+   * `unresolved` and is retried whole on the next ~2s drain tick.
+   */
+  async resolveRedrive(conv, sessionId, message, refusedSessionId) {
+    const ocId = message.opencode_message_id ?? null;
+    if (refusedSessionId) {
+      this.clearRedriveUnresolved(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_redispatched");
+      return "dispatch";
+    }
+    const messages = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
+    if (messages == null || messages.length === 0) {
+      return this.resolveRedriveUnresolved(conv, message);
+    }
+    const state = messageRunState(messages, ocId ?? "");
+    if (state === "done" || state === "failed") {
+      return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
+    }
+    if (state === "running" || state === "queued") {
+      const ongoing = await isSessionOngoing(this.port, sessionId);
+      if (ongoing === true) {
+        return this.reattachRedrive(conv, sessionId, message, ocId);
+      }
+      if (ongoing === false) {
+        this.clearRedriveUnresolved(message.id);
+        void this.postSignal(conv.id, message.id, "redrive_redispatched");
+        return "dispatch";
+      }
+      return this.resolveRedriveUnresolved(conv, message);
+    }
+    this.clearRedriveUnresolved(message.id);
+    void this.postSignal(conv.id, message.id, "redrive_redispatched");
+    return "dispatch";
+  }
+  /**
+   * The `reattached` outcome (Task 3.3): the prior turn is STILL ONGOING per
+   * opencode's own status map — undo the false reclaim instead of starting a
+   * second turn.
+   */
+  async reattachRedrive(conv, sessionId, message, ocId) {
+    let anchorMs;
+    const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
+    if (!Number.isNaN(parsed)) {
+      anchorMs = parsed;
+    } else {
+      anchorMs = this.now();
+      this.log({
+        level: "error",
+        message: `Re-drive: message ${message.id.slice(0, 8)} has null/unparseable processing_started_at (${String(message.processing_started_at)}) \u2014 anchoring the watcher's absolute-age ceiling to now (defensive)`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+    }
+    const title = await this.resolveSessionTitle(sessionId, conv.id);
+    try {
+      await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
+    } catch (err) {
+      if (err instanceof ChannelAuthError) throw err;
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return "unresolved";
+    }
+    this.clearRedriveUnresolved(message.id);
+    this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
+    this.dispatched.add(message.id);
+    this.readopted.add(message.id);
+    this.ensureWatcherRunning(sessionId);
+    const watchedForMs = this.now() - anchorMs;
+    void this.postSignal(conv.id, message.id, "redrive_reattached", {
+      watched_for_ms: watchedForMs
+    });
+    this.log({
+      level: "warn",
+      message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) was wrongly reclaimed to pending while its turn was still running (watched ${watchedForMs}ms) \u2014 restored to processing instead of re-dispatching`,
+      conversation_id: conv.id,
+      message_id: message.id
+    });
+    return "reattached";
+  }
+  /**
+   * The `settled` outcome (Task 3.2): the prior turn already finished (or
+   * errored) while nobody was watching — deliver/report it instead of re-running.
+   * Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
+   * (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
+   * drain, same as any other non-auth failure).
+   */
+  async settleRedrive(conv, sessionId, message, ocId, messages, state) {
+    try {
+      if (state === "done") {
+        const title = await this.resolveSessionTitle(sessionId, conv.id);
+        const usage = messageUsage(messages, ocId ?? "");
+        this.log({
+          level: "info",
+          message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        await this.markDone(conv.id, message.id, sessionId, ocId, title, usage);
+      } else {
+        const error2 = messageError(messages, ocId ?? "") ?? void 0;
+        const usage = messageUsage(messages, ocId ?? "");
+        const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
+        this.log({
+          level: "error",
+          message: `Re-drive: message ${message.id.slice(0, 8)} errored while its row was wrongly reclaimed to pending \u2014 marking failed instead of re-dispatching: ${error2 ?? "(no error text)"}`,
+          conversation_id: conv.id,
+          message_id: message.id
+        });
+        await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
+      }
+    } catch (err) {
+      if (err instanceof ChannelAuthError) throw err;
+      this.log({
+        level: "warn",
+        message: `Re-drive: failed to report message ${message.id.slice(0, 8)} ${state} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conv.id,
+        message_id: message.id
+      });
+      return "unresolved";
+    }
+    this.clearRedriveUnresolved(message.id);
+    void this.postSignal(conv.id, message.id, "redrive_settled");
+    return "settled";
+  }
+  /**
+   * The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
+   * observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
+   * A `pending` row is invisible to every cron arm (all require `status =
+   * 'processing'`), so an indefinitely-`unresolved` row would be stranded with
+   * nothing driving it — bound it to the existing `pausedMaxWaitMs` window
+   * (reusing the knob, not a new constant) and take `dispatch` once elapsed.
+   */
+  resolveRedriveUnresolved(conv, message) {
+    const now = this.now();
+    const since = this.redriveUnresolvedSince.get(message.id);
+    if (since !== void 0 && now - since >= this.pausedMaxWaitMs) {
+      this.clearRedriveUnresolved(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_redispatched");
+      return "dispatch";
+    }
+    if (since === void 0) {
+      this.redriveUnresolvedSince.set(message.id, now);
+    }
+    if (!this.redriveUnresolvedSignalled.has(message.id)) {
+      this.redriveUnresolvedSignalled.add(message.id);
+      void this.postSignal(conv.id, message.id, "redrive_unresolved");
+    }
+    return "unresolved";
+  }
+  /** Clear both `unresolved`-bound trackers for a row (any non-`unresolved` outcome). */
+  clearRedriveUnresolved(messageId) {
+    this.redriveUnresolvedSince.delete(messageId);
+    this.redriveUnresolvedSignalled.delete(messageId);
+  }
   /**
    * Record that `sessionId` is no longer a valid binding for `conversationId`
    * (#553). Keyed by conversation and hard-capped, so it cannot grow with the
@@ -3965,9 +4208,10 @@ var ChannelDriver = class _ChannelDriver {
    * opencode reports ACTIVELY `running` is watched to completion (its liveness
    * heartbeat keeps the cron off its row), while a re-adopted turn that is paused
    * awaiting a human — or queued/unreachable — is still bounded by `deadline` and
-   * handed to the cron. The old "the `deadline` must settle before the ~15-min
-   * cron or they double-drive" reasoning is superseded: liveness now settles the
-   * actively-running case; `deadline` settles the rest. `dispatchedAt` stays `now`
+   * handed to the cron. Real invariant (#965): the cron MAY reclaim a row this
+   * runner still holds; a reclaimed row that already ran is never re-dispatched
+   * while opencode reports its turn ongoing (readopt's own gate here, and the
+   * `pending`-row re-drive fence, `resolveRedrive`). `dispatchedAt` stays `now`
    * (only the appear-guard uses it).
    *
    * `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
@@ -4869,7 +5113,8 @@ var ChannelDriver = class _ChannelDriver {
       opencode_model: row.opencode_model,
       source_message_id: row.source_message_id,
       slack_user_id: row.slack_user_id,
-      attachments: row.attachments ?? null
+      attachments: row.attachments ?? null,
+      opencode_message_id: row.opencode_message_id
     };
   }
   /**
@@ -6121,10 +6366,15 @@ async function handleAuthError(state, error2) {
 }
 async function driveChannels(state, driver) {
   let idlePolls = 0;
+  let idleMs = 0;
   let consecutiveDrainFailures = 0;
+  let unreachableMs = 0;
   let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
   let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
   while (state.running) {
+    const cycleStartedAtMs = performance.now();
+    let idleThisCycle = false;
+    let unreachableThisCycle = false;
     if (state.connection?.reconnecting && state.connection.reconnectPromise) {
       logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
       if (state.interactive) displayStatus(state);
@@ -6140,17 +6390,22 @@ async function driveChannels(state, driver) {
     try {
       const processed = await driver.drainPending();
       consecutiveDrainFailures = 0;
+      unreachableMs = 0;
       state.messageCount += processed;
       const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
       lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
       const appliedFiles = driver.fileSyncActivity().appliedFiles;
-      const fileActivity = carriedOverFileSync || appliedFiles !== lastSeenAppliedFiles;
+      const filesApplied = appliedFiles !== lastSeenAppliedFiles;
+      const fileActivity = carriedOverFileSync || filesApplied;
       lastSeenAppliedFiles = appliedFiles;
+      if (filesApplied) state.claudeUsageRearm?.();
       if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
         idlePolls = 0;
+        idleMs = 0;
         if (processed > 0 && state.interactive) displayStatus(state);
       } else if (state.idleTimeout !== null) {
         idlePolls++;
+        idleThisCycle = true;
         if (idlePolls === 1) {
           logActivity(state, {
             type: "info",
@@ -6176,8 +6431,10 @@ async function driveChannels(state, driver) {
       if (state.interactive) displayStatus(state);
       if (driver.hasInFlightWatchers()) {
         consecutiveDrainFailures = 0;
+        unreachableMs = 0;
       } else if (state.idleTimeout !== null) {
         consecutiveDrainFailures++;
+        unreachableThisCycle = true;
         if (consecutiveDrainFailures === 1) {
           logActivity(state, {
             type: "info",
@@ -6188,25 +6445,22 @@ async function driveChannels(state, driver) {
       }
     }
     await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
-    if (state.idleTimeout !== null && consecutiveDrainFailures >= 2) {
-      const unreachableMs = consecutiveDrainFailures * CHANNEL_POLL_INTERVAL_MS;
-      if (unreachableMs > state.idleTimeout * 1e3) {
-        logActivity(state, {
-          type: "info",
-          level: "warn",
-          message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
-        });
-        if (state.interactive) displayStatus(state);
-        break;
-      }
+    const cycleMs = performance.now() - cycleStartedAtMs;
+    if (idleThisCycle) idleMs += cycleMs;
+    if (unreachableThisCycle) unreachableMs += cycleMs;
+    if (state.idleTimeout !== null && consecutiveDrainFailures >= 2 && unreachableMs > state.idleTimeout * 1e3) {
+      logActivity(state, {
+        type: "info",
+        level: "warn",
+        message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
+      });
+      if (state.interactive) displayStatus(state);
+      break;
     }
-    if (state.idleTimeout !== null && idlePolls >= 2) {
-      const idleMs = idlePolls * CHANNEL_POLL_INTERVAL_MS;
-      if (idleMs > state.idleTimeout * 1e3) {
-        logActivity(state, { type: "info", message: "Idle timeout reached" });
-        if (state.interactive) displayStatus(state);
-        break;
-      }
+    if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
+      logActivity(state, { type: "info", message: "Idle timeout reached" });
+      if (state.interactive) displayStatus(state);
+      break;
     }
   }
 }
@@ -6285,6 +6539,9 @@ function scheduleSessionCleanup(state, driver, options) {
   );
   state.sessionCleanupTimers.push(interval, firstSweep);
 }
+function claudeUsageFailureStreakSuffix(consecutiveFailures) {
+  return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
+}
 function scheduleClaudeUsageReporting(state, options) {
   const { mode, warnings } = resolveClaudeUsageReportingMode(
     options.claudeUsageReporting,
@@ -6303,13 +6560,26 @@ function scheduleClaudeUsageReporting(state, options) {
       level: "debug",
       message: "Claude usage reporting is off (--claude-usage-reporting off)"
     });
-    return;
+    return null;
   }
   let consecutiveFailures = 0;
+  let armed = false;
+  let rearmRequested = false;
   const scheduleNextTick = () => {
+    armed = true;
+    rearmRequested = false;
     state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
   };
-  const tick = async (isFirst) => {
+  const rearm = () => {
+    if (armed) {
+      rearmRequested = true;
+      return;
+    }
+    rearmRequested = false;
+    armed = true;
+    state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
+  };
+  const tick = async (isProbe) => {
     try {
       const usage = await getClaudeUsage();
       const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
@@ -6331,8 +6601,8 @@ function scheduleClaudeUsageReporting(state, options) {
         consecutiveFailures++;
         logActivity(state, {
           type: "info",
-          level: consecutiveFailures === 1 ? "warn" : "debug",
-          message: `Failed to report Claude usage: ${result.error}`
+          level: claudeUsageFailureLogLevel(consecutiveFailures),
+          message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
         });
       }
       scheduleNextTick();
@@ -6345,12 +6615,14 @@ function scheduleClaudeUsageReporting(state, options) {
             message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
           });
           scheduleNextTick();
-        } else if (isFirst) {
+        } else if (isProbe) {
           logActivity(state, {
             type: "info",
             level: "debug",
             message: `Claude usage reporting: ${error2.message}`
           });
+          armed = false;
+          if (rearmRequested) rearm();
         } else {
           logActivity(state, {
             type: "info",
@@ -6364,14 +6636,16 @@ function scheduleClaudeUsageReporting(state, options) {
         const message = error2 instanceof Error ? error2.message : String(error2);
         logActivity(state, {
           type: "info",
-          level: consecutiveFailures === 1 ? "warn" : "debug",
-          message: `Claude usage reporting failed: ${message}`
+          level: claudeUsageFailureLogLevel(consecutiveFailures),
+          message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
         });
         scheduleNextTick();
       }
     }
   };
+  armed = true;
   state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
+  return rearm;
 }
 async function notifyOffline(state) {
   if (!state.agentId || !state.authHeader) return;
@@ -6412,6 +6686,7 @@ async function cleanup(state, opts = {}) {
     clearTimeout(state.claudeUsageTimer);
     state.claudeUsageTimer = null;
   }
+  state.claudeUsageRearm = null;
   if (opts.graceful && state.channelDriver) {
     state.channelDriver.stop();
     log2(state, "Draining in-flight channel work before shutdown...");
@@ -6493,6 +6768,7 @@ async function run(options) {
     lastProxiedActivityAt: null,
     sessionCleanupTimers: [],
     claudeUsageTimer: null,
+    claudeUsageRearm: null,
     authHeader: ""
   };
   setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
@@ -6572,6 +6848,7 @@ async function run(options) {
         console.log(chalk6.dim("Or run `evident login` for interactive authentication"));
         blank();
         process.exit(1);
+        return;
       }
       blank();
       console.log(chalk6.yellow("You are not logged in to Evident."));
@@ -6616,6 +6893,7 @@ async function run(options) {
         } else {
           printError(resolved.error || "Failed to resolve runner ID from key");
           process.exit(1);
+          return;
         }
       } else {
         printError(
@@ -6629,6 +6907,7 @@ async function run(options) {
         );
         blank();
         process.exit(1);
+        return;
       }
     }
     telemetry.info(
@@ -6876,7 +7155,7 @@ async function run(options) {
       throw error2;
     }
     scheduleSessionCleanup(state, channelDriver, options);
-    scheduleClaudeUsageReporting(state, options);
+    state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
     if (!interactive || state.json) {
       log2(state, "Driving channel messages...");
     }
@@ -6961,7 +7240,7 @@ program.command("run").description("Connect to Evident and process messages").op
   []
 ).option(
   "--tunnel-ready-file <path>",
-  "Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
+  "Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
 ).action(
   (options) => {
     run({