@evident-ai/cli 3.5.2-dev.06654ec → 3.5.2-dev.1ca1af3

package/dist/index.js

@@ -1322,7 +1322,7 @@ import ora4 from "ora";
 import { select as select4 } from "@inquirer/prompts";
 
 // src/lib/telemetry.ts
-var CLI_VERSION = (true ? "3.5.2-dev.06654ec" : void 0) ?? process.env.npm_package_version ?? "unknown";
+var CLI_VERSION = (true ? "3.5.2-dev.1ca1af3" : void 0) ?? process.env.npm_package_version ?? "unknown";
 function getCliVersion() {
   return CLI_VERSION;
 }
@@ -2054,12 +2054,23 @@ function buildSessionDbRecoveryActivity(record) {
         metadata: withoutContractFields(record),
         message: `This runner started without restoring session history because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions.${replication}`
       };
-    case "session_db_boot_refused":
-      return {
-        level,
-        metadata: withoutContractFields(record),
-        message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
-      };
+    case "session_db_boot_refused": {
+      const message = (() => {
+        switch (record.reason) {
+          case "local_discard_failed":
+            return `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`;
+          case "replica_separation_unproven":
+            return `This runner did not come online because its damaged session database could not be safely separated from its active backup. The SESSION-DB-REPLICA-SEPARATION-UNVERIFIED boot log contains the partial-quarantine moved and remaining counts. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them.`;
+          case "verification_timeout_before_separation":
+            return "This runner did not come online because session database verification timed out after detecting corruption, before replica separation was attempted. No replica progress is claimed. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
+          case "verification_timeout_during_separation":
+            return "This runner did not come online because session database verification timed out while replica separation was in progress. Separation completion and object removal were not proven. See SESSION-DB-VERIFY-TIMEOUT in the runner boot logs before retrying.";
+          default:
+            return "This runner did not come online because session database recovery could not be completed. Inspect the runner boot logs for the refusal reason before retrying.";
+        }
+      })();
+      return { level, metadata: withoutContractFields(record), message };
+    }
     case "schema_provenance_mismatch":
       return {
         level,
@@ -2345,8 +2356,12 @@ function errorOf(m) {
   if (!m || typeof m !== "object") return void 0;
   return m.info?.error ?? m.error;
 }
+function isTerminalStopAnswer(m) {
+  return completedOf(m) != null && errorOf(m) == null && finishOf(m) === "stop";
+}
 function isAssistantInFlight(m) {
   if (completedOf(m) == null) return true;
+  if (errorOf(m) != null) return false;
   return finishOf(m) === "tool-calls";
 }
 async function getSessionMessages(port, sessionId, client) {
@@ -2793,30 +2808,41 @@ function findAssistantReplyAfter(messages, userMessageId) {
 function findLastAssistantReplyFor(messages, userMessageId) {
   if (!messages || messages.length === 0) return null;
   let lastCorrelated = null;
-  let lastNonErrored = null;
   for (let i = messages.length - 1; i >= 0; i--) {
     const m = messages[i];
     if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
-    if (lastCorrelated === null) lastCorrelated = m;
-    if (errorOf(m) == null) {
-      lastNonErrored = m;
-      break;
+    lastCorrelated = m;
+    break;
+  }
+  if (lastCorrelated) {
+    if (errorOf(lastCorrelated) == null) return lastCorrelated;
+    for (let i = messages.length - 1; i >= 0; i--) {
+      const m = messages[i];
+      if (roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId && isTerminalStopAnswer(m)) {
+        return m;
+      }
     }
+    return lastCorrelated;
   }
-  if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
   const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
   if (userIndex === -1) return null;
   let last = null;
-  let lastOk = null;
+  let lastAssistantIndex = -1;
   for (let i = userIndex + 1; i < messages.length; i++) {
     const role = roleOf2(messages[i]);
     if (role === "user") break;
     if (role === "assistant") {
       last = messages[i];
-      if (errorOf(messages[i]) == null) lastOk = messages[i];
+      lastAssistantIndex = i;
     }
   }
-  return lastOk ?? last;
+  if (last === null || errorOf(last) == null) return last;
+  for (let i = lastAssistantIndex; i > userIndex; i--) {
+    const m = messages[i];
+    if (roleOf2(m) === "user") break;
+    if (roleOf2(m) === "assistant" && isTerminalStopAnswer(m)) return m;
+  }
+  return last;
 }
 function collectSubagentSessions(messages, userMessageId) {
   if (!messages || messages.length === 0) return [];
@@ -2922,8 +2948,9 @@ function attributeTaskCallUsage(messages, windows) {
 }
 function sumAssistantUsage(messages) {
   if (!messages || messages.length === 0) return null;
-  const nonErrored = messages.filter((message) => errorOf(message) == null);
-  const selected = nonErrored.length > 0 ? nonErrored : messages;
+  const hasTrailingError = errorOf(messages[messages.length - 1]) != null;
+  const hasTerminalStopAnswer = messages.some(isTerminalStopAnswer);
+  const selected = hasTrailingError && hasTerminalStopAnswer ? messages.filter((message) => errorOf(message) == null) : messages;
   let sawAnyUsage = false;
   let inputSum = 0;
   let outputSum = 0;
@@ -2979,11 +3006,9 @@ function messageUsage(messages, userMessageId) {
   const byParentAll = messages.filter(
     (m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
   );
-  const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
-  const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
   let correlated;
-  if (byParent.length > 0) {
-    correlated = byParent;
+  if (byParentAll.length > 0) {
+    correlated = byParentAll;
   } else {
     const reply = findAssistantReplyAfter(messages, userMessageId);
     correlated = reply ? [reply] : [];
@@ -3029,14 +3054,38 @@ function messageError(messages, userMessageId) {
   const reply = findLastAssistantReplyFor(messages, userMessageId);
   const error2 = errorOf(reply);
   if (error2 == null) return null;
-  if (typeof error2 === "string") return error2;
-  if (typeof error2 === "object") {
+  let rendered;
+  if (typeof error2 === "string") {
+    rendered = error2;
+  } else if (typeof error2 === "object") {
     const e = error2;
     const dataMessage = e.data?.message;
-    if (typeof dataMessage === "string") return dataMessage;
-    if (typeof e.message === "string") return e.message;
+    if (typeof dataMessage === "string") {
+      rendered = dataMessage;
+    } else if (typeof e.message === "string") {
+      rendered = e.message;
+    } else {
+      rendered = "The agent run failed.";
+    }
+  } else {
+    rendered = "The agent run failed.";
   }
-  return "The agent run failed.";
+  const toolPart = Array.isArray(reply?.parts) ? reply.parts.find(
+    (part) => part && typeof part === "object" && part.type === "tool" && part.state?.status === "error"
+  ) : void 0;
+  return typeof toolPart?.tool === "string" && toolPart.tool.length > 0 ? `${rendered} (tool "${toolPart.tool}")` : rendered;
+}
+function describeAbandonedReplyShape(messages, userMessageId) {
+  const reply = findLastAssistantReplyFor(messages, userMessageId);
+  if (!reply) return null;
+  const parts = Array.isArray(reply.parts) ? reply.parts : [];
+  const lastTool = [...parts].reverse().find((part) => part?.type === "tool");
+  return {
+    partTypes: parts.slice(-12).map((part) => typeof part?.type === "string" ? part.type : "(unknown)"),
+    droppedPartCount: Math.max(0, parts.length - 12),
+    lastToolName: typeof lastTool?.tool === "string" && lastTool.tool.length > 0 ? lastTool.tool : null,
+    lastToolStatus: typeof lastTool?.state?.status === "string" && lastTool.state.status.length > 0 ? lastTool.state.status : null
+  };
 }
 function isAbortedTerminalReply(messages, userMessageId) {
   const reply = findLastAssistantReplyFor(messages, userMessageId);
@@ -4320,9 +4369,12 @@ function runSynchroniser(args, opts) {
 
 // src/lib/opencode/session-db-boot.ts
 var SESSION_DB_RESTORE_TIMEOUT_MS = 3e5;
-var SESSION_DB_VERIFY_TIMEOUT_MS = 12e4;
+var SESSION_DB_VERIFY_TIMEOUT_MS = 36e4;
 var SESSION_DB_SYNCHRONISER_TIMEOUT_MS = 12e4;
-var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 120;
+var SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS = 40;
+var SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS = 30;
+var SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS = 120;
+var SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS = 90;
 function commandError(result) {
   return result.stderr.trim() || `process exited with code ${result.code ?? "null"}`;
 }
@@ -4633,20 +4685,37 @@ async function verifySessionDb(options, configPath, env) {
       timeoutMs: SESSION_DB_VERIFY_TIMEOUT_MS,
       env: {
         ...env,
-        // The synchroniser reads this value in SECONDS. Keep this at 120, not
-        // 120_000, so the walkback gives up before the outer process bound.
+        // The synchroniser reads all four values in SECONDS.
         EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS: String(
           SESSION_DB_VERIFY_WALKBACK_BUDGET_SECONDS
+        ),
+        EVIDENT_SESSION_DB_RESTORE_POINTS_TIMEOUT_SECONDS: String(
+          SESSION_DB_VERIFY_RESTORE_POINTS_TIMEOUT_SECONDS
+        ),
+        EVIDENT_SESSION_DB_RESTORE_CANDIDATE_TIMEOUT_SECONDS: String(
+          SESSION_DB_VERIFY_RESTORE_CANDIDATE_TIMEOUT_SECONDS
+        ),
+        EVIDENT_SESSION_DB_DISPOSAL_BUDGET_SECONDS: String(
+          SESSION_DB_VERIFY_DISPOSAL_BUDGET_SECONDS
         )
       }
     });
     logSynchroniserDiagnostics(result, options);
     if (result.timedOut || result.code === 124 || result.code === 137) {
+      if (result.stderr.includes("SESSION-DB-INTEGRITY-FAILED")) {
+        const reason = result.stderr.includes("SESSION-DB-REPLICA-SIZE") ? "verification_timeout_during_separation" : "verification_timeout_before_separation";
+        options.log(
+          `SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; refusing to start with the restored opencode.db (${reason})`,
+          "error"
+        );
+        reportRecord("verify", "session_db_boot_refused", reason, null, options);
+        return { verifyFatal: true, verifyFatalReason: reason };
+      }
       options.log(
         `SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_TIMEOUT_MS}ms deadline; continuing with the restored opencode.db as-is, unverified`,
         "warn"
       );
-      return false;
+      return { verifyFatal: false };
     }
     if (result.code === 34) {
       reportRecord(
@@ -4656,14 +4725,14 @@ async function verifySessionDb(options, configPath, env) {
         null,
         options
       );
-      return true;
+      return { verifyFatal: true };
     }
     if (result.code === 33) {
       options.log(
         "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated.",
         "warn"
       );
-      return false;
+      return { verifyFatal: false };
     }
     if (result.code !== 0) {
       options.log(
@@ -4671,13 +4740,13 @@ async function verifySessionDb(options, configPath, env) {
         "warn"
       );
     }
-    return false;
+    return { verifyFatal: false };
   }
   options.log(
     "skipping session-DB verification: this boot's session DB was not proven safe to replicate",
     "debug"
   );
-  return false;
+  return { verifyFatal: false };
 }
 function fileExists(path) {
   try {
@@ -4764,7 +4833,7 @@ async function restoreAndVerifySessionDb(options) {
   if (options.noReplicateMarker && fileExists(options.noReplicateMarker)) {
     return { verifyFatal: false };
   }
-  return { verifyFatal: await verifySessionDb(options, configPath, env) };
+  return verifySessionDb(options, configPath, env);
 }
 
 // src/lib/opencode/session-db-provenance.ts
@@ -7016,6 +7085,9 @@ var SUBAGENT_TELEMETRY_FETCH_TIMEOUT_MS = 2e3;
 var SESSION_ERROR_STREAM_HEALTHY_MS = 5e3;
 var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
 var MAX_BUFFERED_SESSION_ERRORS = 256;
+var SESSION_ERROR_RETENTION_TTL_MS = 6e4;
+var SESSION_STORE_UNWRITABLE_ERROR = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
+var SESSION_ERROR_RETAINED_DETAIL = "The runner's session database could not be written, so this turn stopped part-way. Runner-observed session error: ";
 var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
 var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
 var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
@@ -7173,6 +7245,7 @@ var ChannelDriver = class _ChannelDriver {
    * with a short TTL so an unmatched session cannot retain an event indefinitely.
    */
   bufferedSessionErrors = /* @__PURE__ */ new Map();
+  retainedSessionErrors = /* @__PURE__ */ new Map();
   /**
    * AUTHORITATIVE local dedup (WI-3): Evident message ids that have been
    * dispatched and are still in-flight. A message in this set is never
@@ -9013,6 +9086,28 @@ var ChannelDriver = class _ChannelDriver {
       this.bufferedSessionErrors.delete(oldest);
     }
   }
+  retainSessionError(event) {
+    this.retainedSessionErrors.delete(event.sessionId);
+    this.retainedSessionErrors.set(event.sessionId, {
+      reason: event.reason,
+      observedAt: this.now()
+    });
+    while (this.retainedSessionErrors.size > MAX_BUFFERED_SESSION_ERRORS) {
+      const oldest = this.retainedSessionErrors.keys().next().value;
+      if (typeof oldest !== "string") break;
+      this.retainedSessionErrors.delete(oldest);
+    }
+  }
+  readRetainedSessionError(sessionId) {
+    const retained = this.retainedSessionErrors.get(sessionId);
+    if (!retained) return null;
+    if (this.now() - retained.observedAt >= SESSION_ERROR_RETENTION_TTL_MS) {
+      this.retainedSessionErrors.delete(sessionId);
+      return null;
+    }
+    this.retainedSessionErrors.delete(sessionId);
+    return retained.reason;
+  }
   /**
    * Build a fresh `SessionWatcher`. Seeds `lastTickAt`/`lastObservedTickAt`
    * EQUAL (#1618) so a watcher whose loop has not started ticking yet is never
@@ -9339,9 +9434,10 @@ var ChannelDriver = class _ChannelDriver {
         return;
       }
       if ([...watcher.inFlight.values()].some((message) => message.started && !message.done)) {
+        this.retainSessionError(event);
         this.log({
-          level: "debug",
-          message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 deferring to transcript polling`,
+          level: "warn",
+          message: `A turn is already running in session ${event.sessionId.slice(0, 8)} \u2014 retaining the session error and deferring to transcript polling: ${event.reason}`,
           conversation_id: watcher.conv.id
         });
         return;
@@ -9371,9 +9467,10 @@ var ChannelDriver = class _ChannelDriver {
       const messages = await this.getSessionMessages(event.sessionId);
       const state = messageRunState(messages, inFlight.opencodeMessageId);
       if (state !== "queued") {
+        this.retainSessionError(event);
         this.log({
-          level: "debug",
-          message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} observed state ${state}; leaving it to transcript polling`,
+          level: "warn",
+          message: `Session error for message ${inFlight.evidentMessageId.slice(0, 8)} in session ${event.sessionId.slice(0, 8)} observed state ${state}; retaining the reason for transcript polling: ${event.reason}`,
           conversation_id: watcher.conv.id,
           message_id: inFlight.evidentMessageId
         });
@@ -9602,7 +9699,14 @@ var ChannelDriver = class _ChannelDriver {
     if (state === "failed") {
       this.anchorDeliveryDeadline(inFlight);
       if (!inFlight.done) {
-        const error2 = messageError(messages, inFlight.opencodeMessageId) ?? void 0;
+        const messageErrorText = messageError(messages, inFlight.opencodeMessageId);
+        const retainedReason = this.readRetainedSessionError(sessionId);
+        const retainedError = retainedReason === null ? null : `${SESSION_ERROR_RETAINED_DETAIL}${retainedReason}`;
+        const storeError = this.sessionStoreHealth?.isUnwritable() ? SESSION_STORE_UNWRITABLE_ERROR : null;
+        const runnerDetails = [retainedError, storeError].filter(
+          (detail) => detail !== null
+        );
+        const error2 = messageErrorText === null ? runnerDetails.length === 0 ? void 0 : runnerDetails.join(" ") : [messageErrorText, ...runnerDetails].join(" ");
         this.log({
           level: "error",
           message: `Message ${inFlight.evidentMessageId.slice(0, 8)} errored \u2014 marking failed: ${error2 ?? "(no error text)"}`,
@@ -10045,7 +10149,9 @@ var ChannelDriver = class _ChannelDriver {
   /** Report a confirmed b2 abandonment that produced no answer text. */
   async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
     const conv = watcher.conv;
-    const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
+    const legacyError = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
+    const replyShape = describeAbandonedReplyShape(messages, inFlight.opencodeMessageId);
+    const error2 = replyShape === null ? legacyError : `${legacyError} ${replyShape.lastToolName === null ? "No tool call was recorded" : `The last tool call was "${replyShape.lastToolName}" with status "${replyShape.lastToolStatus ?? "(unknown)"}"`}; the reply's final part types were [${replyShape.partTypes.join(", ")}]${replyShape.droppedPartCount > 0 ? ` (${replyShape.droppedPartCount} earlier part types omitted)` : ""}.`;
     this.anchorDeliveryDeadline(inFlight);
     this.log({
       level: "warn",
@@ -10172,7 +10278,7 @@ var ChannelDriver = class _ChannelDriver {
    */
   async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
     const conv = watcher.conv;
-    const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
+    const error2 = SESSION_STORE_UNWRITABLE_ERROR;
     this.log({
       level: "error",
       message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
@@ -14483,6 +14589,7 @@ async function run(options) {
       openCodeLogTail.stop();
     };
     let sessionDbVerifyFatal = false;
+    let sessionDbVerifyFatalReason;
     if (!options.restoreSessionDb) {
       log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
     } else {
@@ -14515,12 +14622,13 @@ async function run(options) {
           reportRecovery: (record) => reportSessionDbRecoveryRecord(state, record)
         });
         sessionDbVerifyFatal = result.verifyFatal;
+        sessionDbVerifyFatalReason = result.verifyFatalReason;
       }
     }
     reportSessionDbRecovery(state);
     if (sessionDbVerifyFatal) {
       throw new Error(
-        "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk"
+        sessionDbVerifyFatalReason === void 0 ? "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk" : `SESSION-DB-VERIFY-TIMEOUT: session DB verification timed out (${sessionDbVerifyFatalReason}); refusing to start`
       );
     }
     applyRunnerOpenCodeConfig({