@evident-ai/cli 3.4.1-dev.aa71e9a → 3.4.1-dev.b45408b

package/README.md

@@ -98,6 +98,10 @@ Options:
   healthy when the runner starts it itself (default: `180`). On expiry the runner
   warns and comes online anyway rather than failing. Env:
   `EVIDENT_OPENCODE_START_TIMEOUT` (seconds).
+- `--litestream-config <path>` — Start `litestream replicate` for the OpenCode
+  session database with this config file. Use this for runner images that persist
+  the session database; the file is produced by `runner-synchroniser litestream-config`.
+  Omit it to disable replication.
 - `--claude-usage-reporting <mode>` — Whether to report the local Claude Code
   subscription's rate-limit usage to Evident, so it shows on the runner page:
   `auto` (default) reports it when a usable Claude Code login is found on this

package/dist/index.js

@@ -722,6 +722,14 @@ function toReportedWindow(window) {
   if (!window) return null;
   return { utilization: window.utilization, resets_at: window.resetsAt };
 }
+function toReportedOwner(snapshot) {
+  if (!snapshot.owner) return null;
+  return {
+    email: snapshot.owner.email,
+    organization_name: snapshot.owner.organizationName,
+    rate_limit_tier: snapshot.owner.rateLimitTier
+  };
+}
 async function reportClaudeUsage(agentId, authHeader, snapshot) {
   try {
     const apiUrl = getApiUrlConfig();
@@ -730,7 +738,8 @@ async function reportClaudeUsage(agentId, authHeader, snapshot) {
       headers: { Authorization: authHeader, "Content-Type": "application/json" },
       body: JSON.stringify({
         five_hour: toReportedWindow(snapshot.fiveHour),
-        seven_day: toReportedWindow(snapshot.sevenDay)
+        seven_day: toReportedWindow(snapshot.sevenDay),
+        owner: toReportedOwner(snapshot)
       }),
       signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
     });
@@ -996,7 +1005,10 @@ import { readFileSync } from "fs";
 import { homedir } from "os";
 import { join } from "path";
 var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
+var CLAUDE_PROFILE_URL = "https://api.anthropic.com/api/oauth/profile";
+var CLAUDE_PROFILE_TIMEOUT_MS = 2e3;
 var KEYCHAIN_SERVICE = "Claude Code-credentials";
+var cachedOwner = null;
 var CLAUDE_CREDENTIALS_SEGMENTS = [".claude", ".credentials.json"];
 function parseClaudeCliCredentials(raw) {
   let parsed;
@@ -1070,6 +1082,47 @@ function toWindow(value) {
   }
   return { utilization: window.utilization, resetsAt };
 }
+function ownerLookupFailure(error2) {
+  const name = error2?.name;
+  return name === "TimeoutError" || name === "AbortError" ? "timed out" : "request failed";
+}
+async function getClaudeUsageOwner(accessToken) {
+  if (cachedOwner?.accessToken === accessToken) {
+    return { owner: cachedOwner.owner, ownerLookupError: null };
+  }
+  try {
+    const response = await fetch(CLAUDE_PROFILE_URL, {
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+        "anthropic-version": "2023-06-01"
+      },
+      signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
+    });
+    if (!response.ok) {
+      return { owner: null, ownerLookupError: `HTTP ${response.status}` };
+    }
+    let body;
+    try {
+      body = await response.json();
+    } catch (error2) {
+      return { owner: null, ownerLookupError: "malformed response" };
+    }
+    const profile = body;
+    if (typeof profile.account?.email !== "string" || !profile.account.email.trim()) {
+      return { owner: null, ownerLookupError: "malformed response" };
+    }
+    const owner = {
+      email: profile.account.email,
+      organizationName: typeof profile.organization?.name === "string" && profile.organization.name.trim() ? profile.organization.name.trim() : null,
+      rateLimitTier: typeof profile.organization?.rate_limit_tier === "string" && profile.organization.rate_limit_tier.trim() ? profile.organization.rate_limit_tier.trim() : null
+    };
+    cachedOwner = { accessToken, owner };
+    return { owner, ownerLookupError: null };
+  } catch (error2) {
+    return { owner: null, ownerLookupError: ownerLookupFailure(error2) };
+  }
+}
 async function getClaudeUsage() {
   const credentials2 = readClaudeCliCredentials();
   if (!credentials2) {
@@ -1089,15 +1142,19 @@ async function getClaudeUsage() {
       Authorization: `Bearer ${credentials2.accessToken}`,
       "Content-Type": "application/json",
       "anthropic-version": "2023-06-01"
-    }
+    },
+    signal: AbortSignal.timeout(CLAUDE_PROFILE_TIMEOUT_MS)
   });
   if (!res.ok) {
     throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
   }
   const body = await res.json();
+  const { owner, ownerLookupError } = await getClaudeUsageOwner(credentials2.accessToken);
   return {
     fiveHour: toWindow(body.five_hour),
-    sevenDay: toWindow(body.seven_day)
+    sevenDay: toWindow(body.seven_day),
+    owner,
+    ownerLookupError
   };
 }
 
@@ -1537,6 +1594,12 @@ function buildSessionDbRecoveryActivity(record) {
         metadata: withoutContractFields(record),
         message: "This runner started with a fresh session database because it could not verify where its session history is backed up. Nothing in the backup was changed; check the runner backup configuration and permissions."
       };
+    case "session_db_boot_refused":
+      return {
+        level,
+        metadata: withoutContractFields(record),
+        message: `This runner did not come online because its damaged session database could not be safely separated from its active backup or proven removed. Backed-up session history remains readable at ${record.quarantine_destination ?? "its original location or the quarantine destination named in the boot logs"}; any local session-database files that remain were left in place and nothing opened or wrote them. See the runner boot logs for SESSION-DB-LOCAL-DISCARD-FAILED details.`
+      };
     default:
       return null;
   }
@@ -1550,7 +1613,8 @@ var OUTCOMES = /* @__PURE__ */ new Set([
   "restore_retried",
   "fresh_session_db",
   "history_rolled_back",
-  "restore_misconfigured"
+  "restore_misconfigured",
+  "session_db_boot_refused"
 ]);
 var STAGES = /* @__PURE__ */ new Set(["restore", "verify"]);
 var SEVERITIES = /* @__PURE__ */ new Set(["warning", "error"]);
@@ -1617,6 +1681,62 @@ function buildOpenCodeVersionWarning(version2) {
 
 // src/lib/opencode/process.ts
 import { execSync, spawn } from "child_process";
+
+// src/lib/process-stop.ts
+async function stopProcessAndWait(child, timeoutMs, sendTerm, sendKill) {
+  if (!child.pid) {
+    return { outcome: "not-running", code: child.exitCode, signal: child.signalCode };
+  }
+  if (child.exitCode !== null || child.signalCode !== null) {
+    return { outcome: "exited", code: child.exitCode, signal: child.signalCode };
+  }
+  return new Promise((resolve3, reject) => {
+    let forced = false;
+    let settled = false;
+    const timer = setTimeout(() => {
+      forced = true;
+      try {
+        sendKill();
+      } catch (error2) {
+        if (error2.code === "ESRCH") {
+          finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
+        } else {
+          fail(error2);
+        }
+      }
+    }, timeoutMs);
+    const finish = (result) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      child.removeListener("exit", onExit);
+      resolve3(result);
+    };
+    const fail = (error2) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      child.removeListener("exit", onExit);
+      reject(error2);
+    };
+    const onExit = (code, signal) => {
+      finish({ outcome: forced ? "killed" : "exited", code, signal });
+    };
+    child.once("exit", onExit);
+    try {
+      sendTerm();
+    } catch (error2) {
+      if (error2.code === "ESRCH") {
+        finish({ outcome: "exited", code: child.exitCode, signal: child.signalCode });
+      } else {
+        fail(error2);
+      }
+      return;
+    }
+  });
+}
+
+// src/lib/opencode/process.ts
 var OPENCODE_PORT_RANGE = [4096, 4097, 4098, 4099, 4100];
 function getProcessCwd(pid) {
   const platform = process.platform;
@@ -1788,23 +1908,20 @@ async function startOpenCode(port) {
   });
   return child;
 }
-function stopOpenCode(opencodeProcess) {
-  if (!opencodeProcess || !opencodeProcess.pid) {
-    return;
-  }
-  try {
+function stopOpenCodeAndWait(opencodeProcess, timeoutMs) {
+  const sendSignal = (signal) => {
     if (process.platform === "win32") {
-      opencodeProcess.kill("SIGTERM");
+      opencodeProcess.kill(signal);
     } else {
-      process.kill(-opencodeProcess.pid, "SIGTERM");
-    }
-  } catch (err) {
-    if (err.code !== "ESRCH") {
-      console.warn(
-        `stopOpenCode: kill failed: ${err instanceof Error ? err.message : String(err)}`
-      );
+      process.kill(-opencodeProcess.pid, signal);
     }
-  }
+  };
+  return stopProcessAndWait(
+    opencodeProcess,
+    timeoutMs,
+    () => sendSignal("SIGTERM"),
+    () => sendSignal("SIGKILL")
+  );
 }
 
 // src/lib/opencode/install.ts
@@ -2387,7 +2504,7 @@ function isPreamblePinnedRunning(messages, userMessageId) {
   return completedOf(reply) != null && finishOf(reply) === "tool-calls";
 }
 function isB2AbandonmentConfirmed(params) {
-  return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
+  return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false && params.rootOngoing === false;
 }
 function isAmbiguousTerminalFinish(m) {
   if (completedOf(m) == null) return false;
@@ -2400,7 +2517,7 @@ function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
   return isAmbiguousTerminalFinish(reply);
 }
 function isAmbiguousFinishResolved(params) {
-  return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
+  return params.sessionOngoing === false || params.sessionOngoing !== true && params.pinnedForMs >= params.maxPinnedMs;
 }
 function messageError(messages, userMessageId) {
   const reply = findLastAssistantReplyFor(messages, userMessageId);
@@ -3182,6 +3299,22 @@ function writeTunnelReadyMarker(path, agentId) {
   }
 }
 
+// src/lib/replication.ts
+import { spawn as spawn2 } from "child_process";
+function startSessionDbReplication(configPath) {
+  return spawn2("litestream", ["replicate", "-config", configPath], {
+    stdio: "inherit"
+  });
+}
+async function stopSessionDbReplication(child, timeoutMs) {
+  return stopProcessAndWait(
+    child,
+    timeoutMs,
+    () => child.kill("SIGTERM"),
+    () => child.kill("SIGKILL")
+  );
+}
+
 // src/lib/openai-usage.ts
 import { readFileSync as readFileSync3 } from "fs";
 import { homedir as homedir2 } from "os";
@@ -5676,6 +5809,7 @@ var ChannelDriver = class _ChannelDriver {
       deliveryDeadlineAnchored: false,
       b2PinnedSinceMs: 0,
       b2LastDescendantCheckMs: 0,
+      b2RootOngoingHeldLogged: false,
       b2AbandonedSignalled: false,
       ambiguousPinnedSinceMs: 0,
       ambiguousResolved: false
@@ -5770,6 +5904,7 @@ var ChannelDriver = class _ChannelDriver {
       deliveryDeadlineAnchored: false,
       b2PinnedSinceMs: 0,
       b2LastDescendantCheckMs: 0,
+      b2RootOngoingHeldLogged: false,
       b2AbandonedSignalled: false,
       ambiguousPinnedSinceMs: 0,
       ambiguousResolved: false
@@ -6123,6 +6258,7 @@ var ChannelDriver = class _ChannelDriver {
       if (snapshotReadable) {
         inFlight.b2PinnedSinceMs = 0;
         inFlight.b2LastDescendantCheckMs = 0;
+        inFlight.b2RootOngoingHeldLogged = false;
         inFlight.b2AbandonedSignalled = false;
       }
     } else {
@@ -6134,11 +6270,15 @@ var ChannelDriver = class _ChannelDriver {
       const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
       if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
         inFlight.b2LastDescendantCheckMs = this.now();
-        const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
+        const [descendantOngoing, rootOngoing] = await Promise.all([
+          this.isAnyDescendantSessionOngoing(sessionId),
+          isSessionOngoing(this.port, sessionId)
+        ]);
         if (isB2AbandonmentConfirmed({
           pinnedForMs,
           minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
-          descendantOngoing
+          descendantOngoing,
+          rootOngoing
         })) {
           inFlight.b2AbandonedSignalled = true;
           this.log({
@@ -6147,12 +6287,26 @@ var ChannelDriver = class _ChannelDriver {
             conversation_id: conv.id,
             message_id: id
           });
+          const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
           void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
-            watched_for_ms: pinnedForMs
+            watched_for_ms: pinnedForMs,
+            finish: reply?.info?.finish ?? reply?.finish,
+            ...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
+            ...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
+            opencode_message_id: inFlight.opencodeMessageId
           });
           await this.settleMessageDone(sessionId, watcher, inFlight, messages);
           return;
         }
+        if (rootOngoing === true && !inFlight.b2RootOngoingHeldLogged) {
+          inFlight.b2RootOngoingHeldLogged = true;
+          this.log({
+            level: "warn",
+            message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with root_ongoing=${rootOngoing} and descendant_ongoing=${descendantOngoing} \u2014 holding until OpenCode confirms the root is idle`,
+            conversation_id: conv.id,
+            message_id: id
+          });
+        }
       }
     }
     const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
@@ -7895,6 +8049,7 @@ var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_
 var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
 var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
 var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
+var CHILD_STOP_TIMEOUT_MS = 1e4;
 function resolveLogLevel(options) {
   const accepted = Object.keys(LOG_LEVELS);
   const validate = (value, source) => {
@@ -8500,7 +8655,17 @@ function scheduleClaudeUsageReporting(state, options) {
     setTimer: (timer) => {
       state.claudeUsageTimer = timer;
     },
-    fetchUsage: getClaudeUsage,
+    fetchUsage: async () => {
+      const usage = await getClaudeUsage();
+      if (usage.ownerLookupError) {
+        logActivity(state, {
+          type: "info",
+          level: "debug",
+          message: `Claude usage owner lookup failed: ${usage.ownerLookupError}`
+        });
+      }
+      return usage;
+    },
     report: (usage) => reportClaudeUsage(state.agentId, state.authHeader, usage),
     isLocalCredentialProblem,
     forcedOnHint: "run `claude` to sign in",
@@ -8672,15 +8837,31 @@ async function cleanup(state, opts = {}) {
   }
   if (state.opencodeProcess) {
     const opencodeProcess = state.opencodeProcess;
-    await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
+    const result = await timeShutdownPhase(
+      state,
+      durations,
+      "opencode_stop",
+      () => stopOpenCodeAndWait(opencodeProcess, CHILD_STOP_TIMEOUT_MS)
+    );
     if (state.interactive) {
-      logActivity(state, { type: "info", message: "Stopped OpenCode process" });
+      logActivity(state, { type: "info", message: `Stopped OpenCode process (${result.outcome})` });
       displayStatus(state);
     } else {
-      log2(state, "Stopped OpenCode process");
+      log2(state, `Stopped OpenCode process (${result.outcome})`);
     }
     state.opencodeProcess = null;
   }
+  if (state.litestreamProcess) {
+    const litestreamProcess = state.litestreamProcess;
+    const result = await timeShutdownPhase(
+      state,
+      durations,
+      "litestream_stop",
+      () => stopSessionDbReplication(litestreamProcess, CHILD_STOP_TIMEOUT_MS)
+    );
+    log2(state, `Stopped litestream replication (${result.outcome})`);
+    state.litestreamProcess = null;
+  }
   return durations;
 }
 async function run(options) {
@@ -8714,6 +8895,7 @@ async function run(options) {
     opencodeConnected: false,
     opencodeVersion: null,
     opencodeProcess: null,
+    litestreamProcess: null,
     connection: null,
     channelDriver: null,
     running: true,
@@ -8978,6 +9160,40 @@ async function run(options) {
       ocSpinner?.fail(error2.message);
       throw error2;
     }
+    if (options.litestreamConfig) {
+      const litestreamProcess = startSessionDbReplication(options.litestreamConfig);
+      state.litestreamProcess = litestreamProcess;
+      let failureHandled = false;
+      const failRunForReplication = (message) => {
+        if (failureHandled || state.shuttingDown || !state.running) return;
+        failureHandled = true;
+        state.shuttingDown = true;
+        logActivity(state, { type: "error", error: message });
+        if (state.interactive) displayStatus(state);
+        void (async () => {
+          try {
+            await cleanup(state);
+            await shutdownTelemetry();
+          } catch (error2) {
+            console.error(
+              `[run] replication failure cleanup failed: ${error2 instanceof Error ? error2.message : String(error2)}`
+            );
+          }
+          process.exit(1);
+        })();
+      };
+      litestreamProcess.on("exit", (code, signal) => {
+        failRunForReplication(
+          `Litestream replication stopped unexpectedly (code: ${code ?? "null"}, signal: ${signal ?? "none"})`
+        );
+      });
+      litestreamProcess.on("error", (error2) => {
+        failRunForReplication(
+          `Litestream replication failed: ${error2 instanceof Error ? error2.message : String(error2)}`
+        );
+      });
+      log2(state, `Started litestream replication with config ${options.litestreamConfig}`);
+    }
     const tunnelSpinner = interactive && !state.json ? ora3("Connecting tunnel...").start() : null;
     const channelDriver = new ChannelDriver({
       agentId: state.agentId,
@@ -9238,6 +9454,9 @@ program.command("run").description("Connect to Evident and process messages").op
 ).option(
   "--tunnel-ready-file <path>",
   "Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
+).option(
+  "--litestream-config <path>",
+  "Replicate the OpenCode session database with `litestream replicate` using this config file (written by the runner image). Omit to disable replication."
 ).action(
   (options) => {
     run({
@@ -9269,7 +9488,8 @@ program.command("run").description("Connect to Evident and process messages").op
       // Raw values — expansion/validation is single-sourced in run.ts's
       // resolveFileSyncDirectories.
       enableFileSyncTo: options.enableFileSyncTo,
-      tunnelReadyFile: options.tunnelReadyFile
+      tunnelReadyFile: options.tunnelReadyFile,
+      litestreamConfig: options.litestreamConfig
     });
   }
 );