@evident-ai/cli 3.4.1-dev.c03b461 → 3.4.1-dev.cb56b3f

package/README.md

@@ -119,6 +119,9 @@ Options:
 - `--opencode-config-overlay <path>` — Apply a runner-provided OpenCode config
   before starting OpenCode. Relative paths are resolved from the working directory;
   an existing `opencode.jsonc` is replaced before `opencode.json`.
+- `--credential-sync-marker <path>` — Own the interval credential sync and write
+  this marker once the shutdown flush has finished, so the runner image's lifecycle
+  hooks can wait on it.
 - `--claude-usage-reporting <mode>` — Whether to report the local Claude Code
   subscription's rate-limit usage to Evident, so it shows on the runner page:
   `auto` (default) reports it when a usable Claude Code login is found on this

package/dist/index.js

@@ -1183,7 +1183,7 @@ async function claudeUsage() {
 }
 
 // src/commands/run.ts
-import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync5 } from "fs";
+import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "fs";
 import { homedir as homedir5 } from "os";
 import { isAbsolute as isAbsolute3, join as join9, parse, resolve as resolvePath2 } from "path";
 import chalk6 from "chalk";
@@ -1728,10 +1728,14 @@ function runSynchroniser(args, opts) {
     let stderr = "";
     let settled = false;
     const timer = {};
+    let abortListener;
+    let spawnListener;
     const finish = (result) => {
       if (settled) return;
       settled = true;
       if (timer.handle) clearTimeout(timer.handle);
+      if (abortListener) opts.signal?.removeEventListener("abort", abortListener);
+      if (spawnListener) child.removeListener("spawn", spawnListener);
       resolve4(result);
     };
     try {
@@ -1757,6 +1761,25 @@ function runSynchroniser(args, opts) {
     child.once("close", (code) => {
       finish({ code, stdout, stderr, timedOut: false });
     });
+    if (opts.signal) {
+      const killChild = () => {
+        if (child.pid === void 0) {
+          if (!spawnListener) {
+            spawnListener = killChild;
+            child.once("spawn", spawnListener);
+          }
+          return;
+        }
+        child.kill("SIGKILL");
+      };
+      abortListener = killChild;
+      if (opts.signal.aborted) {
+        abortListener();
+      } else {
+        opts.signal.addEventListener("abort", abortListener, { once: true });
+        if (opts.signal.aborted) abortListener();
+      }
+    }
     timer.handle = setTimeout(
       () => {
         child.kill("SIGKILL");
@@ -3844,8 +3867,8 @@ function connectTunnel(options) {
       try {
         message = JSON.parse(data.toString());
       } catch (error2) {
-        const errorMessage = error2 instanceof Error ? error2.message : "Unknown error";
-        onError?.(`Failed to handle message: ${errorMessage}`);
+        const errorMessage2 = error2 instanceof Error ? error2.message : "Unknown error";
+        onError?.(`Failed to handle message: ${errorMessage2}`);
         return;
       }
       if (isStreamFrame(message)) {
@@ -5619,7 +5642,7 @@ var ChannelDriver = class _ChannelDriver {
           this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
           break;
         }
-        const errorMessage = err instanceof Error ? err.message : String(err);
+        const errorMessage2 = err instanceof Error ? err.message : String(err);
         this.sessions.delete(conv.id);
         this.supersede(conv.id, sessionId);
         this.log({
@@ -5628,7 +5651,7 @@ var ChannelDriver = class _ChannelDriver {
           conversation_id: conv.id,
           message_id: message.id
         });
-        await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
+        await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
           this.log({
             level: "warn",
             message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
@@ -5639,7 +5662,7 @@ var ChannelDriver = class _ChannelDriver {
         });
         this.log({
           level: "error",
-          message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage}`,
+          message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage2}`,
           conversation_id: conv.id,
           message_id: message.id
         });
@@ -5660,14 +5683,14 @@ var ChannelDriver = class _ChannelDriver {
         this.unconfirmedDispatchFailures.delete(message.id);
         this.sessions.delete(conv.id);
         this.supersede(conv.id, sessionId);
-        const errorMessage = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
+        const errorMessage2 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
         this.log({
           level: "error",
-          message: errorMessage,
+          message: errorMessage2,
           conversation_id: conv.id,
           message_id: message.id
         });
-        await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
+        await this.markFailed(conv.id, message.id, null, errorMessage2).catch((markErr) => {
           this.log({
             level: "warn",
             message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
@@ -7631,14 +7654,14 @@ var ChannelDriver = class _ChannelDriver {
         this.unconfirmedDispatchFailures.delete(row.id);
         this.sessions.delete(readoptConv.id);
         this.supersede(readoptConv.id, sessionId);
-        const errorMessage = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
+        const errorMessage2 = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
         this.log({
           level: "error",
-          message: errorMessage,
+          message: errorMessage2,
           conversation_id: row.conversation_id,
           message_id: row.id
         });
-        await this.markFailed(row.conversation_id, row.id, null, errorMessage).catch((markErr) => {
+        await this.markFailed(row.conversation_id, row.id, null, errorMessage2).catch((markErr) => {
           this.log({
             level: "warn",
             message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
@@ -9079,6 +9102,241 @@ function applyRunnerOpenCodeConfig({
   log3(`Applied runner OpenCode config ${source} to ${join8(cwd, target)}`);
 }
 
+// src/lib/credential-sync.ts
+import { renameSync, writeFileSync as writeFileSync5 } from "fs";
+var CREDENTIAL_SYNC_TICK_TIMEOUT_MS = 3e4;
+var CREDENTIAL_FLUSH_DEADLINE_MS = 8e3;
+var CREDENTIAL_FLUSH_ABORT_GRACE_MS = 500;
+var DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS = 60;
+var STORES = ["claude", "opencode"];
+var MAX_FLUSH_PASSES = 2;
+function outcomesWith(outcome) {
+  return { claude: outcome, opencode: outcome };
+}
+function errorMessage(error2) {
+  return error2 instanceof Error ? error2.message : String(error2);
+}
+function waitForSettlement(promise, timeoutMs) {
+  return new Promise((resolve4) => {
+    let settled = false;
+    const timer = setTimeout(() => finish(false), Math.max(0, timeoutMs));
+    const finish = (value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve4(value);
+    };
+    promise.then(
+      () => finish(true),
+      () => finish(true)
+    );
+  });
+}
+function writeMarker(markerPath, outcomes, log3) {
+  const body = `${STORES.map((store) => `${store}=${outcomes[store]}`).join("\n")}
+`;
+  const temporaryPath = `${markerPath}.tmp`;
+  try {
+    writeFileSync5(temporaryPath, body, { mode: 384 });
+    renameSync(temporaryPath, markerPath);
+  } catch (error2) {
+    log3(`CREDENTIAL-FLUSH-MARKER-UNWRITABLE: ${markerPath}: ${errorMessage(error2)}`, "warn");
+  }
+}
+function intervalSeconds(env, log3) {
+  const raw = env.CREDS_SYNC_INTERVAL;
+  if (raw === void 0 || /^[1-9][0-9]*$/.test(raw) && Number.isSafeInteger(Number(raw))) {
+    return raw === void 0 ? DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS : Number(raw);
+  }
+  log3(
+    `CREDENTIAL-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${raw}' is not a positive integer; using ${DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS}s`,
+    "warn"
+  );
+  return DEFAULT_CREDENTIAL_SYNC_INTERVAL_SECONDS;
+}
+async function runFlushStore(store, env, synchroniserRunner, log3, deadlineAt) {
+  const remainingMs = deadlineAt - Date.now();
+  if (remainingMs <= 0) return { outcome: "timeout", orphaned: false };
+  const controller = new AbortController();
+  let result;
+  let failed = false;
+  const completion = Promise.resolve().then(
+    () => synchroniserRunner(["sync-once", store], {
+      timeoutMs: remainingMs,
+      env,
+      signal: controller.signal
+    })
+  ).then(
+    (value) => {
+      result = value;
+    },
+    (error2) => {
+      failed = true;
+      log3(`CREDENTIAL-FLUSH-ERROR: ${store}: ${errorMessage(error2)}`, "warn");
+    }
+  );
+  const abortTimer = setTimeout(() => controller.abort(), remainingMs);
+  const settledBeforeDeadline = await waitForSettlement(completion, remainingMs);
+  clearTimeout(abortTimer);
+  if (!settledBeforeDeadline) {
+    controller.abort();
+    const settledAfterAbort = await waitForSettlement(completion, CREDENTIAL_FLUSH_ABORT_GRACE_MS);
+    if (!settledAfterAbort) return { outcome: "timeout", orphaned: true };
+    return { outcome: "timeout", orphaned: false };
+  }
+  if (failed || !result) return { outcome: "failed", orphaned: false };
+  if (result.timedOut || Date.now() >= deadlineAt) {
+    return { outcome: "timeout", orphaned: false };
+  }
+  return { outcome: result.code === 0 ? "ok" : "failed", orphaned: false };
+}
+function createCredentialSync({
+  markerPath,
+  env,
+  log: log3,
+  synchroniserRunner = runSynchroniser
+}) {
+  const persistenceDisabled = !env.PERSISTENCE_BUCKET;
+  let disabled = persistenceDisabled;
+  let armed = false;
+  let stopped = false;
+  let timer;
+  let inFlight;
+  let activeTickAbort;
+  let lastTickFailed;
+  let flushPromise;
+  const scheduleTick = (intervalMs, startTick2) => {
+    if (stopped) return;
+    timer = setTimeout(() => {
+      timer = void 0;
+      startTick2();
+    }, intervalMs);
+  };
+  const startTick = (intervalMs) => {
+    if (stopped) return;
+    const controller = new AbortController();
+    activeTickAbort = controller;
+    const tick = (async () => {
+      const outcomes = {
+        claude: "failed",
+        opencode: "failed"
+      };
+      for (const store of STORES) {
+        if (controller.signal.aborted) break;
+        try {
+          const result = await synchroniserRunner(["sync-once", store], {
+            timeoutMs: CREDENTIAL_SYNC_TICK_TIMEOUT_MS,
+            env,
+            signal: controller.signal
+          });
+          outcomes[store] = !result.timedOut && result.code === 0 ? "ok" : "failed";
+        } catch (error2) {
+          outcomes[store] = "failed";
+          log3(`CREDENTIAL-SYNC-TICK-ERROR: ${store}: ${errorMessage(error2)}`, "debug");
+        }
+      }
+      const failed = STORES.some((store) => outcomes[store] === "failed");
+      log3(
+        `CREDENTIAL-SYNC-TICK: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
+        "debug"
+      );
+      if (failed && lastTickFailed !== true) {
+        log3(
+          "CREDENTIAL-SYNC-FAILED: an interval credential sync failed; retrying on the next tick",
+          "warn"
+        );
+      } else if (!failed && lastTickFailed === true) {
+        log3("CREDENTIAL-SYNC-RECOVERED: interval credential sync succeeded again", "warn");
+      }
+      lastTickFailed = failed;
+    })().finally(() => {
+      if (activeTickAbort === controller) activeTickAbort = void 0;
+      if (inFlight === tick) inFlight = void 0;
+      scheduleTick(intervalMs, () => startTick(intervalMs));
+    });
+    inFlight = tick;
+  };
+  const performFlush = async () => {
+    stopped = true;
+    if (timer) {
+      clearTimeout(timer);
+      timer = void 0;
+    }
+    const deadlineAt = Date.now() + CREDENTIAL_FLUSH_DEADLINE_MS;
+    if (inFlight) {
+      const settled = await waitForSettlement(inFlight, CREDENTIAL_FLUSH_DEADLINE_MS);
+      if (!settled) {
+        activeTickAbort?.abort();
+        const settledAfterAbort = await waitForSettlement(
+          inFlight,
+          CREDENTIAL_FLUSH_ABORT_GRACE_MS
+        );
+        if (!settledAfterAbort) {
+          log3(
+            "CREDENTIAL-FLUSH-ORPHANED-TICK: a sync-once child could not be proven settled; skipping the boundary flush rather than becoming a second writer",
+            "warn"
+          );
+          return { outcomes: outcomesWith("timeout"), orphaned: true };
+        }
+      }
+    }
+    if (disabled) return { outcomes: outcomesWith("skipped"), orphaned: false };
+    const outcomes = outcomesWith("timeout");
+    for (const store of STORES) {
+      const result = await runFlushStore(store, env, synchroniserRunner, log3, deadlineAt);
+      if (result.orphaned) {
+        log3(
+          "CREDENTIAL-FLUSH-ORPHANED-CHILD: a flush sync-once child could not be proven settled; refusing another flush pass rather than becoming a second writer",
+          "warn"
+        );
+        return { outcomes: outcomesWith("timeout"), orphaned: true };
+      }
+      outcomes[store] = result.outcome;
+    }
+    return { outcomes, orphaned: false };
+  };
+  let flushPasses = 0;
+  let lastFlush;
+  return {
+    arm() {
+      if (stopped || armed) return;
+      armed = true;
+      if (persistenceDisabled) {
+        disabled = true;
+        log3(
+          "CREDENTIAL-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot",
+          "warn"
+        );
+        return;
+      }
+      disabled = false;
+      const intervalMs = intervalSeconds(env, log3) * 1e3;
+      scheduleTick(intervalMs, () => startTick(intervalMs));
+    },
+    async stopAndFlush(publish) {
+      let result;
+      const runningFlush = flushPromise;
+      if (runningFlush) {
+        result = await runningFlush;
+      } else if (flushPasses >= MAX_FLUSH_PASSES || lastFlush?.orphaned) {
+        result = lastFlush ?? { outcomes: outcomesWith("timeout"), orphaned: true };
+      } else {
+        flushPasses++;
+        const currentFlush = performFlush();
+        flushPromise = currentFlush;
+        try {
+          result = await currentFlush;
+          lastFlush = result;
+        } finally {
+          if (flushPromise === currentFlush) flushPromise = void 0;
+        }
+      }
+      if (publish) writeMarker(markerPath, result.outcomes, log3);
+      return result.outcomes;
+    }
+  };
+}
+
 // src/commands/run.ts
 var MAX_ACTIVITY_LOG_ENTRIES = 10;
 var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
@@ -9418,8 +9676,8 @@ async function driveChannels(state, driver) {
         state.running = false;
         break;
       }
-      const errorMessage = error2 instanceof Error ? error2.message : String(error2);
-      logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
+      const errorMessage2 = error2 instanceof Error ? error2.message : String(error2);
+      logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage2}` });
       if (state.interactive) displayStatus(state);
       if (driver.hasInFlightWatchers()) {
         consecutiveDrainFailures = 0;
@@ -9889,21 +10147,39 @@ async function cleanup(state, opts = {}) {
     clearTimeout(state.resourceUsageTimer);
     state.resourceUsageTimer = null;
   }
+  const credentialSync = state.credentialSync;
+  const flushCredentials = credentialSync ? async (phase, publish) => {
+    await timeShutdownPhase(state, durations, phase, async () => {
+      const outcomes = await credentialSync.stopAndFlush(publish);
+      const level = Object.values(outcomes).every((outcome) => outcome === "ok") ? "info" : "warn";
+      log2(
+        state,
+        `Credential flush: claude=${outcomes.claude}, opencode=${outcomes.opencode}`,
+        level
+      );
+    });
+  } : void 0;
+  let drainSettled = true;
   if (opts.graceful && state.channelDriver) {
     state.channelDriver.stop();
+  }
+  if (flushCredentials) {
+    await flushCredentials("credential_flush", !(opts.graceful && state.channelDriver));
+  }
+  if (opts.graceful && state.channelDriver) {
     log2(state, "Draining in-flight channel work before shutdown...");
     if (state.interactive) {
       logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
       displayStatus(state);
     }
     const driver = state.channelDriver;
-    const settled = await timeShutdownPhase(
+    drainSettled = await timeShutdownPhase(
       state,
       durations,
       "drain",
       () => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
     );
-    if (!settled) {
+    if (!drainSettled) {
       logActivity(state, {
         type: "info",
         message: "Shutdown drain timed out with work still in flight \u2014 leaving it for restart recovery"
@@ -9911,6 +10187,9 @@ async function cleanup(state, opts = {}) {
       if (state.interactive) displayStatus(state);
     }
   }
+  if (opts.graceful && state.channelDriver && flushCredentials && drainSettled) {
+    await flushCredentials("credential_flush_final", true);
+  }
   await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
   if (state.connection) {
     const connection = state.connection;
@@ -9997,9 +10276,23 @@ async function run(options) {
     openaiUsageTimer: null,
     openaiUsageRearm: null,
     resourceUsageTimer: null,
+    credentialSync: null,
     authHeader: ""
   };
   setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
+  if (options.credentialSyncMarker) {
+    state.credentialSync = createCredentialSync({
+      markerPath: options.credentialSyncMarker,
+      env: process.env,
+      log: (message, level = "info") => {
+        if (level === "error") {
+          logActivity(state, { type: "error", error: message });
+        } else {
+          logActivity(state, { type: "info", level, message });
+        }
+      }
+    });
+  }
   if (fileSyncDirectories.length > 0) {
     log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
   } else {
@@ -10207,6 +10500,7 @@ async function run(options) {
       await restoreCredentialStores(credentialContext);
       if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
     }
+    state.credentialSync?.arm();
     let sessionDbVerifyFatal = false;
     if (!options.restoreSessionDb) {
       log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
@@ -10276,7 +10570,7 @@ async function run(options) {
       state.opencodeVersion = oc.version;
       if (options.opencodePidFile && oc.process?.pid !== void 0) {
         try {
-          writeFileSync5(options.opencodePidFile, `${oc.process.pid}
+          writeFileSync6(options.opencodePidFile, `${oc.process.pid}
 `, { mode: 384 });
           chmodSync3(options.opencodePidFile, 384);
         } catch (error2) {
@@ -10392,7 +10686,7 @@ async function run(options) {
           });
           try {
             if (litestreamProcess.pid !== void 0) {
-              writeFileSync5(options.litestreamPidFile, `${litestreamProcess.pid}
+              writeFileSync6(options.litestreamPidFile, `${litestreamProcess.pid}
 `, {
                 mode: 384
               });
@@ -10722,6 +11016,9 @@ program.command("run").description("Connect to Evident and process messages").op
 ).option(
   "--opencode-config-overlay <path>",
   "Apply this runner-provided OpenCode config before starting OpenCode."
+).option(
+  "--credential-sync-marker <path>",
+  "Own the interval credential sync and write this marker once the shutdown flush has finished, so the runner image's lifecycle hooks can wait on it."
 ).action(
   (options) => {
     run({
@@ -10760,7 +11057,8 @@ program.command("run").description("Connect to Evident and process messages").op
       sessionDbNoReplicateMarker: options.sessionDbNoReplicateMarker,
       restoreSessionDb: options.restoreSessionDb,
       restoreRunnerCredentials: options.restoreRunnerCredentials,
-      opencodeConfigOverlay: options.opencodeConfigOverlay
+      opencodeConfigOverlay: options.opencodeConfigOverlay,
+      credentialSyncMarker: options.credentialSyncMarker
     });
   }
 );