@evident-ai/runner-synchroniser 3.4.1-dev.15755a4 → 3.4.1-dev.1856549

package/README.md

@@ -86,6 +86,13 @@ a warning `restore_retried` record before the retry's result is known. Writing i
 and never changes this command's exit code. The CLI reader owns this record contract: add a new
 `v` rather than repurposing a field.
 
+MicroVM boot-shell give-ups use that same JSONL contract and path. The hook truncates the
+report at the start of each `/run`, so a later startup decision supersedes an earlier one;
+`/resume` preserves the current report. Their `replication_suspended` field is `true`, meaning
+that start is not backing up its new session history. The normal mapper defaults the field to
+`false`; the MicroVM `--fresh-db-fallback` outcome is also `true` because it cannot safely
+replicate that boot.
+
 #### Positionals
 
 | Positional                       | Meaning                                                                                                                                |
@@ -119,6 +126,9 @@ marks a runtime that intentionally has no retry budget and will therefore boot f
 otherwise-transient failure. Each flag may appear once, does not change the command's exit code,
 and is rejected when repeated.
 
+On the MicroVM's one-attempt path, `--fresh-db-fallback` also means the fresh database does not
+replicate during that boot.
+
 #### Outcome → exit code
 
 | Outcome                                        | Code | When                                                                        |
@@ -138,6 +148,9 @@ and is rejected when repeated.
 | `fatal(misconfig)`                              | `30` | no object store while persistence is enabled, or the probe failed on attempt 2   |
 | `fatal(deliberate)`                             | `30` | attempt 2, `crash`                                                                |
 
+Fatal exit codes 30 and 34 are reported directly to `POST /v1/runners/self/startup-failure`
+before the CLI starts. Reporting is best-effort and never changes the exit code.
+
 `31` also guarantees local debris (`opencode.db`, `-wal`, `-shm`) is discarded, one `try` per path, at the classifier's single return point. Note `32` isn't purely "transient" — `recovered` shares it with `retryTransient` even though a `recovered` outcome already mutated S3.
 
 The probe is a `list`, not a `get`: S3 answers a wrong bucket name with `NoSuchBucket`, also a 404, so a `get`-based probe would read a misconfigured bucket as healthy and unlock recovery against it.
@@ -217,7 +230,7 @@ what it is.
 - **`clear`** — deletes every key under `<prefix>/opencode.db/` passing `isDeletableReplicaKey`; that guard, not the `list()` prefix, is the boundary (IAM grants `s3:DeleteObject*` bucket-wide). One `try` per key. A **partial** clear still reports `recovered`/`32`. Cost: all saved history, unconditionally.
 - **`crash`** — first in the switch, no S3 mutation even considered → `fatal(deliberate)`/`30` → `entrypoint.sh` `die`s → task replaced → **crash loop** until an operator intervenes.
 
-#### The measured real-world key layout (litestream 0.5.13)
+#### The measured real-world key layout (litestream 0.5.13 historical sample)
 
 `parseLtxKey` (`src/replica-keys.ts`) expects
 `<prefix>/opencode.db/<level:04d>/<minTxid>-<maxTxid>.ltx` — **no `ltx/` path segment**,

package/dist/cli.js

@@ -51944,7 +51944,9 @@ function resolveConfig(env4) {
       env4.EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS,
       DEFAULT_WALKBACK_BUDGET_SECONDS
     ),
-    hasModelApiKey: nonEmpty(env4.ANTHROPIC_API_KEY) !== null || nonEmpty(env4.OPENAI_API_KEY) !== null
+    hasModelApiKey: nonEmpty(env4.ANTHROPIC_API_KEY) !== null || nonEmpty(env4.OPENAI_API_KEY) !== null,
+    runnerKey: nonEmpty(env4.EVIDENT_RUNNER_KEY) ?? nonEmpty(env4.EVIDENT_AGENT_KEY),
+    apiUrl: nonEmpty(env4.EVIDENT_API_URL)
   };
 }
 function storeOf(config, name) {
@@ -52330,7 +52332,7 @@ function renderLitestreamConfig(config) {
 # (no static keys here).
 #
 # INVARIANT (see README "Single-writer invariant"): exactly ONE writer per S3
-# prefix \u2014 v0.5.13 does NOT enforce a server-side lease, so never run two
+# prefix \u2014 we do not enable Litestream's server-side lease, so never run two
 # replicators on one prefix. Only opencode.db is replicated.
 
 dbs:
@@ -62682,7 +62684,8 @@ function base(at2, stage) {
     quarantine_failed_objects: null,
     quarantined_bytes: null,
     verified_restore_point: null,
-    restore_points_tried: null
+    restore_points_tried: null,
+    replication_suspended: false
   };
 }
 function recordRestoreOutcome(outcome, at2, exitCode, attempt, recoveryOccurred, freshDbFallback = false) {
@@ -62693,7 +62696,8 @@ function recordRestoreOutcome(outcome, at2, exitCode, attempt, recoveryOccurred,
       ...record,
       outcome: "fresh_session_db",
       severity: "error",
-      reason: "retry_budget_exhausted"
+      reason: "retry_budget_exhausted",
+      replication_suspended: true
     } : {
       ...record,
       outcome: "restore_retried",
@@ -62934,6 +62938,49 @@ async function walkBack(params) {
   };
 }
 
+// src/startup-failure-report.ts
+init_esm_shims();
+var STARTUP_FAILURE_REPORT_TIMEOUT_MS = 5e3;
+var MAX_RESPONSE_BODY_LENGTH = 1e3;
+async function reportStartupFailure(report, config, deps) {
+  if (config.apiUrl === null || config.runnerKey === null) {
+    deps.log(
+      `Startup failure was not reported to Evident because ${config.apiUrl === null ? "EVIDENT_API_URL" : "a runner key"} is unavailable.`
+    );
+    return;
+  }
+  const endpoint = `${config.apiUrl.replace(/\/+$/, "").replace(/\/v1$/, "")}/v1/runners/self/startup-failure`;
+  try {
+    const response = await deps.fetchImpl(endpoint, {
+      method: "POST",
+      headers: {
+        Authorization: `SandboxKey ${config.runnerKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        classification: report.classification,
+        stage: report.stage,
+        litestream_exit_code: report.litestreamExitCode,
+        attempt: report.attempt,
+        restore_points_tried: report.restorePointsTried,
+        report_id: deps.randomUUID()
+      }),
+      // The container is about to exit, so a long wait only delays ECS replacement.
+      signal: AbortSignal.timeout(STARTUP_FAILURE_REPORT_TIMEOUT_MS)
+    });
+    if (!response.ok) {
+      const body = (await response.text()).slice(0, MAX_RESPONSE_BODY_LENGTH);
+      deps.log(
+        `WARNING: fatal session-DB startup failure report was rejected (${response.status}): ${body}`
+      );
+    }
+  } catch (error) {
+    deps.log(
+      `WARNING: could not report the fatal session-DB startup failure to Evident (${describeError(error)}); the failure is still shown above and in this machine's logs.`
+    );
+  }
+}
+
 // src/self-stop.ts
 init_esm_shims();
 function isComplete(credentials) {
@@ -63275,6 +63322,20 @@ async function commandSessionDbClassify(args, config, deps) {
     log
   });
   log(describeSessionDbClassification(outcome, { bucket: config.bucket, region: config.region }));
+  if (outcome.kind === "fatal") {
+    await reportStartupFailure(
+      {
+        classification: outcome.cause === "misconfig" ? "session_db_restore_misconfigured" : "session_db_restore_crash_chosen",
+        stage: "restore",
+        litestreamExitCode: args.exitCode,
+        attempt: args.attempt,
+        restorePointsTried: null
+      },
+      config,
+      { fetchImpl: deps.fetchImpl ?? fetch, randomUUID: deps.randomUUID ?? crypto.randomUUID, log }
+    );
+    return sessionDbExitCode(outcome);
+  }
   const record = recordRestoreOutcome(
     outcome,
     (deps.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
@@ -63304,6 +63365,20 @@ async function commandSessionDbVerify(litestreamConfigPath, config, deps) {
     budgetSeconds: config.walkbackBudgetSeconds
   });
   log(describeSessionDbVerification(outcome));
+  if (outcome.kind === "exhausted" && sessionDbVerifyExitCode(outcome) === 34) {
+    await reportStartupFailure(
+      {
+        classification: outcome.localDb.kind === "discardFailed" ? "session_db_local_discard_failed" : "session_db_separation_unproven",
+        stage: "verify",
+        litestreamExitCode: null,
+        attempt: null,
+        restorePointsTried: outcome.candidatesTried
+      },
+      config,
+      { fetchImpl: deps.fetchImpl ?? fetch, randomUUID: deps.randomUUID ?? crypto.randomUUID, log }
+    );
+    return sessionDbVerifyExitCode(outcome);
+  }
   const record = recordVerifyOutcome(outcome, (deps.now ?? (() => /* @__PURE__ */ new Date()))().toISOString());
   if (record !== null)
     await appendSessionDbRecoveryRecord(record, config.sessionDbRecoveryReportPath, fileOps, log);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evident-ai/runner-synchroniser",
-  "version": "3.4.1-dev.15755a4",
+  "version": "3.4.1-dev.1856549",
   "description": "Restores and syncs the Evident runner's OpenCode credential stores (and litestream config) to an object store, so a runner survives task replacement with almost no state loss.",
   "type": "module",
   "main": "./dist/cli.js",