@evident-ai/runner-cdk 3.4.1-dev.3adba63 → 3.4.1-dev.51df4de

package/README.md

@@ -97,7 +97,7 @@ the construct's own dogfooding consumer.
 | `cluster`, `securityGroup`, `taskRole`, `logGroup`, `replicaBucket` | interfaces | Shared infra you create and pass in. |
 | `image` | `ecs.ContainerImage` | Your runner image. |
 | `agentSecret`, `wakerSecret` | `secretsmanager.ISecret` | Any Secrets Manager secret. |
-| `availableSecretKeys` | `Set<string>` | Keys present in the agent secret. Requires `EVIDENT_AGENT_KEY` + `GH_TOKEN`; MCP credentials (`BRAVE_API_KEY`, `CLOUDFLARE_API_TOKEN`, `NEON_API_KEY`) are injected only when listed. |
+| `availableSecretKeys` | `Set<string>` | Keys present in the agent secret. Every listed key is injected; `EVIDENT_AGENT_KEY` and `GH_TOKEN` are also injected for compatibility with adopters that do not enumerate their secret. |
 
 ### Two deliberate "no default" choices
 
@@ -162,7 +162,7 @@ and the agent installs on first use.
 
 | MicroVM prop | Omitted behavior |
 | --- | --- |
-| `runnerSecret` | No GitHub or MCP credentials are exported at `/run`. |
+| `runnerSecret` | No GitHub or MCP credentials are exported at `/run`. When supplied, `/run` reads the JSON secret and exports every non-empty value whose key is a valid environment-variable name. |
 | `runnerOpencodeConfigPath` | OpenCode uses the baked project configuration. Relative paths resolve from the workspace; absolute paths resolve in the image. |
 | `gitUserName` / `gitUserEmail` | The hook uses the Evident bot defaults for git identity. |
 

package/dist/controller-lambda/handler.js

@@ -42866,7 +42866,7 @@ Reference: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.ht
 // ../../node_modules/.pnpm/@aws-sdk+signature-v4-multi-region@3.996.42/node_modules/@aws-sdk/signature-v4-multi-region/dist-cjs/index.js
 var require_dist_cjs23 = __commonJS({
   "../../node_modules/.pnpm/@aws-sdk+signature-v4-multi-region@3.996.42/node_modules/@aws-sdk/signature-v4-multi-region/dist-cjs/index.js"(exports2) {
-    var { SignatureV4: SignatureV43, signatureV4aContainer } = require_dist_cjs17();
+    var { SignatureV4: SignatureV43, signatureV4aContainer } = require_dist_cjs2();
     var signatureV4CrtContainer = {
       CrtSignerV4: null
     };

package/dist/evident-scale-to-zero-construct.d.ts

@@ -61,13 +61,25 @@ export type EvidentScaleToZeroConstructProps = {
     replicaBucket: s3.IBucket;
     image: ecs.ContainerImage;
     /**
-     * Container secrets. Requires EVIDENT_AGENT_KEY + GH_TOKEN; the MCP
-     * credentials BRAVE_API_KEY / CLOUDFLARE_API_TOKEN / NEON_API_KEY are injected
-     * only if present (see availableSecretKeys).
+     * Container secrets. `availableSecretKeys` are injected by name; EVIDENT_AGENT_KEY
+     * and GH_TOKEN remain available to external adopters that do not enumerate keys.
+     *
+     * `agentSecret` always supplies EVIDENT_AGENT_KEY. When `capabilitySecret` is
+     * ALSO given (#1868 WI-2), every capability key (GH_TOKEN included) is injected
+     * from THAT secret instead, and `availableSecretKeys` is ignored — the two
+     * deploy targets (this one and the MicroVM stack) then read the same
+     * capability keys from the same shared secret rather than each carrying its
+     * own copy. When `capabilitySecret` is absent, behavior is unchanged: GH_TOKEN
+     * plus every `availableSecretKeys` entry, all from `agentSecret` — an external
+     * adopter passing only `agentSecret`/`availableSecretKeys` is unaffected.
      */
     agentSecret: secretsmanager.ISecret;
-    /** Key names present in the agent secret; gates the optional MCP creds. */
+    /** Key names present in the agent secret. Every listed key is injected into the container. */
     availableSecretKeys: Set<string>;
+    /** The shared capability secret (#1868 WI-2). See `agentSecret`'s doc above. */
+    capabilitySecret?: secretsmanager.ISecret;
+    /** Key names present in `capabilitySecret`. Ignored when `capabilitySecret` is absent. */
+    capabilityKeys?: Set<string>;
     /** Waker secret (EVIDENT_WAKE_SECRET), consumed by the waker at runtime. */
     wakerSecret: secretsmanager.ISecret;
 };

package/dist/evident-scale-to-zero-construct.js

@@ -61,7 +61,7 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
     service;
     constructor(scope, id, props) {
         super(scope, id);
-        const { agentName, evidentAgentId, gitRepo, gitBranch, idleTimeoutSeconds, cpu, memoryLimitMiB, resourcePrefix = DEFAULT_RESOURCE_PREFIX, envName, evidentApiUrl, evidentTunnelUrl, extraEnvironment, cluster, securityGroup, taskRole, logGroup, replicaBucket, image, agentSecret, availableSecretKeys, wakerSecret, } = props;
+        const { agentName, evidentAgentId, gitRepo, gitBranch, idleTimeoutSeconds, cpu, memoryLimitMiB, resourcePrefix = DEFAULT_RESOURCE_PREFIX, envName, evidentApiUrl, evidentTunnelUrl, extraEnvironment, cluster, securityGroup, taskRole, logGroup, replicaBucket, image, agentSecret, availableSecretKeys, capabilitySecret, capabilityKeys, wakerSecret, } = props;
         this.replicaPrefix = `agents/${evidentAgentId}`;
         // A plain STRING (not service.serviceName) so the container env is static and
         // has no construct-ordering dependency on the service.
@@ -127,16 +127,22 @@ class EvidentScaleToZeroConstruct extends constructs_1.Construct {
             ...extraEnvironment,
         };
         const secrets = {
+            // Identifies the runner to Evident. Always agentSecret — capabilitySecret
+            // never carries this key (#1868 WI-2: the two are a deliberate split, not
+            // a duplication of the same data).
             EVIDENT_AGENT_KEY: ecs.Secret.fromSecretsManager(agentSecret, 'EVIDENT_AGENT_KEY'),
-            GH_TOKEN: ecs.Secret.fromSecretsManager(agentSecret, 'GH_TOKEN'),
         };
-        // Optional MCP creds (consumed by opencode.runner.jsonc): BRAVE_API_KEY →
-        // brave-search, CLOUDFLARE_API_TOKEN → the Cloudflare MCP, NEON_API_KEY →
-        // the Neon MCP (read-only). Injected only if present in the SOPS file — a
-        // referenced-but-absent key fails ECS task startup, so an operator can
-        // enable an MCP by just adding its key + redeploy.
-        for (const key of ['BRAVE_API_KEY', 'CLOUDFLARE_API_TOKEN', 'NEON_API_KEY']) {
-            if (availableSecretKeys.has(key)) {
+        if (capabilitySecret !== undefined) {
+            for (const key of capabilityKeys ?? new Set()) {
+                secrets[key] = ecs.Secret.fromSecretsManager(capabilitySecret, key);
+            }
+        }
+        else {
+            // Legacy shape, unchanged: GH_TOKEN (required to clone this construct's
+            // workspace at boot) plus every availableSecretKeys entry, all from
+            // agentSecret — what every external adopter still gets.
+            secrets.GH_TOKEN = ecs.Secret.fromSecretsManager(agentSecret, 'GH_TOKEN');
+            for (const key of availableSecretKeys) {
                 secrets[key] = ecs.Secret.fromSecretsManager(agentSecret, key);
             }
         }

package/dist/microvm/construct.js

@@ -124,7 +124,11 @@ class EvidentMicrovmConstruct extends constructs_1.Construct {
             ...(props.runnerOpencodeConfigPath
                 ? { RUNNER_OPENCODE_CONFIG: props.runnerOpencodeConfigPath }
                 : {}),
-            ...(props.runnerSecret ? { RUNNER_SECRET_ARN: props.runnerSecret.secretArn } : {}),
+            ...(props.runnerSecret
+                ? {
+                    RUNNER_SECRET_ARN: props.runnerSecret.secretArn,
+                }
+                : {}),
             ...(props.gitUserName ? { GIT_USER_NAME: props.gitUserName } : {}),
             ...(props.gitUserEmail ? { GIT_USER_EMAIL: props.gitUserEmail } : {}),
         };

package/dist/microvm/controller/shape-catalogue.d.ts

@@ -1,6 +1,6 @@
 /**
  * Parses the `MICROVM_SHAPES` environment variable (a JSON array, written by
- * CloudFormation from `infrastructure/evident-microvm/src/shapes.ts` at
+ * CloudFormation from `aws/runner-cdk/src/microvm/shapes.ts` at
  * deploy time — see D6/D7 in the plan) into a `ShapeCatalogue` the pure
  * decision core can query, without that core ever touching `process.env`
  * itself.

package/dist/microvm/controller/shape-catalogue.js

@@ -1,7 +1,7 @@
 "use strict";
 /**
  * Parses the `MICROVM_SHAPES` environment variable (a JSON array, written by
- * CloudFormation from `infrastructure/evident-microvm/src/shapes.ts` at
+ * CloudFormation from `aws/runner-cdk/src/microvm/shapes.ts` at
  * deploy time — see D6/D7 in the plan) into a `ShapeCatalogue` the pure
  * decision core can query, without that core ever touching `process.env`
  * itself.

package/dist/microvm-image-context/Dockerfile

@@ -171,10 +171,10 @@ ENV HOME=/home/runner
 # permanent no-op and every VM from this snapshot shares one machine id. Empty
 # is the correct unset state — it is the absence of an identity, so nothing
 # per-VM-unique enters the shared snapshot.
-RUN mkdir -p /var/lib/runner-pg /var/lib/dbus \
+RUN mkdir -p /var/lib/runner-pg /var/lib/dbus /home/runner/.local/state/evident \
   && install -o runner -g runner -m 0644 /dev/null /etc/machine-id \
   && install -o runner -g runner -m 0644 /dev/null /var/lib/dbus/machine-id \
-  && chown -R runner:runner /var/lib/runner-pg /home/runner
+   && chown -R runner:runner /var/lib/runner-pg /home/runner
 COPY --chown=10001:10001 repo ${WORKSPACE}
 WORKDIR ${WORKSPACE}
 USER runner

package/dist/microvm-image-context/hooks/common.sh

@@ -24,6 +24,8 @@ CONTEXT_FILE="/dev/shm/evident-run-context"
 TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
 OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
 LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
+CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
+CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
 
 # Where the runner's credential store lives inside the durable-state bucket.
 # The BUCKET is the same for every VM from an image version, so the stack bakes
@@ -50,14 +52,49 @@ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
 # earlier boot cannot silently disable replication for the VM's whole life.
 SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
 
-log() { echo "[hook:$(basename "$0")] $*"; }
-warn() { echo "[hook:$(basename "$0")] $*" >&2; }
-error() { echo "[hook:$(basename "$0")] ERROR: $*" >&2; }
+hook_name() { printf '%s' "${0##*/}"; }
+log() { echo "[hook:$(hook_name)] $*"; }
+warn() { echo "[hook:$(hook_name)] $*" >&2; }
+error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
+
+session_db_recovery_report_path() {
+  local path="${EVIDENT_SESSION_DB_RECOVERY_REPORT:-}"
+  path="${path#"${path%%[![:space:]]*}"}"
+  path="${path%"${path##*[![:space:]]}"}"
+  if [ -n "${path}" ]; then
+    printf '%s\n' "${path}"
+    return 0
+  fi
+  local home="${HOME:-}"
+  home="${home#"${home%%[![:space:]]*}"}"
+  home="${home%"${home##*[![:space:]]}"}"
+  [ -n "${home}" ] || return 1
+  printf '%s\n' "${home}/.local/state/evident/session-db-recovery.jsonl"
+}
+
+report_session_db_giveup() {
+  local reason="$1" exit_code="${2:-null}" outcome path at
+  case "${reason}" in
+    synchroniser_config_unresolved|synchroniser_config_unevaluable|synchroniser_config_incomplete|litestream_config_unavailable|classification_fatal) outcome="restore_misconfigured" ;;
+    restore_deadline_exceeded|restore_tool_unusable|classification_unrecognised) outcome="fresh_session_db" ;;
+    *) warn "SESSION-DB-RECOVERY-REPORT-UNKNOWN-REASON: ${reason}"; return 0 ;;
+  esac
+  if ! path="$(session_db_recovery_report_path)"; then
+    warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
+    return 0
+  fi
+  at="${EPOCHREALTIME:-}"
+  [ -n "${at}" ] || warn "SESSION-DB-RECOVERY-REPORT-TIMESTAMP-UNAVAILABLE"
+  printf '{"v":1,"event":"session_db_recovery","at":"%s","stage":"restore","outcome":"%s","severity":"error","reason":"%s","litestream_exit_code":%s,"attempt":null,"replica_objects":null,"replica_bytes":null,"quarantine_destination":null,"quarantined_objects":null,"quarantine_failed_objects":null,"quarantined_bytes":null,"verified_restore_point":null,"restore_points_tried":null,"replication_suspended":true}\n' "${at}" "${outcome}" "${reason}" "${exit_code}" >>"${path}" || warn "SESSION-DB-RECOVERY-REPORT-APPEND-FAILED"
+  return 0
+}
 
 # Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
 # corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
 # `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
-# unusable)/32 (retry) — see `runner/synchroniser/src/cli.ts`'s own
+# unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
+# exhausted, replica separated and local disposed) / 34 (could not prove
+# separation or disposal) — see `runner/synchroniser/src/cli.ts`'s own
 # comment for what each means, not restated here. Any OTHER non-zero status
 # means the tool itself broke, which is the only case worth an ERROR here —
 # EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
@@ -83,7 +120,7 @@ run_synchroniser() {
   [ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
   "${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
   case "${rc}" in
-    0 | 10 | 30 | 31 | 32) ;;
+    0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
     124 | 137)
       [ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
       ;;
@@ -270,19 +307,26 @@ bounded_restore_call() {
 }
 
 fetch_runner_secret() {
-  local step_started_s="$1" remaining rc=0 payload value stderr_file started_ms populated=0 missing=0
+  local step_started_s="$1" remaining rc=0 payload stderr_file started_ms populated=0 skipped=0 key value
   if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
     log "runner secret is not configured; continuing without GitHub and MCP credentials"
     return 0
   fi
+  # From here, RUNNER_SECRET_ARN IS configured: every failure branch below
+  # returns 1, not 0. Process env is fixed at spawn — nothing later in this
+  # VM's life can retry a fetch that fails here — so a VM that was promised
+  # these credentials and never got them is contrary evidence it can do the
+  # job it was launched for, exactly the class check_runner_key already treats
+  # as fatal (#1997): fail /run now and let the lifecycle machinery replace
+  # this VM, rather than run the whole session silently degraded.
   remaining="$(remaining_credential_budget "${step_started_s}")"
   if [ "${remaining}" -lt 1 ]; then
     warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
-    return 0
+    return 1
   fi
   if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
     warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
-    return 0
+    return 1
   fi
   started_ms="$(now_ms)"
   payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
@@ -290,33 +334,31 @@ fetch_runner_secret() {
   if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
     warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
     rm -f "${stderr_file}"
-    return 0
+    return 1
   fi
   if [ "${rc}" -ne 0 ]; then
     warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
     rm -f "${stderr_file}"
-    return 0
+    return 1
   fi
   rm -f "${stderr_file}"
   if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
     warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
     return 0
   fi
-  local -a populated_keys=()
-  for key in GH_TOKEN BRAVE_API_KEY CLOUDFLARE_API_TOKEN NEON_API_KEY; do
-    value="$(jq -r --arg k "${key}" '.[$k] // empty' <<<"${payload}")"
-    if [ -n "${value}" ]; then
-      export "${key}=${value}"
-      populated=$((populated + 1))
-      populated_keys+=("${key}")
-    else
-      missing=$((missing + 1))
+  while IFS= read -r -d '' key && IFS= read -r -d '' value; do
+    if [[ ! "${key}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
+      warn "RUNNER-SECRET-KEY-SKIPPED: ${key@Q} is not a valid environment variable name"
+      skipped=$((skipped + 1))
+      continue
     fi
-  done
+    export "${key}=${value}"
+    populated=$((populated + 1))
+  done < <(jq -j 'to_entries[] | select(.value | type == "string" and length > 0) | .key, "\u0000", .value, "\u0000"' <<<"${payload}")
   if [ "${populated}" -eq 0 ]; then
-    warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-microvm/README.md"
+    warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md"
   else
-    log "RUNNER-SECRET-OK: populated ${populated_keys[*]}; ${missing} allow-listed keys empty or absent"
+    log "RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names"
   fi
   return 0
 }
@@ -328,7 +370,10 @@ restore_credentials() {
 
   load_state_config || return 1
   # This shares the existing bounded window so /run's worst-case duration does not grow.
-  fetch_runner_secret "${step_started_s}"
+  # Fatal when RUNNER_SECRET_ARN is configured (see fetch_runner_secret's own
+  # comment); a no-ARN self-hosted runner already returned 0 above and never
+  # reaches this `||`.
+  fetch_runner_secret "${step_started_s}" || return 1
   bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
   bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
 
@@ -352,7 +397,7 @@ restore_credentials() {
 (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
 ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
 a model provider fails until one is connected. See 'Seeding a credential store' in \
-infrastructure/evident-microvm/README.md."
+infrastructure/evident-runner/MICROVM.md."
         ;;
       124 | 137)
         warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
@@ -482,6 +527,35 @@ prewarm_litestream() {
   log "pre-warming ${binary} in the background"
 }
 
+# Reads the aws CLI v2 install tree into the page cache, in the background, so
+# fetch_runner_secret's first `aws secretsmanager get-secret-value` does not
+# pay first-touch I/O on the critical path.
+#
+# A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
+# enough here: v2 ships as a real Python distribution (~7,500 files under the
+# resolved binary's own directory), and a cold invocation demand-pages a
+# scattered set of them (botocore's endpoints.json/partitions.json, service
+# model JSON, shared libs) — one boot measured #1997's fetch at 7,243ms cold
+# vs ~400ms warm, an ~18x gap the litestream read-ahead trick alone cannot
+# close. `find -exec cat` walks that whole tree instead of one file.
+prewarm_aws_cli() {
+  local binary tree
+  binary="$(command -v aws 2>/dev/null || true)"
+  if [ -z "${binary}" ]; then
+    warn "aws CLI is not on PATH; skipping the boot pre-warm"
+    return 0
+  fi
+  tree="$(dirname "$(readlink -f "${binary}")")"
+
+  (
+    local started_ms rc=0
+    started_ms="$(now_ms)"
+    find "${tree}" -type f -exec cat {} + >/dev/null 2>&1 || rc=$?
+    log_elapsed_since aws-cli-prewarm "${started_ms}" "${rc}"
+  ) &
+  log "pre-warming ${tree} in the background"
+}
+
 # Generates ${LITESTREAM_CONFIG_FILE} from runner-synchroniser's own renderer
 # — the SAME config module the credential restore/sync already goes through,
 # so there is no second copy of the bucket/prefix/path logic to drift
@@ -595,6 +669,24 @@ restore_session_db() {
   # suspend/resume and a /run retry, so a marker left by an earlier boot must
   # never silently disable replication for the rest of this VM's life.
   rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
+  local report_path
+  if report_path="$(session_db_recovery_report_path)"; then
+    # The default report directory is provisioned in the image, so this is a
+    # no-op there; a shared-path override (EVIDENT_SESSION_DB_RECOVERY_REPORT)
+    # can name a directory that isn't, and the typed synchroniser writer
+    # already creates it for that same override (session-db-recovery-report.ts).
+    # Best-effort only: report_session_db_giveup below still warns and
+    # continues if this didn't leave a writable path. `${path%/*}` leaves a
+    # bare relative filename (no `/`) unchanged rather than reducing to `.`
+    # like dirname(1) — without the explicit check, mkdir would create a
+    # directory AT the report path itself.
+    local report_dir="${report_path%/*}"
+    [ "${report_dir}" = "${report_path}" ] && report_dir="."
+    [ -d "${report_dir}" ] || mkdir -p "${report_dir}" 2>/dev/null || true
+    : >"${report_path}" || warn "SESSION-DB-RECOVERY-REPORT-TRUNCATE-FAILED"
+  else
+    warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
+  fi
 
   # Config resolved ONCE here, in the synchroniser (config.ts owns
   # OPENCODE_DB_PATH; no second copy of that path here), exactly as
@@ -611,6 +703,7 @@ restore_session_db() {
   log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
   if [ "${env_rc}" -ne 0 ]; then
     mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
+    report_session_db_giveup synchroniser_config_unresolved
     return 0
   fi
   # Guarded for the SAME reason as the substitution above, which is easy to miss:
@@ -621,6 +714,7 @@ restore_session_db() {
   # it is the whole lifecycle transition, so Q3 makes it a give-up instead.
   eval "${synchroniser_env}" || {
     mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
+    report_session_db_giveup synchroniser_config_unevaluable
     return 0
   }
 
@@ -635,6 +729,7 @@ restore_session_db() {
   # be actively wrong, not merely absent).
   if [ -z "${OPENCODE_DB_PATH+x}" ]; then
     mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
+    report_session_db_giveup synchroniser_config_incomplete
     return 0
   fi
 
@@ -652,6 +747,7 @@ restore_session_db() {
 
   if ! ensure_litestream_config; then
     mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
+    report_session_db_giveup litestream_config_unavailable
     return 0
   fi
 
@@ -670,12 +766,14 @@ restore_session_db() {
     124 | 137)
       discard_session_db_debris
       mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
+      report_session_db_giveup restore_deadline_exceeded "${restore_rc}"
       return 0
       ;;
     125 | 126 | 127)
       error "litestream restore could not even run (timeout exited ${restore_rc})"
       discard_session_db_debris
       mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
+      report_session_db_giveup restore_tool_unusable "${restore_rc}"
       return 0
       ;;
   esac
@@ -706,11 +804,13 @@ restore_session_db() {
       # fresh rather than fail /run.
       discard_session_db_debris
       mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
+      report_session_db_giveup classification_fatal
       ;;
     *)
       # run_synchroniser already logged the "tool broke" ERROR for this.
       discard_session_db_debris
       mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
+      report_session_db_giveup classification_unrecognised
       ;;
   esac
 
@@ -718,6 +818,99 @@ restore_session_db() {
 }
 # --- Session DB restore (end) -----------------------------------------------
 
+# --- Session DB integrity verification (#1868 WI-4) --------------------------
+#
+# The restore loop above only proves litestream could WRITE a file; it never
+# proves the file is intact (#1345: a leaf-page-corrupt-but-openable DB
+# re-restores unchanged forever). `session-db-verify` runs a real `PRAGMA
+# integrity_check` and, on failure, walks retained restore points back until
+# one passes — the MicroVM side of the identical check
+# runner/docker-images/fargate/entrypoint.sh already runs after ITS restore
+# loop, through the same runner-synchroniser CLI.
+
+# Bounded by `timed_synchroniser`, not an external `timeout` around
+# `run_synchroniser` itself (a shell function, not an exported binary — that
+# would fail with rc 127, silently mis-triggering the allowlist's "tool
+# broke" branch). The grace period is `CREDENTIAL_RESTORE_KILL_GRACE_SECONDS`
+# (2s): `run_synchroniser`'s `timeout -k` hardcodes that one constant for
+# every `timed_synchroniser` caller, not a value unique to this step, so the
+# real worst case is DEADLINE + 2s, once — accounted for in
+# hook-scripts.test.ts's budget-ladder test.
+#
+# 4s, NOT measured on this platform: #930's synchroniser-call sample put a
+# single call at ~1.2s (a node cold start), and this ONE call also runs a
+# PRAGMA integrity_check that scales with the restored DB's size, which #930
+# never exercised. The 124/137 branch below is the DESIGNED-FOR outcome on a
+# large DB, not an edge case — SYNCHRONISER-TIMING op=session-db-verify is
+# what should actually size this once real boots report it.
+SESSION_DB_VERIFY_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_VERIFY_DEADLINE_SECONDS:-4}"
+
+verify_session_db() {
+  # Guards mirror start_litestream's first three, in the same order and for
+  # the same reason: verifying a DB this boot already decided not to
+  # replicate (or never restored at all) changes nothing about how /run
+  # proceeds, and would spend budget only to report on a moot outcome.
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    log "skipping session-DB verification: persistence is disabled"
+    return 0
+  fi
+
+  if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
+    log "skipping session-DB verification: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
+    return 0
+  fi
+
+  if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
+    log "skipping session-DB verification: no usable ${LITESTREAM_CONFIG_FILE}"
+    return 0
+  fi
+
+  # Exported below this step's own deadline so the CLI's in-process walkback
+  # loop gives up on its own before the process-group kill lands — its 180s
+  # default (config.ts) is far outside this one step's slice of the hook's
+  # SIGTERM budget.
+  local verify_rc=0
+  EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS="${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
+    timed_synchroniser session-db-verify "${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
+    session-db-verify "${LITESTREAM_CONFIG_FILE}" || verify_rc=$?
+
+  case "${verify_rc}" in
+    0) ;; # verified intact, or nothing to verify yet — the CLI already logged which
+    33)
+      # Integrity exhausted, but the classifier already proved the corrupt
+      # replica separated and the local copy disposed of — booting with a
+      # fresh DB and a new backup chain is safe, exactly like ECS's own `33)`.
+      log "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated."
+      ;;
+    34)
+      # The ONE deliberate exception to "nothing about the session DB may
+      # ever fail /run" (Q3): a 34 means separation/disposal could NOT be
+      # proven, so continuing would hand opencode a DB it may not be safe to
+      # open or write — the same evidence-quality bar `check_runner_key`
+      # already applies one step earlier in this hook (contrary evidence,
+      # not absent evidence, is what's fatal). The CLI already logged
+      # SESSION-DB-REPLICA-SEPARATION-UNVERIFIED / SESSION-DB-LOCAL-DISCARD-FAILED.
+      error "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk, so nothing will be started (see the ERROR above)."
+      return 1
+      ;;
+    124 | 137)
+      # The designed-for outcome on a large DB (see the deadline comment
+      # above), not a broken tool: continue with the restored DB exactly as
+      # ECS's own `*)` branch does for an unexpected code.
+      warn "SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_DEADLINE_SECONDS}s deadline; continuing with the restored opencode.db as-is, unverified"
+      ;;
+    *)
+      # run_synchroniser already logged the "tool broke" ERROR for this. A
+      # broken verifier must not turn a boot that works today into a
+      # crash-loop.
+      warn "SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${verify_rc}, which is none of its documented answers; continuing with the restored opencode.db as-is"
+      ;;
+  esac
+
+  return 0
+}
+# --- Session DB integrity verification (end) ---------------------------------
+
 # `kill -0` answers "does this pid exist", which is not the question any caller
 # here is asking. A process that has exited but has not been reaped — a zombie —
 # still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
@@ -810,6 +1003,7 @@ regenerate_machine_id() {
 tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
 opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
 litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
+creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
 
 # `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
 # and an interpolation of a missing field is still a non-empty string, so a
@@ -1030,6 +1224,155 @@ kill_litestream() {
 }
 # --- litestream replicate (end) ----------------------------------------------
 
+# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
+#
+# sync_credentials (above) covers the three boundary flushes /run's restore,
+# /suspend and /terminate already call. What it does NOT cover is a VM that
+# runs for a long time between those boundaries: a provider re-authenticated
+# through the proxied UI hours into a run would sit unflushed until the next
+# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
+# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
+# own sync_credentials_loop is the ECS side of the identical gap; this is the
+# same fix, backgrounded the same way as start_opencode/start_litestream so it
+# outlives this hook process, `( … ) &` rather than `setsid`: a plain
+# backgrounded subshell is reparented to init and keeps running once its
+# parent hook script exits (verified: PPID=1, still alive, with no controlling
+# terminal in this image to send it a stray SIGHUP), and it inherits every
+# function this file defines, so it can call run_synchroniser directly with no
+# re-exec.
+
+# Bounded confirmation window `stop_credential_sync` polls after signalling the
+# loop, sized against the SIGTERM budget ladder (#812 WI-4's
+# hook-scripts.test.ts): /terminate's own steps already use 49 of the 55s
+# ceiling, leaving 6s of headroom — this matches the *_KILL_GRACE_SECONDS
+# convention (CREDENTIAL_RESTORE_KILL_GRACE_SECONDS,
+# SESSION_DB_RESTORE_KILL_GRACE_SECONDS, both 2s) rather than a longer
+# drain-style wait, since the loop's current child is one fast
+# `run_synchroniser sync-once` call (#930: ~1.2s measured), not a writer
+# needing a graceful drain.
+CREDS_SYNC_STOP_WAIT_SECONDS=2
+
+# Best-effort per tick, exactly like sync_credentials above: a failed tick
+# must never end the loop, or a single transient S3 error would silently
+# disable sync for the rest of the VM's life.
+start_credential_sync() {
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
+    return 0
+  fi
+
+  if creds_sync_is_running; then
+    warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
+    return 0
+  fi
+
+  # CREDS_SYNC_INTERVAL is exported by restore_session_db's `eval "$(run_synchroniser env)"`
+  # on /run (config.ts's own default is 60s), but that eval can fail or be skipped by an
+  # earlier give-up — never leave the loop unbound under set -u for a value with a safe,
+  # named fallback (unlike OPENCODE_DB_PATH, a guessed sync cadence is not actively wrong).
+  local interval="${CREDS_SYNC_INTERVAL:-60}"
+  if [ -z "${CREDS_SYNC_INTERVAL:-}" ]; then
+    warn "CREDS-SYNC-INTERVAL-DEFAULTED: CREDS_SYNC_INTERVAL was not set by run_synchroniser env; using ${interval}s"
+  fi
+
+  rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
+
+  (
+    # Releases the fds this subshell inherited from the hook process before
+    # settling in for the VM's whole remaining life: nothing here writes to
+    # them (every synchroniser call already redirects its own), so there is
+    # no reason to keep holding the hook's original stdout/stderr open. A
+    # long-lived process that instead inherited a pipe's write end (a test
+    # harness reading the hook's own output, for one) would keep that pipe
+    # from ever reporting EOF — testing-guide.mdc's own lesson, and the same
+    # reason start_opencode/start_litestream never inherit stdio either. That
+    # redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
+    # failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
+    # surfaced by stop_credential_sync, which DOES have live stdio.
+    exec >/dev/null 2>&1 </dev/null
+
+    # A TERM this subshell receives (from stop_credential_sync, below) only
+    # kills THIS wrapper by default — its currently-running child (`sleep`,
+    # or a `run_synchroniser sync-once` call) is a separate process that
+    # would otherwise be orphaned and keep running, free to upload STALE
+    # credentials to S3 after the boundary flush that /suspend and
+    # /terminate perform immediately following the stop. Tracking the
+    # current child explicitly and forwarding the signal closes that race.
+    creds_sync_child_pid=""
+    trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
+
+    while true; do
+      sleep "${interval}" &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null
+      creds_sync_child_pid=""
+
+      run_synchroniser sync-once claude &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
+      creds_sync_child_pid=""
+
+      run_synchroniser sync-once opencode &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
+      creds_sync_child_pid=""
+    done
+  ) &
+
+  echo $! >"${CREDS_SYNC_PID_FILE}"
+  log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
+}
+
+# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
+# that it and its current child are actually gone before returning: /suspend
+# and /terminate start their own boundary flush immediately after this call,
+# and an orphaned in-flight sync-once surviving past that point can overwrite
+# fresher credentials with stale ones. The TERM trap inside the loop (above)
+# forwards the signal to its current child almost instantly — this poll is a
+# defensive confirmation, not the primary mechanism, so it stays short; a
+# SIGKILL backstop covers a child that ignores TERM entirely.
+#
+# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
+# must say what it found (development-workflow.mdc) — a stopped-before-called
+# loop and a died-on-its-own loop are different facts an operator needs told
+# apart, not the same "nothing to stop" line.
+stop_credential_sync() {
+  if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
+    log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${CREDS_SYNC_PID_FILE}")"
+  if ! process_is_alive "${pid}"; then
+    rm -f "${CREDS_SYNC_PID_FILE}"
+    warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
+    return 0
+  fi
+
+  kill -TERM "${pid}" 2>/dev/null || true
+  rm -f "${CREDS_SYNC_PID_FILE}"
+
+  local waited_ms=0
+  while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
+    sleep 0.1
+    waited_ms=$((waited_ms + 100))
+  done
+
+  if process_is_alive "${pid}"; then
+    kill -KILL "${pid}" 2>/dev/null || true
+    warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
+  fi
+
+  if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
+    warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
+    rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
+  fi
+
+  log "CREDS-SYNC-STOPPED: pid=${pid}"
+}
+# --- credential sync loop (end) -----------------------------------------------
+
 # --- flush_session_db (#812 WI-4) -------------------------------------------
 #
 # The checked, synchronous flush /suspend and /terminate need before they

package/dist/microvm-image-context/hooks/resume

@@ -5,7 +5,10 @@
 # session-DB replicator /suspend stopped before the snapshot (#812 WI-4):
 # litestream does not survive a suspend/resume freeze on this design (Q2) —
 # /suspend stops it and /resume starts a fresh one, the same pattern already
-# proven for the tunnel.
+# proven for the tunnel. The interval credential sync loop (#1868 WI-3) is the
+# same story one function over: /suspend stops it too, so a resumed VM that
+# never restarted it here would never sync credentials again for the rest of
+# its life.
 set -euo pipefail
 
 # shellcheck source=./common.sh
@@ -46,6 +49,7 @@ fi
 # already-running) handle the rest — this needs no logic of its own.
 load_state_config || warn "could not resolve durable-state config; the session DB will not resume replicating"
 start_litestream
+start_credential_sync
 
 # Diagnostic-only, unlike /run's gate: a failed resume costs the user their
 # whole session, so this never exits — it only converts a silent "resumed but

package/dist/microvm-image-context/hooks/run

@@ -28,6 +28,7 @@ cleanup() {
   stop_tunnel || warn "stop_tunnel failed while cleaning up"
   stop_opencode || warn "stop_opencode failed while cleaning up"
   kill_litestream || warn "kill_litestream failed while cleaning up"
+  stop_credential_sync || warn "stop_credential_sync failed while cleaning up"
 }
 trap cleanup EXIT
 
@@ -53,8 +54,12 @@ printf '%s\n' "${state_prefix}" >"${STATE_PREFIX_FILE}"
 
 # 3 — start reading the litestream binary NOW so that read overlaps step 4,
 # which needs a different binary (node), instead of landing inside step 5, which
-# start_opencode has to wait for. Backgrounded and never waited on.
+# start_opencode has to wait for. Backgrounded and never waited on. Same for the
+# aws CLI: step 4's runner-secret fetch is its first-ever invocation in this
+# VM's life, and #1997 measured that cold first touch alone blowing the whole
+# credential-restore deadline.
 prewarm_litestream
+prewarm_aws_cli
 
 # 4 — credential stores, restored before anything that reads them starts. Fatal
 # only if persistence itself is unavailable (an unset bucket/prefix, or a broken
@@ -79,13 +84,22 @@ check_runner_key "${runner_key}" "${api_url}" || exit 1
 restore_session_db
 log "session DB restore done ${SECONDS}s into the hook"
 
-# 7 — apply the overlay before OpenCode resolves its project configuration.
+# 7 — integrity-check the restored DB (#1868 WI-4), after the config file
+# exists (verify_session_db reads it) and before anything opens the DB —
+# the only window in which that's true. The ONE step in this sequence that
+# can still fail /run past the runner-key gate: a 34 means the corrupt DB's
+# separation/disposal could not be proven safe (see verify_session_db's own
+# comment for why that's a deliberate exception to "nothing about the
+# session DB may ever fail /run").
+verify_session_db || exit 1
+
+# 8 — apply the overlay before OpenCode resolves its project configuration.
 apply_runner_opencode_config
 
-# 8 — configure git after credentials are restored and before agent shells start.
+# 9 — configure git after credentials are restored and before agent shells start.
 configure_github_access
 
-# 9 — opencode. Started here, not at build time: a warm process in the shared
+# 10 — opencode. Started here, not at build time: a warm process in the shared
 # snapshot would carry its installation id and database into every VM. Not
 # waited on: a slow opencode boot is not a reason to fail /run (the tunnel CLI
 # auto-starts opencode when it finds none healthy,
@@ -93,14 +107,19 @@ configure_github_access
 # reclaims a runner that never comes online).
 start_opencode
 
-# 10 — begin replicating the session DB (#812 WI-3), now that opencode has
+# 11 — begin replicating the session DB (#812 WI-3), now that opencode has
 # opened it and before any work can arrive over the tunnel. Bare, like
 # restore_session_db above: start_litestream never returns non-zero (every
 # guard inside it is its own `return 0`), so there is nothing here for
 # `set -e` to abort on.
 start_litestream
 
-# 11 — the first per-VM identity on the wire. The subshell's umask makes the file
+# 11a — the interval credential sync (#1868 WI-3), matching ECS's own
+# post-litestream position. Bare for the same reason: every guard inside
+# start_credential_sync is its own `return 0`.
+start_credential_sync
+
+# 12 — the first per-VM identity on the wire. The subshell's umask makes the file
 # unreadable to anyone else from the moment it exists, before the key is in it.
 (
   umask 077

package/dist/microvm-image-context/hooks/suspend

@@ -8,6 +8,9 @@ set -euo pipefail
 # shellcheck source=./common.sh
 source "$(dirname "$0")/common.sh"
 
+# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
+# flush must not race each other on the same credential stores.
+stop_credential_sync
 sync_credentials
 stop_tunnel
 

package/dist/microvm-image-context/hooks/terminate

@@ -8,6 +8,9 @@ set -uo pipefail
 # shellcheck source=./common.sh
 source "$(dirname "$0")/common.sh"
 
+# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
+# flush must not race each other on the same credential stores.
+stop_credential_sync
 sync_credentials
 stop_tunnel
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evident-ai/runner-cdk",
-  "version": "3.4.1-dev.3adba63",
+  "version": "3.4.1-dev.51df4de",
   "description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",