@evident-ai/runner-cdk 0.1.1-dev.da70cd4 → 3.4.1-dev.0ef5061

package/dist/microvm-image-context/hooks/common.sh

@@ -0,0 +1,957 @@
+#!/usr/bin/env bash
+# Sourced by every hook script. Nothing here runs at image build time.
+#
+# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
+# (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
+# runner/synchroniser/src/shell-contract.test.ts — which derives what
+# `run_synchroniser` below must handle from the subcommands this shell calls.
+
+# Installed from npm by the image (docker/Dockerfile's ARG
+# RUNNER_SYNCHRONISER_VERSION) and resolved off PATH here. The Dockerfile's
+# required-binary assertion is the loud failure if it is missing.
+SYNCHRONISER="runner-synchroniser"
+OPENCODE_PORT="${OPENCODE_PORT:-4096}"
+
+# A hook script keeps no memory between invocations, but /resume must re-dial
+# with the runner key /run was given, so /run leaves it here. Deliberate, on two
+# axes: tmpfs means it never reaches the block device, and the SHARED image
+# snapshot is taken at build time — long before /run — so the key cannot enter
+# the artefact every VM boots from. And at 0600 owned by uid 10001 it is
+# readable by nobody who could not already read it out of the tunnel process's
+# /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
+# shellcheck disable=SC2034  # read by the scripts that source this file
+CONTEXT_FILE="/dev/shm/evident-run-context"
+TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
+OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
+LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
+CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
+CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
+
+# Where the runner's credential store lives inside the durable-state bucket.
+# The BUCKET is the same for every VM from an image version, so the stack bakes
+# it into the image environment; the PREFIX selects one runner's store, so it is
+# per-VM and can only arrive in the /run payload. /run leaves it here for the
+# hooks that flush back to it. A file of its own rather than a line of
+# CONTEXT_FILE: it is not a secret, so /suspend never has to read the runner key
+# to find out where to write.
+STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
+
+# The generated litestream.yml (#812). tmpfs for the same two reasons as the
+# files above: it must never reach the block device (Q6, `docs`), and it
+# survives suspend/resume, which is what lets `/resume` start litestream again
+# with no regeneration cost. The CLI regenerates it only when absent or empty.
+LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
+
+# Set by the CLI when session-DB restore or verification could not prove this
+# boot safe to replicate. The teardown hooks read it before flushing, and
+# `/terminate` removes it with the rest of the per-VM state.
+SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
+
+hook_name() { printf '%s' "${0##*/}"; }
+log() { echo "[hook:$(hook_name)] $*"; }
+warn() { echo "[hook:$(hook_name)] $*" >&2; }
+error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
+
+# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
+# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
+# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
+# unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
+# exhausted, replica separated and local disposed) / 34 (could not prove
+# separation or disposal) — see `runner/synchroniser/src/cli.ts`'s own
+# comment for what each means, not restated here. Any OTHER non-zero status
+# means the tool itself broke, which is the only case worth an ERROR here.
+run_synchroniser() {
+  local rc=0
+  "${SYNCHRONISER}" "$@" || rc=$?
+  case "${rc}" in
+    0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
+    *) error "synchroniser '$*' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
+  esac
+  return "${rc}"
+}
+
+# Wall-clock milliseconds. ${EPOCHREALTIME} is a bash 5 builtin (the image is
+# node:22-bookworm-slim → bash 5.2) rather than `date +%s%3N`, which has to
+# exec /bin/date inside the caller's command substitution: measured on a loaded
+# box, that exec put ~190ms of its own cost INSIDE the window being measured,
+# against ~23ms for this — overhead charged to the very number this exists to
+# learn. The remaining ~23ms is the command substitution's subshell, kept
+# because removing it means an out-parameter global, and ~2% of a ~1.2s call
+# biases the reading generous, which is the safe direction for sizing a
+# deadline.
+#
+# The `[.,]` is not paranoia: ${EPOCHREALTIME} renders its decimal separator
+# from LC_NUMERIC, so a comma locale would otherwise silently produce garbage
+# here. Stripping either turns it into whole microseconds, which /1000 makes
+# milliseconds.
+#
+# Never fails its caller: a timing line is diagnostics, and hardening a
+# currently-working path is worse than the gap it closes (#931). The `date`
+# fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
+# `log_elapsed_since` turns into `elapsed_ms=unknown` rather than aborting a
+# hook under `set -e` — and rather than a 0 that would read as a fast healthy call.
+now_ms() {
+  local now="${EPOCHREALTIME:-}"
+  if [ -n "${now}" ]; then
+    now="${now/[.,]/}"
+    echo "$((now / 1000))"
+    return 0
+  fi
+  date +%s%3N 2>/dev/null || echo 0
+}
+
+# Reports what one pre-warm COST, in a deliberately machine-greppable line
+# (`op=`, `elapsed_ms=`, `rc=`) so "how long does this actually take in the
+# fleet?" is a log query rather than another spike. `at_s` is where the step
+# landed on the hook's own SECONDS clock.
+#
+# `unknown`, never a number, when either end failed to read the clock (now_ms's
+# 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
+# very fast healthy call, which would quietly bias the fleet-wide sizing data
+# this line exists to produce, in the DANGEROUS direction (a deadline sized too
+# tight). A non-numeric value drops out of an aggregate instead of poisoning it.
+log_elapsed_since() {
+  local op="$1" started_ms="$2" rc="$3"
+
+  local finished_ms elapsed_ms="unknown"
+  finished_ms="$(now_ms)"
+  if [ "${started_ms}" -gt 0 ] && [ "${finished_ms}" -gt 0 ]; then
+    elapsed_ms="$((finished_ms - started_ms))"
+  fi
+
+  log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
+}
+
+# Exports what `runner-synchroniser` resolves its object-store location from
+# (runner/synchroniser/src/config.ts). It treats either being empty as
+# "persistence disabled" and then reports every restore as a WARNING it still
+# exits 0 for — so an unset value has to be caught HERE, where it can still be
+# told apart from "the store is simply empty".
+load_state_config() {
+  if [ -z "${LITESTREAM_BUCKET:-}" ]; then
+    error "LITESTREAM_BUCKET is unset; the image is missing the durable-state bucket the stack bakes in"
+    return 1
+  fi
+
+  LITESTREAM_PREFIX="$(cat "${STATE_PREFIX_FILE}" 2>/dev/null || true)"
+  if [ -z "${LITESTREAM_PREFIX}" ]; then
+    error "no durable-state prefix at ${STATE_PREFIX_FILE}; /run records the payload's state_prefix there before anything can be restored or flushed"
+    return 1
+  fi
+
+  export LITESTREAM_BUCKET LITESTREAM_PREFIX
+
+  # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
+  # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
+  # is just LITESTREAM_BUCKET. The CLI's replicator and flush_session_db (#812
+  # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
+  # directly, so every entry point agrees on one predicate regardless of which
+  # earlier step in THIS hook process set it; /resume, /suspend and /terminate
+  # all use this same function before any teardown decision.
+  export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
+}
+
+# Best-effort by design: the teardown hooks must still complete even when a
+# credential flush cannot happen.
+sync_credentials() {
+  load_state_config || return 0
+  run_synchroniser sync-once claude || true
+  run_synchroniser sync-once opencode || true
+}
+
+# --- boot pre-warms ----------------------------------------------------------
+
+# Reads the ~30 MB litestream binary into the page cache, in the background, so
+# the FIRST exec of it does not pay that read on the critical path.
+#
+# The pre-warm overlaps the CLI's node startup and auth round trip, so the first
+# litestream operation does not also pay the binary's cold page-cache read.
+# Boot measurements put ~6.6-7.3s before the litestream version line, of which
+# only ~0.2s is accounted for by the two node calls in between — the remainder
+# is INFERRED to be this read, never measured.
+# The `litestream-prewarm` and `litestream-version` timings are what settle it
+# on the next boot.
+#
+# Nothing waits on it and nothing reads its output, so if that inference is
+# wrong this costs one backgrounded `cat`. Deleting the `litestream version`
+# diagnostic instead would save nothing: `litestream restore` two calls later
+# pays the identical read.
+prewarm_litestream() {
+  local binary
+  binary="$(command -v litestream 2>/dev/null || true)"
+  if [ -z "${binary}" ]; then
+    warn "litestream is not on PATH; skipping the boot pre-warm"
+    return 0
+  fi
+
+  # `cat` rather than a throwaway `litestream version`: a sequential read gets
+  # the whole file with readahead, where an exec demand-pages it.
+  (
+    local started_ms rc=0
+    started_ms="$(now_ms)"
+    cat "${binary}" >/dev/null 2>&1 || rc=$?
+    log_elapsed_since litestream-prewarm "${started_ms}" "${rc}"
+  ) &
+  log "pre-warming ${binary} in the background"
+}
+
+# Reads the aws CLI v2 install tree into the page cache, in the background, so
+# the CLI's first runner-secret fetch does not pay first-touch I/O. The read
+# overlaps `evident run`'s node startup and auth round trip rather than a hook
+# deadline.
+#
+# A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
+# enough here: v2 ships as a real Python distribution (~7,500 files under the
+# resolved binary's own directory), and a cold invocation demand-pages a
+# scattered set of them (botocore's endpoints.json/partitions.json, service
+# model JSON, shared libs) — one boot measured #1997's fetch at 7,243ms cold
+# vs ~400ms warm, an ~18x gap the litestream read-ahead trick alone cannot
+# close. `find -exec cat` walks that whole tree instead of one file.
+prewarm_aws_cli() {
+  local binary tree
+  binary="$(command -v aws 2>/dev/null || true)"
+  if [ -z "${binary}" ]; then
+    warn "aws CLI is not on PATH; skipping the boot pre-warm"
+    return 0
+  fi
+  tree="$(dirname "$(readlink -f "${binary}")")"
+
+  (
+    local started_ms rc=0
+    started_ms="$(now_ms)"
+    find "${tree}" -type f -exec cat {} + >/dev/null 2>&1 || rc=$?
+    log_elapsed_since aws-cli-prewarm "${started_ms}" "${rc}"
+  ) &
+  log "pre-warming ${tree} in the background"
+}
+
+# `kill -0` answers "does this pid exist", which is not the question any caller
+# here is asking. A process that has exited but has not been reaped — a zombie —
+# still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
+# normal case for everything these hooks start: `start_tunnel` and its CLI child
+# background a process that outlives the hook, the hook shell must return so AWS
+# gets its 200, and PID 1 in this image is a bare node hook server with no init
+# (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
+# the rest of the VM's life, which made `stop_tunnel` burn its whole budget
+# SIGKILLing a corpse and made a stale pid file suppress the next
+# `start_tunnel`, leaving a resumed VM permanently offline (#718).
+#
+# This is zombie-safe, NOT identity-safe: a recycled pid reads R/S and is still
+# called alive, so the SIGKILL backstop could in principle hit an unrelated
+# process. Pre-existing limitation of the pid-file approach, out of scope here —
+# named so nobody over-trusts the helper (src/image/hook-scripts.test.ts).
+#
+# Also sets PROC_DEAD_REASON on every return path — "gone" for a `kill -0`
+# miss, "zombie" for an unreaped `Z`, reset to empty on the alive path so a
+# stale value from a previous call can never be read — for the caller that
+# needs to say WHICH death it was (stop_tunnel's outcome line), never for this
+# function itself: see the silence note below. File-scope init (below) is what
+# keeps a read of it safe under `set -u` before this has ever run.
+PROC_DEAD_REASON=""
+
+process_is_alive() {
+  if ! kill -0 "$1" 2>/dev/null; then
+    PROC_DEAD_REASON="gone"
+    return 1
+  fi
+
+  # `/proc/<pid>/status` rather than `/proc/<pid>/stat`, whose fields cannot be
+  # split safely when a comm contains a space or a paren. If the state cannot be
+  # read at all, keep the `kill -0` answer: absent evidence is not "dead", and
+  # reading it as dead would SIGTERM-and-forget a live, still-draining CLI.
+  # Three fields, not two: `read` hands the whole remainder to its LAST variable,
+  # so a trailing `_` is what keeps the state letter out of `Z (zombie)`.
+  #
+  # That deferral is silent HERE on purpose: `stop_tunnel` polls this 10x/s, so a
+  # warning inside it would bury the log. The flag lets `is_running` say it once
+  # per guard instead — absent evidence must warn as well as defer
+  # (development-workflow.mdc).
+  PROC_STATE_UNREADABLE=0
+  local state
+  read -r _ state _ < <(grep -m1 '^State:' "/proc/$1/status" 2>/dev/null) || {
+    PROC_STATE_UNREADABLE=1
+    PROC_DEAD_REASON="" # unreadable /proc still means "alive" below, not a dead reason
+    return 0
+  }
+
+  if [ "${state}" = "Z" ]; then
+    PROC_DEAD_REASON="zombie"
+    return 1
+  fi
+
+  PROC_DEAD_REASON=""
+  return 0
+}
+
+is_running() {
+  [ -s "$1" ] || return 1
+
+  local pid
+  pid="$(cat "$1")"
+  process_is_alive "${pid}" || return 1
+
+  # The one state in which this whole helper is back to being a bare `kill -0`
+  # (a `hidepid=` mount, a `/proc` that is not mounted): every liveness answer
+  # silently reverts to #718 — a corpse reads as alive, so suspend SIGKILLs it
+  # after the full wait and the next start is suppressed for the VM's life.
+  [ "${PROC_STATE_UNREADABLE}" = 0 ] \
+    || warn "could not read /proc/${pid}/status; falling back to 'kill -0', which reports an exited-but-unreaped process as alive (#718)"
+
+  return 0
+}
+
+# The snapshot bakes one machine id into every VM launched from this image
+# version, so /run replaces it. Best-effort per file: the hook runs as uid 10001
+# and these live in root-owned directories, so a VM that kept the snapshot's id
+# must still boot — but it says so, because it is a state worth being able to see.
+regenerate_machine_id() {
+  local machine_id
+  machine_id="$(tr -d '-' </proc/sys/kernel/random/uuid)"
+
+  for path in /etc/machine-id /var/lib/dbus/machine-id; do
+    printf '%s\n' "${machine_id}" >"${path}" 2>/dev/null \
+      || warn "could not rewrite ${path}; this VM keeps the snapshot's machine id"
+  done
+}
+
+tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
+opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
+litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
+creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
+
+# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
+# and an interpolation of a missing field is still a non-empty string, so a
+# payload with no runner_key would sail through as the literal "null".
+payload_is_complete() {
+  jq -e '
+    (.runner_key | type == "string" and length > 0)
+    and (.endpoints.api | type == "string" and length > 0)
+    and (.endpoints.tunnel | type == "string" and length > 0)
+    and (.state_prefix | type == "string" and length > 0)
+  ' >/dev/null 2>&1
+}
+
+stop_opencode() {
+  if ! opencode_is_running; then
+    # Clear the file here too, for the same reason stop_tunnel does: opencode
+    # has exit paths that never reach this function (a crash, an OOM kill), so
+    # the pid file routinely outlives the process it names, and leaving it is
+    # #718 again the moment anything starts reaping orphans and the pid is
+    # recycled.
+    rm -f "${OPENCODE_PID_FILE}"
+    log "no opencode to stop"
+    return 0
+  fi
+
+  kill -TERM "$(cat "${OPENCODE_PID_FILE}")" 2>/dev/null || true
+  rm -f "${OPENCODE_PID_FILE}"
+  log "opencode stopped"
+}
+
+# The ceiling stop_opencode_and_wait (below) waits before its SIGKILL backstop.
+# /terminate can afford a real drain — the VM is being torn down either way —
+# unlike /run's cleanup, which cannot (stop_opencode's own cheapness above is
+# load-bearing for that budget, §4, and must not grow a wait).
+OPENCODE_STOP_WAIT_SECONDS="${EVIDENT_OPENCODE_STOP_WAIT_SECONDS:-5}"
+
+# The graceful stop /terminate needs (#812 WI-4): SIGTERM, poll for exit,
+# SIGKILL backstop, clear the pid file. A NEW function rather than growing
+# stop_opencode itself, for the reason above — mirrors stop_tunnel's and
+# stop_litestream's shape, including PROC_DEAD_REASON's one deciding read
+# after the poll loop (never inside it, which would bury the log at 10x/s).
+# This is half of the ordered-shutdown invariant (plan §6): /terminate must
+# stop opencode BEFORE flush_session_db's final sync, or litestream could
+# snapshot while opencode is still writing its WAL and the last session
+# writes would be missing from S3.
+stop_opencode_and_wait() {
+  if ! opencode_is_running; then
+    rm -f "${OPENCODE_PID_FILE}"
+    log "no opencode to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${OPENCODE_PID_FILE}")"
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( OPENCODE_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "opencode ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${OPENCODE_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${OPENCODE_PID_FILE}"
+  log "opencode stopped ${outcome}"
+}
+
+# The graceful wait before the SIGKILL backstop in stop_litestream, below.
+# litestream's own sync is normally sub-second, so 10s is generous headroom —
+# chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
+LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
+
+# The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
+# one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
+# would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
+# operator needs told apart, not a flatter clean/killed binary. /suspend and
+# /terminate call this: the SIGTERM it sends IS the checked flush they need.
+stop_litestream() {
+  if ! litestream_is_running; then
+    rm -f "${LITESTREAM_PID_FILE}"
+    log "no litestream to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${LITESTREAM_PID_FILE}")"
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( LITESTREAM_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "litestream ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${LITESTREAM_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${LITESTREAM_PID_FILE}"
+  log "litestream stopped ${outcome}"
+}
+
+# The cheap stop: one SIGTERM, one rm, one log line, NO poll loop — `/run`'s
+# own cleanup calls this, never stop_litestream, because that path only runs
+# after /run has already FAILED: no session happened yet, so a graceful final
+# sync is worth nothing against a SIGTERM budget that cannot afford another
+# wait on top of stop_tunnel's own. A distinct function rather than a mode
+# argument on stop_litestream:
+# stop_tunnel's own comment argues against an argument a future caller can get
+# wrong. litestream_is_running is the same O(1) check stop_litestream's own
+# no-op branch uses, not a poll — /run's cleanup trap fires this before
+# the replicator start is ever reached whenever an earlier step failed, and an
+# operator reading that log must not be told a stop signal went to a process
+# that never started.
+kill_litestream() {
+  if ! litestream_is_running; then
+    rm -f "${LITESTREAM_PID_FILE}"
+    log "no litestream to stop"
+    return 0
+  fi
+
+  kill -TERM "$(cat "${LITESTREAM_PID_FILE}")" 2>/dev/null || true
+  rm -f "${LITESTREAM_PID_FILE}"
+  log "litestream stop signalled (no wait)"
+}
+# --- litestream replicate (end) ----------------------------------------------
+
+# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
+#
+# sync_credentials (above) covers the three boundary flushes /run's restore,
+# /suspend and /terminate already call. What it does NOT cover is a VM that
+# runs for a long time between those boundaries: a provider re-authenticated
+# through the proxied UI hours into a run would sit unflushed until the next
+# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
+# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
+# own sync_credentials_loop is the ECS side of the identical gap; this is the
+# same fix, backgrounded like the other long-lived services so it
+# outlives this hook process, `( … ) &` rather than `setsid`: a plain
+# backgrounded subshell is reparented to init and keeps running once its
+# parent hook script exits (verified: PPID=1, still alive, with no controlling
+# terminal in this image to send it a stray SIGHUP), and it inherits every
+# function this file defines, so it can call run_synchroniser directly with no
+# re-exec.
+
+# Bounded confirmation window `stop_credential_sync` polls after signalling the
+# loop. The loop's current child is one fast `run_synchroniser sync-once` call,
+# so this stays a short backstop rather than a graceful drain.
+CREDS_SYNC_STOP_WAIT_SECONDS=2
+
+# Best-effort per tick, exactly like sync_credentials above: a failed tick
+# must never end the loop, or a single transient S3 error would silently
+# disable sync for the rest of the VM's life.
+start_credential_sync() {
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
+    return 0
+  fi
+
+  if creds_sync_is_running; then
+    warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
+    return 0
+  fi
+
+  # This is the same environment input runner-synchroniser validates. The
+  # variable is normally absent, so that ordinary case uses the documented
+  # default without a warning; a present invalid value is named and rejected.
+  local interval="${CREDS_SYNC_INTERVAL:-60}"
+  if [ -n "${CREDS_SYNC_INTERVAL+x}" ] && [[ ! "${CREDS_SYNC_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
+    warn "CREDS-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${CREDS_SYNC_INTERVAL}' is not a positive integer; using 60s"
+    interval=60
+  fi
+
+  rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
+
+  (
+    # Releases the fds this subshell inherited from the hook process before
+    # settling in for the VM's whole remaining life: nothing here writes to
+    # them (every synchroniser call already redirects its own), so there is
+    # no reason to keep holding the hook's original stdout/stderr open. A
+    # long-lived process that instead inherited a pipe's write end (a test
+    # harness reading the hook's own output, for one) would keep that pipe
+    # from ever reporting EOF — testing-guide.mdc's own lesson, and the same
+    # reason the long-lived services never inherit stdio either. That
+    # redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
+    # failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
+    # surfaced by stop_credential_sync, which DOES have live stdio.
+    exec >/dev/null 2>&1 </dev/null
+
+    # A TERM this subshell receives (from stop_credential_sync, below) only
+    # kills THIS wrapper by default — its currently-running child (`sleep`,
+    # or a `run_synchroniser sync-once` call) is a separate process that
+    # would otherwise be orphaned and keep running, free to upload STALE
+    # credentials to S3 after the boundary flush that /suspend and
+    # /terminate perform immediately following the stop. Tracking the
+    # current child explicitly and forwarding the signal closes that race.
+    creds_sync_child_pid=""
+    trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
+
+    while true; do
+      sleep "${interval}" &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null
+      creds_sync_child_pid=""
+
+      run_synchroniser sync-once claude &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
+      creds_sync_child_pid=""
+
+      run_synchroniser sync-once opencode &
+      creds_sync_child_pid=$!
+      wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
+      creds_sync_child_pid=""
+    done
+  ) &
+
+  echo $! >"${CREDS_SYNC_PID_FILE}"
+  log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
+}
+
+# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
+# that it and its current child are actually gone before returning: /suspend
+# and /terminate start their own boundary flush immediately after this call,
+# and an orphaned in-flight sync-once surviving past that point can overwrite
+# fresher credentials with stale ones. The TERM trap inside the loop (above)
+# forwards the signal to its current child almost instantly — this poll is a
+# defensive confirmation, not the primary mechanism, so it stays short; a
+# SIGKILL backstop covers a child that ignores TERM entirely.
+#
+# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
+# must say what it found (development-workflow.mdc) — a stopped-before-called
+# loop and a died-on-its-own loop are different facts an operator needs told
+# apart, not the same "nothing to stop" line.
+stop_credential_sync() {
+  if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
+    log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${CREDS_SYNC_PID_FILE}")"
+  if ! process_is_alive "${pid}"; then
+    rm -f "${CREDS_SYNC_PID_FILE}"
+    warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
+    return 0
+  fi
+
+  kill -TERM "${pid}" 2>/dev/null || true
+  rm -f "${CREDS_SYNC_PID_FILE}"
+
+  local waited_ms=0
+  while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
+    sleep 0.1
+    waited_ms=$((waited_ms + 100))
+  done
+
+  if process_is_alive "${pid}"; then
+    kill -KILL "${pid}" 2>/dev/null || true
+    warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
+  fi
+
+  if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
+    warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
+    rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
+  fi
+
+  log "CREDS-SYNC-STOPPED: pid=${pid}"
+}
+# --- credential sync loop (end) -----------------------------------------------
+
+# --- flush_session_db (#812 WI-4) -------------------------------------------
+#
+# The checked, synchronous flush /suspend and /terminate need before they
+# finish tearing down. Never fatal, for the same reason every session-DB
+# function in this file is: /suspend and /terminate must complete their own
+# teardown regardless of whether S3 could be reached.
+
+# Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
+# own sync is normally sub-second (M5, the plan's grounding); 10s is generous
+# headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
+# measured worst case. The SIGKILL backstop gives the flush its own two-second
+# grace after `timeout` sends SIGTERM.
+SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
+SESSION_DB_FLUSH_KILL_GRACE_SECONDS=2
+
+# Guards mirror the CLI replicator's first three exactly (disabled -> marker ->
+# config) — a flush must never attempt work those guards would have refused
+# to start in the first place. PERSISTENCE_BUCKET is the SAME variable
+# the CLI reads, so both agree regardless of which earlier step in
+# THIS process set it (load_state_config above).
+#
+# Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
+# trigger — and only once it is confirmed stopped does the synchronous -once
+# flush run, as a second, checked pass. Running -once while the daemon is
+# still alive would put two writers on one prefix, the single-writer
+# invariant this whole feature exists to protect (Q8) — this ordering is not
+# negotiable.
+#
+# If the daemon has already died (a missing or stale pid file), that IS the
+# detection this needs: SESSION-DB-REPLICATOR-DIED names it loudly instead of
+# silently skipping straight to the flush (development-workflow.mdc: every
+# recovery branch emits a server-visible signal). The flush still runs either
+# way — it is what actually gets any unreplicated writes to S3 before the
+# caller's next step.
+flush_session_db() {
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing to flush."
+    return 0
+  fi
+
+  if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
+    log "skipping session-DB flush: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
+    return 0
+  fi
+
+  if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
+    error "no usable ${LITESTREAM_CONFIG_FILE}; cannot flush the session DB"
+    return 0
+  fi
+
+  if litestream_is_running; then
+    stop_litestream
+  else
+    warn "SESSION-DB-REPLICATOR-DIED: no live litestream at ${LITESTREAM_PID_FILE}; flushing one-shot anyway"
+  fi
+
+  # Only reached with the daemon confirmed stopped, or never running — never
+  # concurrently with it (see above).
+  local flush_rc=0
+  timeout -k "${SESSION_DB_FLUSH_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
+    litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
+
+  if [ "${flush_rc}" -eq 0 ]; then
+    log "SESSION-DB-FINAL-FLUSH: succeeded, ${SECONDS}s into the hook"
+  else
+    warn "SESSION-DB-FINAL-FLUSH: litestream replicate -once exited ${flush_rc}, ${SECONDS}s into the hook; some writes since the last sync may not have reached S3"
+  fi
+}
+# --- flush_session_db (end) --------------------------------------------------
+
+# How long the guest CLI runs with no activity before it exits itself
+# (`evident run --idle-timeout`, apps/cli/src/commands/run.ts), which is what
+# turns a truly-abandoned VM into the clean-offline POST that lets Evident
+# suspend it (#732). Sized from the measured cost of guessing wrong rather than
+# the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
+# ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
+# napping a VM whose user comes straight back costs ~10 s — against ~$0.30/h
+# that is cheap enough that the balance sits far nearer the floor than the
+# ceiling. Not AT the floor, though: the CLI's idle detector needs 2 clear poll
+# cycles (`run.ts`'s `idlePolls >= 2`, ≥4 s of real time), so a value near that
+# would spend more time suspending/resuming than idle.
+# ECS's waker uses 900 s instead only because *its* cold start is far slower
+# than this VM's ~2 s resume — not evidence this default should match it.
+#
+# NOT part of the hook's own teardown budget (TUNNEL_STOP_WAIT_SECONDS et al.,
+# below): it governs the CLI's lifetime long after the hook has already
+# returned its 200, so it is deliberately outside that arithmetic.
+#
+# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
+# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
+# idle-timeout flag, so an operator who knows one knows the other. A
+# non-numeric override must never silently DROP the flag — that degrades to
+# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
+# warns and falls back to the default instead.
+#
+# Deliberately NOT in the /run payload yet: doing so would touch the doorbell
+# contract in four coordinated places for no capability today. #716 shipped
+# the swarm/per-user provisioner but explicitly scoped control-plane
+# idle/suspend policy OUT (its own "Phase 3" — see the issue's "Out of
+# Scope"), so a per-pool idle policy is still a clean follow-up, not
+# something #716 already provides a home for.
+IDLE_TIMEOUT_SECONDS=120
+if [ -n "${EVIDENT_IDLE_TIMEOUT_SECONDS:-}" ]; then
+  if [[ "${EVIDENT_IDLE_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then
+    IDLE_TIMEOUT_SECONDS="${EVIDENT_IDLE_TIMEOUT_SECONDS}"
+  else
+    warn "EVIDENT_IDLE_TIMEOUT_SECONDS='${EVIDENT_IDLE_TIMEOUT_SECONDS}' is not numeric; using the default ${IDLE_TIMEOUT_SECONDS}s instead of leaving the VM always-on"
+  fi
+fi
+
+# `setsid` so the tunnel outlives this hook: the script must return so AWS gets
+# its 200, while the tunnel keeps serving.
+start_tunnel() {
+  local runner_key="$1" api_url="$2" tunnel_url="$3"
+  local restore_runner_credentials="${4:-false}" restore_history="${5:-false}"
+  local -a credential_flags=()
+  local -a session_db_flags=()
+  local -a opencode_config_flags=()
+
+  # The explicit fourth argument is set only by /run; /resume uses the default
+  # so a resumed VM never restores credentials over stores it already has.
+  if [ "${restore_runner_credentials}" = true ]; then
+    credential_flags=(--restore-runner-credentials)
+  fi
+
+  # Only a fresh /run asks the CLI to restore session history. /resume keeps
+  # the database from the snapshot and must not restore over it.
+  if [ "${restore_history}" = true ]; then
+    session_db_flags=(--restore-session-db)
+  fi
+
+  if [ -n "${RUNNER_OPENCODE_CONFIG:-}" ]; then
+    opencode_config_flags=(--opencode-config-overlay "${RUNNER_OPENCODE_CONFIG}")
+  fi
+
+  if tunnel_is_running; then
+    warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
+    return 0
+  fi
+
+  # The key travels in the child's environment ONLY: argv is world-readable
+  # through /proc.
+  # --enable-file-sync-to is what makes the UI's "connect a provider account"
+  # flow reach this VM: without it the CLI declines every queued file and the
+  # page can only say "this runner isn't accepting files". There is no shell
+  # into a MicroVM, so it is the ONLY way to seed model credentials on a runner
+  # that is already up — the S3 credential store is read at /run and /resume,
+  # never mid-life. The allow-list stays a single directory, as on ECS
+  # (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
+  # extra entry widens what Evident can write into a VM that executes agent
+  # code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
+  # an unset one abort rather than silently allow-list "/.claude".
+  EVIDENT_RUNNER_KEY="${runner_key}" setsid evident run \
+    --port "${OPENCODE_PORT}" \
+    --endpoint "${api_url}" \
+    --tunnel "${tunnel_url}" \
+    --opencode-pid-file "${OPENCODE_PID_FILE}" \
+    --litestream-config "${LITESTREAM_CONFIG_FILE}" \
+    --litestream-pid-file "${LITESTREAM_PID_FILE}" \
+    --session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
+    "${credential_flags[@]}" \
+    "${session_db_flags[@]}" \
+    "${opencode_config_flags[@]}" \
+    --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
+    --enable-file-sync-to "${HOME}/.claude" &
+
+  echo $! >"${TUNNEL_PID_FILE}"
+  log "tunnel started (pid $(cat "${TUNNEL_PID_FILE}"))"
+}
+
+# The worst case `evident run` can take to shut down gracefully on SIGTERM, in
+# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS,
+# apps/cli/src/commands/run.ts) + 2 s offline POST (notifyAgentDisconnected,
+# apps/cli/src/commands/agent-lookup.ts) + 5 s telemetry flush
+# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts). Each of the three is bounded there, so
+# this is a ceiling rather than a typical cost — an idle suspend finishes in a
+# couple of seconds. This is the ONE place the budget is written down; the CLI
+# only points back here, because restating it in three places produced #657.
+#
+# DOCUMENTATION ONLY — nothing is derived from this any more. The wait below
+# used to be pinned above it, which is the reasoning that talked #699 into 40 s;
+# since #718 an exited CLI is noticed on the first poll, so the wait is a
+# backstop chosen on its own merits and the two numbers are unrelated. Nothing
+# cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
+# three bounds cited above, so if one of them moves, update it here.
+# shellcheck disable=SC2034  # documentation; deliberately read by nothing
+CLI_SHUTDOWN_CEILING_SECONDS=32
+
+# How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
+# Since #718 this binds ONLY for a CLI that is still draining: one that has
+# already exited is detected on the first 0.1 s poll whatever this says, because
+# process_is_alive no longer mistakes its unreaped corpse for a live process.
+# That is what takes a routine suspend from ~41 s back to the ~2 s it measures.
+#
+# So 10 is chosen, not derived, and it sits in the middle of a real range. The
+# measured shutdown was 1255 ms of drain inside a 2322 ms total (#697): 5 s is
+# only ~2.2x that and re-arms #657's complaint of a kill landing mid-drain,
+# while 30 s never truncates a legitimate drain but rebuilds the headroom
+# problem #699 created — 41 s against a 60 s lifecycleTimeout leaves 19 s for
+# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
+# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
+# before us = 20 s, inside the ~55 s the in-VM server allows this script
+# (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
+# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
+# overrunning that fails the whole lifecycle transition, which is worse than the
+# kill this replaces. The residual cost is a drain longer than 10 s being
+# truncated; that work stays `processing` server-side and is re-adopted on the
+# next start (ADR-0046).
+#
+# src/image/hook-scripts.test.ts holds both that this stays small and that it
+# fits the hook timeout, and times a real SIGKILL against the override to prove
+# the loop HONOURS it rather than a hardcoded deadline. The env override is for
+# tests only, so they need not burn 10 s of wall clock.
+TUNNEL_STOP_WAIT_SECONDS="${EVIDENT_TUNNEL_STOP_WAIT_SECONDS:-10}"
+
+stop_tunnel() {
+  if ! tunnel_is_running; then
+    # Clear the file here too, not only on the path below: `evident run` has
+    # self-exit paths that never reach this function (auth expired, idle
+    # timeout, a crash), so the pid file routinely outlives the process it
+    # names. Leaving it means every later guard re-reads a dead pid — harmless
+    # while process_is_alive agrees it is dead, and #718 again the moment
+    # anything starts reaping orphans and the pid gets recycled.
+    rm -f "${TUNNEL_PID_FILE}"
+    log "no tunnel to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${TUNNEL_PID_FILE}")"
+  # One drain length for every caller: /suspend and /terminate always run over
+  # a connected tunnel, and /run's and /resume's own cleanup traps only reach a
+  # live tunnel here when it was never started (the cheap no-op branch above) —
+  # there is no path left where a just-spawned, not-yet-connected tunnel is the
+  # one being stopped, so there is nothing left to special-case.
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( TUNNEL_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  # This call, not the loop's, is the deciding read: it is what the `if` below
+  # branches on, so PROC_DEAD_REASON is fresh by construction. A corpse can
+  # still be reaped between the loop's last poll and this call, so the reason
+  # named below is what THIS call saw, not necessarily what the loop saw.
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "tunnel ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${TUNNEL_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${TUNNEL_PID_FILE}"
+  log "tunnel stopped ${outcome}"
+}
+
+# --- check_runner_key (#1172) ------------------------------------------------
+#
+# Answers exactly one question before opencode/the tunnel start spending this
+# boot's SIGTERM budget on a key that cannot work: "does the runner key in this
+# payload authenticate against Evident?" Delegates entirely to the CLI's own
+# `evident status --json` (apps/cli/src/commands/status.ts) rather than
+# reimplementing its auth logic here — that command's `reason` field is the
+# published contract this function reads, and its own header states the
+# absent-vs-contrary distinction this function must honour.
+#
+# Keyed on the JSON `reason`, NEVER on the exit code: an older CLI without a
+# `status` subcommand exits 1 from Commander's own "unknown command" handling,
+# which under an exit-code mapping would read as "the key was rejected" and
+# destroy every VM on a hook/CLI version skew — precisely the hazard this
+# image's README's "version skew" section flags. So no parseable JSON line
+# means no verdict, whatever the exit code says.
+#
+# Returns 0 for positive AND absent evidence (network/timeout/5xx/404/old-CLI/
+# crash) — the caller continues either way — and 1 ONLY for contrary evidence
+# (401, another 4xx, or no credentials resolved at all): the key is actually
+# wrong, not merely untested (development-workflow.mdc's "never fail a gate on
+# absent evidence"). Every branch logs a named, greppable RUNNER-KEY-* signal
+# so a caller that treats the return value as fatal (or doesn't) still leaves
+# a trace of which branch fired.
+#
+# A 404 is deliberately NOT contrary: it means the endpoint has no /me route,
+# so the key was never tested. Treating it as a rejection destroyed every VM
+# for a day when the run payload shipped a bare origin instead of the CLI's
+# /v1-prefixed one.
+check_runner_key() {
+  local runner_key="$1" api_url="$2"
+
+  # status.ts's own exit-code contract (its header comment) means this exits
+  # non-zero on EVERY branch except `ok`; guard the capture so an expected
+  # non-zero does not abort this function under the caller's `set -e` before
+  # the case below ever runs. Only stdout is
+  # captured: status.ts's own contract is one parseable JSON line and nothing
+  # else there, and — like litestream's stderr elsewhere in this file — its
+  # stderr is left to reach CloudWatch directly rather than being folded in,
+  # since a stray stderr line ahead of the JSON would otherwise break `jq`.
+  local response rc=0
+  response="$(EVIDENT_API_URL="${api_url}" EVIDENT_RUNNER_KEY="${runner_key}" evident status --json)" || rc=$?
+
+  # `-z` as well as jq's own exit code, and not as belt-and-braces: on EMPTY
+  # input jq has no value to report on, so `jq -e` exits 0 with empty output
+  # rather than its documented 4 (verified on jq 1.6, the image's). Empty
+  # stdout is exactly the old-CLI-skew and missing-binary shape this branch
+  # exists for, so keying only on the exit code sent precisely those cases to
+  # the unrecognised-reason arm below — same safe return, but a log line
+  # blaming the API for an odd answer instead of naming the CLI skew, which is
+  # the one thing an operator needs told here.
+  local reason
+  reason="$(printf '%s' "${response}" | jq -er '.reason' 2>/dev/null)" || reason=""
+  if [ -z "${reason}" ]; then
+    warn "RUNNER-KEY-UNVERIFIABLE: 'evident status --json' exited ${rc} with no parseable JSON on stdout (an old CLI without this subcommand, a missing binary, or a crash — see its stderr above, if any); the key was NOT validated, continuing anyway"
+    return 0
+  fi
+  local detail
+  detail="$(printf '%s' "${response}" | jq -er '.error // empty' 2>/dev/null)" || detail=""
+
+  case "${reason}" in
+    ok)
+      log "RUNNER-KEY-OK: evident status confirmed the runner key against ${api_url}"
+      return 0
+      ;;
+    unauthorized | no_credentials | http_error)
+      error "RUNNER-KEY-REJECTED: evident status reason=${reason} against ${api_url}${detail:+: ${detail}}"
+      return 1
+      ;;
+    unreachable)
+      warn "RUNNER-KEY-UNREACHABLE: could not reach ${api_url} to validate the runner key; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+    endpoint_not_found)
+      # Absent evidence, NOT contrary: a 404 means this endpoint has no /me
+      # route (typically a bare origin where the CLI wants the /v1-prefixed
+      # one), which says nothing about the key. Loud, because the runner will
+      # keep failing every API call until the endpoint is fixed.
+      warn "RUNNER-KEY-ENDPOINT-NOT-FOUND: ${api_url} has no /me route${detail:+: ${detail}}; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+    *)
+      warn "RUNNER-KEY-UNKNOWN-REASON: evident status returned an unrecognised reason='${reason}'; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+  esac
+}
+# --- check_runner_key (end) ---------------------------------------------------