@evident-ai/runner-cdk 0.1.1-dev.0fe02fb → 0.1.1-dev.25e01b7

package/dist/microvm-image-context/hooks/common.sh

@@ -0,0 +1,1255 @@
+#!/usr/bin/env bash
+# Sourced by every hook script. Nothing here runs at image build time.
+#
+# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
+# (packages/runner-image/entrypoint.sh is the first), so both are held to it by
+# packages/runner-synchroniser/src/shell-contract.test.ts — which derives what
+# `run_synchroniser` below must handle from the subcommands this shell calls.
+
+# Installed from npm by the image (docker/Dockerfile's ARG
+# RUNNER_SYNCHRONISER_VERSION) and resolved off PATH here. The Dockerfile's
+# required-binary assertion is the loud failure if it is missing.
+SYNCHRONISER="runner-synchroniser"
+OPENCODE_PORT="${OPENCODE_PORT:-4096}"
+
+# A hook script keeps no memory between invocations, but /resume must re-dial
+# with the runner key /run was given, so /run leaves it here. Deliberate, on two
+# axes: tmpfs means it never reaches the block device, and the SHARED image
+# snapshot is taken at build time — long before /run — so the key cannot enter
+# the artefact every VM boots from. And at 0600 owned by uid 10001 it is
+# readable by nobody who could not already read it out of the tunnel process's
+# /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
+# shellcheck disable=SC2034  # read by the scripts that source this file
+CONTEXT_FILE="/dev/shm/evident-run-context"
+TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
+OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
+LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
+
+# Where the runner's credential store lives inside the durable-state bucket.
+# The BUCKET is the same for every VM from an image version, so the stack bakes
+# it into the image environment; the PREFIX selects one runner's store, so it is
+# per-VM and can only arrive in the /run payload. /run leaves it here for the
+# hooks that flush back to it. A file of its own rather than a line of
+# CONTEXT_FILE: it is not a secret, so /suspend never has to read the runner key
+# to find out where to write.
+STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
+
+# The generated litestream.yml (#812). tmpfs for the same two reasons as the
+# files above: it must never reach the block device (Q6, `docs`), and it
+# survives suspend/resume, which is what lets `/resume` start litestream again
+# with no regeneration cost. Regenerated only when absent or empty
+# (`ensure_litestream_config`, below).
+LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
+
+# Set when a `session-db-classify` answer of 31 (replica unusable), or a
+# restore this hook gave up on for its own reasons (a truncated `timeout`, a
+# broken tool), means this boot must NOT start `litestream replicate` — doing
+# so would let a partial/fresh local DB overwrite a replica this boot never
+# proved is safe to write over. Cleared at the top of every `restore_session_db`
+# call (i.e. every `/run`) before any other decision, so a marker left by an
+# earlier boot cannot silently disable replication for the VM's whole life.
+SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
+
+log() { echo "[hook:$(basename "$0")] $*"; }
+warn() { echo "[hook:$(basename "$0")] $*" >&2; }
+error() { echo "[hook:$(basename "$0")] ERROR: $*" >&2; }
+
+# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
+# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
+# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
+# unusable)/32 (retry) — see `packages/runner-synchroniser/src/cli.ts`'s own
+# comment for what each means, not restated here. Any OTHER non-zero status
+# means the tool itself broke, which is the only case worth an ERROR here —
+# EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
+# that is an intentional bound firing, not a broken tool, so the caller
+# classifies it instead (restore_credentials, #930). A trailing
+# `--evident-deadline=N` (stripped below before forwarding to the CLI, and
+# never produced by anything but timed_synchroniser) is how a caller opts in;
+# every other caller — sync_credentials, restore_session_db's `env` /
+# `litestream-config` / `session-db-classify`, and this file's own contract
+# test — passes none, so 124/137 there still means a genuine external SIGKILL
+# (e.g. an OOM kill) and must keep producing the ERROR below.
+run_synchroniser() {
+  local -a call_args=("$@")
+  local deadline=""
+  local last=$(( ${#call_args[@]} - 1 ))
+  if [ "${last}" -ge 0 ] && [[ "${call_args[last]}" == --evident-deadline=* ]]; then
+    deadline="${call_args[last]#--evident-deadline=}"
+    call_args=("${call_args[@]:0:${last}}")
+  fi
+
+  local rc=0
+  local -a launcher=()
+  [ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
+  "${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
+  case "${rc}" in
+    0 | 10 | 30 | 31 | 32) ;;
+    124 | 137)
+      [ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
+      ;;
+    *) error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
+  esac
+  return "${rc}"
+}
+
+# Wall-clock milliseconds. ${EPOCHREALTIME} is a bash 5 builtin (the image is
+# node:22-bookworm-slim → bash 5.2) rather than `date +%s%3N`, which has to
+# exec /bin/date inside the caller's command substitution: measured on a loaded
+# box, that exec put ~190ms of its own cost INSIDE the window being measured,
+# against ~23ms for this — overhead charged to the very number this exists to
+# learn. The remaining ~23ms is the command substitution's subshell, kept
+# because removing it means an out-parameter global, and ~2% of a ~1.2s call
+# biases the reading generous, which is the safe direction for sizing a
+# deadline.
+#
+# The `[.,]` is not paranoia: ${EPOCHREALTIME} renders its decimal separator
+# from LC_NUMERIC, so a comma locale would otherwise silently produce garbage
+# here. Stripping either turns it into whole microseconds, which /1000 makes
+# milliseconds.
+#
+# Never fails its caller: a timing line is diagnostics, and hardening a
+# currently-working path is worse than the gap it closes (#931). The `date`
+# fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
+# timed_synchroniser turns into `elapsed_ms=unknown` rather than aborting /run
+# under `set -e` — and rather than a 0 that would read as a fast healthy call.
+now_ms() {
+  local now="${EPOCHREALTIME:-}"
+  if [ -n "${now}" ]; then
+    now="${now/[.,]/}"
+    echo "$((now / 1000))"
+    return 0
+  fi
+  date +%s%3N 2>/dev/null || echo 0
+}
+
+# Reports what one step COST, in a deliberately machine-greppable line (`op=`,
+# `elapsed_ms=`, `rc=`) so "how long does this actually take in the fleet?" is a
+# log query rather than another spike. `at_s` is where the step landed on the
+# hook's own SECONDS clock, which is what the pre-opencode budget in
+# hook-scripts.test.ts is derived against.
+#
+# Callable directly, not only through timed_synchroniser below, because the
+# steps that dominate the pre-opencode window CANNOT be wrapped: their stdout is
+# captured (`x="$(run_synchroniser env)"`, later `eval`'d), so a `log` line
+# emitted inside the substitution would be evaluated as configuration.
+#
+# `unknown`, never a number, when either end failed to read the clock (now_ms's
+# 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
+# very fast healthy call, which would quietly bias the fleet-wide sizing data
+# this line exists to produce, in the DANGEROUS direction (a deadline sized too
+# tight). A non-numeric value drops out of an aggregate instead of poisoning it.
+log_elapsed_since() {
+  local op="$1" started_ms="$2" rc="$3"
+
+  local finished_ms elapsed_ms="unknown"
+  finished_ms="$(now_ms)"
+  if [ "${started_ms}" -gt 0 ] && [ "${finished_ms}" -gt 0 ]; then
+    elapsed_ms="$((finished_ms - started_ms))"
+  fi
+
+  log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
+}
+
+# Runs ONE synchroniser call and reports what it COST, without changing what it
+# MEANS: the call's own status is returned untouched, so every caller's
+# `|| return 1` / `|| status=$?` keeps working exactly as before.
+#
+# `deadline` is an explicit parameter, forwarded to run_synchroniser as a
+# trailing `--evident-deadline=N` (never a global or dynamically-scoped
+# variable, #930) — every current caller (restore_credentials) always has a
+# positive remaining-budget value by the time it calls this, having already
+# decided to skip the call entirely otherwise.
+timed_synchroniser() {
+  local op="$1" deadline="$2"
+  shift 2
+
+  local started_ms rc=0
+  started_ms="$(now_ms)"
+  run_synchroniser "$@" "--evident-deadline=${deadline}" || rc=$?
+  log_elapsed_since "${op}" "${started_ms}" "${rc}"
+  return "${rc}"
+}
+
+# Exports what `runner-synchroniser` resolves its object-store location from
+# (packages/runner-synchroniser/src/config.ts). It treats either being empty as
+# "persistence disabled" and then reports every restore as a WARNING it still
+# exits 0 for — so an unset value has to be caught HERE, where it can still be
+# told apart from "the store is simply empty".
+load_state_config() {
+  if [ -z "${LITESTREAM_BUCKET:-}" ]; then
+    error "LITESTREAM_BUCKET is unset; the image is missing the durable-state bucket the stack bakes in"
+    return 1
+  fi
+
+  LITESTREAM_PREFIX="$(cat "${STATE_PREFIX_FILE}" 2>/dev/null || true)"
+  if [ -z "${LITESTREAM_PREFIX}" ]; then
+    error "no durable-state prefix at ${STATE_PREFIX_FILE}; /run records the payload's state_prefix there before anything can be restored or flushed"
+    return 1
+  fi
+
+  export LITESTREAM_BUCKET LITESTREAM_PREFIX
+
+  # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
+  # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
+  # is just LITESTREAM_BUCKET. start_litestream and flush_session_db (#812
+  # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
+  # directly, so every entry point agrees on one predicate regardless of which
+  # earlier step in THIS hook process set it: restore_session_db's
+  # `eval "$(run_synchroniser env)"` on /run, or this function on /resume,
+  # /suspend and /terminate, none of which ever call the synchroniser before
+  # needing the answer.
+  export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
+}
+
+# /run starts no LLM task — it restores state, starts opencode and dials the
+# tunnel, none of which needs a model credential. Which credential a turn needs
+# depends on the model that turn's message asks for, so that question has no
+# single boot-time answer; it is answered per message, not here. What IS still
+# fatal is credential persistence being unavailable at all (an unset bucket/
+# prefix, or a broken synchroniser bundle) — that would silently drop refreshed
+# tokens on every suspend, so those two restores stay `|| return 1`.
+# `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
+# a log line, since the runtime forwards no hook stderr to the doorbell caller.
+
+# How long restore_credentials' shared step budget is, covering all THREE of
+# its synchroniser calls (restore claude, restore opencode, model-auth-ready)
+# as one window rather than a timeout apiece — a fixed per-call cap at the same
+# total would false-fire on the ordinary case of one slow call (a slow S3 GET),
+# which is precisely the boot this exists to keep healthy. From #930's 3-boot
+# sample: the step measured ~3-5s total (~1.2s per call, three node cold
+# starts of a 1.8 MB bundle) — a WEAK estimate this file's own
+# SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
+# routine slow call boots this VM with no model credentials until an operator
+# reconnects it; too loose and a hung call burns more of the hook's own
+# SIGTERM budget before the VM is destroyed mid-boot anyway. The env override
+# is for tests only, so they need not burn wall clock.
+CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECONDS:-8}"
+# The SIGKILL backstop `timeout -k` applies after its own SIGTERM, exactly like
+# SESSION_DB_RESTORE_KILL_GRACE_SECONDS above (a bare `timeout` only SIGTERMs,
+# and a call that ignored it would be unbounded again). Hard ceiling on the
+# step: DEADLINE + this = 10s, once — not per call.
+CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
+
+# What is left of the shared step budget, in whole seconds, `step_started_s`
+# seconds after it began. `SECONDS` (a bash builtin with no failure mode,
+# unlike `now_ms`) truncates, so a step that began at true time `s0` can read
+# less elapsed time than actually passed — the `- 1` restores the invariant
+# that no call is granted more than DEADLINE + GRACE from step start.
+remaining_credential_budget() {
+  local step_started_s="$1"
+  echo $(( CREDENTIAL_RESTORE_DEADLINE_SECONDS - (SECONDS - step_started_s) - 1 ))
+}
+
+# Runs one of the step's bounded calls: skips it with a named warn if the
+# shared budget is already exhausted, treats a timeout (124/137) as the
+# non-fatal warn-and-continue D1 approved, and keeps every other non-zero
+# fatal exactly as restore_credentials always has (`|| return 1` aborts /run
+# before opencode and the tunnel start).
+bounded_restore_call() {
+  local op="$1" step_started_s="$2"
+  shift 2
+
+  local remaining
+  remaining="$(remaining_credential_budget "${step_started_s}")"
+  if [ "${remaining}" -lt 1 ]; then
+    warn "CREDENTIAL-RESTORE-SKIPPED: ${op} skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
+    return 0
+  fi
+
+  local rc=0
+  timed_synchroniser "${op}" "${remaining}" "$@" || rc=$?
+  case "${rc}" in
+    0) ;;
+    124 | 137)
+      warn "CREDENTIAL-RESTORE-TIMEOUT: ${op} did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
+      ;;
+    *) return 1 ;;
+  esac
+  return 0
+}
+
+restore_credentials() {
+  local step_started_ms step_started_s
+  step_started_ms="$(now_ms)"
+  step_started_s="${SECONDS}"
+
+  load_state_config || return 1
+  bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
+  bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
+
+  # `restore` exits 0 whether it restored, found nothing, or discarded a corrupt
+  # object, so the synchroniser's own predicate is the only reliable answer to
+  # "can this VM authenticate a model right now". Its status is captured, not
+  # used as a condition, so it can never become this function's return status —
+  # `run` calls this bare (no `||`) under `set -euo pipefail`, and any
+  # non-zero return here would abort the hook before opencode and the tunnel
+  # ever start.
+  local auth_status=0 auth_remaining
+  auth_remaining="$(remaining_credential_budget "${step_started_s}")"
+  if [ "${auth_remaining}" -lt 1 ]; then
+    warn "CREDENTIAL-RESTORE-SKIPPED: model-auth-ready skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
+  else
+    timed_synchroniser model-auth-ready "${auth_remaining}" model-auth-ready || auth_status=$?
+    case "${auth_status}" in
+      0) ;; # some model auth is configured; the two restores above already logged what they found
+      10)
+        warn "no model credentials under s3://${LITESTREAM_BUCKET}/${LITESTREAM_PREFIX}/ \
+(neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
+ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
+a model provider fails until one is connected. See 'Seeding a credential store' in \
+infrastructure/evident-microvm/README.md."
+        ;;
+      124 | 137)
+        warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
+        ;;
+      *)
+        # run_synchroniser has already `error`ed the tool-broke line for a
+        # broken tool; any other unrecognised code lands here too. Do not
+        # claim there are no credentials — absent evidence is not contrary
+        # evidence (development-workflow.mdc).
+        warn "could not determine whether this VM has model credentials"
+        ;;
+    esac
+  fi
+  # The step total, in the SAME greppable shape as the per-call lines above, so
+  # one query answers both "what does credential restore cost?" and "which of
+  # its three calls cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
+  # + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
+  # comment above that constant) rather than per call. `rc=0` is deliberate
+  # here, not an absence of failure modes: a timed-out call already warned by
+  # name above and is non-fatal by design (D1); every failure that DOES abort
+  # this step has already returned 1 above.
+  log_elapsed_since credential-restore-step "${step_started_ms}" 0
+  return 0
+}
+
+# Best-effort by design: /suspend must still drop the tunnel and /terminate must
+# still clean up, so a flush that cannot happen is loud but never fatal.
+sync_credentials() {
+  load_state_config || return 0
+  run_synchroniser sync-once claude || true
+  run_synchroniser sync-once opencode || true
+}
+
+# --- Session DB restore (#812 WI-2) ----------------------------------------
+#
+# Restores opencode.db from S3 before opencode opens it — the read half only.
+# Writes NOTHING to S3 itself; replication back to S3 is started separately by
+# `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
+# `restore_session_db` are the MicroVM side of the same contract
+# packages/runner-image/entrypoint.sh's inlined restore already speaks, going
+# through the SAME runner-synchroniser CLI.
+
+# Reads the ~30 MB litestream binary into the page cache, in the background, so
+# the FIRST exec of it does not pay that read on the critical path.
+#
+# Why it is on the critical path at all: restore_session_db must finish before
+# start_opencode, and HOOK_SIGTERM_SECONDS is a POINT on the hook's SECONDS
+# clock, so a second spent here is a second taken from the rest of /run — and a
+# /run that overruns is SIGTERMed mid-boot and its VM destroyed. (Until #1172
+# the binding point was the tighter in-hook opencode readiness deadline; that
+# deadline is gone, the hook's own SIGTERM is not.)
+# Boot measurements put ~6.6-7.3s between the credential restore and the
+# litestream version line, of which only ~0.2s is accounted for by the two node
+# calls in between — the remainder is INFERRED to be this read, never measured.
+# The `litestream-prewarm` and `litestream-version` timings are what settle it
+# on the next boot.
+#
+# Nothing waits on it and nothing reads its output, so if that inference is
+# wrong this costs one backgrounded `cat`. Deleting the `litestream version`
+# diagnostic instead would save nothing: `litestream restore` two calls later
+# pays the identical read.
+prewarm_litestream() {
+  local binary
+  binary="$(command -v litestream 2>/dev/null || true)"
+  if [ -z "${binary}" ]; then
+    warn "litestream is not on PATH; skipping the boot pre-warm"
+    return 0
+  fi
+
+  # `cat` rather than a throwaway `litestream version`: a sequential read gets
+  # the whole file with readahead, where an exec demand-pages it.
+  (
+    local started_ms rc=0
+    started_ms="$(now_ms)"
+    cat "${binary}" >/dev/null 2>&1 || rc=$?
+    log_elapsed_since litestream-prewarm "${started_ms}" "${rc}"
+  ) &
+  log "pre-warming ${binary} in the background"
+}
+
+# Generates ${LITESTREAM_CONFIG_FILE} from runner-synchroniser's own renderer
+# — the SAME config module the credential restore/sync already goes through,
+# so there is no second copy of the bucket/prefix/path logic to drift
+# (litestream-config.ts's own header makes the same point). Only when absent
+# or empty, so `/resume` (WI-4) restarting litestream after a suspend/resume
+# snapshot — /dev/shm intact — pays nothing to regenerate it.
+#
+# Returns 1 on failure — fatal to the CALLER, never to the hook: every caller
+# (restore_session_db here, start_litestream in WI-3) must still let opencode
+# and the tunnel start regardless.
+ensure_litestream_config() {
+  if [ -s "${LITESTREAM_CONFIG_FILE}" ]; then
+    return 0
+  fi
+
+  # Timed OUTSIDE the substitution, not with timed_synchroniser: this call's
+  # stdout IS the config, so a timing line emitted inside `$(...)` would land in
+  # litestream.yml.
+  local rendered config_started_ms config_rc=0
+  config_started_ms="$(now_ms)"
+  rendered="$(run_synchroniser litestream-config)" || config_rc=$?
+  log_elapsed_since litestream-config "${config_started_ms}" "${config_rc}"
+  if [ "${config_rc}" -ne 0 ]; then
+    error "could not generate ${LITESTREAM_CONFIG_FILE}: runner-synchroniser litestream-config failed (see the error above)"
+    return 1
+  fi
+  printf '%s\n' "${rendered}" >"${LITESTREAM_CONFIG_FILE}" || {
+    error "could not write ${LITESTREAM_CONFIG_FILE}"
+    return 1
+  }
+
+  # Q7's diagnostic, for whoever can read this VM's CloudWatch log: which way
+  # litestream resolved its AWS region, and whether it can even see the
+  # names it would need to (never the VALUES of any OTHER AWS_* variable —
+  # AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN may be present here).
+  local region_empty="yes"
+  if printf '%s\n' "${rendered}" | grep -Eq '^ *region: *[^[:space:]]'; then
+    region_empty="no"
+  fi
+  # Hoisted out of the log line below so it can be timed: this is the FIRST exec
+  # of the litestream binary on the boot, and the pre-warm /run fires (see
+  # prewarm_litestream) is aimed squarely at what this number measures.
+  local version_started_ms litestream_version
+  version_started_ms="$(now_ms)"
+  litestream_version="$(litestream version 2>/dev/null || echo 'unknown')"
+  log_elapsed_since litestream-version "${version_started_ms}" 0
+  log "litestream ${litestream_version}; AWS_REGION=${AWS_REGION:-<unset>} AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-<unset>}; rendered litestream.yml region empty: ${region_empty}"
+}
+
+# Writes the marker AND names the reason, in one call, so the marker can never
+# appear silently. Every give-up path in restore_session_db calls this EXCEPT
+# the classifier's 31 (#1106 — see that branch), and no success path ever does.
+# It is what a later `/resume`/`start_litestream` (WI-3) reads to skip
+# replicating this boot: a local DB that is only PARTIALLY restored must never
+# be allowed to overwrite a replica this boot never proved safe to write over.
+# A merely FRESH local DB is not that danger — litestream continues the txid
+# chain — which is precisely why 31 no longer belongs here.
+mark_no_replicate() {
+  : >"${SESSION_DB_NO_REPLICATE_MARKER}"
+  warn "SESSION-DB-NO-REPLICATE: $1"
+}
+
+# Removes local session-DB debris a give-up path leaves behind. Called on
+# EVERY give-up path and NO success path: skipping it on a give-up is a real
+# bug (a truncated file left in place makes opencode fail to open a malformed
+# database, which fails /run outright — worse than the history loss this
+# discards); calling it on a success path would delete a good restore. One
+# `rm -f` per path, not a single command, so a failure removing one does not
+# skip the other two.
+discard_session_db_debris() {
+  rm -f "${OPENCODE_DB_PATH}" "${OPENCODE_DB_PATH}-wal" "${OPENCODE_DB_PATH}-shm"
+}
+
+# How long restore_session_db waits for ONE `litestream restore` before giving
+# up. From M4 (see the plan's §2): ~50 MB/s extrapolated from the x86_64 ECS
+# container's own live replica, so 7s covers roughly 300-350 MB — an
+# EXTRAPOLATION, not a MicroVM measurement (different arch, network path and
+# credential source); the boot line ensure_litestream_config logs, plus the
+# ${SECONDS} stamps in /run, are how it gets measured here for real.
+#
+# 7 rather than the 8 it was: hook-scripts.test.ts's pre-opencode ladder held an
+# allowance of 4s for the rest of this step, which live boots measured at
+# 6.7-7.4s, so the ladder passed while real boots blew the deadline. Correcting
+# that allowance is what takes this second — the remedy the ladder's own failure
+# message prescribes, and the one that leaves the SIGTERM worst case untouched.
+# The cost is real and unmeasured on this platform: ~50 MB less restorable
+# replica before a truncation that silently loses history. A fresh
+# state_prefix starts at zero, so this is generous for a long time, but
+# RoutingStrategy is 'per_user' only (routing-strategy.ts) and every prefix is
+# deterministic and permanent per (pool, routing key) — so a long-lived
+# runner's opencode.db grows monotonically across VM generations, exactly
+# like the ~1 GB ECS one M4 measured, and will eventually hit this ceiling.
+# Past it the restore is truncated (SESSION-DB-RESTORE-TRUNCATED, below) and
+# this boot skips replication — silent, permanent history loss for that
+# runner unless somebody reads the log. Not solved here (WI-0 Task 0.2).
+# The env override is for tests only, so they need not burn 7s of wall clock.
+SESSION_DB_RESTORE_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_RESTORE_DEADLINE_SECONDS:-7}"
+# The SIGKILL backstop `timeout -k` applies after its own SIGTERM: `timeout`
+# alone only SIGTERMs, and a litestream that ignored it would be unbounded
+# again. Hard ceiling on this step: DEADLINE + this = 9s.
+SESSION_DB_RESTORE_KILL_GRACE_SECONDS=2
+
+# The one bounded restore attempt, run before start_opencode (opencode opens
+# the DB the moment it starts, so this is the only place it can happen).
+# Every branch returns 0 — /run calls this bare under `set -euo pipefail`,
+# exactly like restore_credentials, and nothing about the session DB may ever
+# fail /run (Q3): a /run that exits 1 fails the whole lifecycle transition
+# and the user gets NO runner at all, which is worse than losing history.
+restore_session_db() {
+  # Cleanup, not a decision, and unconditionally first: /dev/shm survives
+  # suspend/resume and a /run retry, so a marker left by an earlier boot must
+  # never silently disable replication for the rest of this VM's life.
+  rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
+
+  # Config resolved ONCE here, in the synchroniser (config.ts owns
+  # OPENCODE_DB_PATH; no second copy of that path here), exactly as
+  # entrypoint.sh's own line does. Unlike ECS's `|| die`, a failure here must
+  # not abort the hook — `x="$(cmd)"` under `set -e` WOULD abort the whole
+  # shell on a failing substitution, which is exactly why it is caught
+  # explicitly rather than left to `set -e`.
+  # Timed OUTSIDE the substitution for the same reason ensure_litestream_config
+  # is: this call's stdout is `eval`'d, so a timing line emitted inside `$(...)`
+  # would be evaluated as configuration.
+  local synchroniser_env env_started_ms env_rc=0
+  env_started_ms="$(now_ms)"
+  synchroniser_env="$(run_synchroniser env)" || env_rc=$?
+  log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
+  if [ "${env_rc}" -ne 0 ]; then
+    mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
+    return 0
+  fi
+  # Guarded for the SAME reason as the substitution above, which is easy to miss:
+  # `eval` returns the status of what it ran, so a malformed line from a skewed
+  # bundle would abort /run right here under `set -e` — before start_opencode,
+  # so the user gets NO runner at all. entrypoint.sh:110 leaves the identical
+  # line bare because on ECS the blast radius is a crash-loop-and-replace; here
+  # it is the whole lifecycle transition, so Q3 makes it a give-up instead.
+  eval "${synchroniser_env}" || {
+    mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
+    return 0
+  }
+
+  # #931's exact lesson, one shell over (packages/runner-image/entrypoint.sh):
+  # the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
+  # 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
+  # would otherwise abort right here under `set -u` the moment
+  # `${OPENCODE_DB_PATH}` is dereferenced below. entrypoint.sh may `die` on
+  # this (ECS just crash-loops and replaces the task); this hook cannot —
+  # Q3 says nothing about the session DB may ever fail /run — so a missing
+  # OPENCODE_DB_PATH is a give-up, not a silent default (a guessed path would
+  # be actively wrong, not merely absent).
+  if [ -z "${OPENCODE_DB_PATH+x}" ]; then
+    mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
+    return 0
+  fi
+
+  # #770's arm, checked FIRST of the decisions and with its OWN message —
+  # never the SESSION-DB-REPLICA-UNUSABLE one below (#764's finding: an
+  # operator grepping that marker must not hit the disabled case).
+  # PERSISTENCE_BUCKET is empty exactly when LITESTREAM_BUCKET or
+  # LITESTREAM_PREFIX is unset (config.ts's persistenceEnabled) — the
+  # IDENTICAL predicate ECS's own litestream-launch gate uses
+  # (entrypoint.sh), so the two images agree on what "disabled" means.
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated."
+    return 0
+  fi
+
+  if ! ensure_litestream_config; then
+    mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
+    return 0
+  fi
+
+  local restore_rc=0 restore_started_ms
+  restore_started_ms="$(now_ms)"
+  timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_RESTORE_DEADLINE_SECONDS}" \
+    litestream restore -config "${LITESTREAM_CONFIG_FILE}" \
+    -if-db-not-exists -if-replica-exists "${OPENCODE_DB_PATH}" || restore_rc=$?
+  log_elapsed_since session-db-restore "${restore_started_ms}" "${restore_rc}"
+
+  # `timeout`'s OWN codes, handled BEFORE classifying: 124/137 are a
+  # TRUNCATED restore, not a corrupt replica — handing them to
+  # session-db-classify would mislabel a slow/large replica as one. 125-127
+  # mean `timeout` (or litestream itself) is broken, not the replica.
+  case "${restore_rc}" in
+    124 | 137)
+      discard_session_db_debris
+      mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
+      return 0
+      ;;
+    125 | 126 | 127)
+      error "litestream restore could not even run (timeout exited ${restore_rc})"
+      discard_session_db_debris
+      mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
+      return 0
+      ;;
+  esac
+
+  local classify_rc=0
+  run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
+  case "${classify_rc}" in
+    0) ;; # restored, or no replica yet — the CLI already logged which
+    31)
+      # The classifier already discarded the local debris (session-db.ts) — and
+      # that is exactly why this is the ONE give-up path that does NOT
+      # mark_no_replicate (#1106). Every other give-up here may be sitting on a
+      # half-restored DB, so the marker stays; a 31 is structurally guaranteed
+      # to be a FRESH one, and replicating it starts a new backup chain instead
+      # of leaving this boot with a zero-width backup window. Same reasoning,
+      # and same one-line change, as packages/runner-image/entrypoint.sh's `31)`.
+      warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and starting a fresh backup chain this boot (see the WARNING above)"
+      ;;
+    32)
+      # The CLI's own contract for 32 is "re-run litestream restore and ask
+      # again", but this hook has budget for exactly ONE attempt (Q4) — name
+      # the deviation so nobody reads this as a bug.
+      discard_session_db_debris
+      mark_no_replicate "session-db-classify asked for another restore attempt (32), but this hook has budget for only one; treating it as a give-up rather than retrying"
+      ;;
+    30)
+      # The CLI already logged its own FATAL line above; Q3 still says boot
+      # fresh rather than fail /run.
+      discard_session_db_debris
+      mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
+      ;;
+    *)
+      # run_synchroniser already logged the "tool broke" ERROR for this.
+      discard_session_db_debris
+      mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
+      ;;
+  esac
+
+  return 0
+}
+# --- Session DB restore (end) -----------------------------------------------
+
+# `kill -0` answers "does this pid exist", which is not the question any caller
+# here is asking. A process that has exited but has not been reaped — a zombie —
+# still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
+# normal case for everything these hooks start: `start_tunnel`/`start_opencode`
+# background a process that outlives the hook, the hook shell must return so AWS
+# gets its 200, and PID 1 in this image is a bare node hook server with no init
+# (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
+# the rest of the VM's life, which made `stop_tunnel` burn its whole budget
+# SIGKILLing a corpse and made a stale pid file suppress the next
+# `start_tunnel`, leaving a resumed VM permanently offline (#718).
+#
+# This is zombie-safe, NOT identity-safe: a recycled pid reads R/S and is still
+# called alive, so the SIGKILL backstop could in principle hit an unrelated
+# process. Pre-existing limitation of the pid-file approach, out of scope here —
+# named so nobody over-trusts the helper (src/image/hook-scripts.test.ts).
+#
+# Also sets PROC_DEAD_REASON on every return path — "gone" for a `kill -0`
+# miss, "zombie" for an unreaped `Z`, reset to empty on the alive path so a
+# stale value from a previous call can never be read — for the caller that
+# needs to say WHICH death it was (stop_tunnel's outcome line), never for this
+# function itself: see the silence note below. File-scope init (below) is what
+# keeps a read of it safe under `set -u` before this has ever run.
+PROC_DEAD_REASON=""
+
+process_is_alive() {
+  if ! kill -0 "$1" 2>/dev/null; then
+    PROC_DEAD_REASON="gone"
+    return 1
+  fi
+
+  # `/proc/<pid>/status` rather than `/proc/<pid>/stat`, whose fields cannot be
+  # split safely when a comm contains a space or a paren. If the state cannot be
+  # read at all, keep the `kill -0` answer: absent evidence is not "dead", and
+  # reading it as dead would SIGTERM-and-forget a live, still-draining CLI.
+  # Three fields, not two: `read` hands the whole remainder to its LAST variable,
+  # so a trailing `_` is what keeps the state letter out of `Z (zombie)`.
+  #
+  # That deferral is silent HERE on purpose: `stop_tunnel` polls this 10x/s, so a
+  # warning inside it would bury the log. The flag lets `is_running` say it once
+  # per guard instead — absent evidence must warn as well as defer
+  # (development-workflow.mdc).
+  PROC_STATE_UNREADABLE=0
+  local state
+  read -r _ state _ < <(grep -m1 '^State:' "/proc/$1/status" 2>/dev/null) || {
+    PROC_STATE_UNREADABLE=1
+    PROC_DEAD_REASON="" # unreadable /proc still means "alive" below, not a dead reason
+    return 0
+  }
+
+  if [ "${state}" = "Z" ]; then
+    PROC_DEAD_REASON="zombie"
+    return 1
+  fi
+
+  PROC_DEAD_REASON=""
+  return 0
+}
+
+is_running() {
+  [ -s "$1" ] || return 1
+
+  local pid
+  pid="$(cat "$1")"
+  process_is_alive "${pid}" || return 1
+
+  # The one state in which this whole helper is back to being a bare `kill -0`
+  # (a `hidepid=` mount, a `/proc` that is not mounted): every liveness answer
+  # silently reverts to #718 — a corpse reads as alive, so suspend SIGKILLs it
+  # after the full wait and the next start is suppressed for the VM's life.
+  [ "${PROC_STATE_UNREADABLE}" = 0 ] \
+    || warn "could not read /proc/${pid}/status; falling back to 'kill -0', which reports an exited-but-unreaped process as alive (#718)"
+
+  return 0
+}
+
+# The snapshot bakes one machine id into every VM launched from this image
+# version, so /run replaces it. Best-effort per file: the hook runs as uid 10001
+# and these live in root-owned directories, so a VM that kept the snapshot's id
+# must still boot — but it says so, because it is a state worth being able to see.
+regenerate_machine_id() {
+  local machine_id
+  machine_id="$(tr -d '-' </proc/sys/kernel/random/uuid)"
+
+  for path in /etc/machine-id /var/lib/dbus/machine-id; do
+    printf '%s\n' "${machine_id}" >"${path}" 2>/dev/null \
+      || warn "could not rewrite ${path}; this VM keeps the snapshot's machine id"
+  done
+}
+
+tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
+opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
+litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
+
+# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
+# and an interpolation of a missing field is still a non-empty string, so a
+# payload with no runner_key would sail through as the literal "null".
+payload_is_complete() {
+  jq -e '
+    (.runner_key | type == "string" and length > 0)
+    and (.endpoints.api | type == "string" and length > 0)
+    and (.endpoints.tunnel | type == "string" and length > 0)
+    and (.state_prefix | type == "string" and length > 0)
+  ' >/dev/null 2>&1
+}
+
+# `setsid` so opencode outlives this hook: the script must return so AWS gets its
+# 200, while opencode keeps serving. The guard makes /run idempotent — a retry
+# after a killed hook must not put a second instance on the same port.
+start_opencode() {
+  if opencode_is_running; then
+    warn "opencode already running (pid $(cat "${OPENCODE_PID_FILE}")); reusing it"
+    return 0
+  fi
+
+  setsid opencode serve \
+    --hostname 127.0.0.1 \
+    --port "${OPENCODE_PORT}" \
+    --print-logs &
+
+  echo $! >"${OPENCODE_PID_FILE}"
+  log "opencode starting (pid $(cat "${OPENCODE_PID_FILE}"))"
+}
+
+stop_opencode() {
+  if ! opencode_is_running; then
+    # Clear the file here too, for the same reason stop_tunnel does: opencode
+    # has exit paths that never reach this function (a crash, an OOM kill), so
+    # the pid file routinely outlives the process it names, and leaving it is
+    # #718 again the moment anything starts reaping orphans and the pid is
+    # recycled.
+    rm -f "${OPENCODE_PID_FILE}"
+    log "no opencode to stop"
+    return 0
+  fi
+
+  kill -TERM "$(cat "${OPENCODE_PID_FILE}")" 2>/dev/null || true
+  rm -f "${OPENCODE_PID_FILE}"
+  log "opencode stopped"
+}
+
+# The ceiling stop_opencode_and_wait (below) waits before its SIGKILL backstop.
+# /terminate can afford a real drain — the VM is being torn down either way —
+# unlike /run's cleanup, which cannot (stop_opencode's own cheapness above is
+# load-bearing for that budget, §4, and must not grow a wait).
+OPENCODE_STOP_WAIT_SECONDS="${EVIDENT_OPENCODE_STOP_WAIT_SECONDS:-5}"
+
+# The graceful stop /terminate needs (#812 WI-4): SIGTERM, poll for exit,
+# SIGKILL backstop, clear the pid file. A NEW function rather than growing
+# stop_opencode itself, for the reason above — mirrors stop_tunnel's and
+# stop_litestream's shape, including PROC_DEAD_REASON's one deciding read
+# after the poll loop (never inside it, which would bury the log at 10x/s).
+# This is half of the ordered-shutdown invariant (plan §6): /terminate must
+# stop opencode BEFORE flush_session_db's final sync, or litestream could
+# snapshot while opencode is still writing its WAL and the last session
+# writes would be missing from S3.
+stop_opencode_and_wait() {
+  if ! opencode_is_running; then
+    rm -f "${OPENCODE_PID_FILE}"
+    log "no opencode to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${OPENCODE_PID_FILE}")"
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( OPENCODE_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "opencode ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${OPENCODE_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${OPENCODE_PID_FILE}"
+  log "opencode stopped ${outcome}"
+}
+
+# --- litestream replicate (#812 WI-3) ---------------------------------------
+#
+# The write half of session-DB persistence. restore_session_db (above) is the
+# read half only and never starts this. Q8 chose one backgrounded, unsupervised
+# `litestream replicate` over a periodic flush or a `replicate -exec` wrapper
+# around opencode: SIGTERM is `replicate`'s own final-sync trigger, so
+# stop_litestream (below) doubles as the checked flush /suspend and /terminate
+# need, and there is no supervisor in this image to hand the process to (no
+# init, no `wait -n` shell that could stay alive after the hook returns).
+
+# The graceful wait before the SIGKILL backstop in stop_litestream, below.
+# litestream's own sync is normally sub-second, so 10s is generous headroom —
+# chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
+LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
+
+# Guards, in this exact order and no other — reordering any of them re-arms the
+# invariant it exists to protect:
+#
+#   1. persistence disabled (#770) — bucket AND prefix both non-empty — is its
+#      OWN arm with its OWN message, checked FIRST, and must never log
+#      SESSION-DB-REPLICA-UNUSABLE: an operator grepping that marker must not
+#      hit the disabled case (#764).
+#   2. the no-replicate marker (restore_session_db above) — set when this boot's
+#      local DB may be only PARTIALLY restored, so starting replicate here would
+#      let it overwrite a replica this boot never proved safe to write over.
+#      A classifier 31 no longer sets it (#1106): that DB is guaranteed fresh,
+#      and skipping replicate for it was what made the history loss unbounded.
+#   3. no usable config — ensure_litestream_config (restore_session_db, above)
+#      already logged why it is missing; this is not a place to retry it.
+#   4. already running — idempotence, mirroring start_tunnel's own guard.
+#
+# Only past all four does it actually spawn, backgrounded with `setsid` for the
+# same reason start_opencode/start_tunnel are: the hook must return so AWS gets
+# its 200, while replicate keeps running. litestream's OWN stderr is left to
+# reach CloudWatch — never redirected — because there is no supervisor in this
+# image, so that is the only channel that can say why a replicator died.
+start_litestream() {
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing is replicated."
+    return 0
+  fi
+
+  if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
+    log "skipping litestream replicate: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
+    return 0
+  fi
+
+  if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
+    error "no usable ${LITESTREAM_CONFIG_FILE}; not starting litestream replicate"
+    return 0
+  fi
+
+  if litestream_is_running; then
+    warn "litestream already running (pid $(cat "${LITESTREAM_PID_FILE}")); reusing it"
+    return 0
+  fi
+
+  setsid litestream replicate -config "${LITESTREAM_CONFIG_FILE}" &
+
+  echo $! >"${LITESTREAM_PID_FILE}"
+  log "litestream replicate starting (pid $(cat "${LITESTREAM_PID_FILE}"))"
+}
+
+# The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
+# one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
+# would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
+# operator needs told apart, not a flatter clean/killed binary. /suspend and
+# /terminate call this: the SIGTERM it sends IS the checked flush they need.
+stop_litestream() {
+  if ! litestream_is_running; then
+    rm -f "${LITESTREAM_PID_FILE}"
+    log "no litestream to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${LITESTREAM_PID_FILE}")"
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( LITESTREAM_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "litestream ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${LITESTREAM_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${LITESTREAM_PID_FILE}"
+  log "litestream stopped ${outcome}"
+}
+
+# The cheap stop: one SIGTERM, one rm, one log line, NO poll loop — `/run`'s
+# own cleanup calls this, never stop_litestream, because that path only runs
+# after /run has already FAILED: no session happened yet, so a graceful final
+# sync is worth nothing against a SIGTERM budget that cannot afford another
+# wait on top of stop_tunnel's own. A distinct function rather than a mode
+# argument on stop_litestream:
+# stop_tunnel's own comment argues against an argument a future caller can get
+# wrong. litestream_is_running is the same O(1) check stop_litestream's own
+# no-op branch uses, not a poll — /run's cleanup trap fires this before
+# start_litestream is ever reached whenever an earlier step failed, and an
+# operator reading that log must not be told a stop signal went to a process
+# that never started.
+kill_litestream() {
+  if ! litestream_is_running; then
+    rm -f "${LITESTREAM_PID_FILE}"
+    log "no litestream to stop"
+    return 0
+  fi
+
+  kill -TERM "$(cat "${LITESTREAM_PID_FILE}")" 2>/dev/null || true
+  rm -f "${LITESTREAM_PID_FILE}"
+  log "litestream stop signalled (no wait)"
+}
+# --- litestream replicate (end) ----------------------------------------------
+
+# --- flush_session_db (#812 WI-4) -------------------------------------------
+#
+# The checked, synchronous flush /suspend and /terminate need before they
+# finish tearing down. Never fatal, for the same reason every session-DB
+# function in this file is: /suspend and /terminate must complete their own
+# teardown regardless of whether S3 could be reached.
+
+# Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
+# own sync is normally sub-second (M5, the plan's grounding); 10s is generous
+# headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
+# measured worst case. The SIGKILL backstop below reuses
+# SESSION_DB_RESTORE_KILL_GRACE_SECONDS rather than its own literal: it is the
+# identical concept (bare `timeout` only SIGTERMs, and a litestream that
+# ignored it would be unbounded again), and there is no reason for the restore
+# and flush paths to ever drift on how long a SIGTERM gets to land.
+SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
+
+# Guards mirror start_litestream's first three exactly (disabled -> marker ->
+# config) — a flush must never attempt work those guards would have refused
+# to start in the first place. PERSISTENCE_BUCKET is the SAME variable
+# start_litestream reads, so both agree regardless of which earlier step in
+# THIS process set it (load_state_config, above, or restore_session_db's
+# eval, on /run).
+#
+# Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
+# trigger — and only once it is confirmed stopped does the synchronous -once
+# flush run, as a second, checked pass. Running -once while the daemon is
+# still alive would put two writers on one prefix, the single-writer
+# invariant this whole feature exists to protect (Q8) — this ordering is not
+# negotiable.
+#
+# If the daemon has already died (a missing or stale pid file), that IS the
+# detection this needs: SESSION-DB-REPLICATOR-DIED names it loudly instead of
+# silently skipping straight to the flush (development-workflow.mdc: every
+# recovery branch emits a server-visible signal). The flush still runs either
+# way — it is what actually gets any unreplicated writes to S3 before the
+# caller's next step.
+flush_session_db() {
+  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
+    warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing to flush."
+    return 0
+  fi
+
+  if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
+    log "skipping session-DB flush: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
+    return 0
+  fi
+
+  if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
+    error "no usable ${LITESTREAM_CONFIG_FILE}; cannot flush the session DB"
+    return 0
+  fi
+
+  if litestream_is_running; then
+    stop_litestream
+  else
+    warn "SESSION-DB-REPLICATOR-DIED: no live litestream at ${LITESTREAM_PID_FILE}; flushing one-shot anyway"
+  fi
+
+  # Only reached with the daemon confirmed stopped, or never running — never
+  # concurrently with it (see above).
+  local flush_rc=0
+  timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
+    litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
+
+  if [ "${flush_rc}" -eq 0 ]; then
+    log "SESSION-DB-FINAL-FLUSH: succeeded, ${SECONDS}s into the hook"
+  else
+    warn "SESSION-DB-FINAL-FLUSH: litestream replicate -once exited ${flush_rc}, ${SECONDS}s into the hook; some writes since the last sync may not have reached S3"
+  fi
+}
+# --- flush_session_db (end) --------------------------------------------------
+
+# How long the guest CLI runs with no activity before it exits itself
+# (`evident run --idle-timeout`, apps/cli/src/commands/run.ts), which is what
+# turns a truly-abandoned VM into the clean-offline POST that lets Evident
+# suspend it (#732). Sized from the measured cost of guessing wrong rather than
+# the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
+# ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
+# napping a VM whose user comes straight back costs ~10 s — against ~$0.30/h
+# that is cheap enough that the balance sits far nearer the floor than the
+# ceiling. Not AT the floor, though: the CLI's idle detector needs 2 clear poll
+# cycles (`run.ts`'s `idlePolls >= 2`, ≥4 s of real time), so a value near that
+# would spend more time suspending/resuming than idle.
+# ECS's waker uses 900 s instead only because *its* cold start is far slower
+# than this VM's ~2 s resume — not evidence this default should match it.
+#
+# NOT part of the hook's own teardown budget (TUNNEL_STOP_WAIT_SECONDS et al.,
+# below): it governs the CLI's lifetime long after the hook has already
+# returned its 200, so it is deliberately outside that arithmetic.
+#
+# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
+# packages/runner-image/entrypoint.sh uses for the ECS runner's own
+# idle-timeout flag, so an operator who knows one knows the other. A
+# non-numeric override must never silently DROP the flag — that degrades to
+# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
+# warns and falls back to the default instead.
+#
+# Deliberately NOT in the /run payload yet: doing so would touch the doorbell
+# contract in four coordinated places for no capability today. #716 shipped
+# the swarm/per-user provisioner but explicitly scoped control-plane
+# idle/suspend policy OUT (its own "Phase 3" — see the issue's "Out of
+# Scope"), so a per-pool idle policy is still a clean follow-up, not
+# something #716 already provides a home for.
+IDLE_TIMEOUT_SECONDS=120
+if [ -n "${EVIDENT_IDLE_TIMEOUT_SECONDS:-}" ]; then
+  if [[ "${EVIDENT_IDLE_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then
+    IDLE_TIMEOUT_SECONDS="${EVIDENT_IDLE_TIMEOUT_SECONDS}"
+  else
+    warn "EVIDENT_IDLE_TIMEOUT_SECONDS='${EVIDENT_IDLE_TIMEOUT_SECONDS}' is not numeric; using the default ${IDLE_TIMEOUT_SECONDS}s instead of leaving the VM always-on"
+  fi
+fi
+
+# `setsid` so the tunnel outlives this hook: the script must return so AWS gets
+# its 200, while the tunnel keeps serving.
+start_tunnel() {
+  local runner_key="$1" api_url="$2" tunnel_url="$3"
+
+  if tunnel_is_running; then
+    warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
+    return 0
+  fi
+
+  # The key travels in the child's environment ONLY: argv is world-readable
+  # through /proc.
+  # --enable-file-sync-to is what makes the UI's "connect a provider account"
+  # flow reach this VM: without it the CLI declines every queued file and the
+  # page can only say "this runner isn't accepting files". There is no shell
+  # into a MicroVM, so it is the ONLY way to seed model credentials on a runner
+  # that is already up — the S3 credential store is read at /run and /resume,
+  # never mid-life. The allow-list stays a single directory, as on ECS
+  # (packages/runner-image/entrypoint.sh): the flag is repeatable, but every
+  # extra entry widens what Evident can write into a VM that executes agent
+  # code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
+  # an unset one abort rather than silently allow-list "/.claude".
+  EVIDENT_RUNNER_KEY="${runner_key}" setsid evident run \
+    --port "${OPENCODE_PORT}" \
+    --endpoint "${api_url}" \
+    --tunnel "${tunnel_url}" \
+    --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
+    --enable-file-sync-to "${HOME}/.claude" &
+
+  echo $! >"${TUNNEL_PID_FILE}"
+  log "tunnel started (pid $(cat "${TUNNEL_PID_FILE}"))"
+}
+
+# The worst case `evident run` can take to shut down gracefully on SIGTERM, in
+# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS,
+# apps/cli/src/commands/run.ts) + 2 s offline POST (notifyAgentDisconnected,
+# apps/cli/src/commands/agent-lookup.ts) + 5 s telemetry flush
+# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts). Each of the three is bounded there, so
+# this is a ceiling rather than a typical cost — an idle suspend finishes in a
+# couple of seconds. This is the ONE place the budget is written down; the CLI
+# only points back here, because restating it in three places produced #657.
+#
+# DOCUMENTATION ONLY — nothing is derived from this any more. The wait below
+# used to be pinned above it, which is the reasoning that talked #699 into 40 s;
+# since #718 an exited CLI is noticed on the first poll, so the wait is a
+# backstop chosen on its own merits and the two numbers are unrelated. Nothing
+# cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
+# three bounds cited above, so if one of them moves, update it here.
+# shellcheck disable=SC2034  # documentation; deliberately read by nothing
+CLI_SHUTDOWN_CEILING_SECONDS=32
+
+# How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
+# Since #718 this binds ONLY for a CLI that is still draining: one that has
+# already exited is detected on the first 0.1 s poll whatever this says, because
+# process_is_alive no longer mistakes its unreaped corpse for a live process.
+# That is what takes a routine suspend from ~41 s back to the ~2 s it measures.
+#
+# So 10 is chosen, not derived, and it sits in the middle of a real range. The
+# measured shutdown was 1255 ms of drain inside a 2322 ms total (#697): 5 s is
+# only ~2.2x that and re-arms #657's complaint of a kill landing mid-drain,
+# while 30 s never truncates a legitimate drain but rebuilds the headroom
+# problem #699 created — 41 s against a 60 s lifecycleTimeout leaves 19 s for
+# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
+# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
+# before us = 20 s, inside the ~55 s the in-VM server allows this script
+# (DEFAULT_TIMEOUT_SECONDS, packages/lambda-microvm-runtime/src/runtime.ts),
+# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
+# overrunning that fails the whole lifecycle transition, which is worse than the
+# kill this replaces. The residual cost is a drain longer than 10 s being
+# truncated; that work stays `processing` server-side and is re-adopted on the
+# next start (ADR-0046).
+#
+# src/image/hook-scripts.test.ts holds both that this stays small and that it
+# fits the hook timeout, and times a real SIGKILL against the override to prove
+# the loop HONOURS it rather than a hardcoded deadline. The env override is for
+# tests only, so they need not burn 10 s of wall clock.
+TUNNEL_STOP_WAIT_SECONDS="${EVIDENT_TUNNEL_STOP_WAIT_SECONDS:-10}"
+
+stop_tunnel() {
+  if ! tunnel_is_running; then
+    # Clear the file here too, not only on the path below: `evident run` has
+    # self-exit paths that never reach this function (auth expired, idle
+    # timeout, a crash), so the pid file routinely outlives the process it
+    # names. Leaving it means every later guard re-reads a dead pid — harmless
+    # while process_is_alive agrees it is dead, and #718 again the moment
+    # anything starts reaping orphans and the pid gets recycled.
+    rm -f "${TUNNEL_PID_FILE}"
+    log "no tunnel to stop"
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${TUNNEL_PID_FILE}")"
+  # One drain length for every caller: /suspend and /terminate always run over
+  # a connected tunnel, and /run's and /resume's own cleanup traps only reach a
+  # live tunnel here when it was never started (the cheap no-op branch above) —
+  # there is no path left where a just-spawned, not-yet-connected tunnel is the
+  # one being stopped, so there is nothing left to special-case.
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local tenths=0
+  local deadline_tenths=$(( TUNNEL_STOP_WAIT_SECONDS * 10 ))
+  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
+    process_is_alive "${pid}" || break
+    sleep 0.1
+    tenths=$(( tenths + 1 ))
+  done
+
+  # This call, not the loop's, is the deciding read: it is what the `if` below
+  # branches on, so PROC_DEAD_REASON is fresh by construction. A corpse can
+  # still be reaped between the loop's last poll and this call, so the reason
+  # named below is what THIS call saw, not necessarily what the loop saw.
+  local outcome
+  if process_is_alive "${pid}"; then
+    warn "tunnel ${pid} ignored SIGTERM; killing"
+    kill -KILL "${pid}" 2>/dev/null || true
+    outcome="after SIGKILL (ignored SIGTERM for ${TUNNEL_STOP_WAIT_SECONDS}s)"
+  else
+    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
+  fi
+
+  rm -f "${TUNNEL_PID_FILE}"
+  log "tunnel stopped ${outcome}"
+}
+
+# --- check_runner_key (#1172) ------------------------------------------------
+#
+# Answers exactly one question before opencode/the tunnel start spending this
+# boot's SIGTERM budget on a key that cannot work: "does the runner key in this
+# payload authenticate against Evident?" Delegates entirely to the CLI's own
+# `evident status --json` (apps/cli/src/commands/status.ts) rather than
+# reimplementing its auth logic here — that command's `reason` field is the
+# published contract this function reads, and its own header states the
+# absent-vs-contrary distinction this function must honour.
+#
+# Keyed on the JSON `reason`, NEVER on the exit code: an older CLI without a
+# `status` subcommand exits 1 from Commander's own "unknown command" handling,
+# which under an exit-code mapping would read as "the key was rejected" and
+# destroy every VM on a hook/CLI version skew — precisely the hazard this
+# image's README's "version skew" section flags. So no parseable JSON line
+# means no verdict, whatever the exit code says.
+#
+# Returns 0 for positive AND absent evidence (network/timeout/5xx/404/old-CLI/
+# crash) — the caller continues either way — and 1 ONLY for contrary evidence
+# (401, another 4xx, or no credentials resolved at all): the key is actually
+# wrong, not merely untested (development-workflow.mdc's "never fail a gate on
+# absent evidence"). Every branch logs a named, greppable RUNNER-KEY-* signal
+# so a caller that treats the return value as fatal (or doesn't) still leaves
+# a trace of which branch fired.
+#
+# A 404 is deliberately NOT contrary: it means the endpoint has no /me route,
+# so the key was never tested. Treating it as a rejection destroyed every VM
+# for a day when the run payload shipped a bare origin instead of the CLI's
+# /v1-prefixed one.
+check_runner_key() {
+  local runner_key="$1" api_url="$2"
+
+  # status.ts's own exit-code contract (its header comment) means this exits
+  # non-zero on EVERY branch except `ok` — guarded exactly like
+  # ensure_litestream_config's `run_synchroniser litestream-config` call,
+  # above, so that expected non-zero does not abort this function under the
+  # caller's `set -e` before the case below ever runs. Only stdout is
+  # captured: status.ts's own contract is one parseable JSON line and nothing
+  # else there, and — like litestream's stderr elsewhere in this file — its
+  # stderr is left to reach CloudWatch directly rather than being folded in,
+  # since a stray stderr line ahead of the JSON would otherwise break `jq`.
+  local response rc=0
+  response="$(EVIDENT_API_URL="${api_url}" EVIDENT_RUNNER_KEY="${runner_key}" evident status --json)" || rc=$?
+
+  # `-z` as well as jq's own exit code, and not as belt-and-braces: on EMPTY
+  # input jq has no value to report on, so `jq -e` exits 0 with empty output
+  # rather than its documented 4 (verified on jq 1.6, the image's). Empty
+  # stdout is exactly the old-CLI-skew and missing-binary shape this branch
+  # exists for, so keying only on the exit code sent precisely those cases to
+  # the unrecognised-reason arm below — same safe return, but a log line
+  # blaming the API for an odd answer instead of naming the CLI skew, which is
+  # the one thing an operator needs told here.
+  local reason
+  reason="$(printf '%s' "${response}" | jq -er '.reason' 2>/dev/null)" || reason=""
+  if [ -z "${reason}" ]; then
+    warn "RUNNER-KEY-UNVERIFIABLE: 'evident status --json' exited ${rc} with no parseable JSON on stdout (an old CLI without this subcommand, a missing binary, or a crash — see its stderr above, if any); the key was NOT validated, continuing anyway"
+    return 0
+  fi
+  local detail
+  detail="$(printf '%s' "${response}" | jq -er '.error // empty' 2>/dev/null)" || detail=""
+
+  case "${reason}" in
+    ok)
+      log "RUNNER-KEY-OK: evident status confirmed the runner key against ${api_url}"
+      return 0
+      ;;
+    unauthorized | no_credentials | http_error)
+      error "RUNNER-KEY-REJECTED: evident status reason=${reason} against ${api_url}${detail:+: ${detail}}"
+      return 1
+      ;;
+    unreachable)
+      warn "RUNNER-KEY-UNREACHABLE: could not reach ${api_url} to validate the runner key; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+    endpoint_not_found)
+      # Absent evidence, NOT contrary: a 404 means this endpoint has no /me
+      # route (typically a bare origin where the CLI wants the /v1-prefixed
+      # one), which says nothing about the key. Loud, because the runner will
+      # keep failing every API call until the endpoint is fixed.
+      warn "RUNNER-KEY-ENDPOINT-NOT-FOUND: ${api_url} has no /me route${detail:+: ${detail}}; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+    *)
+      warn "RUNNER-KEY-UNKNOWN-REASON: evident status returned an unrecognised reason='${reason}'; the key was NOT validated, continuing anyway"
+      return 0
+      ;;
+  esac
+}
+# --- check_runner_key (end) ---------------------------------------------------