@evident-ai/runner-cdk 3.5.2-dev.8d3aa06 → 3.5.2-dev.bbd799b

package/dist/microvm-image-context/hooks/common.sh

@@ -1,905 +0,0 @@
-#!/usr/bin/env bash
-# Sourced by every hook script. Nothing here runs at image build time.
-#
-# These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
-# (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
-# runner/synchroniser/src/shell-contract.test.ts — which derives what
-# `run_synchroniser` below must handle from the subcommands this shell calls.
-
-# Installed from npm by the image (docker/Dockerfile's ARG
-# RUNNER_SYNCHRONISER_VERSION) and resolved off PATH here. The Dockerfile's
-# required-binary assertion is the loud failure if it is missing.
-SYNCHRONISER="runner-synchroniser"
-OPENCODE_PORT="${OPENCODE_PORT:-4096}"
-
-# A hook script keeps no memory between invocations, but /resume must re-dial
-# with the runner key /run was given, so /run leaves it here. Deliberate, on two
-# axes: tmpfs means it never reaches the block device, and the SHARED image
-# snapshot is taken at build time — long before /run — so the key cannot enter
-# the artefact every VM boots from. And at 0600 owned by uid 10001 it is
-# readable by nobody who could not already read it out of the tunnel process's
-# /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
-# shellcheck disable=SC2034  # read by the scripts that source this file
-CONTEXT_FILE="/dev/shm/evident-run-context"
-
-# The runtime injects MICROVM_ID only into /run, never /resume. This tmpfs file
-# survives suspend/resume so /resume can restore the id for every fresh CLI
-# process to self-report and acknowledge a fulfilled recycle request (#1906).
-# shellcheck disable=SC2034  # read by the scripts that source this file
-MICROVM_ID_FILE="/dev/shm/evident-microvm-id"
-TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
-OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
-LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
-
-# Where the runner's credential store lives inside the durable-state bucket.
-# The BUCKET is the same for every VM from an image version, so the stack bakes
-# it into the image environment; the PREFIX selects one runner's store, so it is
-# per-VM and can only arrive in the /run payload. /run leaves it here for the
-# hooks that flush back to it. A file of its own rather than a line of
-# CONTEXT_FILE: it is not a secret, so /suspend never has to read the runner key
-# to find out where to write.
-STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
-
-# The generated litestream.yml (#812). tmpfs for the same two reasons as the
-# files above: it must never reach the block device (Q6, `docs`), and it
-# survives suspend/resume, which is what lets `/resume` start litestream again
-# with no regeneration cost. The CLI regenerates it only when absent or empty.
-LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
-
-# Set by the CLI when session-DB restore or verification could not prove this
-# boot safe to replicate. The teardown hooks read it before flushing, and
-# `/terminate` removes it with the rest of the per-VM state.
-SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
-
-# Completion evidence for the CLI-owned credential flush. It lives in tmpfs,
-# is removed before every handshake, and is also removed by /terminate.
-CREDENTIAL_FLUSH_MARKER_FILE="/dev/shm/evident-credential-flush"
-
-hook_name() { printf '%s' "${0##*/}"; }
-log() { echo "[hook:$(hook_name)] $*"; }
-warn() { echo "[hook:$(hook_name)] $*" >&2; }
-error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
-
-# Returns the CLI's own exit code. `restore` logs domain outcomes and returns 0;
-# a non-zero status means the tool itself failed. `sync-once` returns 40 for
-# `failed`, `hashFailed`, and `localInvalid` (credentials not persisted), and 0
-# for every other outcome. The predicates answer "no" with 10, and
-# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
-# unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
-# exhausted, replica separated and local disposed) / 34 (could not prove
-# separation or disposal). Any status outside a command's contractual answers
-# means the tool itself broke, which is the only case worth an ERROR here.
-run_synchroniser() {
-  local rc=0
-  "${SYNCHRONISER}" "$@" || rc=$?
-  case "${rc}" in
-    0 | 10 | 30 | 31 | 32 | 33 | 34 | 40) ;;
-    *) error "synchroniser '$*' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
-  esac
-  return "${rc}"
-}
-
-# Wall-clock milliseconds. ${EPOCHREALTIME} is a bash 5 builtin (the image is
-# node:22-bookworm-slim → bash 5.2) rather than `date +%s%3N`, which has to
-# exec /bin/date inside the caller's command substitution: measured on a loaded
-# box, that exec put ~190ms of its own cost INSIDE the window being measured,
-# against ~23ms for this — overhead charged to the very number this exists to
-# learn. The remaining ~23ms is the command substitution's subshell, kept
-# because removing it means an out-parameter global, and ~2% of a ~1.2s call
-# biases the reading generous, which is the safe direction for sizing a
-# deadline.
-#
-# The `[.,]` is not paranoia: ${EPOCHREALTIME} renders its decimal separator
-# from LC_NUMERIC, so a comma locale would otherwise silently produce garbage
-# here. Stripping either turns it into whole microseconds, which /1000 makes
-# milliseconds.
-#
-# Never fails its caller: a timing line is diagnostics, and hardening a
-# currently-working path is worse than the gap it closes (#931). The `date`
-# fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
-# `log_elapsed_since` turns into `elapsed_ms=unknown` rather than aborting a
-# hook under `set -e` — and rather than a 0 that would read as a fast healthy call.
-now_ms() {
-  local now="${EPOCHREALTIME:-}"
-  if [ -n "${now}" ]; then
-    now="${now/[.,]/}"
-    echo "$((now / 1000))"
-    return 0
-  fi
-  date +%s%3N 2>/dev/null || echo 0
-}
-
-# Reports what one pre-warm COST, in a deliberately machine-greppable line
-# (`op=`, `elapsed_ms=`, `rc=`) so "how long does this actually take in the
-# fleet?" is a log query rather than another spike. `at_s` is where the step
-# landed on the hook's own SECONDS clock.
-#
-# `unknown`, never a number, when either end failed to read the clock (now_ms's
-# 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
-# very fast healthy call, which would quietly bias the fleet-wide sizing data
-# this line exists to produce, in the DANGEROUS direction (a deadline sized too
-# tight). A non-numeric value drops out of an aggregate instead of poisoning it.
-log_elapsed_since() {
-  local op="$1" started_ms="$2" rc="$3"
-
-  local finished_ms elapsed_ms="unknown"
-  finished_ms="$(now_ms)"
-  if [ "${started_ms}" -gt 0 ] && [ "${finished_ms}" -gt 0 ]; then
-    elapsed_ms="$((finished_ms - started_ms))"
-  fi
-
-  log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
-}
-
-# Exports what `runner-synchroniser` resolves its object-store location from
-# (runner/synchroniser/src/config.ts). It treats either being empty as
-# "persistence disabled" and then reports every restore as a WARNING it still
-# exits 0 for — so an unset value has to be caught HERE, where it can still be
-# told apart from "the store is simply empty".
-load_state_config() {
-  if [ -z "${LITESTREAM_BUCKET:-}" ]; then
-    error "LITESTREAM_BUCKET is unset; the image is missing the durable-state bucket the stack bakes in"
-    return 1
-  fi
-
-  LITESTREAM_PREFIX="$(cat "${STATE_PREFIX_FILE}" 2>/dev/null || true)"
-  if [ -z "${LITESTREAM_PREFIX}" ]; then
-    error "no durable-state prefix at ${STATE_PREFIX_FILE}; /run records the payload's state_prefix there before anything can be restored or flushed"
-    return 1
-  fi
-
-  export LITESTREAM_BUCKET LITESTREAM_PREFIX
-
-  # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
-  # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
-  # is just LITESTREAM_BUCKET. The CLI's replicator and flush_session_db (#812
-  # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
-  # directly, so every entry point agrees on one predicate regardless of which
-  # earlier step in THIS hook process set it; /resume, /suspend and /terminate
-  # all use this same function before any teardown decision.
-  export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
-}
-
-# Best-effort by design: the teardown hooks must still complete even when a
-# credential flush cannot happen.
-sync_credentials() {
-  load_state_config || return 0
-  run_synchroniser sync-once claude || true
-  run_synchroniser sync-once opencode || true
-}
-
-# --- boot pre-warms ----------------------------------------------------------
-
-# Reads the ~30 MB litestream binary into the page cache, in the background, so
-# the FIRST exec of it does not pay that read on the critical path.
-#
-# The pre-warm overlaps the CLI's node startup and auth round trip, so the first
-# litestream operation does not also pay the binary's cold page-cache read.
-# Boot measurements put ~6.6-7.3s before the litestream version line, of which
-# only ~0.2s is accounted for by the two node calls in between — the remainder
-# is INFERRED to be this read, never measured.
-# The `litestream-prewarm` and `litestream-version` timings are what settle it
-# on the next boot.
-#
-# Nothing waits on it and nothing reads its output, so if that inference is
-# wrong this costs one backgrounded `cat`. Deleting the `litestream version`
-# diagnostic instead would save nothing: `litestream restore` two calls later
-# pays the identical read.
-prewarm_litestream() {
-  local binary
-  binary="$(command -v litestream 2>/dev/null || true)"
-  if [ -z "${binary}" ]; then
-    warn "litestream is not on PATH; skipping the boot pre-warm"
-    return 0
-  fi
-
-  # `cat` rather than a throwaway `litestream version`: a sequential read gets
-  # the whole file with readahead, where an exec demand-pages it.
-  (
-    local started_ms rc=0
-    started_ms="$(now_ms)"
-    cat "${binary}" >/dev/null 2>&1 || rc=$?
-    log_elapsed_since litestream-prewarm "${started_ms}" "${rc}"
-  ) &
-  log "pre-warming ${binary} in the background"
-}
-
-# Reads the aws CLI v2 install tree into the page cache, in the background, so
-# the CLI's first runner-secret fetch does not pay first-touch I/O. The read
-# overlaps `evident run`'s node startup and auth round trip rather than a hook
-# deadline.
-#
-# A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
-# enough here: v2 ships as a real Python distribution (~7,500 files under the
-# resolved binary's own directory), and a cold invocation demand-pages a
-# scattered set of them (botocore's endpoints.json/partitions.json, service
-# model JSON, shared libs) — one boot measured #1997's fetch at 7,243ms cold
-# vs ~400ms warm, an ~18x gap the litestream read-ahead trick alone cannot
-# close. `find -exec cat` walks that whole tree instead of one file.
-prewarm_aws_cli() {
-  local binary tree
-  binary="$(command -v aws 2>/dev/null || true)"
-  if [ -z "${binary}" ]; then
-    warn "aws CLI is not on PATH; skipping the boot pre-warm"
-    return 0
-  fi
-  tree="$(dirname "$(readlink -f "${binary}")")"
-
-  (
-    local started_ms rc=0
-    started_ms="$(now_ms)"
-    find "${tree}" -type f -exec cat {} + >/dev/null 2>&1 || rc=$?
-    log_elapsed_since aws-cli-prewarm "${started_ms}" "${rc}"
-  ) &
-  log "pre-warming ${tree} in the background"
-}
-
-# `kill -0` answers "does this pid exist", which is not the question any caller
-# here is asking. A process that has exited but has not been reaped — a zombie —
-# still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
-# normal case for everything these hooks start: `start_tunnel` and its CLI child
-# background a process that outlives the hook, the hook shell must return so AWS
-# gets its 200, and PID 1 in this image is a bare node hook server with no init
-# (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
-# the rest of the VM's life, which made `stop_tunnel` burn its whole budget
-# SIGKILLing a corpse and made a stale pid file suppress the next
-# `start_tunnel`, leaving a resumed VM permanently offline (#718).
-#
-# This is zombie-safe, NOT identity-safe: a recycled pid reads R/S and is still
-# called alive, so the SIGKILL backstop could in principle hit an unrelated
-# process. Pre-existing limitation of the pid-file approach, out of scope here —
-# named so nobody over-trusts the helper (src/image/hook-scripts.test.ts).
-#
-# Also sets PROC_DEAD_REASON on every return path — "gone" for a `kill -0`
-# miss, "zombie" for an unreaped `Z`, reset to empty on the alive path so a
-# stale value from a previous call can never be read — for the caller that
-# needs to say WHICH death it was (stop_tunnel's outcome line), never for this
-# function itself: see the silence note below. File-scope init (below) is what
-# keeps a read of it safe under `set -u` before this has ever run.
-PROC_DEAD_REASON=""
-
-process_is_alive() {
-  if ! kill -0 "$1" 2>/dev/null; then
-    PROC_DEAD_REASON="gone"
-    return 1
-  fi
-
-  # `/proc/<pid>/status` rather than `/proc/<pid>/stat`, whose fields cannot be
-  # split safely when a comm contains a space or a paren. If the state cannot be
-  # read at all, keep the `kill -0` answer: absent evidence is not "dead", and
-  # reading it as dead would SIGTERM-and-forget a live, still-draining CLI.
-  # Three fields, not two: `read` hands the whole remainder to its LAST variable,
-  # so a trailing `_` is what keeps the state letter out of `Z (zombie)`.
-  #
-  # That deferral is silent HERE on purpose: `stop_tunnel` polls this 10x/s, so a
-  # warning inside it would bury the log. The flag lets `is_running` say it once
-  # per guard instead — absent evidence must warn as well as defer
-  # (development-workflow.mdc).
-  PROC_STATE_UNREADABLE=0
-  local state
-  read -r _ state _ < <(grep -m1 '^State:' "/proc/$1/status" 2>/dev/null) || {
-    PROC_STATE_UNREADABLE=1
-    PROC_DEAD_REASON="" # unreadable /proc still means "alive" below, not a dead reason
-    return 0
-  }
-
-  if [ "${state}" = "Z" ]; then
-    PROC_DEAD_REASON="zombie"
-    return 1
-  fi
-
-  PROC_DEAD_REASON=""
-  return 0
-}
-
-is_running() {
-  [ -s "$1" ] || return 1
-
-  local pid
-  pid="$(cat "$1")"
-  process_is_alive "${pid}" || return 1
-
-  # The one state in which this whole helper is back to being a bare `kill -0`
-  # (a `hidepid=` mount, a `/proc` that is not mounted): every liveness answer
-  # silently reverts to #718 — a corpse reads as alive, so suspend SIGKILLs it
-  # after the full wait and the next start is suppressed for the VM's life.
-  [ "${PROC_STATE_UNREADABLE}" = 0 ] \
-    || warn "could not read /proc/${pid}/status; falling back to 'kill -0', which reports an exited-but-unreaped process as alive (#718)"
-
-  return 0
-}
-
-# The snapshot bakes one machine id into every VM launched from this image
-# version, so /run replaces it. Best-effort per file: the hook runs as uid 10001
-# and these live in root-owned directories, so a VM that kept the snapshot's id
-# must still boot — but it says so, because it is a state worth being able to see.
-regenerate_machine_id() {
-  local machine_id
-  machine_id="$(tr -d '-' </proc/sys/kernel/random/uuid)"
-
-  for path in /etc/machine-id /var/lib/dbus/machine-id; do
-    printf '%s\n' "${machine_id}" >"${path}" 2>/dev/null \
-      || warn "could not rewrite ${path}; this VM keeps the snapshot's machine id"
-  done
-}
-
-tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
-opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
-litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
-
-# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
-# and an interpolation of a missing field is still a non-empty string, so a
-# payload with no runner_key would sail through as the literal "null".
-payload_is_complete() {
-  jq -e '
-    (.runner_key | type == "string" and length > 0)
-    and (.endpoints.api | type == "string" and length > 0)
-    and (.endpoints.tunnel | type == "string" and length > 0)
-    and (.state_prefix | type == "string" and length > 0)
-  ' >/dev/null 2>&1
-}
-
-stop_opencode() {
-  if ! opencode_is_running; then
-    # Clear the file here too, for the same reason stop_tunnel does: opencode
-    # has exit paths that never reach this function (a crash, an OOM kill), so
-    # the pid file routinely outlives the process it names, and leaving it is
-    # #718 again the moment anything starts reaping orphans and the pid is
-    # recycled.
-    rm -f "${OPENCODE_PID_FILE}"
-    log "no opencode to stop"
-    return 0
-  fi
-
-  kill -TERM "$(cat "${OPENCODE_PID_FILE}")" 2>/dev/null || true
-  rm -f "${OPENCODE_PID_FILE}"
-  log "opencode stopped"
-}
-
-# The ceiling stop_opencode_and_wait (below) waits before its SIGKILL backstop.
-# /terminate can afford a real drain — the VM is being torn down either way —
-# unlike /run's cleanup, which cannot (stop_opencode's own cheapness above is
-# load-bearing for that budget, §4, and must not grow a wait).
-OPENCODE_STOP_WAIT_SECONDS="${EVIDENT_OPENCODE_STOP_WAIT_SECONDS:-5}"
-
-# The graceful stop /terminate needs (#812 WI-4): SIGTERM, poll for exit,
-# SIGKILL backstop, clear the pid file. A NEW function rather than growing
-# stop_opencode itself, for the reason above — mirrors stop_tunnel's and
-# stop_litestream's shape, including PROC_DEAD_REASON's one deciding read
-# after the poll loop (never inside it, which would bury the log at 10x/s).
-# This is half of the ordered-shutdown invariant (plan §6): /terminate must
-# stop opencode BEFORE flush_session_db's final sync, or litestream could
-# snapshot while opencode is still writing its WAL and the last session
-# writes would be missing from S3.
-stop_opencode_and_wait() {
-  if ! opencode_is_running; then
-    rm -f "${OPENCODE_PID_FILE}"
-    log "no opencode to stop"
-    return 0
-  fi
-
-  local pid
-  pid="$(cat "${OPENCODE_PID_FILE}")"
-  kill -TERM "${pid}" 2>/dev/null || true
-
-  local tenths=0
-  local deadline_tenths=$(( OPENCODE_STOP_WAIT_SECONDS * 10 ))
-  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
-    process_is_alive "${pid}" || break
-    sleep 0.1
-    tenths=$(( tenths + 1 ))
-  done
-
-  local outcome
-  if process_is_alive "${pid}"; then
-    warn "opencode ${pid} ignored SIGTERM; killing"
-    kill -KILL "${pid}" 2>/dev/null || true
-    outcome="after SIGKILL (ignored SIGTERM for ${OPENCODE_STOP_WAIT_SECONDS}s)"
-  else
-    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
-  fi
-
-  rm -f "${OPENCODE_PID_FILE}"
-  log "opencode stopped ${outcome}"
-}
-
-# The graceful wait before the SIGKILL backstop in stop_litestream, below.
-# litestream's own sync is normally sub-second, so 10s is generous headroom —
-# chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
-LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
-
-# The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
-# one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
-# would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
-# operator needs told apart, not a flatter clean/killed binary. /suspend and
-# /terminate call this: the SIGTERM it sends IS the checked flush they need.
-stop_litestream() {
-  if ! litestream_is_running; then
-    rm -f "${LITESTREAM_PID_FILE}"
-    log "no litestream to stop"
-    return 0
-  fi
-
-  local pid
-  pid="$(cat "${LITESTREAM_PID_FILE}")"
-  kill -TERM "${pid}" 2>/dev/null || true
-
-  local tenths=0
-  local deadline_tenths=$(( LITESTREAM_STOP_WAIT_SECONDS * 10 ))
-  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
-    process_is_alive "${pid}" || break
-    sleep 0.1
-    tenths=$(( tenths + 1 ))
-  done
-
-  local outcome
-  if process_is_alive "${pid}"; then
-    warn "litestream ${pid} ignored SIGTERM; killing"
-    kill -KILL "${pid}" 2>/dev/null || true
-    outcome="after SIGKILL (ignored SIGTERM for ${LITESTREAM_STOP_WAIT_SECONDS}s)"
-  else
-    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
-  fi
-
-  rm -f "${LITESTREAM_PID_FILE}"
-  log "litestream stopped ${outcome}"
-}
-
-# The cheap stop: one SIGTERM, one rm, one log line, NO poll loop — `/run`'s
-# own cleanup calls this, never stop_litestream, because that path only runs
-# after /run has already FAILED: no session happened yet, so a graceful final
-# sync is worth nothing against a SIGTERM budget that cannot afford another
-# wait on top of stop_tunnel's own. A distinct function rather than a mode
-# argument on stop_litestream:
-# stop_tunnel's own comment argues against an argument a future caller can get
-# wrong. litestream_is_running is the same O(1) check stop_litestream's own
-# no-op branch uses, not a poll — /run's cleanup trap fires this before
-# the replicator start is ever reached whenever an earlier step failed, and an
-# operator reading that log must not be told a stop signal went to a process
-# that never started.
-kill_litestream() {
-  if ! litestream_is_running; then
-    rm -f "${LITESTREAM_PID_FILE}"
-    log "no litestream to stop"
-    return 0
-  fi
-
-  kill -TERM "$(cat "${LITESTREAM_PID_FILE}")" 2>/dev/null || true
-  rm -f "${LITESTREAM_PID_FILE}"
-  log "litestream stop signalled (no wait)"
-}
-# --- litestream replicate (end) ----------------------------------------------
-
-# --- flush_session_db (#812 WI-4) -------------------------------------------
-#
-# The checked, synchronous flush /suspend and /terminate need before they
-# finish tearing down. Never fatal, for the same reason every session-DB
-# function in this file is: /suspend and /terminate must complete their own
-# teardown regardless of whether S3 could be reached.
-
-# Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
-# own sync is normally sub-second (M5, the plan's grounding); 10s is generous
-# headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
-# measured worst case. The SIGKILL backstop gives the flush its own two-second
-# grace after `timeout` sends SIGTERM.
-SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
-SESSION_DB_FLUSH_KILL_GRACE_SECONDS=2
-
-# Guards mirror the CLI replicator's first three exactly (disabled -> marker ->
-# config) — a flush must never attempt work those guards would have refused
-# to start in the first place. PERSISTENCE_BUCKET is the SAME variable
-# the CLI reads, so both agree regardless of which earlier step in
-# THIS process set it (load_state_config above).
-#
-# Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
-# trigger — and only once it is confirmed stopped does the synchronous -once
-# flush run, as a second, checked pass. Running -once while the daemon is
-# still alive would put two writers on one prefix, the single-writer
-# invariant this whole feature exists to protect (Q8) — this ordering is not
-# negotiable.
-#
-# If the daemon has already died (a missing or stale pid file), that IS the
-# detection this needs: SESSION-DB-REPLICATOR-DIED names it loudly instead of
-# silently skipping straight to the flush (development-workflow.mdc: every
-# recovery branch emits a server-visible signal). The flush still runs either
-# way — it is what actually gets any unreplicated writes to S3 before the
-# caller's next step.
-flush_session_db() {
-  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
-    warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing to flush."
-    return 0
-  fi
-
-  if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
-    log "skipping session-DB flush: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
-    return 0
-  fi
-
-  if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
-    error "no usable ${LITESTREAM_CONFIG_FILE}; cannot flush the session DB"
-    return 0
-  fi
-
-  if litestream_is_running; then
-    stop_litestream
-  else
-    warn "SESSION-DB-REPLICATOR-DIED: no live litestream at ${LITESTREAM_PID_FILE}; flushing one-shot anyway"
-  fi
-
-  # Only reached with the daemon confirmed stopped, or never running — never
-  # concurrently with it (see above).
-  local flush_rc=0
-  timeout -k "${SESSION_DB_FLUSH_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
-    litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
-
-  if [ "${flush_rc}" -eq 0 ]; then
-    log "SESSION-DB-FINAL-FLUSH: succeeded, ${SECONDS}s into the hook"
-  else
-    warn "SESSION-DB-FINAL-FLUSH: litestream replicate -once exited ${flush_rc}, ${SECONDS}s into the hook; some writes since the last sync may not have reached S3"
-  fi
-}
-# --- flush_session_db (end) --------------------------------------------------
-
-# How long the guest CLI runs with no activity before it exits itself
-# (`evident run --idle-timeout`), which is what
-# turns a truly-abandoned VM into the clean-offline POST that lets Evident
-# suspend it (#732). Sized from the measured cost of guessing wrong rather than
-# the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
-# ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
-# napping a VM whose user comes straight back costs ~10 s — against the
-# legacy conservative baseline input of ~$0.30/h, AWS can burst a loaded VM
-# to a 16 GB / 8 vCPU peak (~$1.06/h at sustained full load—a worst-case
-# ceiling, not an expectation); that is cheap enough that the balance
-# sits far nearer the floor than the ceiling. Not AT the floor, though: the
-# CLI's idle detector needs 2 clear poll cycles (≥4 s of real time), so a value
-# near that would spend more time
-# suspending/resuming than idle.
-# ECS's waker uses 900 s instead only because *its* cold start is far slower
-# than this VM's ~2 s resume — not evidence this default should match it.
-#
-# NOT part of the hook's own teardown budget (TUNNEL_STOP_WAIT_SECONDS et al.,
-# below): it governs the CLI's lifetime long after the hook has already
-# returned its 200, so it is deliberately outside that arithmetic.
-#
-# EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
-# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
-# idle-timeout flag, so an operator who knows one knows the other. A
-# non-numeric override must never silently DROP the flag — that degrades to
-# an always-on VM burning ~$7.17/day at baseline, or up to ~$25.49/day at
-# sustained full peak as load increases (a worst-case ceiling, not an
-# expectation), exactly the bug this closes — so it
-# warns and falls back to the default instead.
-#
-# Deliberately NOT in the /run payload yet: doing so would touch the doorbell
-# contract in four coordinated places for no capability today. #716 shipped
-# the swarm/per-user provisioner but explicitly scoped control-plane
-# idle/suspend policy OUT (its own "Phase 3" — see the issue's "Out of
-# Scope"), so a per-pool idle policy is still a clean follow-up, not
-# something #716 already provides a home for.
-IDLE_TIMEOUT_SECONDS=120
-if [ -n "${EVIDENT_IDLE_TIMEOUT_SECONDS:-}" ]; then
-  if [[ "${EVIDENT_IDLE_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then
-    IDLE_TIMEOUT_SECONDS="${EVIDENT_IDLE_TIMEOUT_SECONDS}"
-  else
-    warn "EVIDENT_IDLE_TIMEOUT_SECONDS='${EVIDENT_IDLE_TIMEOUT_SECONDS}' is not numeric; using the default ${IDLE_TIMEOUT_SECONDS}s instead of leaving the VM always-on"
-  fi
-fi
-
-# `setsid` so the tunnel outlives this hook: the script must return so AWS gets
-# its 200, while the tunnel keeps serving.
-start_tunnel() {
-  local runner_key="$1" api_url="$2" tunnel_url="$3"
-  local restore_runner_credentials="${4:-false}" restore_history="${5:-false}"
-  local -a credential_flags=()
-  local -a session_db_flags=()
-  local -a opencode_config_flags=()
-
-  # The explicit fourth argument is set only by /run; /resume uses the default
-  # so a resumed VM never restores credentials over stores it already has.
-  if [ "${restore_runner_credentials}" = true ]; then
-    credential_flags=(--restore-runner-credentials)
-  fi
-
-  # Only a fresh /run asks the CLI to restore session history. /resume keeps
-  # the database from the snapshot and must not restore over it.
-  if [ "${restore_history}" = true ]; then
-    session_db_flags=(--restore-session-db)
-  fi
-
-  if [ -n "${RUNNER_OPENCODE_CONFIG:-}" ]; then
-    opencode_config_flags=(--opencode-config-overlay "${RUNNER_OPENCODE_CONFIG}")
-  fi
-
-  if tunnel_is_running; then
-    warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
-    return 0
-  fi
-
-  # The key travels in the child's environment ONLY: argv is world-readable
-  # through /proc.
-  # --enable-file-sync-to is what makes the UI's "connect a provider account"
-  # flow reach this VM: without it the CLI declines every queued file and the
-  # page can only say "this runner isn't accepting files". There is no shell
-  # into a MicroVM, so it is the ONLY way to seed model credentials on a runner
-  # that is already up — the S3 credential store is read at /run and /resume,
-  # never mid-life. The allow-list stays a single directory, as on ECS
-  # (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
-  # extra entry widens what Evident can write into a VM that executes agent
-  # code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
-  # an unset one abort rather than silently allow-list "/.claude".
-  EVIDENT_RUNNER_KEY="${runner_key}" setsid evident run \
-    --port "${OPENCODE_PORT}" \
-    --endpoint "${api_url}" \
-    --tunnel "${tunnel_url}" \
-    --opencode-pid-file "${OPENCODE_PID_FILE}" \
-    --litestream-config "${LITESTREAM_CONFIG_FILE}" \
-    --litestream-pid-file "${LITESTREAM_PID_FILE}" \
-    --session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
-    --credential-sync-marker "${CREDENTIAL_FLUSH_MARKER_FILE}" \
-    "${credential_flags[@]}" \
-    "${session_db_flags[@]}" \
-    "${opencode_config_flags[@]}" \
-    --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
-    --enable-file-sync-to "${HOME}/.claude" &
-
-  echo $! >"${TUNNEL_PID_FILE}"
-  log "tunnel started (pid $(cat "${TUNNEL_PID_FILE}"))"
-}
-
-# The worst case `evident run` can take to shut down gracefully on SIGTERM, in
-# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS) + 2 s offline
-# POST (notifyAgentDisconnected) + 5 s telemetry flush
-# (TELEMETRY_SHUTDOWN_TIMEOUT_MS) + 8.5 s pre-drain credential flush
-# + 8.5 s post-drain credential flush. Each phase is bounded there, so this is
-# a ceiling rather than a typical cost — an idle suspend finishes in a couple
-# of seconds. This is the ONE place the hand-maintained budget is written down;
-# a guard in the source repository keeps this number in step with the CLI's
-# declared bounds.
-#
-# DOCUMENTATION ONLY — nothing is derived from this value. It is the hand-maintained
-# sum of the five bounded phases above (25 + 2 + 5 + 8.5 + 8.5); update it here if
-# any of those bounds changes.
-# shellcheck disable=SC2034  # documentation; deliberately read by nothing
-CLI_SHUTDOWN_CEILING_SECONDS=49
-
-# How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
-# Since #718 this binds ONLY for a CLI that is still draining: one that has
-# already exited is detected on the first 0.1 s poll whatever this says, because
-# process_is_alive no longer mistakes its unreaped corpse for a live process.
-# That is what takes a routine suspend from ~41 s back to the ~2 s it measures.
-#
-# So 10 is chosen, not derived, and it sits in the middle of a real range. The
-# measured shutdown was 1255 ms of drain inside a 2322 ms total (#697): 5 s is
-# only ~2.2x that and re-arms #657's complaint of a kill landing mid-drain,
-# while 30 s never truncates a legitimate drain but rebuilds the headroom
-# problem #699 created — 41 s against a 60 s lifecycleTimeout leaves 19 s for
-# everything else in the hook. 10 s is ~4x the measured shutdown and gives a
-# provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
-# before us = 20 s, inside the ~55 s the in-VM server allows this script
-# (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
-# itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
-# overrunning that fails the whole lifecycle transition, which is worse than the
-# kill this replaces. The residual cost is a drain longer than 10 s being
-# truncated; that work stays `processing` server-side and is re-adopted on the
-# next start (ADR-0046).
-#
-# src/image/hook-scripts.test.ts holds both that this stays small and that it
-# fits the hook timeout, and times a real SIGKILL against the override to prove
-# the loop HONOURS it rather than a hardcoded deadline. The env override is for
-# tests only, so they need not burn 10 s of wall clock.
-TUNNEL_STOP_WAIT_SECONDS="${EVIDENT_TUNNEL_STOP_WAIT_SECONDS:-10}"
-
-stop_tunnel() {
-  if ! tunnel_is_running; then
-    # Clear the file here too, not only on the path below: `evident run` has
-    # self-exit paths that never reach this function (auth expired, idle
-    # timeout, a crash), so the pid file routinely outlives the process it
-    # names. Leaving it means every later guard re-reads a dead pid — harmless
-    # while process_is_alive agrees it is dead, and #718 again the moment
-    # anything starts reaping orphans and the pid gets recycled.
-    rm -f "${TUNNEL_PID_FILE}"
-    log "no tunnel to stop"
-    return 0
-  fi
-
-  local pid
-  pid="$(cat "${TUNNEL_PID_FILE}")"
-  # One drain length for every caller: /suspend and /terminate always run over
-  # a connected tunnel, and /run's and /resume's own cleanup traps only reach a
-  # live tunnel here when it was never started (the cheap no-op branch above) —
-  # there is no path left where a just-spawned, not-yet-connected tunnel is the
-  # one being stopped, so there is nothing left to special-case.
-  kill -TERM "${pid}" 2>/dev/null || true
-
-  local tenths=0
-  local deadline_tenths=$(( TUNNEL_STOP_WAIT_SECONDS * 10 ))
-  while [ "${tenths}" -lt "${deadline_tenths}" ]; do
-    process_is_alive "${pid}" || break
-    sleep 0.1
-    tenths=$(( tenths + 1 ))
-  done
-
-  # This call, not the loop's, is the deciding read: it is what the `if` below
-  # branches on, so PROC_DEAD_REASON is fresh by construction. A corpse can
-  # still be reaped between the loop's last poll and this call, so the reason
-  # named below is what THIS call saw, not necessarily what the loop saw.
-  local outcome
-  if process_is_alive "${pid}"; then
-    warn "tunnel ${pid} ignored SIGTERM; killing"
-    kill -KILL "${pid}" 2>/dev/null || true
-    outcome="after SIGKILL (ignored SIGTERM for ${TUNNEL_STOP_WAIT_SECONDS}s)"
-  else
-    outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
-  fi
-
-  rm -f "${TUNNEL_PID_FILE}"
-  log "tunnel stopped ${outcome}"
-}
-
-# The CLI owns the interval loop and its boundary flush. Two seconds of slack
-# over the CLI's 8s flush deadline keeps this marker handshake inside the 55s
-# hook ceiling while leaving the fallback flush and tunnel stop budget intact.
-#
-# This bounds only the CLI's pre-drain flush, which always runs first and
-# unconditionally (run.ts's cleanup(), before the channel-work drain) — not
-# the CLI's post-drain second pass, which can take up to
-# SHUTDOWN_DRAIN_TIMEOUT_MS longer than this wait covers. A drain that
-# consumes the whole window loses the SECOND pass, not the credential
-# guarantee itself: the pre-drain flush already persisted everything on disk
-# at signal time, exactly what the old bash `sync_credentials` guaranteed in
-# one synchronous call — so the worst case here is no worse than before this
-# handshake existed, never a fresh data-loss window. See
-# docs/decisions/0063-microvm-boot-orchestration-in-cli.md's two-phase-flush
-# section for the full reasoning.
-CREDENTIAL_FLUSH_WAIT_SECONDS="${EVIDENT_CREDENTIAL_FLUSH_WAIT_SECONDS:-10}"
-
-# Remove the previous answer, signal the same CLI that stop_tunnel handles, and
-# wait for either its marker or its death. A fallback sync runs only after the
-# CLI is known to be absent, never alongside a live CLI that may still write.
-stop_runner_and_flush_credentials() {
-  rm -f "${CREDENTIAL_FLUSH_MARKER_FILE}"
-
-  if ! tunnel_is_running; then
-    warn "CREDS-FLUSH-NO-RUNNER: no live CLI at handshake entry"
-    stop_tunnel
-    sync_credentials
-    return 0
-  fi
-
-  local pid
-  pid="$(cat "${TUNNEL_PID_FILE}")"
-  kill -TERM "${pid}" 2>/dev/null || true
-
-  local waited_ms=0 marker_found=false runner_exited=false
-  while [ "${waited_ms}" -lt $((CREDENTIAL_FLUSH_WAIT_SECONDS * 1000)) ]; do
-    if [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
-      marker_found=true
-      break
-    fi
-    if ! process_is_alive "${pid}"; then
-      runner_exited=true
-      break
-    fi
-    sleep 0.1
-    waited_ms=$((waited_ms + 100))
-  done
-
-  # The CLI can publish the marker and exit inside one poll tick. A final
-  # marker check after a death break preserves that answer instead of falling
-  # through to the fallback.
-  if [ "${runner_exited}" = true ] && [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
-    marker_found=true
-    runner_exited=false
-  fi
-
-  if [ "${marker_found}" = true ]; then
-    local failures
-    failures="$(grep -Ev '^(claude|opencode)=ok$' "${CREDENTIAL_FLUSH_MARKER_FILE}" || true)"
-    if [ -n "${failures}" ]; then
-      warn "CREDS-FLUSH-HAD-FAILURES: ${failures//$'\n'/ }"
-    else
-      log "CREDS-FLUSH-OK"
-    fi
-    stop_tunnel
-    return 0
-  fi
-
-  if [ "${runner_exited}" = true ]; then
-    warn "CREDS-FLUSH-RUNNER-EXITED: CLI exited without a completion marker"
-    stop_tunnel
-    sync_credentials
-    return 0
-  fi
-
-  warn "CREDS-FLUSH-TIMEOUT: live CLI did not write a completion marker within ${CREDENTIAL_FLUSH_WAIT_SECONDS}s"
-  stop_tunnel
-}
-
-# --- check_runner_key (#1172) ------------------------------------------------
-#
-# Answers exactly one question before opencode/the tunnel start spending this
-# boot's SIGTERM budget on a key that cannot work: "does the runner key in this
-# payload authenticate against Evident?" Delegates entirely to the CLI's own
-# `evident status --json` rather than
-# reimplementing its auth logic here — that command's `reason` field is the
-# published contract this function reads, and its own header states the
-# absent-vs-contrary distinction this function must honour.
-#
-# Keyed on the JSON `reason`, NEVER on the exit code: an older CLI without a
-# `status` subcommand exits 1 from Commander's own "unknown command" handling,
-# which under an exit-code mapping would read as "the key was rejected" and
-# destroy every VM on a hook/CLI version skew — precisely the hazard this
-# image's README's "version skew" section flags. So no parseable JSON line
-# means no verdict, whatever the exit code says.
-#
-# Returns 0 for positive AND absent evidence (network/timeout/5xx/404/old-CLI/
-# crash) — the caller continues either way — and 1 ONLY for contrary evidence
-# (401, another 4xx, or no credentials resolved at all): the key is actually
-# wrong, not merely untested (development-workflow.mdc's "never fail a gate on
-# absent evidence"). Every branch logs a named, greppable RUNNER-KEY-* signal
-# so a caller that treats the return value as fatal (or doesn't) still leaves
-# a trace of which branch fired.
-#
-# A 404 is deliberately NOT contrary: it means the endpoint has no /me route,
-# so the key was never tested. Treating it as a rejection destroyed every VM
-# for a day when the run payload shipped a bare origin instead of the CLI's
-# /v1-prefixed one.
-check_runner_key() {
-  local runner_key="$1" api_url="$2"
-
-  # status.ts's own exit-code contract (its header comment) means this exits
-  # non-zero on EVERY branch except `ok`; guard the capture so an expected
-  # non-zero does not abort this function under the caller's `set -e` before
-  # the case below ever runs. Only stdout is
-  # captured: status.ts's own contract is one parseable JSON line and nothing
-  # else there, and — like litestream's stderr elsewhere in this file — its
-  # stderr is left to reach CloudWatch directly rather than being folded in,
-  # since a stray stderr line ahead of the JSON would otherwise break `jq`.
-  local response rc=0
-  response="$(EVIDENT_API_URL="${api_url}" EVIDENT_RUNNER_KEY="${runner_key}" evident status --json)" || rc=$?
-
-  # `-z` as well as jq's own exit code, and not as belt-and-braces: on EMPTY
-  # input jq has no value to report on, so `jq -e` exits 0 with empty output
-  # rather than its documented 4 (verified on jq 1.6, the image's). Empty
-  # stdout is exactly the old-CLI-skew and missing-binary shape this branch
-  # exists for, so keying only on the exit code sent precisely those cases to
-  # the unrecognised-reason arm below — same safe return, but a log line
-  # blaming the API for an odd answer instead of naming the CLI skew, which is
-  # the one thing an operator needs told here.
-  local reason
-  reason="$(printf '%s' "${response}" | jq -er '.reason' 2>/dev/null)" || reason=""
-  if [ -z "${reason}" ]; then
-    warn "RUNNER-KEY-UNVERIFIABLE: 'evident status --json' exited ${rc} with no parseable JSON on stdout (an old CLI without this subcommand, a missing binary, or a crash — see its stderr above, if any); the key was NOT validated, continuing anyway"
-    return 0
-  fi
-  local detail
-  detail="$(printf '%s' "${response}" | jq -er '.error // empty' 2>/dev/null)" || detail=""
-
-  case "${reason}" in
-    ok)
-      log "RUNNER-KEY-OK: evident status confirmed the runner key against ${api_url}"
-      return 0
-      ;;
-    unauthorized | no_credentials | http_error)
-      error "RUNNER-KEY-REJECTED: evident status reason=${reason} against ${api_url}${detail:+: ${detail}}"
-      return 1
-      ;;
-    unreachable)
-      warn "RUNNER-KEY-UNREACHABLE: could not reach ${api_url} to validate the runner key; the key was NOT validated, continuing anyway"
-      return 0
-      ;;
-    endpoint_not_found)
-      # Absent evidence, NOT contrary: a 404 means this endpoint has no /me
-      # route (typically a bare origin where the CLI wants the /v1-prefixed
-      # one), which says nothing about the key. Loud, because the runner will
-      # keep failing every API call until the endpoint is fixed.
-      warn "RUNNER-KEY-ENDPOINT-NOT-FOUND: ${api_url} has no /me route${detail:+: ${detail}}; the key was NOT validated, continuing anyway"
-      return 0
-      ;;
-    *)
-      warn "RUNNER-KEY-UNKNOWN-REASON: evident status returned an unrecognised reason='${reason}'; the key was NOT validated, continuing anyway"
-      return 0
-      ;;
-  esac
-}
-# --- check_runner_key (end) ---------------------------------------------------