@evident-ai/runner-cdk 3.4.1-dev.d4a79de → 3.4.1-dev.f80aa1c

package/dist/microvm-image-context/hooks/common.sh

@@ -30,8 +30,6 @@ MICROVM_ID_FILE="/dev/shm/evident-microvm-id"
 TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
 OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
 LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
-CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
-CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
 
 # Where the runner's credential store lives inside the durable-state bucket.
 # The BUCKET is the same for every VM from an image version, so the stack bakes
@@ -53,6 +51,10 @@ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
 # `/terminate` removes it with the rest of the per-VM state.
 SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
 
+# Completion evidence for the CLI-owned credential flush. It lives in tmpfs,
+# is removed before every handshake, and is also removed by /terminate.
+CREDENTIAL_FLUSH_MARKER_FILE="/dev/shm/evident-credential-flush"
+
 hook_name() { printf '%s' "${0##*/}"; }
 log() { echo "[hook:$(hook_name)] $*"; }
 warn() { echo "[hook:$(hook_name)] $*" >&2; }
@@ -324,7 +326,6 @@ regenerate_machine_id() {
 tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
 opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
 litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
-creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
 
 # `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
 # and an interpolation of a missing field is still a non-empty string, so a
@@ -469,149 +470,6 @@ kill_litestream() {
 }
 # --- litestream replicate (end) ----------------------------------------------
 
-# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
-#
-# sync_credentials (above) covers the three boundary flushes /run's restore,
-# /suspend and /terminate already call. What it does NOT cover is a VM that
-# runs for a long time between those boundaries: a provider re-authenticated
-# through the proxied UI hours into a run would sit unflushed until the next
-# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
-# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
-# own sync_credentials_loop is the ECS side of the identical gap; this is the
-# same fix, backgrounded like the other long-lived services so it
-# outlives this hook process, `( … ) &` rather than `setsid`: a plain
-# backgrounded subshell is reparented to init and keeps running once its
-# parent hook script exits (verified: PPID=1, still alive, with no controlling
-# terminal in this image to send it a stray SIGHUP), and it inherits every
-# function this file defines, so it can call run_synchroniser directly with no
-# re-exec.
-
-# Bounded confirmation window `stop_credential_sync` polls after signalling the
-# loop. The loop's current child is one fast `run_synchroniser sync-once` call,
-# so this stays a short backstop rather than a graceful drain.
-CREDS_SYNC_STOP_WAIT_SECONDS=2
-
-# Best-effort per tick, exactly like sync_credentials above: a failed tick
-# must never end the loop, or a single transient S3 error would silently
-# disable sync for the rest of the VM's life.
-start_credential_sync() {
-  if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
-    warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
-    return 0
-  fi
-
-  if creds_sync_is_running; then
-    warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
-    return 0
-  fi
-
-  # This is the same environment input runner-synchroniser validates. The
-  # variable is normally absent, so that ordinary case uses the documented
-  # default without a warning; a present invalid value is named and rejected.
-  local interval="${CREDS_SYNC_INTERVAL:-60}"
-  if [ -n "${CREDS_SYNC_INTERVAL+x}" ] && [[ ! "${CREDS_SYNC_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
-    warn "CREDS-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${CREDS_SYNC_INTERVAL}' is not a positive integer; using 60s"
-    interval=60
-  fi
-
-  rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
-
-  (
-    # Releases the fds this subshell inherited from the hook process before
-    # settling in for the VM's whole remaining life: nothing here writes to
-    # them (every synchroniser call already redirects its own), so there is
-    # no reason to keep holding the hook's original stdout/stderr open. A
-    # long-lived process that instead inherited a pipe's write end (a test
-    # harness reading the hook's own output, for one) would keep that pipe
-    # from ever reporting EOF — testing-guide.mdc's own lesson, and the same
-    # reason the long-lived services never inherit stdio either. That
-    # redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
-    # failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
-    # surfaced by stop_credential_sync, which DOES have live stdio.
-    exec >/dev/null 2>&1 </dev/null
-
-    # A TERM this subshell receives (from stop_credential_sync, below) only
-    # kills THIS wrapper by default — its currently-running child (`sleep`,
-    # or a `run_synchroniser sync-once` call) is a separate process that
-    # would otherwise be orphaned and keep running, free to upload STALE
-    # credentials to S3 after the boundary flush that /suspend and
-    # /terminate perform immediately following the stop. Tracking the
-    # current child explicitly and forwarding the signal closes that race.
-    creds_sync_child_pid=""
-    trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
-
-    while true; do
-      sleep "${interval}" &
-      creds_sync_child_pid=$!
-      wait "${creds_sync_child_pid}" 2>/dev/null
-      creds_sync_child_pid=""
-
-      run_synchroniser sync-once claude &
-      creds_sync_child_pid=$!
-      wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
-      creds_sync_child_pid=""
-
-      run_synchroniser sync-once opencode &
-      creds_sync_child_pid=$!
-      wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
-      creds_sync_child_pid=""
-    done
-  ) &
-
-  echo $! >"${CREDS_SYNC_PID_FILE}"
-  log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
-}
-
-# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
-# that it and its current child are actually gone before returning: /suspend
-# and /terminate start their own boundary flush immediately after this call,
-# and an orphaned in-flight sync-once surviving past that point can overwrite
-# fresher credentials with stale ones. The TERM trap inside the loop (above)
-# forwards the signal to its current child almost instantly — this poll is a
-# defensive confirmation, not the primary mechanism, so it stays short; a
-# SIGKILL backstop covers a child that ignores TERM entirely.
-#
-# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
-# must say what it found (development-workflow.mdc) — a stopped-before-called
-# loop and a died-on-its-own loop are different facts an operator needs told
-# apart, not the same "nothing to stop" line.
-stop_credential_sync() {
-  if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
-    log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
-    return 0
-  fi
-
-  local pid
-  pid="$(cat "${CREDS_SYNC_PID_FILE}")"
-  if ! process_is_alive "${pid}"; then
-    rm -f "${CREDS_SYNC_PID_FILE}"
-    warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
-    return 0
-  fi
-
-  kill -TERM "${pid}" 2>/dev/null || true
-  rm -f "${CREDS_SYNC_PID_FILE}"
-
-  local waited_ms=0
-  while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
-    sleep 0.1
-    waited_ms=$((waited_ms + 100))
-  done
-
-  if process_is_alive "${pid}"; then
-    kill -KILL "${pid}" 2>/dev/null || true
-    warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
-  fi
-
-  if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
-    warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
-    rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
-  fi
-
-  log "CREDS-SYNC-STOPPED: pid=${pid}"
-}
-# --- credential sync loop (end) -----------------------------------------------
-
 # --- flush_session_db (#812 WI-4) -------------------------------------------
 #
 # The checked, synchronous flush /suspend and /terminate need before they
@@ -688,11 +546,14 @@ flush_session_db() {
 # suspend it (#732). Sized from the measured cost of guessing wrong rather than
 # the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
 # ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
-# napping a VM whose user comes straight back costs ~10 s — against ~$0.30/h
-# that is cheap enough that the balance sits far nearer the floor than the
-# ceiling. Not AT the floor, though: the CLI's idle detector needs 2 clear poll
-# cycles (`run.ts`'s `idlePolls >= 2`, ≥4 s of real time), so a value near that
-# would spend more time suspending/resuming than idle.
+# napping a VM whose user comes straight back costs ~10 s — against the
+# legacy conservative baseline input of ~$0.30/h, AWS can burst a loaded VM
+# to a 16 GB / 8 vCPU peak (~$1.06/h at sustained full load—a worst-case
+# ceiling, not an expectation); that is cheap enough that the balance
+# sits far nearer the floor than the ceiling. Not AT the floor, though: the
+# CLI's idle detector needs 2 clear poll cycles (`run.ts`'s `idlePolls >= 2`,
+# ≥4 s of real time), so a value near that would spend more time
+# suspending/resuming than idle.
 # ECS's waker uses 900 s instead only because *its* cold start is far slower
 # than this VM's ~2 s resume — not evidence this default should match it.
 #
@@ -704,7 +565,9 @@ flush_session_db() {
 # runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
 # idle-timeout flag, so an operator who knows one knows the other. A
 # non-numeric override must never silently DROP the flag — that degrades to
-# an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
+# an always-on VM burning ~$7.17/day at baseline, or up to ~$25.49/day at
+# sustained full peak as load increases (a worst-case ceiling, not an
+# expectation), exactly the bug this closes — so it
 # warns and falls back to the default instead.
 #
 # Deliberately NOT in the /run payload yet: doing so would touch the doorbell
@@ -772,6 +635,7 @@ start_tunnel() {
     --litestream-config "${LITESTREAM_CONFIG_FILE}" \
     --litestream-pid-file "${LITESTREAM_PID_FILE}" \
     --session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
+    --credential-sync-marker "${CREDENTIAL_FLUSH_MARKER_FILE}" \
     "${credential_flags[@]}" \
     "${session_db_flags[@]}" \
     "${opencode_config_flags[@]}" \
@@ -786,19 +650,17 @@ start_tunnel() {
 # whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS,
 # apps/cli/src/commands/run.ts) + 2 s offline POST (notifyAgentDisconnected,
 # apps/cli/src/commands/agent-lookup.ts) + 5 s telemetry flush
-# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts). Each of the three is bounded there, so
-# this is a ceiling rather than a typical cost — an idle suspend finishes in a
-# couple of seconds. This is the ONE place the budget is written down; the CLI
-# only points back here, because restating it in three places produced #657.
+# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts) + 8.5 s pre-drain credential flush
+# + 8.5 s post-drain credential flush. Each phase is bounded there, so this is
+# a ceiling rather than a typical cost — an idle suspend finishes in a couple
+# of seconds. This is the ONE place the hand-maintained budget is written down;
+# the CLI points back here when its bounds change.
 #
-# DOCUMENTATION ONLY — nothing is derived from this any more. The wait below
-# used to be pinned above it, which is the reasoning that talked #699 into 40 s;
-# since #718 an exited CLI is noticed on the first poll, so the wait is a
-# backstop chosen on its own merits and the two numbers are unrelated. Nothing
-# cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
-# three bounds cited above, so if one of them moves, update it here.
+# DOCUMENTATION ONLY — nothing is derived from this value. It is the hand-maintained
+# sum of the five bounded phases above (25 + 2 + 5 + 8.5 + 8.5); update it here if
+# any of those bounds changes.
 # shellcheck disable=SC2034  # documentation; deliberately read by nothing
-CLI_SHUTDOWN_CEILING_SECONDS=32
+CLI_SHUTDOWN_CEILING_SECONDS=49
 
 # How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
 # Since #718 this binds ONLY for a CLI that is still draining: one that has
@@ -874,6 +736,85 @@ stop_tunnel() {
   log "tunnel stopped ${outcome}"
 }
 
+# The CLI owns the interval loop and its boundary flush. Two seconds of slack
+# over the CLI's 8s flush deadline keeps this marker handshake inside the 55s
+# hook ceiling while leaving the fallback flush and tunnel stop budget intact.
+#
+# This bounds only the CLI's pre-drain flush, which always runs first and
+# unconditionally (run.ts's cleanup(), before the channel-work drain) — not
+# the CLI's post-drain second pass, which can take up to
+# SHUTDOWN_DRAIN_TIMEOUT_MS longer than this wait covers. A drain that
+# consumes the whole window loses the SECOND pass, not the credential
+# guarantee itself: the pre-drain flush already persisted everything on disk
+# at signal time, exactly what the old bash `sync_credentials` guaranteed in
+# one synchronous call — so the worst case here is no worse than before this
+# handshake existed, never a fresh data-loss window. See
+# docs/decisions/0063-microvm-boot-orchestration-in-cli.md's two-phase-flush
+# section for the full reasoning.
+CREDENTIAL_FLUSH_WAIT_SECONDS="${EVIDENT_CREDENTIAL_FLUSH_WAIT_SECONDS:-10}"
+
+# Remove the previous answer, signal the same CLI that stop_tunnel handles, and
+# wait for either its marker or its death. A fallback sync runs only after the
+# CLI is known to be absent, never alongside a live CLI that may still write.
+stop_runner_and_flush_credentials() {
+  rm -f "${CREDENTIAL_FLUSH_MARKER_FILE}"
+
+  if ! tunnel_is_running; then
+    warn "CREDS-FLUSH-NO-RUNNER: no live CLI at handshake entry"
+    stop_tunnel
+    sync_credentials
+    return 0
+  fi
+
+  local pid
+  pid="$(cat "${TUNNEL_PID_FILE}")"
+  kill -TERM "${pid}" 2>/dev/null || true
+
+  local waited_ms=0 marker_found=false runner_exited=false
+  while [ "${waited_ms}" -lt $((CREDENTIAL_FLUSH_WAIT_SECONDS * 1000)) ]; do
+    if [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
+      marker_found=true
+      break
+    fi
+    if ! process_is_alive "${pid}"; then
+      runner_exited=true
+      break
+    fi
+    sleep 0.1
+    waited_ms=$((waited_ms + 100))
+  done
+
+  # The CLI can publish the marker and exit inside one poll tick. A final
+  # marker check after a death break preserves that answer instead of falling
+  # through to the fallback.
+  if [ "${runner_exited}" = true ] && [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
+    marker_found=true
+    runner_exited=false
+  fi
+
+  if [ "${marker_found}" = true ]; then
+    local failures
+    failures="$(grep -Ev '^(claude|opencode)=ok$' "${CREDENTIAL_FLUSH_MARKER_FILE}" || true)"
+    if [ -n "${failures}" ]; then
+      warn "CREDS-FLUSH-HAD-FAILURES: ${failures//$'\n'/ }"
+    else
+      log "CREDS-FLUSH-OK"
+    fi
+    stop_tunnel
+    return 0
+  fi
+
+  if [ "${runner_exited}" = true ]; then
+    warn "CREDS-FLUSH-RUNNER-EXITED: CLI exited without a completion marker"
+    stop_tunnel
+    sync_credentials
+    return 0
+  fi
+
+  warn "CREDS-FLUSH-TIMEOUT: live CLI did not write a completion marker within ${CREDENTIAL_FLUSH_WAIT_SECONDS}s"
+  stop_tunnel
+}
+
 # --- check_runner_key (#1172) ------------------------------------------------
 #
 # Answers exactly one question before opencode/the tunnel start spending this

package/dist/microvm-image-context/hooks/resume

@@ -2,10 +2,8 @@
 #
 # Re-dial with the identity /run left behind. There is no step that could fetch
 # a fresh runner key, so the one from /run is what resumes. The CLI delegated by
-# start_tunnel owns the session-DB replicator's post-resume start. The interval
-# credential sync loop (#1868 WI-3) stays here because /suspend stops it too, so
-# a resumed VM that never restarted it here would never sync credentials again
-# for the rest of its life.
+# start_tunnel owns the credential loop and the session-DB replicator's
+# post-resume start.
 set -euo pipefail
 
 # shellcheck source=./common.sh
@@ -51,7 +49,6 @@ fi
 # replication" rather than a failed resume. The CLI's own guards handle the
 # rest — this needs no logic of its own.
 load_state_config || warn "could not resolve durable-state config; the session DB will not resume replicating"
-start_credential_sync
 
 # Diagnostic-only, unlike /run's gate: a failed resume costs the user their
 # whole session, so this never exits — it only converts a silent "resumed but

package/dist/microvm-image-context/hooks/run

@@ -28,7 +28,6 @@ cleanup() {
   stop_tunnel || warn "stop_tunnel failed while cleaning up"
   stop_opencode || warn "stop_opencode failed while cleaning up"
   kill_litestream || warn "kill_litestream failed while cleaning up"
-  stop_credential_sync || warn "stop_credential_sync failed while cleaning up"
 }
 trap cleanup EXIT
 
@@ -73,18 +72,14 @@ load_state_config || exit 1
 # it warns and this VM still boots.
 check_runner_key "${runner_key}" "${api_url}" || exit 1
 
-# 6 — the interval credential sync (#1868 WI-3). It stays before the context file
-# is written. Bare: every guard inside start_credential_sync is its own `return 0`.
-start_credential_sync
-
-# 7 — preserve the VM id across suspend/resume. The runtime always supplies it
+# 6 — preserve the VM id across suspend/resume. The runtime always supplies it
 # for a /run hook, and the subshell keeps the persisted value protected.
 (
   umask 077
   printf '%s\n' "${MICROVM_ID}" >"${MICROVM_ID_FILE}"
 )
 
-# 8 — the first per-VM identity on the wire. The subshell's umask makes the file
+# 7 — the first per-VM identity on the wire. The subshell's umask makes the file
 # unreadable to anyone else from the moment it exists, before the key is in it.
 (
   umask 077

package/dist/microvm-image-context/hooks/suspend

@@ -8,11 +8,10 @@ set -euo pipefail
 # shellcheck source=./common.sh
 source "$(dirname "$0")/common.sh"
 
-# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
-# flush must not race each other on the same credential stores.
-stop_credential_sync
-sync_credentials
-stop_tunnel
+# load_state_config exports PERSISTENCE_BUCKET, which flush_session_db below
+# requires; the CLI flush completes, or is proven absent, before teardown continues.
+load_state_config || warn "could not resolve durable-state config; neither the credential flush nor the session-DB flush below can run"
+stop_runner_and_flush_credentials
 
 # opencode is deliberately NOT stopped here (#812 WI-4) — it is inside the
 # snapshot and must resume with it — so a turn still in flight may not be

package/dist/microvm-image-context/hooks/terminate

@@ -8,11 +8,10 @@ set -uo pipefail
 # shellcheck source=./common.sh
 source "$(dirname "$0")/common.sh"
 
-# Stopped BEFORE the boundary flush (#1868 WI-3): the interval loop and this
-# flush must not race each other on the same credential stores.
-stop_credential_sync
-sync_credentials
-stop_tunnel
+# load_state_config exports PERSISTENCE_BUCKET, which flush_session_db below
+# requires; the CLI flush completes, or is proven absent, before teardown continues.
+load_state_config || warn "could not resolve durable-state config; neither the credential flush nor the session-DB flush below can run"
+stop_runner_and_flush_credentials
 
 # Writers stopped BEFORE litestream's final sync (#812 WI-4, the
 # ordered-shutdown invariant, plan §6): otherwise litestream could snapshot
@@ -32,6 +31,6 @@ flush_session_db
 # this hook runs under `set -u` (above), so an unbound one aborts the script
 # AT THIS LINE, leaving the runner key sitting in CONTEXT_FILE on a VM that is
 # being torn down. That is the one thing this line exists to prevent.
-rm -f "${CONTEXT_FILE}" "${SESSION_DB_NO_REPLICATE_MARKER}"
+rm -f "${CONTEXT_FILE}" "${SESSION_DB_NO_REPLICATE_MARKER}" "${CREDENTIAL_FLUSH_MARKER_FILE}"
 
 exit 0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evident-ai/runner-cdk",
-  "version": "3.4.1-dev.d4a79de",
+  "version": "3.4.1-dev.f80aa1c",
   "description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",