@evident-ai/runner-cdk 0.1.0 → 0.1.1-dev.04f2155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/README.md +75 -18
  2. package/dist/controller-lambda/handler.js +50523 -0
  3. package/dist/evident-scale-to-zero-construct.js +3 -4
  4. package/dist/index.d.ts +4 -0
  5. package/dist/index.js +14 -1
  6. package/dist/microvm/constants.d.ts +7 -0
  7. package/dist/microvm/constants.js +35 -0
  8. package/dist/microvm/construct.d.ts +87 -0
  9. package/dist/microvm/construct.js +253 -0
  10. package/dist/microvm/controller/doorbell.d.ts +73 -0
  11. package/dist/microvm/controller/doorbell.js +107 -0
  12. package/dist/microvm/controller/handle-doorbell.d.ts +27 -0
  13. package/dist/microvm/controller/handle-doorbell.js +480 -0
  14. package/dist/microvm/controller/microvm-client.d.ts +75 -0
  15. package/dist/microvm/controller/microvm-client.js +7 -0
  16. package/dist/microvm/controller/shape-catalogue.d.ts +64 -0
  17. package/dist/microvm/controller/shape-catalogue.js +108 -0
  18. package/dist/microvm/controller/throttle-retry.d.ts +11 -0
  19. package/dist/microvm/controller/throttle-retry.js +27 -0
  20. package/dist/microvm/image/stage-context.d.ts +33 -0
  21. package/dist/microvm/image/stage-context.js +148 -0
  22. package/dist/microvm/shapes.d.ts +72 -0
  23. package/dist/microvm/shapes.js +93 -0
  24. package/dist/microvm-image-context/Dockerfile +224 -0
  25. package/dist/microvm-image-context/hook-server.js +290 -0
  26. package/dist/microvm-image-context/hooks/common.sh +1359 -0
  27. package/dist/microvm-image-context/hooks/resume +79 -0
  28. package/dist/microvm-image-context/hooks/run +117 -0
  29. package/dist/microvm-image-context/hooks/suspend +19 -0
  30. package/dist/microvm-image-context/hooks/terminate +34 -0
  31. package/package.json +11 -4
@@ -0,0 +1,1359 @@
1
+ #!/usr/bin/env bash
2
+ # Sourced by every hook script. Nothing here runs at image build time.
3
+ #
4
+ # These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
5
+ # (packages/runner-image/entrypoint.sh is the first), so both are held to it by
6
+ # packages/runner-synchroniser/src/shell-contract.test.ts — which derives what
7
+ # `run_synchroniser` below must handle from the subcommands this shell calls.
8
+
9
+ # Installed from npm by the image (docker/Dockerfile's ARG
10
+ # RUNNER_SYNCHRONISER_VERSION) and resolved off PATH here. The Dockerfile's
11
+ # required-binary assertion is the loud failure if it is missing.
12
+ SYNCHRONISER="runner-synchroniser"
13
+ OPENCODE_PORT="${OPENCODE_PORT:-4096}"
14
+
15
+ # A hook script keeps no memory between invocations, but /resume must re-dial
16
+ # with the runner key /run was given, so /run leaves it here. Deliberate, on two
17
+ # axes: tmpfs means it never reaches the block device, and the SHARED image
18
+ # snapshot is taken at build time — long before /run — so the key cannot enter
19
+ # the artefact every VM boots from. And at 0600 owned by uid 10001 it is
20
+ # readable by nobody who could not already read it out of the tunnel process's
21
+ # /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
22
+ # shellcheck disable=SC2034 # read by the scripts that source this file
23
+ CONTEXT_FILE="/dev/shm/evident-run-context"
24
+ TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
25
+ OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
26
+ LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
27
+
28
+ # Where the runner's credential store lives inside the durable-state bucket.
29
+ # The BUCKET is the same for every VM from an image version, so the stack bakes
30
+ # it into the image environment; the PREFIX selects one runner's store, so it is
31
+ # per-VM and can only arrive in the /run payload. /run leaves it here for the
32
+ # hooks that flush back to it. A file of its own rather than a line of
33
+ # CONTEXT_FILE: it is not a secret, so /suspend never has to read the runner key
34
+ # to find out where to write.
35
+ STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
36
+
37
+ # The generated litestream.yml (#812). tmpfs for the same two reasons as the
38
+ # files above: it must never reach the block device (Q6, `docs`), and it
39
+ # survives suspend/resume, which is what lets `/resume` start litestream again
40
+ # with no regeneration cost. Regenerated only when absent or empty
41
+ # (`ensure_litestream_config`, below).
42
+ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
43
+
44
+ # Set when a `session-db-classify` answer of 31 (replica unusable), or a
45
+ # restore this hook gave up on for its own reasons (a truncated `timeout`, a
46
+ # broken tool), means this boot must NOT start `litestream replicate` — doing
47
+ # so would let a partial/fresh local DB overwrite a replica this boot never
48
+ # proved is safe to write over. Cleared at the top of every `restore_session_db`
49
+ # call (i.e. every `/run`) before any other decision, so a marker left by an
50
+ # earlier boot cannot silently disable replication for the VM's whole life.
51
+ SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
52
+
53
+ log() { echo "[hook:$(basename "$0")] $*"; }
54
+ warn() { echo "[hook:$(basename "$0")] $*" >&2; }
55
+ error() { echo "[hook:$(basename "$0")] ERROR: $*" >&2; }
56
+
57
+ # Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
58
+ # corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
59
+ # `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
60
+ # unusable)/32 (retry) — see `packages/runner-synchroniser/src/cli.ts`'s own
61
+ # comment for what each means, not restated here. Any OTHER non-zero status
62
+ # means the tool itself broke, which is the only case worth an ERROR here —
63
+ # EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
64
+ # that is an intentional bound firing, not a broken tool, so the caller
65
+ # classifies it instead (restore_credentials, #930). A trailing
66
+ # `--evident-deadline=N` (stripped below before forwarding to the CLI, and
67
+ # never produced by anything but timed_synchroniser) is how a caller opts in;
68
+ # every other caller — sync_credentials, restore_session_db's `env` /
69
+ # `litestream-config` / `session-db-classify`, and this file's own contract
70
+ # test — passes none, so 124/137 there still means a genuine external SIGKILL
71
+ # (e.g. an OOM kill) and must keep producing the ERROR below.
72
+ run_synchroniser() {
73
+ local -a call_args=("$@")
74
+ local deadline=""
75
+ local last=$(( ${#call_args[@]} - 1 ))
76
+ if [ "${last}" -ge 0 ] && [[ "${call_args[last]}" == --evident-deadline=* ]]; then
77
+ deadline="${call_args[last]#--evident-deadline=}"
78
+ call_args=("${call_args[@]:0:${last}}")
79
+ fi
80
+
81
+ local rc=0
82
+ local -a launcher=()
83
+ [ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
84
+ "${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
85
+ case "${rc}" in
86
+ 0 | 10 | 30 | 31 | 32) ;;
87
+ 124 | 137)
88
+ [ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
89
+ ;;
90
+ *) error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
91
+ esac
92
+ return "${rc}"
93
+ }
94
+
95
+ # Wall-clock milliseconds. ${EPOCHREALTIME} is a bash 5 builtin (the image is
96
+ # node:22-bookworm-slim → bash 5.2) rather than `date +%s%3N`, which has to
97
+ # exec /bin/date inside the caller's command substitution: measured on a loaded
98
+ # box, that exec put ~190ms of its own cost INSIDE the window being measured,
99
+ # against ~23ms for this — overhead charged to the very number this exists to
100
+ # learn. The remaining ~23ms is the command substitution's subshell, kept
101
+ # because removing it means an out-parameter global, and ~2% of a ~1.2s call
102
+ # biases the reading generous, which is the safe direction for sizing a
103
+ # deadline.
104
+ #
105
+ # The `[.,]` is not paranoia: ${EPOCHREALTIME} renders its decimal separator
106
+ # from LC_NUMERIC, so a comma locale would otherwise silently produce garbage
107
+ # here. Stripping either turns it into whole microseconds, which /1000 makes
108
+ # milliseconds.
109
+ #
110
+ # Never fails its caller: a timing line is diagnostics, and hardening a
111
+ # currently-working path is worse than the gap it closes (#931). The `date`
112
+ # fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
113
+ # timed_synchroniser turns into `elapsed_ms=unknown` rather than aborting /run
114
+ # under `set -e` — and rather than a 0 that would read as a fast healthy call.
115
+ now_ms() {
116
+ local now="${EPOCHREALTIME:-}"
117
+ if [ -n "${now}" ]; then
118
+ now="${now/[.,]/}"
119
+ echo "$((now / 1000))"
120
+ return 0
121
+ fi
122
+ date +%s%3N 2>/dev/null || echo 0
123
+ }
124
+
125
+ # Reports what one step COST, in a deliberately machine-greppable line (`op=`,
126
+ # `elapsed_ms=`, `rc=`) so "how long does this actually take in the fleet?" is a
127
+ # log query rather than another spike. `at_s` is where the step landed on the
128
+ # hook's own SECONDS clock, which is what the pre-opencode budget in
129
+ # hook-scripts.test.ts is derived against.
130
+ #
131
+ # Callable directly, not only through timed_synchroniser below, because the
132
+ # steps that dominate the pre-opencode window CANNOT be wrapped: their stdout is
133
+ # captured (`x="$(run_synchroniser env)"`, later `eval`'d), so a `log` line
134
+ # emitted inside the substitution would be evaluated as configuration.
135
+ #
136
+ # `unknown`, never a number, when either end failed to read the clock (now_ms's
137
+ # 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
138
+ # very fast healthy call, which would quietly bias the fleet-wide sizing data
139
+ # this line exists to produce, in the DANGEROUS direction (a deadline sized too
140
+ # tight). A non-numeric value drops out of an aggregate instead of poisoning it.
141
+ log_elapsed_since() {
142
+ local op="$1" started_ms="$2" rc="$3"
143
+
144
+ local finished_ms elapsed_ms="unknown"
145
+ finished_ms="$(now_ms)"
146
+ if [ "${started_ms}" -gt 0 ] && [ "${finished_ms}" -gt 0 ]; then
147
+ elapsed_ms="$((finished_ms - started_ms))"
148
+ fi
149
+
150
+ log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
151
+ }
152
+
153
+ # Runs ONE synchroniser call and reports what it COST, without changing what it
154
+ # MEANS: the call's own status is returned untouched, so every caller's
155
+ # `|| return 1` / `|| status=$?` keeps working exactly as before.
156
+ #
157
+ # `deadline` is an explicit parameter, forwarded to run_synchroniser as a
158
+ # trailing `--evident-deadline=N` (never a global or dynamically-scoped
159
+ # variable, #930) — every current caller (restore_credentials) always has a
160
+ # positive remaining-budget value by the time it calls this, having already
161
+ # decided to skip the call entirely otherwise.
162
+ timed_synchroniser() {
163
+ local op="$1" deadline="$2"
164
+ shift 2
165
+
166
+ local started_ms rc=0
167
+ started_ms="$(now_ms)"
168
+ run_synchroniser "$@" "--evident-deadline=${deadline}" || rc=$?
169
+ log_elapsed_since "${op}" "${started_ms}" "${rc}"
170
+ return "${rc}"
171
+ }
172
+
173
+ # Exports what `runner-synchroniser` resolves its object-store location from
174
+ # (packages/runner-synchroniser/src/config.ts). It treats either being empty as
175
+ # "persistence disabled" and then reports every restore as a WARNING it still
176
+ # exits 0 for — so an unset value has to be caught HERE, where it can still be
177
+ # told apart from "the store is simply empty".
178
+ load_state_config() {
179
+ if [ -z "${LITESTREAM_BUCKET:-}" ]; then
180
+ error "LITESTREAM_BUCKET is unset; the image is missing the durable-state bucket the stack bakes in"
181
+ return 1
182
+ fi
183
+
184
+ LITESTREAM_PREFIX="$(cat "${STATE_PREFIX_FILE}" 2>/dev/null || true)"
185
+ if [ -z "${LITESTREAM_PREFIX}" ]; then
186
+ error "no durable-state prefix at ${STATE_PREFIX_FILE}; /run records the payload's state_prefix there before anything can be restored or flushed"
187
+ return 1
188
+ fi
189
+
190
+ export LITESTREAM_BUCKET LITESTREAM_PREFIX
191
+
192
+ # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
193
+ # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
194
+ # is just LITESTREAM_BUCKET. start_litestream and flush_session_db (#812
195
+ # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
196
+ # directly, so every entry point agrees on one predicate regardless of which
197
+ # earlier step in THIS hook process set it: restore_session_db's
198
+ # `eval "$(run_synchroniser env)"` on /run, or this function on /resume,
199
+ # /suspend and /terminate, none of which ever call the synchroniser before
200
+ # needing the answer.
201
+ export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
202
+ }
203
+
204
+ # /run starts no LLM task — it restores state, starts opencode and dials the
205
+ # tunnel, none of which needs a model credential. Which credential a turn needs
206
+ # depends on the model that turn's message asks for, so that question has no
207
+ # single boot-time answer; it is answered per message, not here. What IS still
208
+ # fatal is credential persistence being unavailable at all (an unset bucket/
209
+ # prefix, or a broken synchroniser bundle) — that would silently drop refreshed
210
+ # tokens on every suspend, so those two restores stay `|| return 1`.
211
+ # `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
212
+ # a log line, since the runtime forwards no hook stderr to the doorbell caller.
213
+
214
+ # How long restore_credentials' shared step budget is, covering the runner-secret
215
+ # fetch plus its three synchroniser calls (restore claude, restore opencode,
216
+ # model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
217
+ # total would false-fire on the ordinary case of one slow call (a slow S3 GET),
218
+ # which is precisely the boot this exists to keep healthy. From #930's 3-boot
219
+ # sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
220
+ # node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
221
+ # SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
222
+ # routine slow call boots this VM with no model credentials until an operator
223
+ # reconnects it; too loose and a hung call burns more of the hook's own
224
+ # SIGTERM budget before the VM is destroyed mid-boot anyway. The env override
225
+ # is for tests only, so they need not burn wall clock.
226
+ CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECONDS:-8}"
227
+ # The SIGKILL backstop `timeout -k` applies after its own SIGTERM, exactly like
228
+ # SESSION_DB_RESTORE_KILL_GRACE_SECONDS above (a bare `timeout` only SIGTERMs,
229
+ # and a call that ignored it would be unbounded again). Hard ceiling on the
230
+ # step: DEADLINE + this = 10s, once — not per call.
231
+ CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
232
+ GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
233
+
234
+ # What is left of the shared step budget, in whole seconds, `step_started_s`
235
+ # seconds after it began. `SECONDS` (a bash builtin with no failure mode,
236
+ # unlike `now_ms`) truncates, so a step that began at true time `s0` can read
237
+ # less elapsed time than actually passed — the `- 1` restores the invariant
238
+ # that no call is granted more than DEADLINE + GRACE from step start.
239
+ remaining_credential_budget() {
240
+ local step_started_s="$1"
241
+ echo $(( CREDENTIAL_RESTORE_DEADLINE_SECONDS - (SECONDS - step_started_s) - 1 ))
242
+ }
243
+
244
+ # Runs one of the step's bounded calls: skips it with a named warn if the
245
+ # shared budget is already exhausted, treats a timeout (124/137) as the
246
+ # non-fatal warn-and-continue D1 approved, and keeps every other non-zero
247
+ # fatal exactly as restore_credentials always has (`|| return 1` aborts /run
248
+ # before opencode and the tunnel start).
249
+ bounded_restore_call() {
250
+ local op="$1" step_started_s="$2"
251
+ shift 2
252
+
253
+ local remaining
254
+ remaining="$(remaining_credential_budget "${step_started_s}")"
255
+ if [ "${remaining}" -lt 1 ]; then
256
+ warn "CREDENTIAL-RESTORE-SKIPPED: ${op} skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
257
+ return 0
258
+ fi
259
+
260
+ local rc=0
261
+ timed_synchroniser "${op}" "${remaining}" "$@" || rc=$?
262
+ case "${rc}" in
263
+ 0) ;;
264
+ 124 | 137)
265
+ warn "CREDENTIAL-RESTORE-TIMEOUT: ${op} did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
266
+ ;;
267
+ *) return 1 ;;
268
+ esac
269
+ return 0
270
+ }
271
+
272
+ fetch_runner_secret() {
273
+ local step_started_s="$1" remaining rc=0 payload value stderr_file started_ms populated=0 missing=0
274
+ if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
275
+ log "runner secret is not configured; continuing without GitHub and MCP credentials"
276
+ return 0
277
+ fi
278
+ remaining="$(remaining_credential_budget "${step_started_s}")"
279
+ if [ "${remaining}" -lt 1 ]; then
280
+ warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
281
+ return 0
282
+ fi
283
+ if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
284
+ warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
285
+ return 0
286
+ fi
287
+ started_ms="$(now_ms)"
288
+ payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
289
+ log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
290
+ if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
291
+ warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
292
+ rm -f "${stderr_file}"
293
+ return 0
294
+ fi
295
+ if [ "${rc}" -ne 0 ]; then
296
+ warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
297
+ rm -f "${stderr_file}"
298
+ return 0
299
+ fi
300
+ rm -f "${stderr_file}"
301
+ if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
302
+ warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
303
+ return 0
304
+ fi
305
+ local -a populated_keys=()
306
+ for key in GH_TOKEN BRAVE_API_KEY CLOUDFLARE_API_TOKEN NEON_API_KEY; do
307
+ value="$(jq -r --arg k "${key}" '.[$k] // empty' <<<"${payload}")"
308
+ if [ -n "${value}" ]; then
309
+ export "${key}=${value}"
310
+ populated=$((populated + 1))
311
+ populated_keys+=("${key}")
312
+ else
313
+ missing=$((missing + 1))
314
+ fi
315
+ done
316
+ if [ "${populated}" -eq 0 ]; then
317
+ warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-microvm/README.md"
318
+ else
319
+ log "RUNNER-SECRET-OK: populated ${populated_keys[*]}; ${missing} allow-listed keys empty or absent"
320
+ fi
321
+ return 0
322
+ }
323
+
324
+ restore_credentials() {
325
+ local step_started_ms step_started_s
326
+ step_started_ms="$(now_ms)"
327
+ step_started_s="${SECONDS}"
328
+
329
+ load_state_config || return 1
330
+ # This shares the existing bounded window so /run's worst-case duration does not grow.
331
+ fetch_runner_secret "${step_started_s}"
332
+ bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
333
+ bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
334
+
335
+ # `restore` exits 0 whether it restored, found nothing, or discarded a corrupt
336
+ # object, so the synchroniser's own predicate is the only reliable answer to
337
+ # "can this VM authenticate a model right now". Its status is captured, not
338
+ # used as a condition, so it can never become this function's return status —
339
+ # `run` calls this bare (no `||`) under `set -euo pipefail`, and any
340
+ # non-zero return here would abort the hook before opencode and the tunnel
341
+ # ever start.
342
+ local auth_status=0 auth_remaining
343
+ auth_remaining="$(remaining_credential_budget "${step_started_s}")"
344
+ if [ "${auth_remaining}" -lt 1 ]; then
345
+ warn "CREDENTIAL-RESTORE-SKIPPED: model-auth-ready skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
346
+ else
347
+ timed_synchroniser model-auth-ready "${auth_remaining}" model-auth-ready || auth_status=$?
348
+ case "${auth_status}" in
349
+ 0) ;; # some model auth is configured; the two restores above already logged what they found
350
+ 10)
351
+ warn "no model credentials under s3://${LITESTREAM_BUCKET}/${LITESTREAM_PREFIX}/ \
352
+ (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
353
+ ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
354
+ a model provider fails until one is connected. See 'Seeding a credential store' in \
355
+ infrastructure/evident-microvm/README.md."
356
+ ;;
357
+ 124 | 137)
358
+ warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
359
+ ;;
360
+ *)
361
+ # run_synchroniser has already `error`ed the tool-broke line for a
362
+ # broken tool; any other unrecognised code lands here too. Do not
363
+ # claim there are no credentials — absent evidence is not contrary
364
+ # evidence (development-workflow.mdc).
365
+ warn "could not determine whether this VM has model credentials"
366
+ ;;
367
+ esac
368
+ fi
369
+ # The step total, in the SAME greppable shape as the per-call lines above, so
370
+ # one query answers both "what does credential restore cost?" and "which of
371
+ # its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
372
+ # + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
373
+ # comment above that constant) rather than per call. `rc=0` is deliberate
374
+ # here, not an absence of failure modes: a timed-out call already warned by
375
+ # name above and is non-fatal by design (D1); every failure that DOES abort
376
+ # this step has already returned 1 above.
377
+ log_elapsed_since credential-restore-step "${step_started_ms}" 0
378
+ return 0
379
+ }
380
+
381
+ apply_runner_opencode_config() {
382
+ if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
383
+ log "runner OpenCode config is not configured; using the baked project config"
384
+ return 0
385
+ fi
386
+ local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
387
+ [[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
388
+ [ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
389
+ if [ ! -f "${source}" ]; then
390
+ error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
391
+ return 0
392
+ fi
393
+ cp "${source}" "${WORKSPACE}/${target}"
394
+ git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
395
+ || warn "could not mark ${target} skip-worktree; it may show as a local change"
396
+ log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
397
+ }
398
+
399
+ configure_github_access() {
400
+ if [ -z "${GH_TOKEN:-}" ]; then
401
+ warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
402
+ return 0
403
+ fi
404
+ export GIT_CONFIG_GLOBAL=/tmp/gitconfig
405
+ if ! : >"${GIT_CONFIG_GLOBAL}" ||
406
+ ! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
407
+ ! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
408
+ ! git config --global init.defaultBranch main ||
409
+ ! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
410
+ ! chmod 0700 /tmp/git-credential-helper.sh ||
411
+ ! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
412
+ warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
413
+ return 0
414
+ fi
415
+ (
416
+ local output rc=0 login repo_url repo
417
+ output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
418
+ if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
419
+ if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
420
+ log "GITHUB-AUTH-OK: ${output}"
421
+ repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
422
+ repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
423
+ [ -n "${repo}" ] || return
424
+ output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
425
+ if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
426
+ [ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
427
+ ) &
428
+ }
429
+
430
+ # Best-effort by design: /suspend must still drop the tunnel and /terminate must
431
+ # still clean up, so a flush that cannot happen is loud but never fatal.
432
+ sync_credentials() {
433
+ load_state_config || return 0
434
+ run_synchroniser sync-once claude || true
435
+ run_synchroniser sync-once opencode || true
436
+ }
437
+
438
+ # --- Session DB restore (#812 WI-2) ----------------------------------------
439
+ #
440
+ # Restores opencode.db from S3 before opencode opens it — the read half only.
441
+ # Writes NOTHING to S3 itself; replication back to S3 is started separately by
442
+ # `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
443
+ # `restore_session_db` are the MicroVM side of the same contract
444
+ # packages/runner-image/entrypoint.sh's inlined restore already speaks, going
445
+ # through the SAME runner-synchroniser CLI.
446
+
447
+ # Reads the ~30 MB litestream binary into the page cache, in the background, so
448
+ # the FIRST exec of it does not pay that read on the critical path.
449
+ #
450
+ # Why it is on the critical path at all: restore_session_db must finish before
451
+ # start_opencode, and HOOK_SIGTERM_SECONDS is a POINT on the hook's SECONDS
452
+ # clock, so a second spent here is a second taken from the rest of /run — and a
453
+ # /run that overruns is SIGTERMed mid-boot and its VM destroyed. (Until #1172
454
+ # the binding point was the tighter in-hook opencode readiness deadline; that
455
+ # deadline is gone, the hook's own SIGTERM is not.)
456
+ # Boot measurements put ~6.6-7.3s between the credential restore and the
457
+ # litestream version line, of which only ~0.2s is accounted for by the two node
458
+ # calls in between — the remainder is INFERRED to be this read, never measured.
459
+ # The `litestream-prewarm` and `litestream-version` timings are what settle it
460
+ # on the next boot.
461
+ #
462
+ # Nothing waits on it and nothing reads its output, so if that inference is
463
+ # wrong this costs one backgrounded `cat`. Deleting the `litestream version`
464
+ # diagnostic instead would save nothing: `litestream restore` two calls later
465
+ # pays the identical read.
466
+ prewarm_litestream() {
467
+ local binary
468
+ binary="$(command -v litestream 2>/dev/null || true)"
469
+ if [ -z "${binary}" ]; then
470
+ warn "litestream is not on PATH; skipping the boot pre-warm"
471
+ return 0
472
+ fi
473
+
474
+ # `cat` rather than a throwaway `litestream version`: a sequential read gets
475
+ # the whole file with readahead, where an exec demand-pages it.
476
+ (
477
+ local started_ms rc=0
478
+ started_ms="$(now_ms)"
479
+ cat "${binary}" >/dev/null 2>&1 || rc=$?
480
+ log_elapsed_since litestream-prewarm "${started_ms}" "${rc}"
481
+ ) &
482
+ log "pre-warming ${binary} in the background"
483
+ }
484
+
485
+ # Generates ${LITESTREAM_CONFIG_FILE} from runner-synchroniser's own renderer
486
+ # — the SAME config module the credential restore/sync already goes through,
487
+ # so there is no second copy of the bucket/prefix/path logic to drift
488
+ # (litestream-config.ts's own header makes the same point). Only when absent
489
+ # or empty, so `/resume` (WI-4) restarting litestream after a suspend/resume
490
+ # snapshot — /dev/shm intact — pays nothing to regenerate it.
491
+ #
492
+ # Returns 1 on failure — fatal to the CALLER, never to the hook: every caller
493
+ # (restore_session_db here, start_litestream in WI-3) must still let opencode
494
+ # and the tunnel start regardless.
495
+ ensure_litestream_config() {
496
+ if [ -s "${LITESTREAM_CONFIG_FILE}" ]; then
497
+ return 0
498
+ fi
499
+
500
+ # Timed OUTSIDE the substitution, not with timed_synchroniser: this call's
501
+ # stdout IS the config, so a timing line emitted inside `$(...)` would land in
502
+ # litestream.yml.
503
+ local rendered config_started_ms config_rc=0
504
+ config_started_ms="$(now_ms)"
505
+ rendered="$(run_synchroniser litestream-config)" || config_rc=$?
506
+ log_elapsed_since litestream-config "${config_started_ms}" "${config_rc}"
507
+ if [ "${config_rc}" -ne 0 ]; then
508
+ error "could not generate ${LITESTREAM_CONFIG_FILE}: runner-synchroniser litestream-config failed (see the error above)"
509
+ return 1
510
+ fi
511
+ printf '%s\n' "${rendered}" >"${LITESTREAM_CONFIG_FILE}" || {
512
+ error "could not write ${LITESTREAM_CONFIG_FILE}"
513
+ return 1
514
+ }
515
+
516
+ # Q7's diagnostic, for whoever can read this VM's CloudWatch log: which way
517
+ # litestream resolved its AWS region, and whether it can even see the
518
+ # names it would need to (never the VALUES of any OTHER AWS_* variable —
519
+ # AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN may be present here).
520
+ local region_empty="yes"
521
+ if printf '%s\n' "${rendered}" | grep -Eq '^ *region: *[^[:space:]]'; then
522
+ region_empty="no"
523
+ fi
524
+ # Hoisted out of the log line below so it can be timed: this is the FIRST exec
525
+ # of the litestream binary on the boot, and the pre-warm /run fires (see
526
+ # prewarm_litestream) is aimed squarely at what this number measures.
527
+ local version_started_ms litestream_version
528
+ version_started_ms="$(now_ms)"
529
+ litestream_version="$(litestream version 2>/dev/null || echo 'unknown')"
530
+ log_elapsed_since litestream-version "${version_started_ms}" 0
531
+ log "litestream ${litestream_version}; AWS_REGION=${AWS_REGION:-<unset>} AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-<unset>}; rendered litestream.yml region empty: ${region_empty}"
532
+ }
533
+
534
+ # Writes the marker AND names the reason, in one call, so the marker can never
535
+ # appear silently. Every give-up path in restore_session_db calls this EXCEPT
536
+ # the classifier's 31 (#1106 — see that branch), and no success path ever does.
537
+ # It is what a later `/resume`/`start_litestream` (WI-3) reads to skip
538
+ # replicating this boot: a local DB that is only PARTIALLY restored must never
539
+ # be allowed to overwrite a replica this boot never proved safe to write over.
540
+ # A merely FRESH local DB is not that danger — litestream continues the txid
541
+ # chain — which is precisely why 31 no longer belongs here.
542
+ mark_no_replicate() {
543
+ : >"${SESSION_DB_NO_REPLICATE_MARKER}"
544
+ warn "SESSION-DB-NO-REPLICATE: $1"
545
+ }
546
+
547
+ # Removes local session-DB debris a give-up path leaves behind. Called on
548
+ # EVERY give-up path and NO success path: skipping it on a give-up is a real
549
+ # bug (a truncated file left in place makes opencode fail to open a malformed
550
+ # database, which fails /run outright — worse than the history loss this
551
+ # discards); calling it on a success path would delete a good restore. One
552
+ # `rm -f` per path, not a single command, so a failure removing one does not
553
+ # skip the other two.
554
+ discard_session_db_debris() {
555
+ rm -f "${OPENCODE_DB_PATH}" "${OPENCODE_DB_PATH}-wal" "${OPENCODE_DB_PATH}-shm"
556
+ }
557
+
558
+ # How long restore_session_db waits for ONE `litestream restore` before giving
559
+ # up. From M4 (see the plan's §2): ~50 MB/s extrapolated from the x86_64 ECS
560
+ # container's own live replica, so 7s covers roughly 300-350 MB — an
561
+ # EXTRAPOLATION, not a MicroVM measurement (different arch, network path and
562
+ # credential source); the boot line ensure_litestream_config logs, plus the
563
+ # ${SECONDS} stamps in /run, are how it gets measured here for real.
564
+ #
565
+ # 7 rather than the 8 it was: hook-scripts.test.ts's pre-opencode ladder held an
566
+ # allowance of 4s for the rest of this step, which live boots measured at
567
+ # 6.7-7.4s, so the ladder passed while real boots blew the deadline. Correcting
568
+ # that allowance is what takes this second — the remedy the ladder's own failure
569
+ # message prescribes, and the one that leaves the SIGTERM worst case untouched.
570
+ # The cost is real and unmeasured on this platform: ~50 MB less restorable
571
+ # replica before a truncation that silently loses history. A fresh
572
+ # state_prefix starts at zero, so this is generous for a long time, but
573
+ # RoutingStrategy is 'per_user' only (routing-strategy.ts) and every prefix is
574
+ # deterministic and permanent per (pool, routing key) — so a long-lived
575
+ # runner's opencode.db grows monotonically across VM generations, exactly
576
+ # like the ~1 GB ECS one M4 measured, and will eventually hit this ceiling.
577
+ # Past it the restore is truncated (SESSION-DB-RESTORE-TRUNCATED, below) and
578
+ # this boot skips replication — silent, permanent history loss for that
579
+ # runner unless somebody reads the log. Not solved here (WI-0 Task 0.2).
580
+ # The env override is for tests only, so they need not burn 7s of wall clock.
581
+ SESSION_DB_RESTORE_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_RESTORE_DEADLINE_SECONDS:-7}"
582
+ # The SIGKILL backstop `timeout -k` applies after its own SIGTERM: `timeout`
583
+ # alone only SIGTERMs, and a litestream that ignored it would be unbounded
584
+ # again. Hard ceiling on this step: DEADLINE + this = 9s.
585
+ SESSION_DB_RESTORE_KILL_GRACE_SECONDS=2
586
+
587
+ # The one bounded restore attempt, run before start_opencode (opencode opens
588
+ # the DB the moment it starts, so this is the only place it can happen).
589
+ # Every branch returns 0 — /run calls this bare under `set -euo pipefail`,
590
+ # exactly like restore_credentials, and nothing about the session DB may ever
591
+ # fail /run (Q3): a /run that exits 1 fails the whole lifecycle transition
592
+ # and the user gets NO runner at all, which is worse than losing history.
593
+ restore_session_db() {
594
+ # Cleanup, not a decision, and unconditionally first: /dev/shm survives
595
+ # suspend/resume and a /run retry, so a marker left by an earlier boot must
596
+ # never silently disable replication for the rest of this VM's life.
597
+ rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
598
+
599
+ # Config resolved ONCE here, in the synchroniser (config.ts owns
600
+ # OPENCODE_DB_PATH; no second copy of that path here), exactly as
601
+ # entrypoint.sh's own line does. Unlike ECS's `|| die`, a failure here must
602
+ # not abort the hook — `x="$(cmd)"` under `set -e` WOULD abort the whole
603
+ # shell on a failing substitution, which is exactly why it is caught
604
+ # explicitly rather than left to `set -e`.
605
+ # Timed OUTSIDE the substitution for the same reason ensure_litestream_config
606
+ # is: this call's stdout is `eval`'d, so a timing line emitted inside `$(...)`
607
+ # would be evaluated as configuration.
608
+ local synchroniser_env env_started_ms env_rc=0
609
+ env_started_ms="$(now_ms)"
610
+ synchroniser_env="$(run_synchroniser env)" || env_rc=$?
611
+ log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
612
+ if [ "${env_rc}" -ne 0 ]; then
613
+ mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
614
+ return 0
615
+ fi
616
+ # Guarded for the SAME reason as the substitution above, which is easy to miss:
617
+ # `eval` returns the status of what it ran, so a malformed line from a skewed
618
+ # bundle would abort /run right here under `set -e` — before start_opencode,
619
+ # so the user gets NO runner at all. entrypoint.sh:110 leaves the identical
620
+ # line bare because on ECS the blast radius is a crash-loop-and-replace; here
621
+ # it is the whole lifecycle transition, so Q3 makes it a give-up instead.
622
+ eval "${synchroniser_env}" || {
623
+ mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
624
+ return 0
625
+ }
626
+
627
+ # #931's exact lesson, one shell over (packages/runner-image/entrypoint.sh):
628
+ # the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
629
+ # 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
630
+ # would otherwise abort right here under `set -u` the moment
631
+ # `${OPENCODE_DB_PATH}` is dereferenced below. entrypoint.sh may `die` on
632
+ # this (ECS just crash-loops and replaces the task); this hook cannot —
633
+ # Q3 says nothing about the session DB may ever fail /run — so a missing
634
+ # OPENCODE_DB_PATH is a give-up, not a silent default (a guessed path would
635
+ # be actively wrong, not merely absent).
636
+ if [ -z "${OPENCODE_DB_PATH+x}" ]; then
637
+ mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
638
+ return 0
639
+ fi
640
+
641
+ # #770's arm, checked FIRST of the decisions and with its OWN message —
642
+ # never the SESSION-DB-REPLICA-UNUSABLE one below (#764's finding: an
643
+ # operator grepping that marker must not hit the disabled case).
644
+ # PERSISTENCE_BUCKET is empty exactly when LITESTREAM_BUCKET or
645
+ # LITESTREAM_PREFIX is unset (config.ts's persistenceEnabled) — the
646
+ # IDENTICAL predicate ECS's own litestream-launch gate uses
647
+ # (entrypoint.sh), so the two images agree on what "disabled" means.
648
+ if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
649
+ warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated."
650
+ return 0
651
+ fi
652
+
653
+ if ! ensure_litestream_config; then
654
+ mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
655
+ return 0
656
+ fi
657
+
658
+ local restore_rc=0 restore_started_ms
659
+ restore_started_ms="$(now_ms)"
660
+ timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_RESTORE_DEADLINE_SECONDS}" \
661
+ litestream restore -config "${LITESTREAM_CONFIG_FILE}" \
662
+ -if-db-not-exists -if-replica-exists "${OPENCODE_DB_PATH}" || restore_rc=$?
663
+ log_elapsed_since session-db-restore "${restore_started_ms}" "${restore_rc}"
664
+
665
+ # `timeout`'s OWN codes, handled BEFORE classifying: 124/137 are a
666
+ # TRUNCATED restore, not a corrupt replica — handing them to
667
+ # session-db-classify would mislabel a slow/large replica as one. 125-127
668
+ # mean `timeout` (or litestream itself) is broken, not the replica.
669
+ case "${restore_rc}" in
670
+ 124 | 137)
671
+ discard_session_db_debris
672
+ mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
673
+ return 0
674
+ ;;
675
+ 125 | 126 | 127)
676
+ error "litestream restore could not even run (timeout exited ${restore_rc})"
677
+ discard_session_db_debris
678
+ mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
679
+ return 0
680
+ ;;
681
+ esac
682
+
683
+ local classify_rc=0
684
+ run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
685
+ case "${classify_rc}" in
686
+ 0) ;; # restored, or no replica yet — the CLI already logged which
687
+ 31)
688
+ # The classifier already discarded the local debris (session-db.ts) — and
689
+ # that is exactly why this is the ONE give-up path that does NOT
690
+ # mark_no_replicate (#1106). Every other give-up here may be sitting on a
691
+ # half-restored DB, so the marker stays; a 31 is structurally guaranteed
692
+ # to be a FRESH one, and replicating it starts a new backup chain instead
693
+ # of leaving this boot with a zero-width backup window. Same reasoning,
694
+ # and same one-line change, as packages/runner-image/entrypoint.sh's `31)`.
695
+ warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and starting a fresh backup chain this boot (see the WARNING above)"
696
+ ;;
697
+ 32)
698
+ # The CLI's own contract for 32 is "re-run litestream restore and ask
699
+ # again", but this hook has budget for exactly ONE attempt (Q4) — name
700
+ # the deviation so nobody reads this as a bug.
701
+ discard_session_db_debris
702
+ mark_no_replicate "session-db-classify asked for another restore attempt (32), but this hook has budget for only one; treating it as a give-up rather than retrying"
703
+ ;;
704
+ 30)
705
+ # The CLI already logged its own FATAL line above; Q3 still says boot
706
+ # fresh rather than fail /run.
707
+ discard_session_db_debris
708
+ mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
709
+ ;;
710
+ *)
711
+ # run_synchroniser already logged the "tool broke" ERROR for this.
712
+ discard_session_db_debris
713
+ mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
714
+ ;;
715
+ esac
716
+
717
+ return 0
718
+ }
719
+ # --- Session DB restore (end) -----------------------------------------------
720
+
721
+ # `kill -0` answers "does this pid exist", which is not the question any caller
722
+ # here is asking. A process that has exited but has not been reaped — a zombie —
723
+ # still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
724
+ # normal case for everything these hooks start: `start_tunnel`/`start_opencode`
725
+ # background a process that outlives the hook, the hook shell must return so AWS
726
+ # gets its 200, and PID 1 in this image is a bare node hook server with no init
727
+ # (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
728
+ # the rest of the VM's life, which made `stop_tunnel` burn its whole budget
729
+ # SIGKILLing a corpse and made a stale pid file suppress the next
730
+ # `start_tunnel`, leaving a resumed VM permanently offline (#718).
731
+ #
732
+ # This is zombie-safe, NOT identity-safe: a recycled pid reads R/S and is still
733
+ # called alive, so the SIGKILL backstop could in principle hit an unrelated
734
+ # process. Pre-existing limitation of the pid-file approach, out of scope here —
735
+ # named so nobody over-trusts the helper (src/image/hook-scripts.test.ts).
736
+ #
737
+ # Also sets PROC_DEAD_REASON on every return path — "gone" for a `kill -0`
738
+ # miss, "zombie" for an unreaped `Z`, reset to empty on the alive path so a
739
+ # stale value from a previous call can never be read — for the caller that
740
+ # needs to say WHICH death it was (stop_tunnel's outcome line), never for this
741
+ # function itself: see the silence note below. File-scope init (below) is what
742
+ # keeps a read of it safe under `set -u` before this has ever run.
743
+ PROC_DEAD_REASON=""
744
+
745
+ process_is_alive() {
746
+ if ! kill -0 "$1" 2>/dev/null; then
747
+ PROC_DEAD_REASON="gone"
748
+ return 1
749
+ fi
750
+
751
+ # `/proc/<pid>/status` rather than `/proc/<pid>/stat`, whose fields cannot be
752
+ # split safely when a comm contains a space or a paren. If the state cannot be
753
+ # read at all, keep the `kill -0` answer: absent evidence is not "dead", and
754
+ # reading it as dead would SIGTERM-and-forget a live, still-draining CLI.
755
+ # Three fields, not two: `read` hands the whole remainder to its LAST variable,
756
+ # so a trailing `_` is what keeps the state letter out of `Z (zombie)`.
757
+ #
758
+ # That deferral is silent HERE on purpose: `stop_tunnel` polls this 10x/s, so a
759
+ # warning inside it would bury the log. The flag lets `is_running` say it once
760
+ # per guard instead — absent evidence must warn as well as defer
761
+ # (development-workflow.mdc).
762
+ PROC_STATE_UNREADABLE=0
763
+ local state
764
+ read -r _ state _ < <(grep -m1 '^State:' "/proc/$1/status" 2>/dev/null) || {
765
+ PROC_STATE_UNREADABLE=1
766
+ PROC_DEAD_REASON="" # unreadable /proc still means "alive" below, not a dead reason
767
+ return 0
768
+ }
769
+
770
+ if [ "${state}" = "Z" ]; then
771
+ PROC_DEAD_REASON="zombie"
772
+ return 1
773
+ fi
774
+
775
+ PROC_DEAD_REASON=""
776
+ return 0
777
+ }
778
+
779
+ is_running() {
780
+ [ -s "$1" ] || return 1
781
+
782
+ local pid
783
+ pid="$(cat "$1")"
784
+ process_is_alive "${pid}" || return 1
785
+
786
+ # The one state in which this whole helper is back to being a bare `kill -0`
787
+ # (a `hidepid=` mount, a `/proc` that is not mounted): every liveness answer
788
+ # silently reverts to #718 — a corpse reads as alive, so suspend SIGKILLs it
789
+ # after the full wait and the next start is suppressed for the VM's life.
790
+ [ "${PROC_STATE_UNREADABLE}" = 0 ] \
791
+ || warn "could not read /proc/${pid}/status; falling back to 'kill -0', which reports an exited-but-unreaped process as alive (#718)"
792
+
793
+ return 0
794
+ }
795
+
796
+ # The snapshot bakes one machine id into every VM launched from this image
797
+ # version, so /run replaces it. Best-effort per file: the hook runs as uid 10001
798
+ # and these live in root-owned directories, so a VM that kept the snapshot's id
799
+ # must still boot — but it says so, because it is a state worth being able to see.
800
+ regenerate_machine_id() {
801
+ local machine_id
802
+ machine_id="$(tr -d '-' </proc/sys/kernel/random/uuid)"
803
+
804
+ for path in /etc/machine-id /var/lib/dbus/machine-id; do
805
+ printf '%s\n' "${machine_id}" >"${path}" 2>/dev/null \
806
+ || warn "could not rewrite ${path}; this VM keeps the snapshot's machine id"
807
+ done
808
+ }
809
+
810
+ tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
811
+ opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
812
+ litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
813
+
814
+ # `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
815
+ # and an interpolation of a missing field is still a non-empty string, so a
816
+ # payload with no runner_key would sail through as the literal "null".
817
+ payload_is_complete() {
818
+ jq -e '
819
+ (.runner_key | type == "string" and length > 0)
820
+ and (.endpoints.api | type == "string" and length > 0)
821
+ and (.endpoints.tunnel | type == "string" and length > 0)
822
+ and (.state_prefix | type == "string" and length > 0)
823
+ ' >/dev/null 2>&1
824
+ }
825
+
826
+ # `setsid` so opencode outlives this hook: the script must return so AWS gets its
827
+ # 200, while opencode keeps serving. The guard makes /run idempotent — a retry
828
+ # after a killed hook must not put a second instance on the same port.
829
+ start_opencode() {
830
+ if opencode_is_running; then
831
+ warn "opencode already running (pid $(cat "${OPENCODE_PID_FILE}")); reusing it"
832
+ return 0
833
+ fi
834
+
835
+ setsid opencode serve \
836
+ --hostname 127.0.0.1 \
837
+ --port "${OPENCODE_PORT}" \
838
+ --print-logs &
839
+
840
+ echo $! >"${OPENCODE_PID_FILE}"
841
+ log "opencode starting (pid $(cat "${OPENCODE_PID_FILE}"))"
842
+ }
843
+
844
+ stop_opencode() {
845
+ if ! opencode_is_running; then
846
+ # Clear the file here too, for the same reason stop_tunnel does: opencode
847
+ # has exit paths that never reach this function (a crash, an OOM kill), so
848
+ # the pid file routinely outlives the process it names, and leaving it is
849
+ # #718 again the moment anything starts reaping orphans and the pid is
850
+ # recycled.
851
+ rm -f "${OPENCODE_PID_FILE}"
852
+ log "no opencode to stop"
853
+ return 0
854
+ fi
855
+
856
+ kill -TERM "$(cat "${OPENCODE_PID_FILE}")" 2>/dev/null || true
857
+ rm -f "${OPENCODE_PID_FILE}"
858
+ log "opencode stopped"
859
+ }
860
+
861
+ # The ceiling stop_opencode_and_wait (below) waits before its SIGKILL backstop.
862
+ # /terminate can afford a real drain — the VM is being torn down either way —
863
+ # unlike /run's cleanup, which cannot (stop_opencode's own cheapness above is
864
+ # load-bearing for that budget, §4, and must not grow a wait).
865
+ OPENCODE_STOP_WAIT_SECONDS="${EVIDENT_OPENCODE_STOP_WAIT_SECONDS:-5}"
866
+
867
+ # The graceful stop /terminate needs (#812 WI-4): SIGTERM, poll for exit,
868
+ # SIGKILL backstop, clear the pid file. A NEW function rather than growing
869
+ # stop_opencode itself, for the reason above — mirrors stop_tunnel's and
870
+ # stop_litestream's shape, including PROC_DEAD_REASON's one deciding read
871
+ # after the poll loop (never inside it, which would bury the log at 10x/s).
872
+ # This is half of the ordered-shutdown invariant (plan §6): /terminate must
873
+ # stop opencode BEFORE flush_session_db's final sync, or litestream could
874
+ # snapshot while opencode is still writing its WAL and the last session
875
+ # writes would be missing from S3.
876
+ stop_opencode_and_wait() {
877
+ if ! opencode_is_running; then
878
+ rm -f "${OPENCODE_PID_FILE}"
879
+ log "no opencode to stop"
880
+ return 0
881
+ fi
882
+
883
+ local pid
884
+ pid="$(cat "${OPENCODE_PID_FILE}")"
885
+ kill -TERM "${pid}" 2>/dev/null || true
886
+
887
+ local tenths=0
888
+ local deadline_tenths=$(( OPENCODE_STOP_WAIT_SECONDS * 10 ))
889
+ while [ "${tenths}" -lt "${deadline_tenths}" ]; do
890
+ process_is_alive "${pid}" || break
891
+ sleep 0.1
892
+ tenths=$(( tenths + 1 ))
893
+ done
894
+
895
+ local outcome
896
+ if process_is_alive "${pid}"; then
897
+ warn "opencode ${pid} ignored SIGTERM; killing"
898
+ kill -KILL "${pid}" 2>/dev/null || true
899
+ outcome="after SIGKILL (ignored SIGTERM for ${OPENCODE_STOP_WAIT_SECONDS}s)"
900
+ else
901
+ outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
902
+ fi
903
+
904
+ rm -f "${OPENCODE_PID_FILE}"
905
+ log "opencode stopped ${outcome}"
906
+ }
907
+
908
+ # --- litestream replicate (#812 WI-3) ---------------------------------------
909
+ #
910
+ # The write half of session-DB persistence. restore_session_db (above) is the
911
+ # read half only and never starts this. Q8 chose one backgrounded, unsupervised
912
+ # `litestream replicate` over a periodic flush or a `replicate -exec` wrapper
913
+ # around opencode: SIGTERM is `replicate`'s own final-sync trigger, so
914
+ # stop_litestream (below) doubles as the checked flush /suspend and /terminate
915
+ # need, and there is no supervisor in this image to hand the process to (no
916
+ # init, no `wait -n` shell that could stay alive after the hook returns).
917
+
918
+ # The graceful wait before the SIGKILL backstop in stop_litestream, below.
919
+ # litestream's own sync is normally sub-second, so 10s is generous headroom —
920
+ # chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
921
+ LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
922
+
923
+ # Guards, in this exact order and no other — reordering any of them re-arms the
924
+ # invariant it exists to protect:
925
+ #
926
+ # 1. persistence disabled (#770) — bucket AND prefix both non-empty — is its
927
+ # OWN arm with its OWN message, checked FIRST, and must never log
928
+ # SESSION-DB-REPLICA-UNUSABLE: an operator grepping that marker must not
929
+ # hit the disabled case (#764).
930
+ # 2. the no-replicate marker (restore_session_db above) — set when this boot's
931
+ # local DB may be only PARTIALLY restored, so starting replicate here would
932
+ # let it overwrite a replica this boot never proved safe to write over.
933
+ # A classifier 31 no longer sets it (#1106): that DB is guaranteed fresh,
934
+ # and skipping replicate for it was what made the history loss unbounded.
935
+ # 3. no usable config — ensure_litestream_config (restore_session_db, above)
936
+ # already logged why it is missing; this is not a place to retry it.
937
+ # 4. already running — idempotence, mirroring start_tunnel's own guard.
938
+ #
939
+ # Only past all four does it actually spawn, backgrounded with `setsid` for the
940
+ # same reason start_opencode/start_tunnel are: the hook must return so AWS gets
941
+ # its 200, while replicate keeps running. litestream's OWN stderr is left to
942
+ # reach CloudWatch — never redirected — because there is no supervisor in this
943
+ # image, so that is the only channel that can say why a replicator died.
944
+ start_litestream() {
945
+ if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
946
+ warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing is replicated."
947
+ return 0
948
+ fi
949
+
950
+ if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
951
+ log "skipping litestream replicate: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
952
+ return 0
953
+ fi
954
+
955
+ if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
956
+ error "no usable ${LITESTREAM_CONFIG_FILE}; not starting litestream replicate"
957
+ return 0
958
+ fi
959
+
960
+ if litestream_is_running; then
961
+ warn "litestream already running (pid $(cat "${LITESTREAM_PID_FILE}")); reusing it"
962
+ return 0
963
+ fi
964
+
965
+ setsid litestream replicate -config "${LITESTREAM_CONFIG_FILE}" &
966
+
967
+ echo $! >"${LITESTREAM_PID_FILE}"
968
+ log "litestream replicate starting (pid $(cat "${LITESTREAM_PID_FILE}"))"
969
+ }
970
+
971
+ # The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
972
+ # one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
973
+ # would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
974
+ # operator needs told apart, not a flatter clean/killed binary. /suspend and
975
+ # /terminate call this: the SIGTERM it sends IS the checked flush they need.
976
+ stop_litestream() {
977
+ if ! litestream_is_running; then
978
+ rm -f "${LITESTREAM_PID_FILE}"
979
+ log "no litestream to stop"
980
+ return 0
981
+ fi
982
+
983
+ local pid
984
+ pid="$(cat "${LITESTREAM_PID_FILE}")"
985
+ kill -TERM "${pid}" 2>/dev/null || true
986
+
987
+ local tenths=0
988
+ local deadline_tenths=$(( LITESTREAM_STOP_WAIT_SECONDS * 10 ))
989
+ while [ "${tenths}" -lt "${deadline_tenths}" ]; do
990
+ process_is_alive "${pid}" || break
991
+ sleep 0.1
992
+ tenths=$(( tenths + 1 ))
993
+ done
994
+
995
+ local outcome
996
+ if process_is_alive "${pid}"; then
997
+ warn "litestream ${pid} ignored SIGTERM; killing"
998
+ kill -KILL "${pid}" 2>/dev/null || true
999
+ outcome="after SIGKILL (ignored SIGTERM for ${LITESTREAM_STOP_WAIT_SECONDS}s)"
1000
+ else
1001
+ outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
1002
+ fi
1003
+
1004
+ rm -f "${LITESTREAM_PID_FILE}"
1005
+ log "litestream stopped ${outcome}"
1006
+ }
1007
+
1008
+ # The cheap stop: one SIGTERM, one rm, one log line, NO poll loop — `/run`'s
1009
+ # own cleanup calls this, never stop_litestream, because that path only runs
1010
+ # after /run has already FAILED: no session happened yet, so a graceful final
1011
+ # sync is worth nothing against a SIGTERM budget that cannot afford another
1012
+ # wait on top of stop_tunnel's own. A distinct function rather than a mode
1013
+ # argument on stop_litestream:
1014
+ # stop_tunnel's own comment argues against an argument a future caller can get
1015
+ # wrong. litestream_is_running is the same O(1) check stop_litestream's own
1016
+ # no-op branch uses, not a poll — /run's cleanup trap fires this before
1017
+ # start_litestream is ever reached whenever an earlier step failed, and an
1018
+ # operator reading that log must not be told a stop signal went to a process
1019
+ # that never started.
1020
+ kill_litestream() {
1021
+ if ! litestream_is_running; then
1022
+ rm -f "${LITESTREAM_PID_FILE}"
1023
+ log "no litestream to stop"
1024
+ return 0
1025
+ fi
1026
+
1027
+ kill -TERM "$(cat "${LITESTREAM_PID_FILE}")" 2>/dev/null || true
1028
+ rm -f "${LITESTREAM_PID_FILE}"
1029
+ log "litestream stop signalled (no wait)"
1030
+ }
1031
+ # --- litestream replicate (end) ----------------------------------------------
1032
+
1033
+ # --- flush_session_db (#812 WI-4) -------------------------------------------
1034
+ #
1035
+ # The checked, synchronous flush /suspend and /terminate need before they
1036
+ # finish tearing down. Never fatal, for the same reason every session-DB
1037
+ # function in this file is: /suspend and /terminate must complete their own
1038
+ # teardown regardless of whether S3 could be reached.
1039
+
1040
+ # Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
1041
+ # own sync is normally sub-second (M5, the plan's grounding); 10s is generous
1042
+ # headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
1043
+ # measured worst case. The SIGKILL backstop below reuses
1044
+ # SESSION_DB_RESTORE_KILL_GRACE_SECONDS rather than its own literal: it is the
1045
+ # identical concept (bare `timeout` only SIGTERMs, and a litestream that
1046
+ # ignored it would be unbounded again), and there is no reason for the restore
1047
+ # and flush paths to ever drift on how long a SIGTERM gets to land.
1048
+ SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
1049
+
1050
+ # Guards mirror start_litestream's first three exactly (disabled -> marker ->
1051
+ # config) — a flush must never attempt work those guards would have refused
1052
+ # to start in the first place. PERSISTENCE_BUCKET is the SAME variable
1053
+ # start_litestream reads, so both agree regardless of which earlier step in
1054
+ # THIS process set it (load_state_config, above, or restore_session_db's
1055
+ # eval, on /run).
1056
+ #
1057
+ # Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
1058
+ # trigger — and only once it is confirmed stopped does the synchronous -once
1059
+ # flush run, as a second, checked pass. Running -once while the daemon is
1060
+ # still alive would put two writers on one prefix, the single-writer
1061
+ # invariant this whole feature exists to protect (Q8) — this ordering is not
1062
+ # negotiable.
1063
+ #
1064
+ # If the daemon has already died (a missing or stale pid file), that IS the
1065
+ # detection this needs: SESSION-DB-REPLICATOR-DIED names it loudly instead of
1066
+ # silently skipping straight to the flush (development-workflow.mdc: every
1067
+ # recovery branch emits a server-visible signal). The flush still runs either
1068
+ # way — it is what actually gets any unreplicated writes to S3 before the
1069
+ # caller's next step.
1070
+ flush_session_db() {
1071
+ if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
1072
+ warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing to flush."
1073
+ return 0
1074
+ fi
1075
+
1076
+ if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
1077
+ log "skipping session-DB flush: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
1078
+ return 0
1079
+ fi
1080
+
1081
+ if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
1082
+ error "no usable ${LITESTREAM_CONFIG_FILE}; cannot flush the session DB"
1083
+ return 0
1084
+ fi
1085
+
1086
+ if litestream_is_running; then
1087
+ stop_litestream
1088
+ else
1089
+ warn "SESSION-DB-REPLICATOR-DIED: no live litestream at ${LITESTREAM_PID_FILE}; flushing one-shot anyway"
1090
+ fi
1091
+
1092
+ # Only reached with the daemon confirmed stopped, or never running — never
1093
+ # concurrently with it (see above).
1094
+ local flush_rc=0
1095
+ timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
1096
+ litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
1097
+
1098
+ if [ "${flush_rc}" -eq 0 ]; then
1099
+ log "SESSION-DB-FINAL-FLUSH: succeeded, ${SECONDS}s into the hook"
1100
+ else
1101
+ warn "SESSION-DB-FINAL-FLUSH: litestream replicate -once exited ${flush_rc}, ${SECONDS}s into the hook; some writes since the last sync may not have reached S3"
1102
+ fi
1103
+ }
1104
+ # --- flush_session_db (end) --------------------------------------------------
1105
+
1106
+ # How long the guest CLI runs with no activity before it exits itself
1107
+ # (`evident run --idle-timeout`, apps/cli/src/commands/run.ts), which is what
1108
+ # turns a truly-abandoned VM into the clean-offline POST that lets Evident
1109
+ # suspend it (#732). Sized from the measured cost of guessing wrong rather than
1110
+ # the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
1111
+ # ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
1112
+ # napping a VM whose user comes straight back costs ~10 s — against ~$0.30/h
1113
+ # that is cheap enough that the balance sits far nearer the floor than the
1114
+ # ceiling. Not AT the floor, though: the CLI's idle detector needs 2 clear poll
1115
+ # cycles (`run.ts`'s `idlePolls >= 2`, ≥4 s of real time), so a value near that
1116
+ # would spend more time suspending/resuming than idle.
1117
+ # ECS's waker uses 900 s instead only because *its* cold start is far slower
1118
+ # than this VM's ~2 s resume — not evidence this default should match it.
1119
+ #
1120
+ # NOT part of the hook's own teardown budget (TUNNEL_STOP_WAIT_SECONDS et al.,
1121
+ # below): it governs the CLI's lifetime long after the hook has already
1122
+ # returned its 200, so it is deliberately outside that arithmetic.
1123
+ #
1124
+ # EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
1125
+ # packages/runner-image/entrypoint.sh uses for the ECS runner's own
1126
+ # idle-timeout flag, so an operator who knows one knows the other. A
1127
+ # non-numeric override must never silently DROP the flag — that degrades to
1128
+ # an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
1129
+ # warns and falls back to the default instead.
1130
+ #
1131
+ # Deliberately NOT in the /run payload yet: doing so would touch the doorbell
1132
+ # contract in four coordinated places for no capability today. #716 shipped
1133
+ # the swarm/per-user provisioner but explicitly scoped control-plane
1134
+ # idle/suspend policy OUT (its own "Phase 3" — see the issue's "Out of
1135
+ # Scope"), so a per-pool idle policy is still a clean follow-up, not
1136
+ # something #716 already provides a home for.
1137
+ IDLE_TIMEOUT_SECONDS=120
1138
+ if [ -n "${EVIDENT_IDLE_TIMEOUT_SECONDS:-}" ]; then
1139
+ if [[ "${EVIDENT_IDLE_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]]; then
1140
+ IDLE_TIMEOUT_SECONDS="${EVIDENT_IDLE_TIMEOUT_SECONDS}"
1141
+ else
1142
+ warn "EVIDENT_IDLE_TIMEOUT_SECONDS='${EVIDENT_IDLE_TIMEOUT_SECONDS}' is not numeric; using the default ${IDLE_TIMEOUT_SECONDS}s instead of leaving the VM always-on"
1143
+ fi
1144
+ fi
1145
+
1146
+ # `setsid` so the tunnel outlives this hook: the script must return so AWS gets
1147
+ # its 200, while the tunnel keeps serving.
1148
+ start_tunnel() {
1149
+ local runner_key="$1" api_url="$2" tunnel_url="$3"
1150
+
1151
+ if tunnel_is_running; then
1152
+ warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
1153
+ return 0
1154
+ fi
1155
+
1156
+ # The key travels in the child's environment ONLY: argv is world-readable
1157
+ # through /proc.
1158
+ # --enable-file-sync-to is what makes the UI's "connect a provider account"
1159
+ # flow reach this VM: without it the CLI declines every queued file and the
1160
+ # page can only say "this runner isn't accepting files". There is no shell
1161
+ # into a MicroVM, so it is the ONLY way to seed model credentials on a runner
1162
+ # that is already up — the S3 credential store is read at /run and /resume,
1163
+ # never mid-life. The allow-list stays a single directory, as on ECS
1164
+ # (packages/runner-image/entrypoint.sh): the flag is repeatable, but every
1165
+ # extra entry widens what Evident can write into a VM that executes agent
1166
+ # code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
1167
+ # an unset one abort rather than silently allow-list "/.claude".
1168
+ EVIDENT_RUNNER_KEY="${runner_key}" setsid evident run \
1169
+ --port "${OPENCODE_PORT}" \
1170
+ --endpoint "${api_url}" \
1171
+ --tunnel "${tunnel_url}" \
1172
+ --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
1173
+ --enable-file-sync-to "${HOME}/.claude" &
1174
+
1175
+ echo $! >"${TUNNEL_PID_FILE}"
1176
+ log "tunnel started (pid $(cat "${TUNNEL_PID_FILE}"))"
1177
+ }
1178
+
1179
+ # The worst case `evident run` can take to shut down gracefully on SIGTERM, in
1180
+ # whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS,
1181
+ # apps/cli/src/commands/run.ts) + 2 s offline POST (notifyAgentDisconnected,
1182
+ # apps/cli/src/commands/agent-lookup.ts) + 5 s telemetry flush
1183
+ # (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts). Each of the three is bounded there, so
1184
+ # this is a ceiling rather than a typical cost — an idle suspend finishes in a
1185
+ # couple of seconds. This is the ONE place the budget is written down; the CLI
1186
+ # only points back here, because restating it in three places produced #657.
1187
+ #
1188
+ # DOCUMENTATION ONLY — nothing is derived from this any more. The wait below
1189
+ # used to be pinned above it, which is the reasoning that talked #699 into 40 s;
1190
+ # since #718 an exited CLI is noticed on the first poll, so the wait is a
1191
+ # backstop chosen on its own merits and the two numbers are unrelated. Nothing
1192
+ # cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
1193
+ # three bounds cited above, so if one of them moves, update it here.
1194
+ # shellcheck disable=SC2034 # documentation; deliberately read by nothing
1195
+ CLI_SHUTDOWN_CEILING_SECONDS=32
1196
+
1197
+ # How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
1198
+ # Since #718 this binds ONLY for a CLI that is still draining: one that has
1199
+ # already exited is detected on the first 0.1 s poll whatever this says, because
1200
+ # process_is_alive no longer mistakes its unreaped corpse for a live process.
1201
+ # That is what takes a routine suspend from ~41 s back to the ~2 s it measures.
1202
+ #
1203
+ # So 10 is chosen, not derived, and it sits in the middle of a real range. The
1204
+ # measured shutdown was 1255 ms of drain inside a 2322 ms total (#697): 5 s is
1205
+ # only ~2.2x that and re-arms #657's complaint of a kill landing mid-drain,
1206
+ # while 30 s never truncates a legitimate drain but rebuilds the headroom
1207
+ # problem #699 created — 41 s against a 60 s lifecycleTimeout leaves 19 s for
1208
+ # everything else in the hook. 10 s is ~4x the measured shutdown and gives a
1209
+ # provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
1210
+ # before us = 20 s, inside the ~55 s the in-VM server allows this script
1211
+ # (DEFAULT_TIMEOUT_SECONDS, packages/lambda-microvm-runtime/src/runtime.ts),
1212
+ # itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
1213
+ # overrunning that fails the whole lifecycle transition, which is worse than the
1214
+ # kill this replaces. The residual cost is a drain longer than 10 s being
1215
+ # truncated; that work stays `processing` server-side and is re-adopted on the
1216
+ # next start (ADR-0046).
1217
+ #
1218
+ # src/image/hook-scripts.test.ts holds both that this stays small and that it
1219
+ # fits the hook timeout, and times a real SIGKILL against the override to prove
1220
+ # the loop HONOURS it rather than a hardcoded deadline. The env override is for
1221
+ # tests only, so they need not burn 10 s of wall clock.
1222
+ TUNNEL_STOP_WAIT_SECONDS="${EVIDENT_TUNNEL_STOP_WAIT_SECONDS:-10}"
1223
+
1224
+ stop_tunnel() {
1225
+ if ! tunnel_is_running; then
1226
+ # Clear the file here too, not only on the path below: `evident run` has
1227
+ # self-exit paths that never reach this function (auth expired, idle
1228
+ # timeout, a crash), so the pid file routinely outlives the process it
1229
+ # names. Leaving it means every later guard re-reads a dead pid — harmless
1230
+ # while process_is_alive agrees it is dead, and #718 again the moment
1231
+ # anything starts reaping orphans and the pid gets recycled.
1232
+ rm -f "${TUNNEL_PID_FILE}"
1233
+ log "no tunnel to stop"
1234
+ return 0
1235
+ fi
1236
+
1237
+ local pid
1238
+ pid="$(cat "${TUNNEL_PID_FILE}")"
1239
+ # One drain length for every caller: /suspend and /terminate always run over
1240
+ # a connected tunnel, and /run's and /resume's own cleanup traps only reach a
1241
+ # live tunnel here when it was never started (the cheap no-op branch above) —
1242
+ # there is no path left where a just-spawned, not-yet-connected tunnel is the
1243
+ # one being stopped, so there is nothing left to special-case.
1244
+ kill -TERM "${pid}" 2>/dev/null || true
1245
+
1246
+ local tenths=0
1247
+ local deadline_tenths=$(( TUNNEL_STOP_WAIT_SECONDS * 10 ))
1248
+ while [ "${tenths}" -lt "${deadline_tenths}" ]; do
1249
+ process_is_alive "${pid}" || break
1250
+ sleep 0.1
1251
+ tenths=$(( tenths + 1 ))
1252
+ done
1253
+
1254
+ # This call, not the loop's, is the deciding read: it is what the `if` below
1255
+ # branches on, so PROC_DEAD_REASON is fresh by construction. A corpse can
1256
+ # still be reaped between the loop's last poll and this call, so the reason
1257
+ # named below is what THIS call saw, not necessarily what the loop saw.
1258
+ local outcome
1259
+ if process_is_alive "${pid}"; then
1260
+ warn "tunnel ${pid} ignored SIGTERM; killing"
1261
+ kill -KILL "${pid}" 2>/dev/null || true
1262
+ outcome="after SIGKILL (ignored SIGTERM for ${TUNNEL_STOP_WAIT_SECONDS}s)"
1263
+ else
1264
+ outcome="cleanly in $(( tenths / 10 )).$(( tenths % 10 ))s (${PROC_DEAD_REASON})"
1265
+ fi
1266
+
1267
+ rm -f "${TUNNEL_PID_FILE}"
1268
+ log "tunnel stopped ${outcome}"
1269
+ }
1270
+
1271
+ # --- check_runner_key (#1172) ------------------------------------------------
1272
+ #
1273
+ # Answers exactly one question before opencode/the tunnel start spending this
1274
+ # boot's SIGTERM budget on a key that cannot work: "does the runner key in this
1275
+ # payload authenticate against Evident?" Delegates entirely to the CLI's own
1276
+ # `evident status --json` (apps/cli/src/commands/status.ts) rather than
1277
+ # reimplementing its auth logic here — that command's `reason` field is the
1278
+ # published contract this function reads, and its own header states the
1279
+ # absent-vs-contrary distinction this function must honour.
1280
+ #
1281
+ # Keyed on the JSON `reason`, NEVER on the exit code: an older CLI without a
1282
+ # `status` subcommand exits 1 from Commander's own "unknown command" handling,
1283
+ # which under an exit-code mapping would read as "the key was rejected" and
1284
+ # destroy every VM on a hook/CLI version skew — precisely the hazard this
1285
+ # image's README's "version skew" section flags. So no parseable JSON line
1286
+ # means no verdict, whatever the exit code says.
1287
+ #
1288
+ # Returns 0 for positive AND absent evidence (network/timeout/5xx/404/old-CLI/
1289
+ # crash) — the caller continues either way — and 1 ONLY for contrary evidence
1290
+ # (401, another 4xx, or no credentials resolved at all): the key is actually
1291
+ # wrong, not merely untested (development-workflow.mdc's "never fail a gate on
1292
+ # absent evidence"). Every branch logs a named, greppable RUNNER-KEY-* signal
1293
+ # so a caller that treats the return value as fatal (or doesn't) still leaves
1294
+ # a trace of which branch fired.
1295
+ #
1296
+ # A 404 is deliberately NOT contrary: it means the endpoint has no /me route,
1297
+ # so the key was never tested. Treating it as a rejection destroyed every VM
1298
+ # for a day when the run payload shipped a bare origin instead of the CLI's
1299
+ # /v1-prefixed one.
1300
+ check_runner_key() {
1301
+ local runner_key="$1" api_url="$2"
1302
+
1303
+ # status.ts's own exit-code contract (its header comment) means this exits
1304
+ # non-zero on EVERY branch except `ok` — guarded exactly like
1305
+ # ensure_litestream_config's `run_synchroniser litestream-config` call,
1306
+ # above, so that expected non-zero does not abort this function under the
1307
+ # caller's `set -e` before the case below ever runs. Only stdout is
1308
+ # captured: status.ts's own contract is one parseable JSON line and nothing
1309
+ # else there, and — like litestream's stderr elsewhere in this file — its
1310
+ # stderr is left to reach CloudWatch directly rather than being folded in,
1311
+ # since a stray stderr line ahead of the JSON would otherwise break `jq`.
1312
+ local response rc=0
1313
+ response="$(EVIDENT_API_URL="${api_url}" EVIDENT_RUNNER_KEY="${runner_key}" evident status --json)" || rc=$?
1314
+
1315
+ # `-z` as well as jq's own exit code, and not as belt-and-braces: on EMPTY
1316
+ # input jq has no value to report on, so `jq -e` exits 0 with empty output
1317
+ # rather than its documented 4 (verified on jq 1.6, the image's). Empty
1318
+ # stdout is exactly the old-CLI-skew and missing-binary shape this branch
1319
+ # exists for, so keying only on the exit code sent precisely those cases to
1320
+ # the unrecognised-reason arm below — same safe return, but a log line
1321
+ # blaming the API for an odd answer instead of naming the CLI skew, which is
1322
+ # the one thing an operator needs told here.
1323
+ local reason
1324
+ reason="$(printf '%s' "${response}" | jq -er '.reason' 2>/dev/null)" || reason=""
1325
+ if [ -z "${reason}" ]; then
1326
+ warn "RUNNER-KEY-UNVERIFIABLE: 'evident status --json' exited ${rc} with no parseable JSON on stdout (an old CLI without this subcommand, a missing binary, or a crash — see its stderr above, if any); the key was NOT validated, continuing anyway"
1327
+ return 0
1328
+ fi
1329
+ local detail
1330
+ detail="$(printf '%s' "${response}" | jq -er '.error // empty' 2>/dev/null)" || detail=""
1331
+
1332
+ case "${reason}" in
1333
+ ok)
1334
+ log "RUNNER-KEY-OK: evident status confirmed the runner key against ${api_url}"
1335
+ return 0
1336
+ ;;
1337
+ unauthorized | no_credentials | http_error)
1338
+ error "RUNNER-KEY-REJECTED: evident status reason=${reason} against ${api_url}${detail:+: ${detail}}"
1339
+ return 1
1340
+ ;;
1341
+ unreachable)
1342
+ warn "RUNNER-KEY-UNREACHABLE: could not reach ${api_url} to validate the runner key; the key was NOT validated, continuing anyway"
1343
+ return 0
1344
+ ;;
1345
+ endpoint_not_found)
1346
+ # Absent evidence, NOT contrary: a 404 means this endpoint has no /me
1347
+ # route (typically a bare origin where the CLI wants the /v1-prefixed
1348
+ # one), which says nothing about the key. Loud, because the runner will
1349
+ # keep failing every API call until the endpoint is fixed.
1350
+ warn "RUNNER-KEY-ENDPOINT-NOT-FOUND: ${api_url} has no /me route${detail:+: ${detail}}; the key was NOT validated, continuing anyway"
1351
+ return 0
1352
+ ;;
1353
+ *)
1354
+ warn "RUNNER-KEY-UNKNOWN-REASON: evident status returned an unrecognised reason='${reason}'; the key was NOT validated, continuing anyway"
1355
+ return 0
1356
+ ;;
1357
+ esac
1358
+ }
1359
+ # --- check_runner_key (end) ---------------------------------------------------