@evident-ai/runner-cdk 3.4.1-dev.51df4de → 3.4.1-dev.59c7df3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,17 +39,12 @@ STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
39
39
  # The generated litestream.yml (#812). tmpfs for the same two reasons as the
40
40
  # files above: it must never reach the block device (Q6, `docs`), and it
41
41
  # survives suspend/resume, which is what lets `/resume` start litestream again
42
- # with no regeneration cost. Regenerated only when absent or empty
43
- # (`ensure_litestream_config`, below).
42
+ # with no regeneration cost. The CLI regenerates it only when absent or empty.
44
43
  LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
45
44
 
46
- # Set when a `session-db-classify` answer of 31 (replica unusable), or a
47
- # restore this hook gave up on for its own reasons (a truncated `timeout`, a
48
- # broken tool), means this boot must NOT start `litestream replicate` — doing
49
- # so would let a partial/fresh local DB overwrite a replica this boot never
50
- # proved is safe to write over. Cleared at the top of every `restore_session_db`
51
- # call (i.e. every `/run`) before any other decision, so a marker left by an
52
- # earlier boot cannot silently disable replication for the VM's whole life.
45
+ # Set by the CLI when session-DB restore or verification could not prove this
46
+ # boot safe to replicate. The teardown hooks read it before flushing, and
47
+ # `/terminate` removes it with the rest of the per-VM state.
53
48
  SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
54
49
 
55
50
  hook_name() { printf '%s' "${0##*/}"; }
@@ -57,38 +52,6 @@ log() { echo "[hook:$(hook_name)] $*"; }
57
52
  warn() { echo "[hook:$(hook_name)] $*" >&2; }
58
53
  error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
59
54
 
60
- session_db_recovery_report_path() {
61
- local path="${EVIDENT_SESSION_DB_RECOVERY_REPORT:-}"
62
- path="${path#"${path%%[![:space:]]*}"}"
63
- path="${path%"${path##*[![:space:]]}"}"
64
- if [ -n "${path}" ]; then
65
- printf '%s\n' "${path}"
66
- return 0
67
- fi
68
- local home="${HOME:-}"
69
- home="${home#"${home%%[![:space:]]*}"}"
70
- home="${home%"${home##*[![:space:]]}"}"
71
- [ -n "${home}" ] || return 1
72
- printf '%s\n' "${home}/.local/state/evident/session-db-recovery.jsonl"
73
- }
74
-
75
- report_session_db_giveup() {
76
- local reason="$1" exit_code="${2:-null}" outcome path at
77
- case "${reason}" in
78
- synchroniser_config_unresolved|synchroniser_config_unevaluable|synchroniser_config_incomplete|litestream_config_unavailable|classification_fatal) outcome="restore_misconfigured" ;;
79
- restore_deadline_exceeded|restore_tool_unusable|classification_unrecognised) outcome="fresh_session_db" ;;
80
- *) warn "SESSION-DB-RECOVERY-REPORT-UNKNOWN-REASON: ${reason}"; return 0 ;;
81
- esac
82
- if ! path="$(session_db_recovery_report_path)"; then
83
- warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
84
- return 0
85
- fi
86
- at="${EPOCHREALTIME:-}"
87
- [ -n "${at}" ] || warn "SESSION-DB-RECOVERY-REPORT-TIMESTAMP-UNAVAILABLE"
88
- printf '{"v":1,"event":"session_db_recovery","at":"%s","stage":"restore","outcome":"%s","severity":"error","reason":"%s","litestream_exit_code":%s,"attempt":null,"replica_objects":null,"replica_bytes":null,"quarantine_destination":null,"quarantined_objects":null,"quarantine_failed_objects":null,"quarantined_bytes":null,"verified_restore_point":null,"restore_points_tried":null,"replication_suspended":true}\n' "${at}" "${outcome}" "${reason}" "${exit_code}" >>"${path}" || warn "SESSION-DB-RECOVERY-REPORT-APPEND-FAILED"
89
- return 0
90
- }
91
-
92
55
  # Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
93
56
  # corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
94
57
  # `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
@@ -96,35 +59,13 @@ report_session_db_giveup() {
96
59
  # exhausted, replica separated and local disposed) / 34 (could not prove
97
60
  # separation or disposal) — see `runner/synchroniser/src/cli.ts`'s own
98
61
  # comment for what each means, not restated here. Any OTHER non-zero status
99
- # means the tool itself broke, which is the only case worth an ERROR here
100
- # EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
101
- # that is an intentional bound firing, not a broken tool, so the caller
102
- # classifies it instead (restore_credentials, #930). A trailing
103
- # `--evident-deadline=N` (stripped below before forwarding to the CLI, and
104
- # never produced by anything but timed_synchroniser) is how a caller opts in;
105
- # every other caller — sync_credentials, restore_session_db's `env` /
106
- # `litestream-config` / `session-db-classify`, and this file's own contract
107
- # test — passes none, so 124/137 there still means a genuine external SIGKILL
108
- # (e.g. an OOM kill) and must keep producing the ERROR below.
62
+ # means the tool itself broke, which is the only case worth an ERROR here.
109
63
  run_synchroniser() {
110
- local -a call_args=("$@")
111
- local deadline=""
112
- local last=$(( ${#call_args[@]} - 1 ))
113
- if [ "${last}" -ge 0 ] && [[ "${call_args[last]}" == --evident-deadline=* ]]; then
114
- deadline="${call_args[last]#--evident-deadline=}"
115
- call_args=("${call_args[@]:0:${last}}")
116
- fi
117
-
118
64
  local rc=0
119
- local -a launcher=()
120
- [ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
121
- "${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
65
+ "${SYNCHRONISER}" "$@" || rc=$?
122
66
  case "${rc}" in
123
67
  0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
124
- 124 | 137)
125
- [ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
126
- ;;
127
- *) error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
68
+ *) error "synchroniser '$*' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
128
69
  esac
129
70
  return "${rc}"
130
71
  }
@@ -147,8 +88,8 @@ run_synchroniser() {
147
88
  # Never fails its caller: a timing line is diagnostics, and hardening a
148
89
  # currently-working path is worse than the gap it closes (#931). The `date`
149
90
  # fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
150
- # timed_synchroniser turns into `elapsed_ms=unknown` rather than aborting /run
151
- # under `set -e` — and rather than a 0 that would read as a fast healthy call.
91
+ # `log_elapsed_since` turns into `elapsed_ms=unknown` rather than aborting a
92
+ # hook under `set -e` — and rather than a 0 that would read as a fast healthy call.
152
93
  now_ms() {
153
94
  local now="${EPOCHREALTIME:-}"
154
95
  if [ -n "${now}" ]; then
@@ -159,16 +100,10 @@ now_ms() {
159
100
  date +%s%3N 2>/dev/null || echo 0
160
101
  }
161
102
 
162
- # Reports what one step COST, in a deliberately machine-greppable line (`op=`,
163
- # `elapsed_ms=`, `rc=`) so "how long does this actually take in the fleet?" is a
164
- # log query rather than another spike. `at_s` is where the step landed on the
165
- # hook's own SECONDS clock, which is what the pre-opencode budget in
166
- # hook-scripts.test.ts is derived against.
167
- #
168
- # Callable directly, not only through timed_synchroniser below, because the
169
- # steps that dominate the pre-opencode window CANNOT be wrapped: their stdout is
170
- # captured (`x="$(run_synchroniser env)"`, later `eval`'d), so a `log` line
171
- # emitted inside the substitution would be evaluated as configuration.
103
+ # Reports what one pre-warm COST, in a deliberately machine-greppable line
104
+ # (`op=`, `elapsed_ms=`, `rc=`) so "how long does this actually take in the
105
+ # fleet?" is a log query rather than another spike. `at_s` is where the step
106
+ # landed on the hook's own SECONDS clock.
172
107
  #
173
108
  # `unknown`, never a number, when either end failed to read the clock (now_ms's
174
109
  # 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
@@ -187,26 +122,6 @@ log_elapsed_since() {
187
122
  log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
188
123
  }
189
124
 
190
- # Runs ONE synchroniser call and reports what it COST, without changing what it
191
- # MEANS: the call's own status is returned untouched, so every caller's
192
- # `|| return 1` / `|| status=$?` keeps working exactly as before.
193
- #
194
- # `deadline` is an explicit parameter, forwarded to run_synchroniser as a
195
- # trailing `--evident-deadline=N` (never a global or dynamically-scoped
196
- # variable, #930) — every current caller (restore_credentials) always has a
197
- # positive remaining-budget value by the time it calls this, having already
198
- # decided to skip the call entirely otherwise.
199
- timed_synchroniser() {
200
- local op="$1" deadline="$2"
201
- shift 2
202
-
203
- local started_ms rc=0
204
- started_ms="$(now_ms)"
205
- run_synchroniser "$@" "--evident-deadline=${deadline}" || rc=$?
206
- log_elapsed_since "${op}" "${started_ms}" "${rc}"
207
- return "${rc}"
208
- }
209
-
210
125
  # Exports what `runner-synchroniser` resolves its object-store location from
211
126
  # (runner/synchroniser/src/config.ts). It treats either being empty as
212
127
  # "persistence disabled" and then reports every restore as a WARNING it still
@@ -228,279 +143,32 @@ load_state_config() {
228
143
 
229
144
  # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
230
145
  # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
231
- # is just LITESTREAM_BUCKET. start_litestream and flush_session_db (#812
146
+ # is just LITESTREAM_BUCKET. The CLI's replicator and flush_session_db (#812
232
147
  # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
233
148
  # directly, so every entry point agrees on one predicate regardless of which
234
- # earlier step in THIS hook process set it: restore_session_db's
235
- # `eval "$(run_synchroniser env)"` on /run, or this function on /resume,
236
- # /suspend and /terminate, none of which ever call the synchroniser before
237
- # needing the answer.
149
+ # earlier step in THIS hook process set it; /resume, /suspend and /terminate
150
+ # all use this same function before any teardown decision.
238
151
  export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
239
152
  }
240
153
 
241
- # /run starts no LLM task it restores state, starts opencode and dials the
242
- # tunnel, none of which needs a model credential. Which credential a turn needs
243
- # depends on the model that turn's message asks for, so that question has no
244
- # single boot-time answer; it is answered per message, not here. What IS still
245
- # fatal is credential persistence being unavailable at all (an unset bucket/
246
- # prefix, or a broken synchroniser bundle) — that would silently drop refreshed
247
- # tokens on every suspend, so those two restores stay `|| return 1`.
248
- # `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
249
- # a log line, since the runtime forwards no hook stderr to the doorbell caller.
250
-
251
- # How long restore_credentials' shared step budget is, covering the runner-secret
252
- # fetch plus its three synchroniser calls (restore claude, restore opencode,
253
- # model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
254
- # total would false-fire on the ordinary case of one slow call (a slow S3 GET),
255
- # which is precisely the boot this exists to keep healthy. From #930's 3-boot
256
- # sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
257
- # node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
258
- # SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
259
- # routine slow call boots this VM with no model credentials until an operator
260
- # reconnects it; too loose and a hung call burns more of the hook's own
261
- # SIGTERM budget before the VM is destroyed mid-boot anyway. The env override
262
- # is for tests only, so they need not burn wall clock.
263
- CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECONDS:-8}"
264
- # The SIGKILL backstop `timeout -k` applies after its own SIGTERM, exactly like
265
- # SESSION_DB_RESTORE_KILL_GRACE_SECONDS above (a bare `timeout` only SIGTERMs,
266
- # and a call that ignored it would be unbounded again). Hard ceiling on the
267
- # step: DEADLINE + this = 10s, once — not per call.
268
- CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
269
- GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
270
-
271
- # What is left of the shared step budget, in whole seconds, `step_started_s`
272
- # seconds after it began. `SECONDS` (a bash builtin with no failure mode,
273
- # unlike `now_ms`) truncates, so a step that began at true time `s0` can read
274
- # less elapsed time than actually passed — the `- 1` restores the invariant
275
- # that no call is granted more than DEADLINE + GRACE from step start.
276
- remaining_credential_budget() {
277
- local step_started_s="$1"
278
- echo $(( CREDENTIAL_RESTORE_DEADLINE_SECONDS - (SECONDS - step_started_s) - 1 ))
279
- }
280
-
281
- # Runs one of the step's bounded calls: skips it with a named warn if the
282
- # shared budget is already exhausted, treats a timeout (124/137) as the
283
- # non-fatal warn-and-continue D1 approved, and keeps every other non-zero
284
- # fatal exactly as restore_credentials always has (`|| return 1` aborts /run
285
- # before opencode and the tunnel start).
286
- bounded_restore_call() {
287
- local op="$1" step_started_s="$2"
288
- shift 2
289
-
290
- local remaining
291
- remaining="$(remaining_credential_budget "${step_started_s}")"
292
- if [ "${remaining}" -lt 1 ]; then
293
- warn "CREDENTIAL-RESTORE-SKIPPED: ${op} skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
294
- return 0
295
- fi
296
-
297
- local rc=0
298
- timed_synchroniser "${op}" "${remaining}" "$@" || rc=$?
299
- case "${rc}" in
300
- 0) ;;
301
- 124 | 137)
302
- warn "CREDENTIAL-RESTORE-TIMEOUT: ${op} did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
303
- ;;
304
- *) return 1 ;;
305
- esac
306
- return 0
307
- }
308
-
309
- fetch_runner_secret() {
310
- local step_started_s="$1" remaining rc=0 payload stderr_file started_ms populated=0 skipped=0 key value
311
- if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
312
- log "runner secret is not configured; continuing without GitHub and MCP credentials"
313
- return 0
314
- fi
315
- # From here, RUNNER_SECRET_ARN IS configured: every failure branch below
316
- # returns 1, not 0. Process env is fixed at spawn — nothing later in this
317
- # VM's life can retry a fetch that fails here — so a VM that was promised
318
- # these credentials and never got them is contrary evidence it can do the
319
- # job it was launched for, exactly the class check_runner_key already treats
320
- # as fatal (#1997): fail /run now and let the lifecycle machinery replace
321
- # this VM, rather than run the whole session silently degraded.
322
- remaining="$(remaining_credential_budget "${step_started_s}")"
323
- if [ "${remaining}" -lt 1 ]; then
324
- warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
325
- return 1
326
- fi
327
- if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
328
- warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
329
- return 1
330
- fi
331
- started_ms="$(now_ms)"
332
- payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
333
- log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
334
- if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
335
- warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
336
- rm -f "${stderr_file}"
337
- return 1
338
- fi
339
- if [ "${rc}" -ne 0 ]; then
340
- warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
341
- rm -f "${stderr_file}"
342
- return 1
343
- fi
344
- rm -f "${stderr_file}"
345
- if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
346
- warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
347
- return 0
348
- fi
349
- while IFS= read -r -d '' key && IFS= read -r -d '' value; do
350
- if [[ ! "${key}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
351
- warn "RUNNER-SECRET-KEY-SKIPPED: ${key@Q} is not a valid environment variable name"
352
- skipped=$((skipped + 1))
353
- continue
354
- fi
355
- export "${key}=${value}"
356
- populated=$((populated + 1))
357
- done < <(jq -j 'to_entries[] | select(.value | type == "string" and length > 0) | .key, "\u0000", .value, "\u0000"' <<<"${payload}")
358
- if [ "${populated}" -eq 0 ]; then
359
- warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md"
360
- else
361
- log "RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names"
362
- fi
363
- return 0
364
- }
365
-
366
- restore_credentials() {
367
- local step_started_ms step_started_s
368
- step_started_ms="$(now_ms)"
369
- step_started_s="${SECONDS}"
370
-
371
- load_state_config || return 1
372
- # This shares the existing bounded window so /run's worst-case duration does not grow.
373
- # Fatal when RUNNER_SECRET_ARN is configured (see fetch_runner_secret's own
374
- # comment); a no-ARN self-hosted runner already returned 0 above and never
375
- # reaches this `||`.
376
- fetch_runner_secret "${step_started_s}" || return 1
377
- bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
378
- bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
379
-
380
- # `restore` exits 0 whether it restored, found nothing, or discarded a corrupt
381
- # object, so the synchroniser's own predicate is the only reliable answer to
382
- # "can this VM authenticate a model right now". Its status is captured, not
383
- # used as a condition, so it can never become this function's return status —
384
- # `run` calls this bare (no `||`) under `set -euo pipefail`, and any
385
- # non-zero return here would abort the hook before opencode and the tunnel
386
- # ever start.
387
- local auth_status=0 auth_remaining
388
- auth_remaining="$(remaining_credential_budget "${step_started_s}")"
389
- if [ "${auth_remaining}" -lt 1 ]; then
390
- warn "CREDENTIAL-RESTORE-SKIPPED: model-auth-ready skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
391
- else
392
- timed_synchroniser model-auth-ready "${auth_remaining}" model-auth-ready || auth_status=$?
393
- case "${auth_status}" in
394
- 0) ;; # some model auth is configured; the two restores above already logged what they found
395
- 10)
396
- warn "no model credentials under s3://${LITESTREAM_BUCKET}/${LITESTREAM_PREFIX}/ \
397
- (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
398
- ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
399
- a model provider fails until one is connected. See 'Seeding a credential store' in \
400
- infrastructure/evident-runner/MICROVM.md."
401
- ;;
402
- 124 | 137)
403
- warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
404
- ;;
405
- *)
406
- # run_synchroniser has already `error`ed the tool-broke line for a
407
- # broken tool; any other unrecognised code lands here too. Do not
408
- # claim there are no credentials — absent evidence is not contrary
409
- # evidence (development-workflow.mdc).
410
- warn "could not determine whether this VM has model credentials"
411
- ;;
412
- esac
413
- fi
414
- # The step total, in the SAME greppable shape as the per-call lines above, so
415
- # one query answers both "what does credential restore cost?" and "which of
416
- # its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
417
- # + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
418
- # comment above that constant) rather than per call. `rc=0` is deliberate
419
- # here, not an absence of failure modes: a timed-out call already warned by
420
- # name above and is non-fatal by design (D1); every failure that DOES abort
421
- # this step has already returned 1 above.
422
- log_elapsed_since credential-restore-step "${step_started_ms}" 0
423
- return 0
424
- }
425
-
426
- apply_runner_opencode_config() {
427
- if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
428
- log "runner OpenCode config is not configured; using the baked project config"
429
- return 0
430
- fi
431
- local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
432
- [[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
433
- [ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
434
- if [ ! -f "${source}" ]; then
435
- error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
436
- return 0
437
- fi
438
- cp "${source}" "${WORKSPACE}/${target}"
439
- git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
440
- || warn "could not mark ${target} skip-worktree; it may show as a local change"
441
- log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
442
- }
443
-
444
- configure_github_access() {
445
- if [ -z "${GH_TOKEN:-}" ]; then
446
- warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
447
- return 0
448
- fi
449
- export GIT_CONFIG_GLOBAL=/tmp/gitconfig
450
- if ! : >"${GIT_CONFIG_GLOBAL}" ||
451
- ! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
452
- ! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
453
- ! git config --global init.defaultBranch main ||
454
- ! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
455
- ! chmod 0700 /tmp/git-credential-helper.sh ||
456
- ! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
457
- warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
458
- return 0
459
- fi
460
- (
461
- local output rc=0 login repo_url repo
462
- output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
463
- if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
464
- if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
465
- log "GITHUB-AUTH-OK: ${output}"
466
- repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
467
- repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
468
- [ -n "${repo}" ] || return
469
- output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
470
- if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
471
- [ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
472
- ) &
473
- }
474
-
475
- # Best-effort by design: /suspend must still drop the tunnel and /terminate must
476
- # still clean up, so a flush that cannot happen is loud but never fatal.
154
+ # Best-effort by design: the teardown hooks must still complete even when a
155
+ # credential flush cannot happen.
477
156
  sync_credentials() {
478
157
  load_state_config || return 0
479
158
  run_synchroniser sync-once claude || true
480
159
  run_synchroniser sync-once opencode || true
481
160
  }
482
161
 
483
- # --- Session DB restore (#812 WI-2) ----------------------------------------
484
- #
485
- # Restores opencode.db from S3 before opencode opens it — the read half only.
486
- # Writes NOTHING to S3 itself; replication back to S3 is started separately by
487
- # `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
488
- # `restore_session_db` are the MicroVM side of the same contract
489
- # runner/docker-images/fargate/entrypoint.sh's inlined restore already speaks, going
490
- # through the SAME runner-synchroniser CLI.
162
+ # --- boot pre-warms ----------------------------------------------------------
491
163
 
492
164
  # Reads the ~30 MB litestream binary into the page cache, in the background, so
493
165
  # the FIRST exec of it does not pay that read on the critical path.
494
166
  #
495
- # Why it is on the critical path at all: restore_session_db must finish before
496
- # start_opencode, and HOOK_SIGTERM_SECONDS is a POINT on the hook's SECONDS
497
- # clock, so a second spent here is a second taken from the rest of /run — and a
498
- # /run that overruns is SIGTERMed mid-boot and its VM destroyed. (Until #1172
499
- # the binding point was the tighter in-hook opencode readiness deadline; that
500
- # deadline is gone, the hook's own SIGTERM is not.)
501
- # Boot measurements put ~6.6-7.3s between the credential restore and the
502
- # litestream version line, of which only ~0.2s is accounted for by the two node
503
- # calls in between — the remainder is INFERRED to be this read, never measured.
167
+ # The pre-warm overlaps the CLI's node startup and auth round trip, so the first
168
+ # litestream operation does not also pay the binary's cold page-cache read.
169
+ # Boot measurements put ~6.6-7.3s before the litestream version line, of which
170
+ # only ~0.2s is accounted for by the two node calls in between — the remainder
171
+ # is INFERRED to be this read, never measured.
504
172
  # The `litestream-prewarm` and `litestream-version` timings are what settle it
505
173
  # on the next boot.
506
174
  #
@@ -528,8 +196,9 @@ prewarm_litestream() {
528
196
  }
529
197
 
530
198
  # Reads the aws CLI v2 install tree into the page cache, in the background, so
531
- # fetch_runner_secret's first `aws secretsmanager get-secret-value` does not
532
- # pay first-touch I/O on the critical path.
199
+ # the CLI's first runner-secret fetch does not pay first-touch I/O. The read
200
+ # overlaps `evident run`'s node startup and auth round trip rather than a hook
201
+ # deadline.
533
202
  #
534
203
  # A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
535
204
  # enough here: v2 ships as a real Python distribution (~7,500 files under the
@@ -556,365 +225,10 @@ prewarm_aws_cli() {
556
225
  log "pre-warming ${tree} in the background"
557
226
  }
558
227
 
559
- # Generates ${LITESTREAM_CONFIG_FILE} from runner-synchroniser's own renderer
560
- # — the SAME config module the credential restore/sync already goes through,
561
- # so there is no second copy of the bucket/prefix/path logic to drift
562
- # (litestream-config.ts's own header makes the same point). Only when absent
563
- # or empty, so `/resume` (WI-4) restarting litestream after a suspend/resume
564
- # snapshot — /dev/shm intact — pays nothing to regenerate it.
565
- #
566
- # Returns 1 on failure — fatal to the CALLER, never to the hook: every caller
567
- # (restore_session_db here, start_litestream in WI-3) must still let opencode
568
- # and the tunnel start regardless.
569
- ensure_litestream_config() {
570
- if [ -s "${LITESTREAM_CONFIG_FILE}" ]; then
571
- return 0
572
- fi
573
-
574
- # Timed OUTSIDE the substitution, not with timed_synchroniser: this call's
575
- # stdout IS the config, so a timing line emitted inside `$(...)` would land in
576
- # litestream.yml.
577
- local rendered config_started_ms config_rc=0
578
- config_started_ms="$(now_ms)"
579
- rendered="$(run_synchroniser litestream-config)" || config_rc=$?
580
- log_elapsed_since litestream-config "${config_started_ms}" "${config_rc}"
581
- if [ "${config_rc}" -ne 0 ]; then
582
- error "could not generate ${LITESTREAM_CONFIG_FILE}: runner-synchroniser litestream-config failed (see the error above)"
583
- return 1
584
- fi
585
- printf '%s\n' "${rendered}" >"${LITESTREAM_CONFIG_FILE}" || {
586
- error "could not write ${LITESTREAM_CONFIG_FILE}"
587
- return 1
588
- }
589
-
590
- # Q7's diagnostic, for whoever can read this VM's CloudWatch log: which way
591
- # litestream resolved its AWS region, and whether it can even see the
592
- # names it would need to (never the VALUES of any OTHER AWS_* variable —
593
- # AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN may be present here).
594
- local region_empty="yes"
595
- if printf '%s\n' "${rendered}" | grep -Eq '^ *region: *[^[:space:]]'; then
596
- region_empty="no"
597
- fi
598
- # Hoisted out of the log line below so it can be timed: this is the FIRST exec
599
- # of the litestream binary on the boot, and the pre-warm /run fires (see
600
- # prewarm_litestream) is aimed squarely at what this number measures.
601
- local version_started_ms litestream_version
602
- version_started_ms="$(now_ms)"
603
- litestream_version="$(litestream version 2>/dev/null || echo 'unknown')"
604
- log_elapsed_since litestream-version "${version_started_ms}" 0
605
- log "litestream ${litestream_version}; AWS_REGION=${AWS_REGION:-<unset>} AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-<unset>}; rendered litestream.yml region empty: ${region_empty}"
606
- }
607
-
608
- # Writes the marker AND names the reason, in one call, so the marker can never
609
- # appear silently. Every give-up path in restore_session_db calls this EXCEPT
610
- # the classifier's 31 (#1106 — see that branch), and no success path ever does.
611
- # It is what a later `/resume`/`start_litestream` (WI-3) reads to skip
612
- # replicating this boot: a local DB that is only PARTIALLY restored must never
613
- # be allowed to overwrite a replica this boot never proved safe to write over.
614
- # A merely FRESH local DB is not that danger — litestream continues the txid
615
- # chain — which is precisely why 31 no longer belongs here.
616
- mark_no_replicate() {
617
- : >"${SESSION_DB_NO_REPLICATE_MARKER}"
618
- warn "SESSION-DB-NO-REPLICATE: $1"
619
- }
620
-
621
- # Removes local session-DB debris a give-up path leaves behind. Called on
622
- # EVERY give-up path and NO success path: skipping it on a give-up is a real
623
- # bug (a truncated file left in place makes opencode fail to open a malformed
624
- # database, which fails /run outright — worse than the history loss this
625
- # discards); calling it on a success path would delete a good restore. One
626
- # `rm -f` per path, not a single command, so a failure removing one does not
627
- # skip the other two.
628
- discard_session_db_debris() {
629
- rm -f "${OPENCODE_DB_PATH}" "${OPENCODE_DB_PATH}-wal" "${OPENCODE_DB_PATH}-shm"
630
- }
631
-
632
- # How long restore_session_db waits for ONE `litestream restore` before giving
633
- # up. From M4 (see the plan's §2): ~50 MB/s extrapolated from the x86_64 ECS
634
- # container's own live replica, so 7s covers roughly 300-350 MB — an
635
- # EXTRAPOLATION, not a MicroVM measurement (different arch, network path and
636
- # credential source); the boot line ensure_litestream_config logs, plus the
637
- # ${SECONDS} stamps in /run, are how it gets measured here for real.
638
- #
639
- # 7 rather than the 8 it was: hook-scripts.test.ts's pre-opencode ladder held an
640
- # allowance of 4s for the rest of this step, which live boots measured at
641
- # 6.7-7.4s, so the ladder passed while real boots blew the deadline. Correcting
642
- # that allowance is what takes this second — the remedy the ladder's own failure
643
- # message prescribes, and the one that leaves the SIGTERM worst case untouched.
644
- # The cost is real and unmeasured on this platform: ~50 MB less restorable
645
- # replica before a truncation that silently loses history. A fresh
646
- # state_prefix starts at zero, so this is generous for a long time, but
647
- # RoutingStrategy is 'per_user' only (routing-strategy.ts) and every prefix is
648
- # deterministic and permanent per (pool, routing key) — so a long-lived
649
- # runner's opencode.db grows monotonically across VM generations, exactly
650
- # like the ~1 GB ECS one M4 measured, and will eventually hit this ceiling.
651
- # Past it the restore is truncated (SESSION-DB-RESTORE-TRUNCATED, below) and
652
- # this boot skips replication — silent, permanent history loss for that
653
- # runner unless somebody reads the log. Not solved here (WI-0 Task 0.2).
654
- # The env override is for tests only, so they need not burn 7s of wall clock.
655
- SESSION_DB_RESTORE_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_RESTORE_DEADLINE_SECONDS:-7}"
656
- # The SIGKILL backstop `timeout -k` applies after its own SIGTERM: `timeout`
657
- # alone only SIGTERMs, and a litestream that ignored it would be unbounded
658
- # again. Hard ceiling on this step: DEADLINE + this = 9s.
659
- SESSION_DB_RESTORE_KILL_GRACE_SECONDS=2
660
-
661
- # The one bounded restore attempt, run before start_opencode (opencode opens
662
- # the DB the moment it starts, so this is the only place it can happen).
663
- # Every branch returns 0 — /run calls this bare under `set -euo pipefail`,
664
- # exactly like restore_credentials, and nothing about the session DB may ever
665
- # fail /run (Q3): a /run that exits 1 fails the whole lifecycle transition
666
- # and the user gets NO runner at all, which is worse than losing history.
667
- restore_session_db() {
668
- # Cleanup, not a decision, and unconditionally first: /dev/shm survives
669
- # suspend/resume and a /run retry, so a marker left by an earlier boot must
670
- # never silently disable replication for the rest of this VM's life.
671
- rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
672
- local report_path
673
- if report_path="$(session_db_recovery_report_path)"; then
674
- # The default report directory is provisioned in the image, so this is a
675
- # no-op there; a shared-path override (EVIDENT_SESSION_DB_RECOVERY_REPORT)
676
- # can name a directory that isn't, and the typed synchroniser writer
677
- # already creates it for that same override (session-db-recovery-report.ts).
678
- # Best-effort only: report_session_db_giveup below still warns and
679
- # continues if this didn't leave a writable path. `${path%/*}` leaves a
680
- # bare relative filename (no `/`) unchanged rather than reducing to `.`
681
- # like dirname(1) — without the explicit check, mkdir would create a
682
- # directory AT the report path itself.
683
- local report_dir="${report_path%/*}"
684
- [ "${report_dir}" = "${report_path}" ] && report_dir="."
685
- [ -d "${report_dir}" ] || mkdir -p "${report_dir}" 2>/dev/null || true
686
- : >"${report_path}" || warn "SESSION-DB-RECOVERY-REPORT-TRUNCATE-FAILED"
687
- else
688
- warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
689
- fi
690
-
691
- # Config resolved ONCE here, in the synchroniser (config.ts owns
692
- # OPENCODE_DB_PATH; no second copy of that path here), exactly as
693
- # entrypoint.sh's own line does. Unlike ECS's `|| die`, a failure here must
694
- # not abort the hook — `x="$(cmd)"` under `set -e` WOULD abort the whole
695
- # shell on a failing substitution, which is exactly why it is caught
696
- # explicitly rather than left to `set -e`.
697
- # Timed OUTSIDE the substitution for the same reason ensure_litestream_config
698
- # is: this call's stdout is `eval`'d, so a timing line emitted inside `$(...)`
699
- # would be evaluated as configuration.
700
- local synchroniser_env env_started_ms env_rc=0
701
- env_started_ms="$(now_ms)"
702
- synchroniser_env="$(run_synchroniser env)" || env_rc=$?
703
- log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
704
- if [ "${env_rc}" -ne 0 ]; then
705
- mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
706
- report_session_db_giveup synchroniser_config_unresolved
707
- return 0
708
- fi
709
- # Guarded for the SAME reason as the substitution above, which is easy to miss:
710
- # `eval` returns the status of what it ran, so a malformed line from a skewed
711
- # bundle would abort /run right here under `set -e` — before start_opencode,
712
- # so the user gets NO runner at all. entrypoint.sh:110 leaves the identical
713
- # line bare because on ECS the blast radius is a crash-loop-and-replace; here
714
- # it is the whole lifecycle transition, so Q3 makes it a give-up instead.
715
- eval "${synchroniser_env}" || {
716
- mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
717
- report_session_db_giveup synchroniser_config_unevaluable
718
- return 0
719
- }
720
-
721
- # #931's exact lesson, one shell over (runner/docker-images/fargate/entrypoint.sh):
722
- # the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
723
- # 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
724
- # would otherwise abort right here under `set -u` the moment
725
- # `${OPENCODE_DB_PATH}` is dereferenced below. entrypoint.sh may `die` on
726
- # this (ECS just crash-loops and replaces the task); this hook cannot —
727
- # Q3 says nothing about the session DB may ever fail /run — so a missing
728
- # OPENCODE_DB_PATH is a give-up, not a silent default (a guessed path would
729
- # be actively wrong, not merely absent).
730
- if [ -z "${OPENCODE_DB_PATH+x}" ]; then
731
- mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
732
- report_session_db_giveup synchroniser_config_incomplete
733
- return 0
734
- fi
735
-
736
- # #770's arm, checked FIRST of the decisions and with its OWN message —
737
- # never the SESSION-DB-REPLICA-UNUSABLE one below (#764's finding: an
738
- # operator grepping that marker must not hit the disabled case).
739
- # PERSISTENCE_BUCKET is empty exactly when LITESTREAM_BUCKET or
740
- # LITESTREAM_PREFIX is unset (config.ts's persistenceEnabled) — the
741
- # IDENTICAL predicate ECS's own litestream-launch gate uses
742
- # (entrypoint.sh), so the two images agree on what "disabled" means.
743
- if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
744
- warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated."
745
- return 0
746
- fi
747
-
748
- if ! ensure_litestream_config; then
749
- mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
750
- report_session_db_giveup litestream_config_unavailable
751
- return 0
752
- fi
753
-
754
- local restore_rc=0 restore_started_ms
755
- restore_started_ms="$(now_ms)"
756
- timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_RESTORE_DEADLINE_SECONDS}" \
757
- litestream restore -config "${LITESTREAM_CONFIG_FILE}" \
758
- -if-db-not-exists -if-replica-exists "${OPENCODE_DB_PATH}" || restore_rc=$?
759
- log_elapsed_since session-db-restore "${restore_started_ms}" "${restore_rc}"
760
-
761
- # `timeout`'s OWN codes, handled BEFORE classifying: 124/137 are a
762
- # TRUNCATED restore, not a corrupt replica — handing them to
763
- # session-db-classify would mislabel a slow/large replica as one. 125-127
764
- # mean `timeout` (or litestream itself) is broken, not the replica.
765
- case "${restore_rc}" in
766
- 124 | 137)
767
- discard_session_db_debris
768
- mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
769
- report_session_db_giveup restore_deadline_exceeded "${restore_rc}"
770
- return 0
771
- ;;
772
- 125 | 126 | 127)
773
- error "litestream restore could not even run (timeout exited ${restore_rc})"
774
- discard_session_db_debris
775
- mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
776
- report_session_db_giveup restore_tool_unusable "${restore_rc}"
777
- return 0
778
- ;;
779
- esac
780
-
781
- local classify_rc=0
782
- run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave --fresh-db-fallback || classify_rc=$?
783
- case "${classify_rc}" in
784
- 0) ;; # restored, or no replica yet — the CLI already logged which
785
- 31)
786
- # The classifier already discarded the local debris (session-db.ts) — and
787
- # that is exactly why this is the ONE give-up path that does NOT
788
- # mark_no_replicate (#1106). Every other give-up here may be sitting on a
789
- # half-restored DB, so the marker stays; a 31 is structurally guaranteed
790
- # to be a FRESH one, and replicating it starts a new backup chain instead
791
- # of leaving this boot with a zero-width backup window. Same reasoning,
792
- # and same one-line change, as runner/docker-images/fargate/entrypoint.sh's `31)`.
793
- warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot (see the WARNING above)"
794
- ;;
795
- 32)
796
- # The CLI's own contract for 32 is "re-run litestream restore and ask
797
- # again", but this hook has budget for exactly ONE attempt (Q4) — name
798
- # the deviation so nobody reads this as a bug.
799
- discard_session_db_debris
800
- mark_no_replicate "session-db-classify asked for another restore attempt (32), but this hook has budget for only one; treating it as a give-up rather than retrying"
801
- ;;
802
- 30)
803
- # The CLI already logged its own FATAL line above; Q3 still says boot
804
- # fresh rather than fail /run.
805
- discard_session_db_debris
806
- mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
807
- report_session_db_giveup classification_fatal
808
- ;;
809
- *)
810
- # run_synchroniser already logged the "tool broke" ERROR for this.
811
- discard_session_db_debris
812
- mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
813
- report_session_db_giveup classification_unrecognised
814
- ;;
815
- esac
816
-
817
- return 0
818
- }
819
- # --- Session DB restore (end) -----------------------------------------------
820
-
821
- # --- Session DB integrity verification (#1868 WI-4) --------------------------
822
- #
823
- # The restore loop above only proves litestream could WRITE a file; it never
824
- # proves the file is intact (#1345: a leaf-page-corrupt-but-openable DB
825
- # re-restores unchanged forever). `session-db-verify` runs a real `PRAGMA
826
- # integrity_check` and, on failure, walks retained restore points back until
827
- # one passes — the MicroVM side of the identical check
828
- # runner/docker-images/fargate/entrypoint.sh already runs after ITS restore
829
- # loop, through the same runner-synchroniser CLI.
830
-
831
- # Bounded by `timed_synchroniser`, not an external `timeout` around
832
- # `run_synchroniser` itself (a shell function, not an exported binary — that
833
- # would fail with rc 127, silently mis-triggering the allowlist's "tool
834
- # broke" branch). The grace period is `CREDENTIAL_RESTORE_KILL_GRACE_SECONDS`
835
- # (2s): `run_synchroniser`'s `timeout -k` hardcodes that one constant for
836
- # every `timed_synchroniser` caller, not a value unique to this step, so the
837
- # real worst case is DEADLINE + 2s, once — accounted for in
838
- # hook-scripts.test.ts's budget-ladder test.
839
- #
840
- # 4s, NOT measured on this platform: #930's synchroniser-call sample put a
841
- # single call at ~1.2s (a node cold start), and this ONE call also runs a
842
- # PRAGMA integrity_check that scales with the restored DB's size, which #930
843
- # never exercised. The 124/137 branch below is the DESIGNED-FOR outcome on a
844
- # large DB, not an edge case — SYNCHRONISER-TIMING op=session-db-verify is
845
- # what should actually size this once real boots report it.
846
- SESSION_DB_VERIFY_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_VERIFY_DEADLINE_SECONDS:-4}"
847
-
848
- verify_session_db() {
849
- # Guards mirror start_litestream's first three, in the same order and for
850
- # the same reason: verifying a DB this boot already decided not to
851
- # replicate (or never restored at all) changes nothing about how /run
852
- # proceeds, and would spend budget only to report on a moot outcome.
853
- if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
854
- log "skipping session-DB verification: persistence is disabled"
855
- return 0
856
- fi
857
-
858
- if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
859
- log "skipping session-DB verification: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
860
- return 0
861
- fi
862
-
863
- if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
864
- log "skipping session-DB verification: no usable ${LITESTREAM_CONFIG_FILE}"
865
- return 0
866
- fi
867
-
868
- # Exported below this step's own deadline so the CLI's in-process walkback
869
- # loop gives up on its own before the process-group kill lands — its 180s
870
- # default (config.ts) is far outside this one step's slice of the hook's
871
- # SIGTERM budget.
872
- local verify_rc=0
873
- EVIDENT_SESSION_DB_WALKBACK_BUDGET_SECONDS="${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
874
- timed_synchroniser session-db-verify "${SESSION_DB_VERIFY_DEADLINE_SECONDS}" \
875
- session-db-verify "${LITESTREAM_CONFIG_FILE}" || verify_rc=$?
876
-
877
- case "${verify_rc}" in
878
- 0) ;; # verified intact, or nothing to verify yet — the CLI already logged which
879
- 33)
880
- # Integrity exhausted, but the classifier already proved the corrupt
881
- # replica separated and the local copy disposed of — booting with a
882
- # fresh DB and a new backup chain is safe, exactly like ECS's own `33)`.
883
- log "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated."
884
- ;;
885
- 34)
886
- # The ONE deliberate exception to "nothing about the session DB may
887
- # ever fail /run" (Q3): a 34 means separation/disposal could NOT be
888
- # proven, so continuing would hand opencode a DB it may not be safe to
889
- # open or write — the same evidence-quality bar `check_runner_key`
890
- # already applies one step earlier in this hook (contrary evidence,
891
- # not absent evidence, is what's fatal). The CLI already logged
892
- # SESSION-DB-REPLICA-SEPARATION-UNVERIFIED / SESSION-DB-LOCAL-DISCARD-FAILED.
893
- error "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk, so nothing will be started (see the ERROR above)."
894
- return 1
895
- ;;
896
- 124 | 137)
897
- # The designed-for outcome on a large DB (see the deadline comment
898
- # above), not a broken tool: continue with the restored DB exactly as
899
- # ECS's own `*)` branch does for an unexpected code.
900
- warn "SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_DEADLINE_SECONDS}s deadline; continuing with the restored opencode.db as-is, unverified"
901
- ;;
902
- *)
903
- # run_synchroniser already logged the "tool broke" ERROR for this. A
904
- # broken verifier must not turn a boot that works today into a
905
- # crash-loop.
906
- warn "SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${verify_rc}, which is none of its documented answers; continuing with the restored opencode.db as-is"
907
- ;;
908
- esac
909
-
910
- return 0
911
- }
912
- # --- Session DB integrity verification (end) ---------------------------------
913
-
914
228
  # `kill -0` answers "does this pid exist", which is not the question any caller
915
229
  # here is asking. A process that has exited but has not been reaped — a zombie —
916
230
  # still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
917
- # normal case for everything these hooks start: `start_tunnel`/`start_opencode`
231
+ # normal case for everything these hooks start: `start_tunnel` and its CLI child
918
232
  # background a process that outlives the hook, the hook shell must return so AWS
919
233
  # gets its 200, and PID 1 in this image is a bare node hook server with no init
920
234
  # (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
@@ -1017,24 +331,6 @@ payload_is_complete() {
1017
331
  ' >/dev/null 2>&1
1018
332
  }
1019
333
 
1020
- # `setsid` so opencode outlives this hook: the script must return so AWS gets its
1021
- # 200, while opencode keeps serving. The guard makes /run idempotent — a retry
1022
- # after a killed hook must not put a second instance on the same port.
1023
- start_opencode() {
1024
- if opencode_is_running; then
1025
- warn "opencode already running (pid $(cat "${OPENCODE_PID_FILE}")); reusing it"
1026
- return 0
1027
- fi
1028
-
1029
- setsid opencode serve \
1030
- --hostname 127.0.0.1 \
1031
- --port "${OPENCODE_PORT}" \
1032
- --print-logs &
1033
-
1034
- echo $! >"${OPENCODE_PID_FILE}"
1035
- log "opencode starting (pid $(cat "${OPENCODE_PID_FILE}"))"
1036
- }
1037
-
1038
334
  stop_opencode() {
1039
335
  if ! opencode_is_running; then
1040
336
  # Clear the file here too, for the same reason stop_tunnel does: opencode
@@ -1099,69 +395,11 @@ stop_opencode_and_wait() {
1099
395
  log "opencode stopped ${outcome}"
1100
396
  }
1101
397
 
1102
- # --- litestream replicate (#812 WI-3) ---------------------------------------
1103
- #
1104
- # The write half of session-DB persistence. restore_session_db (above) is the
1105
- # read half only and never starts this. Q8 chose one backgrounded, unsupervised
1106
- # `litestream replicate` over a periodic flush or a `replicate -exec` wrapper
1107
- # around opencode: SIGTERM is `replicate`'s own final-sync trigger, so
1108
- # stop_litestream (below) doubles as the checked flush /suspend and /terminate
1109
- # need, and there is no supervisor in this image to hand the process to (no
1110
- # init, no `wait -n` shell that could stay alive after the hook returns).
1111
-
1112
398
  # The graceful wait before the SIGKILL backstop in stop_litestream, below.
1113
399
  # litestream's own sync is normally sub-second, so 10s is generous headroom —
1114
400
  # chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
1115
401
  LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
1116
402
 
1117
- # Guards, in this exact order and no other — reordering any of them re-arms the
1118
- # invariant it exists to protect:
1119
- #
1120
- # 1. persistence disabled (#770) — bucket AND prefix both non-empty — is its
1121
- # OWN arm with its OWN message, checked FIRST, and must never log
1122
- # SESSION-DB-REPLICA-UNUSABLE: an operator grepping that marker must not
1123
- # hit the disabled case (#764).
1124
- # 2. the no-replicate marker (restore_session_db above) — set when this boot's
1125
- # local DB may be only PARTIALLY restored, so starting replicate here would
1126
- # let it overwrite a replica this boot never proved safe to write over.
1127
- # A classifier 31 no longer sets it (#1106): that DB is guaranteed fresh,
1128
- # and skipping replicate for it was what made the history loss unbounded.
1129
- # 3. no usable config — ensure_litestream_config (restore_session_db, above)
1130
- # already logged why it is missing; this is not a place to retry it.
1131
- # 4. already running — idempotence, mirroring start_tunnel's own guard.
1132
- #
1133
- # Only past all four does it actually spawn, backgrounded with `setsid` for the
1134
- # same reason start_opencode/start_tunnel are: the hook must return so AWS gets
1135
- # its 200, while replicate keeps running. litestream's OWN stderr is left to
1136
- # reach CloudWatch — never redirected — because there is no supervisor in this
1137
- # image, so that is the only channel that can say why a replicator died.
1138
- start_litestream() {
1139
- if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
1140
- warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing is replicated."
1141
- return 0
1142
- fi
1143
-
1144
- if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
1145
- log "skipping litestream replicate: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
1146
- return 0
1147
- fi
1148
-
1149
- if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
1150
- error "no usable ${LITESTREAM_CONFIG_FILE}; not starting litestream replicate"
1151
- return 0
1152
- fi
1153
-
1154
- if litestream_is_running; then
1155
- warn "litestream already running (pid $(cat "${LITESTREAM_PID_FILE}")); reusing it"
1156
- return 0
1157
- fi
1158
-
1159
- setsid litestream replicate -config "${LITESTREAM_CONFIG_FILE}" &
1160
-
1161
- echo $! >"${LITESTREAM_PID_FILE}"
1162
- log "litestream replicate starting (pid $(cat "${LITESTREAM_PID_FILE}"))"
1163
- }
1164
-
1165
403
  # The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
1166
404
  # one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
1167
405
  # would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
@@ -1208,7 +446,7 @@ stop_litestream() {
1208
446
  # stop_tunnel's own comment argues against an argument a future caller can get
1209
447
  # wrong. litestream_is_running is the same O(1) check stop_litestream's own
1210
448
  # no-op branch uses, not a poll — /run's cleanup trap fires this before
1211
- # start_litestream is ever reached whenever an earlier step failed, and an
449
+ # the replicator start is ever reached whenever an earlier step failed, and an
1212
450
  # operator reading that log must not be told a stop signal went to a process
1213
451
  # that never started.
1214
452
  kill_litestream() {
@@ -1233,7 +471,7 @@ kill_litestream() {
1233
471
  # suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
1234
472
  # loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
1235
473
  # own sync_credentials_loop is the ECS side of the identical gap; this is the
1236
- # same fix, backgrounded the same way as start_opencode/start_litestream so it
474
+ # same fix, backgrounded like the other long-lived services so it
1237
475
  # outlives this hook process, `( … ) &` rather than `setsid`: a plain
1238
476
  # backgrounded subshell is reparented to init and keeps running once its
1239
477
  # parent hook script exits (verified: PPID=1, still alive, with no controlling
@@ -1242,14 +480,8 @@ kill_litestream() {
1242
480
  # re-exec.
1243
481
 
1244
482
  # Bounded confirmation window `stop_credential_sync` polls after signalling the
1245
- # loop, sized against the SIGTERM budget ladder (#812 WI-4's
1246
- # hook-scripts.test.ts): /terminate's own steps already use 49 of the 55s
1247
- # ceiling, leaving 6s of headroom — this matches the *_KILL_GRACE_SECONDS
1248
- # convention (CREDENTIAL_RESTORE_KILL_GRACE_SECONDS,
1249
- # SESSION_DB_RESTORE_KILL_GRACE_SECONDS, both 2s) rather than a longer
1250
- # drain-style wait, since the loop's current child is one fast
1251
- # `run_synchroniser sync-once` call (#930: ~1.2s measured), not a writer
1252
- # needing a graceful drain.
483
+ # loop. The loop's current child is one fast `run_synchroniser sync-once` call,
484
+ # so this stays a short backstop rather than a graceful drain.
1253
485
  CREDS_SYNC_STOP_WAIT_SECONDS=2
1254
486
 
1255
487
  # Best-effort per tick, exactly like sync_credentials above: a failed tick
@@ -1266,13 +498,13 @@ start_credential_sync() {
1266
498
  return 0
1267
499
  fi
1268
500
 
1269
- # CREDS_SYNC_INTERVAL is exported by restore_session_db's `eval "$(run_synchroniser env)"`
1270
- # on /run (config.ts's own default is 60s), but that eval can fail or be skipped by an
1271
- # earlier give-up never leave the loop unbound under set -u for a value with a safe,
1272
- # named fallback (unlike OPENCODE_DB_PATH, a guessed sync cadence is not actively wrong).
501
+ # This is the same environment input runner-synchroniser validates. The
502
+ # variable is normally absent, so that ordinary case uses the documented
503
+ # default without a warning; a present invalid value is named and rejected.
1273
504
  local interval="${CREDS_SYNC_INTERVAL:-60}"
1274
- if [ -z "${CREDS_SYNC_INTERVAL:-}" ]; then
1275
- warn "CREDS-SYNC-INTERVAL-DEFAULTED: CREDS_SYNC_INTERVAL was not set by run_synchroniser env; using ${interval}s"
505
+ if [ -n "${CREDS_SYNC_INTERVAL+x}" ] && [[ ! "${CREDS_SYNC_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
506
+ warn "CREDS-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${CREDS_SYNC_INTERVAL}' is not a positive integer; using 60s"
507
+ interval=60
1276
508
  fi
1277
509
 
1278
510
  rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
@@ -1285,7 +517,7 @@ start_credential_sync() {
1285
517
  # long-lived process that instead inherited a pipe's write end (a test
1286
518
  # harness reading the hook's own output, for one) would keep that pipe
1287
519
  # from ever reporting EOF — testing-guide.mdc's own lesson, and the same
1288
- # reason start_opencode/start_litestream never inherit stdio either. That
520
+ # reason the long-lived services never inherit stdio either. That
1289
521
  # redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
1290
522
  # failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
1291
523
  # surfaced by stop_credential_sync, which DOES have live stdio.
@@ -1383,19 +615,16 @@ stop_credential_sync() {
1383
615
  # Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
1384
616
  # own sync is normally sub-second (M5, the plan's grounding); 10s is generous
1385
617
  # headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
1386
- # measured worst case. The SIGKILL backstop below reuses
1387
- # SESSION_DB_RESTORE_KILL_GRACE_SECONDS rather than its own literal: it is the
1388
- # identical concept (bare `timeout` only SIGTERMs, and a litestream that
1389
- # ignored it would be unbounded again), and there is no reason for the restore
1390
- # and flush paths to ever drift on how long a SIGTERM gets to land.
618
+ # measured worst case. The SIGKILL backstop gives the flush its own two-second
619
+ # grace after `timeout` sends SIGTERM.
1391
620
  SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
621
+ SESSION_DB_FLUSH_KILL_GRACE_SECONDS=2
1392
622
 
1393
- # Guards mirror start_litestream's first three exactly (disabled -> marker ->
623
+ # Guards mirror the CLI replicator's first three exactly (disabled -> marker ->
1394
624
  # config) — a flush must never attempt work those guards would have refused
1395
625
  # to start in the first place. PERSISTENCE_BUCKET is the SAME variable
1396
- # start_litestream reads, so both agree regardless of which earlier step in
1397
- # THIS process set it (load_state_config, above, or restore_session_db's
1398
- # eval, on /run).
626
+ # the CLI reads, so both agree regardless of which earlier step in
627
+ # THIS process set it (load_state_config above).
1399
628
  #
1400
629
  # Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
1401
630
  # trigger — and only once it is confirmed stopped does the synchronous -once
@@ -1435,7 +664,7 @@ flush_session_db() {
1435
664
  # Only reached with the daemon confirmed stopped, or never running — never
1436
665
  # concurrently with it (see above).
1437
666
  local flush_rc=0
1438
- timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
667
+ timeout -k "${SESSION_DB_FLUSH_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
1439
668
  litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
1440
669
 
1441
670
  if [ "${flush_rc}" -eq 0 ]; then
@@ -1490,6 +719,26 @@ fi
1490
719
  # its 200, while the tunnel keeps serving.
1491
720
  start_tunnel() {
1492
721
  local runner_key="$1" api_url="$2" tunnel_url="$3"
722
+ local restore_runner_credentials="${4:-false}" restore_history="${5:-false}"
723
+ local -a credential_flags=()
724
+ local -a session_db_flags=()
725
+ local -a opencode_config_flags=()
726
+
727
+ # The explicit fourth argument is set only by /run; /resume uses the default
728
+ # so a resumed VM never restores credentials over stores it already has.
729
+ if [ "${restore_runner_credentials}" = true ]; then
730
+ credential_flags=(--restore-runner-credentials)
731
+ fi
732
+
733
+ # Only a fresh /run asks the CLI to restore session history. /resume keeps
734
+ # the database from the snapshot and must not restore over it.
735
+ if [ "${restore_history}" = true ]; then
736
+ session_db_flags=(--restore-session-db)
737
+ fi
738
+
739
+ if [ -n "${RUNNER_OPENCODE_CONFIG:-}" ]; then
740
+ opencode_config_flags=(--opencode-config-overlay "${RUNNER_OPENCODE_CONFIG}")
741
+ fi
1493
742
 
1494
743
  if tunnel_is_running; then
1495
744
  warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
@@ -1512,6 +761,13 @@ start_tunnel() {
1512
761
  --port "${OPENCODE_PORT}" \
1513
762
  --endpoint "${api_url}" \
1514
763
  --tunnel "${tunnel_url}" \
764
+ --opencode-pid-file "${OPENCODE_PID_FILE}" \
765
+ --litestream-config "${LITESTREAM_CONFIG_FILE}" \
766
+ --litestream-pid-file "${LITESTREAM_PID_FILE}" \
767
+ --session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
768
+ "${credential_flags[@]}" \
769
+ "${session_db_flags[@]}" \
770
+ "${opencode_config_flags[@]}" \
1515
771
  --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
1516
772
  --enable-file-sync-to "${HOME}/.claude" &
1517
773
 
@@ -1644,10 +900,9 @@ check_runner_key() {
1644
900
  local runner_key="$1" api_url="$2"
1645
901
 
1646
902
  # status.ts's own exit-code contract (its header comment) means this exits
1647
- # non-zero on EVERY branch except `ok` guarded exactly like
1648
- # ensure_litestream_config's `run_synchroniser litestream-config` call,
1649
- # above, so that expected non-zero does not abort this function under the
1650
- # caller's `set -e` before the case below ever runs. Only stdout is
903
+ # non-zero on EVERY branch except `ok`; guard the capture so an expected
904
+ # non-zero does not abort this function under the caller's `set -e` before
905
+ # the case below ever runs. Only stdout is
1651
906
  # captured: status.ts's own contract is one parseable JSON line and nothing
1652
907
  # else there, and — like litestream's stderr elsewhere in this file — its
1653
908
  # stderr is left to reach CloudWatch directly rather than being folded in,
@@ -1,14 +1,11 @@
1
1
  #!/usr/bin/env bash
2
2
  #
3
3
  # Re-dial with the identity /run left behind. There is no step that could fetch
4
- # a fresh runner key, so the one from /run is what resumes. Also restarts the
5
- # session-DB replicator /suspend stopped before the snapshot (#812 WI-4):
6
- # litestream does not survive a suspend/resume freeze on this design (Q2)
7
- # /suspend stops it and /resume starts a fresh one, the same pattern already
8
- # proven for the tunnel. The interval credential sync loop (#1868 WI-3) is the
9
- # same story one function over: /suspend stops it too, so a resumed VM that
10
- # never restarted it here would never sync credentials again for the rest of
11
- # its life.
4
+ # a fresh runner key, so the one from /run is what resumes. The CLI delegated by
5
+ # start_tunnel owns the session-DB replicator's post-resume start. The interval
6
+ # credential sync loop (#1868 WI-3) stays here because /suspend stops it too, so
7
+ # a resumed VM that never restarted it here would never sync credentials again
8
+ # for the rest of its life.
12
9
  set -euo pipefail
13
10
 
14
11
  # shellcheck source=./common.sh
@@ -44,11 +41,9 @@ fi
44
41
 
45
42
  # Tolerant, never fatal: a resume that fails costs the user their whole
46
43
  # session, so a broken durable-state config degrades to "no session-DB
47
- # restart" rather than a failed resume (the same Q3 reasoning /run's
48
- # session-DB step follows). start_litestream's own guards (disabled/marker/
49
- # already-running) handle the rest — this needs no logic of its own.
44
+ # replication" rather than a failed resume. The CLI's own guards handle the
45
+ # rest this needs no logic of its own.
50
46
  load_state_config || warn "could not resolve durable-state config; the session DB will not resume replicating"
51
- start_litestream
52
47
  start_credential_sync
53
48
 
54
49
  # Diagnostic-only, unlike /run's gate: a failed resume costs the user their
@@ -52,24 +52,20 @@ regenerate_machine_id
52
52
  # reads it, and left behind for the hooks that flush back to the same prefix.
53
53
  printf '%s\n' "${state_prefix}" >"${STATE_PREFIX_FILE}"
54
54
 
55
- # 3 — start reading the litestream binary NOW so that read overlaps step 4,
56
- # which needs a different binary (node), instead of landing inside step 5, which
57
- # start_opencode has to wait for. Backgrounded and never waited on. Same for the
58
- # aws CLI: step 4's runner-secret fetch is its first-ever invocation in this
59
- # VM's life, and #1997 measured that cold first touch alone blowing the whole
60
- # credential-restore deadline.
55
+ # 3 — start reading the litestream and aws CLI binaries NOW. The reads run in
56
+ # the background and overlap the CLI's own node startup and auth round trip,
57
+ # keeping their cold first touches off the path to the first credential fetch.
61
58
  prewarm_litestream
62
59
  prewarm_aws_cli
63
60
 
64
- # 4 — credential stores, restored before anything that reads them starts. Fatal
65
- # only if persistence itself is unavailable (an unset bucket/prefix, or a broken
66
- # synchroniser bundle) — a VM with no model credentials yet still boots.
67
- restore_credentials
61
+ # 4 — export the durable-state config before anything that reads it. Fatal,
62
+ # because the CLI and the teardown hooks must agree on this runner's state prefix.
63
+ load_state_config || exit 1
68
64
 
69
65
  # 5 — does the runner key in this payload actually authenticate against
70
- # Evident? Checked here, before opencode or the tunnel spend any of this boot's
71
- # SIGTERM budget on a key that cannot work, and before anything has started
72
- # that cleanup would need to tear down. Fatal ONLY on contrary evidence
66
+ # Evident? Checked here, before the tunnel spends any of this boot's time on a key
67
+ # that cannot work, and before anything has started that cleanup would need to
68
+ # tear down. Fatal ONLY on contrary evidence
73
69
  # (check_runner_key's own contract, common.sh): a key the API actively rejects
74
70
  # cannot work regardless, and failing here costs ~5s against the ~10 minutes a
75
71
  # runner that can never connect would otherwise burn before the lifecycle cron
@@ -77,49 +73,11 @@ restore_credentials
77
73
  # it warns and this VM still boots.
78
74
  check_runner_key "${runner_key}" "${api_url}" || exit 1
79
75
 
80
- # 6 — the OpenCode session DB (#812 WI-2), restored before opencode opens it —
81
- # the only place this can happen. Bare, like restore_credentials above:
82
- # restore_session_db never returns non-zero (Q3 — nothing about the session
83
- # DB may ever fail /run), so there is deliberately no `||` here to catch.
84
- restore_session_db
85
- log "session DB restore done ${SECONDS}s into the hook"
86
-
87
- # 7 — integrity-check the restored DB (#1868 WI-4), after the config file
88
- # exists (verify_session_db reads it) and before anything opens the DB —
89
- # the only window in which that's true. The ONE step in this sequence that
90
- # can still fail /run past the runner-key gate: a 34 means the corrupt DB's
91
- # separation/disposal could not be proven safe (see verify_session_db's own
92
- # comment for why that's a deliberate exception to "nothing about the
93
- # session DB may ever fail /run").
94
- verify_session_db || exit 1
95
-
96
- # 8 — apply the overlay before OpenCode resolves its project configuration.
97
- apply_runner_opencode_config
98
-
99
- # 9 — configure git after credentials are restored and before agent shells start.
100
- configure_github_access
101
-
102
- # 10 — opencode. Started here, not at build time: a warm process in the shared
103
- # snapshot would carry its installation id and database into every VM. Not
104
- # waited on: a slow opencode boot is not a reason to fail /run (the tunnel CLI
105
- # auto-starts opencode when it finds none healthy,
106
- # apps/cli/src/commands/ensure-opencode.ts, and the api-worker lifecycle cron
107
- # reclaims a runner that never comes online).
108
- start_opencode
109
-
110
- # 11 — begin replicating the session DB (#812 WI-3), now that opencode has
111
- # opened it and before any work can arrive over the tunnel. Bare, like
112
- # restore_session_db above: start_litestream never returns non-zero (every
113
- # guard inside it is its own `return 0`), so there is nothing here for
114
- # `set -e` to abort on.
115
- start_litestream
116
-
117
- # 11a — the interval credential sync (#1868 WI-3), matching ECS's own
118
- # post-litestream position. Bare for the same reason: every guard inside
119
- # start_credential_sync is its own `return 0`.
76
+ # 6 — the interval credential sync (#1868 WI-3). It stays before the context file
77
+ # is written. Bare: every guard inside start_credential_sync is its own `return 0`.
120
78
  start_credential_sync
121
79
 
122
- # 12 — the first per-VM identity on the wire. The subshell's umask makes the file
80
+ # 7 — the first per-VM identity on the wire. The subshell's umask makes the file
123
81
  # unreadable to anyone else from the moment it exists, before the key is in it.
124
82
  (
125
83
  umask 077
@@ -131,6 +89,6 @@ start_credential_sync
131
89
  # attempt cap (apps/cli/src/lib/tunnel/connection.ts), and a control-plane
132
90
  # reclaim pass (the api-worker lifecycle cron) is what decides whether a
133
91
  # runner that never connects gets suspended.
134
- start_tunnel "${runner_key}" "${api_url}" "${tunnel_url}"
92
+ start_tunnel "${runner_key}" "${api_url}" "${tunnel_url}" true true
135
93
 
136
94
  run_succeeded=true
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@evident-ai/runner-cdk",
3
- "version": "3.4.1-dev.51df4de",
3
+ "version": "3.4.1-dev.59c7df3",
4
4
  "description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",