@evident-ai/runner-cdk 0.1.1-dev.da88f8e → 3.4.1-dev.0ef5061

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,8 +2,8 @@
2
2
  # Sourced by every hook script. Nothing here runs at image build time.
3
3
  #
4
4
  # These hooks are the SECOND shell speaking the runner-synchroniser CLI contract
5
- # (packages/runner-image/entrypoint.sh is the first), so both are held to it by
6
- # packages/runner-synchroniser/src/shell-contract.test.ts — which derives what
5
+ # (runner/docker-images/fargate/entrypoint.sh is the first), so both are held to it by
6
+ # runner/synchroniser/src/shell-contract.test.ts — which derives what
7
7
  # `run_synchroniser` below must handle from the subcommands this shell calls.
8
8
 
9
9
  # Installed from npm by the image (docker/Dockerfile's ARG
@@ -24,6 +24,8 @@ CONTEXT_FILE="/dev/shm/evident-run-context"
24
24
  TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
25
25
  OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
26
26
  LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
27
+ CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
28
+ CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
27
29
 
28
30
  # Where the runner's credential store lives inside the durable-state bucket.
29
31
  # The BUCKET is the same for every VM from an image version, so the stack bakes
@@ -37,57 +39,33 @@ STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
37
39
  # The generated litestream.yml (#812). tmpfs for the same two reasons as the
38
40
  # files above: it must never reach the block device (Q6, `docs`), and it
39
41
  # survives suspend/resume, which is what lets `/resume` start litestream again
40
- # with no regeneration cost. Regenerated only when absent or empty
41
- # (`ensure_litestream_config`, below).
42
+ # with no regeneration cost. The CLI regenerates it only when absent or empty.
42
43
  LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
43
44
 
44
- # Set when a `session-db-classify` answer of 31 (replica unusable), or a
45
- # restore this hook gave up on for its own reasons (a truncated `timeout`, a
46
- # broken tool), means this boot must NOT start `litestream replicate` — doing
47
- # so would let a partial/fresh local DB overwrite a replica this boot never
48
- # proved is safe to write over. Cleared at the top of every `restore_session_db`
49
- # call (i.e. every `/run`) before any other decision, so a marker left by an
50
- # earlier boot cannot silently disable replication for the VM's whole life.
45
+ # Set by the CLI when session-DB restore or verification could not prove this
46
+ # boot safe to replicate. The teardown hooks read it before flushing, and
47
+ # `/terminate` removes it with the rest of the per-VM state.
51
48
  SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
52
49
 
53
- log() { echo "[hook:$(basename "$0")] $*"; }
54
- warn() { echo "[hook:$(basename "$0")] $*" >&2; }
55
- error() { echo "[hook:$(basename "$0")] ERROR: $*" >&2; }
50
+ hook_name() { printf '%s' "${0##*/}"; }
51
+ log() { echo "[hook:$(hook_name)] $*"; }
52
+ warn() { echo "[hook:$(hook_name)] $*" >&2; }
53
+ error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
56
54
 
57
55
  # Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
58
56
  # corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
59
57
  # `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
60
- # unusable)/32 (retry) see `packages/runner-synchroniser/src/cli.ts`'s own
58
+ # unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
59
+ # exhausted, replica separated and local disposed) / 34 (could not prove
60
+ # separation or disposal) — see `runner/synchroniser/src/cli.ts`'s own
61
61
  # comment for what each means, not restated here. Any OTHER non-zero status
62
- # means the tool itself broke, which is the only case worth an ERROR here
63
- # EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
64
- # that is an intentional bound firing, not a broken tool, so the caller
65
- # classifies it instead (restore_credentials, #930). A trailing
66
- # `--evident-deadline=N` (stripped below before forwarding to the CLI, and
67
- # never produced by anything but timed_synchroniser) is how a caller opts in;
68
- # every other caller — sync_credentials, restore_session_db's `env` /
69
- # `litestream-config` / `session-db-classify`, and this file's own contract
70
- # test — passes none, so 124/137 there still means a genuine external SIGKILL
71
- # (e.g. an OOM kill) and must keep producing the ERROR below.
62
+ # means the tool itself broke, which is the only case worth an ERROR here.
72
63
  run_synchroniser() {
73
- local -a call_args=("$@")
74
- local deadline=""
75
- local last=$(( ${#call_args[@]} - 1 ))
76
- if [ "${last}" -ge 0 ] && [[ "${call_args[last]}" == --evident-deadline=* ]]; then
77
- deadline="${call_args[last]#--evident-deadline=}"
78
- call_args=("${call_args[@]:0:${last}}")
79
- fi
80
-
81
64
  local rc=0
82
- local -a launcher=()
83
- [ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
84
- "${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
65
+ "${SYNCHRONISER}" "$@" || rc=$?
85
66
  case "${rc}" in
86
- 0 | 10 | 30 | 31 | 32) ;;
87
- 124 | 137)
88
- [ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
89
- ;;
90
- *) error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
67
+ 0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
68
+ *) error "synchroniser '$*' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
91
69
  esac
92
70
  return "${rc}"
93
71
  }
@@ -110,8 +88,8 @@ run_synchroniser() {
110
88
  # Never fails its caller: a timing line is diagnostics, and hardening a
111
89
  # currently-working path is worse than the gap it closes (#931). The `date`
112
90
  # fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
113
- # timed_synchroniser turns into `elapsed_ms=unknown` rather than aborting /run
114
- # under `set -e` — and rather than a 0 that would read as a fast healthy call.
91
+ # `log_elapsed_since` turns into `elapsed_ms=unknown` rather than aborting a
92
+ # hook under `set -e` — and rather than a 0 that would read as a fast healthy call.
115
93
  now_ms() {
116
94
  local now="${EPOCHREALTIME:-}"
117
95
  if [ -n "${now}" ]; then
@@ -122,16 +100,10 @@ now_ms() {
122
100
  date +%s%3N 2>/dev/null || echo 0
123
101
  }
124
102
 
125
- # Reports what one step COST, in a deliberately machine-greppable line (`op=`,
126
- # `elapsed_ms=`, `rc=`) so "how long does this actually take in the fleet?" is a
127
- # log query rather than another spike. `at_s` is where the step landed on the
128
- # hook's own SECONDS clock, which is what the pre-opencode budget in
129
- # hook-scripts.test.ts is derived against.
130
- #
131
- # Callable directly, not only through timed_synchroniser below, because the
132
- # steps that dominate the pre-opencode window CANNOT be wrapped: their stdout is
133
- # captured (`x="$(run_synchroniser env)"`, later `eval`'d), so a `log` line
134
- # emitted inside the substitution would be evaluated as configuration.
103
+ # Reports what one pre-warm COST, in a deliberately machine-greppable line
104
+ # (`op=`, `elapsed_ms=`, `rc=`) so "how long does this actually take in the
105
+ # fleet?" is a log query rather than another spike. `at_s` is where the step
106
+ # landed on the hook's own SECONDS clock.
135
107
  #
136
108
  # `unknown`, never a number, when either end failed to read the clock (now_ms's
137
109
  # 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
@@ -150,28 +122,8 @@ log_elapsed_since() {
150
122
  log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
151
123
  }
152
124
 
153
- # Runs ONE synchroniser call and reports what it COST, without changing what it
154
- # MEANS: the call's own status is returned untouched, so every caller's
155
- # `|| return 1` / `|| status=$?` keeps working exactly as before.
156
- #
157
- # `deadline` is an explicit parameter, forwarded to run_synchroniser as a
158
- # trailing `--evident-deadline=N` (never a global or dynamically-scoped
159
- # variable, #930) — every current caller (restore_credentials) always has a
160
- # positive remaining-budget value by the time it calls this, having already
161
- # decided to skip the call entirely otherwise.
162
- timed_synchroniser() {
163
- local op="$1" deadline="$2"
164
- shift 2
165
-
166
- local started_ms rc=0
167
- started_ms="$(now_ms)"
168
- run_synchroniser "$@" "--evident-deadline=${deadline}" || rc=$?
169
- log_elapsed_since "${op}" "${started_ms}" "${rc}"
170
- return "${rc}"
171
- }
172
-
173
125
  # Exports what `runner-synchroniser` resolves its object-store location from
174
- # (packages/runner-synchroniser/src/config.ts). It treats either being empty as
126
+ # (runner/synchroniser/src/config.ts). It treats either being empty as
175
127
  # "persistence disabled" and then reports every restore as a WARNING it still
176
128
  # exits 0 for — so an unset value has to be caught HERE, where it can still be
177
129
  # told apart from "the store is simply empty".
@@ -191,167 +143,32 @@ load_state_config() {
191
143
 
192
144
  # Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
193
145
  # both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
194
- # is just LITESTREAM_BUCKET. start_litestream and flush_session_db (#812
146
+ # is just LITESTREAM_BUCKET. The CLI's replicator and flush_session_db (#812
195
147
  # WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
196
148
  # directly, so every entry point agrees on one predicate regardless of which
197
- # earlier step in THIS hook process set it: restore_session_db's
198
- # `eval "$(run_synchroniser env)"` on /run, or this function on /resume,
199
- # /suspend and /terminate, none of which ever call the synchroniser before
200
- # needing the answer.
149
+ # earlier step in THIS hook process set it; /resume, /suspend and /terminate
150
+ # all use this same function before any teardown decision.
201
151
  export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
202
152
  }
203
153
 
204
- # /run starts no LLM task it restores state, starts opencode and dials the
205
- # tunnel, none of which needs a model credential. Which credential a turn needs
206
- # depends on the model that turn's message asks for, so that question has no
207
- # single boot-time answer; it is answered per message, not here. What IS still
208
- # fatal is credential persistence being unavailable at all (an unset bucket/
209
- # prefix, or a broken synchroniser bundle) — that would silently drop refreshed
210
- # tokens on every suspend, so those two restores stay `|| return 1`.
211
- # `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
212
- # a log line, since the runtime forwards no hook stderr to the doorbell caller.
213
-
214
- # How long restore_credentials' shared step budget is, covering all THREE of
215
- # its synchroniser calls (restore claude, restore opencode, model-auth-ready)
216
- # as one window rather than a timeout apiece — a fixed per-call cap at the same
217
- # total would false-fire on the ordinary case of one slow call (a slow S3 GET),
218
- # which is precisely the boot this exists to keep healthy. From #930's 3-boot
219
- # sample: the step measured ~3-5s total (~1.2s per call, three node cold
220
- # starts of a 1.8 MB bundle) — a WEAK estimate this file's own
221
- # SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
222
- # routine slow call boots this VM with no model credentials until an operator
223
- # reconnects it; too loose and a hung call burns more of the hook's own
224
- # SIGTERM budget before the VM is destroyed mid-boot anyway. The env override
225
- # is for tests only, so they need not burn wall clock.
226
- CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECONDS:-8}"
227
- # The SIGKILL backstop `timeout -k` applies after its own SIGTERM, exactly like
228
- # SESSION_DB_RESTORE_KILL_GRACE_SECONDS above (a bare `timeout` only SIGTERMs,
229
- # and a call that ignored it would be unbounded again). Hard ceiling on the
230
- # step: DEADLINE + this = 10s, once — not per call.
231
- CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
232
-
233
- # What is left of the shared step budget, in whole seconds, `step_started_s`
234
- # seconds after it began. `SECONDS` (a bash builtin with no failure mode,
235
- # unlike `now_ms`) truncates, so a step that began at true time `s0` can read
236
- # less elapsed time than actually passed — the `- 1` restores the invariant
237
- # that no call is granted more than DEADLINE + GRACE from step start.
238
- remaining_credential_budget() {
239
- local step_started_s="$1"
240
- echo $(( CREDENTIAL_RESTORE_DEADLINE_SECONDS - (SECONDS - step_started_s) - 1 ))
241
- }
242
-
243
- # Runs one of the step's bounded calls: skips it with a named warn if the
244
- # shared budget is already exhausted, treats a timeout (124/137) as the
245
- # non-fatal warn-and-continue D1 approved, and keeps every other non-zero
246
- # fatal exactly as restore_credentials always has (`|| return 1` aborts /run
247
- # before opencode and the tunnel start).
248
- bounded_restore_call() {
249
- local op="$1" step_started_s="$2"
250
- shift 2
251
-
252
- local remaining
253
- remaining="$(remaining_credential_budget "${step_started_s}")"
254
- if [ "${remaining}" -lt 1 ]; then
255
- warn "CREDENTIAL-RESTORE-SKIPPED: ${op} skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
256
- return 0
257
- fi
258
-
259
- local rc=0
260
- timed_synchroniser "${op}" "${remaining}" "$@" || rc=$?
261
- case "${rc}" in
262
- 0) ;;
263
- 124 | 137)
264
- warn "CREDENTIAL-RESTORE-TIMEOUT: ${op} did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
265
- ;;
266
- *) return 1 ;;
267
- esac
268
- return 0
269
- }
270
-
271
- restore_credentials() {
272
- local step_started_ms step_started_s
273
- step_started_ms="$(now_ms)"
274
- step_started_s="${SECONDS}"
275
-
276
- load_state_config || return 1
277
- bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
278
- bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
279
-
280
- # `restore` exits 0 whether it restored, found nothing, or discarded a corrupt
281
- # object, so the synchroniser's own predicate is the only reliable answer to
282
- # "can this VM authenticate a model right now". Its status is captured, not
283
- # used as a condition, so it can never become this function's return status —
284
- # `run` calls this bare (no `||`) under `set -euo pipefail`, and any
285
- # non-zero return here would abort the hook before opencode and the tunnel
286
- # ever start.
287
- local auth_status=0 auth_remaining
288
- auth_remaining="$(remaining_credential_budget "${step_started_s}")"
289
- if [ "${auth_remaining}" -lt 1 ]; then
290
- warn "CREDENTIAL-RESTORE-SKIPPED: model-auth-ready skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
291
- else
292
- timed_synchroniser model-auth-ready "${auth_remaining}" model-auth-ready || auth_status=$?
293
- case "${auth_status}" in
294
- 0) ;; # some model auth is configured; the two restores above already logged what they found
295
- 10)
296
- warn "no model credentials under s3://${LITESTREAM_BUCKET}/${LITESTREAM_PREFIX}/ \
297
- (neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
298
- ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
299
- a model provider fails until one is connected. See 'Seeding a credential store' in \
300
- infrastructure/evident-microvm/README.md."
301
- ;;
302
- 124 | 137)
303
- warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
304
- ;;
305
- *)
306
- # run_synchroniser has already `error`ed the tool-broke line for a
307
- # broken tool; any other unrecognised code lands here too. Do not
308
- # claim there are no credentials — absent evidence is not contrary
309
- # evidence (development-workflow.mdc).
310
- warn "could not determine whether this VM has model credentials"
311
- ;;
312
- esac
313
- fi
314
- # The step total, in the SAME greppable shape as the per-call lines above, so
315
- # one query answers both "what does credential restore cost?" and "which of
316
- # its three calls cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
317
- # + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
318
- # comment above that constant) rather than per call. `rc=0` is deliberate
319
- # here, not an absence of failure modes: a timed-out call already warned by
320
- # name above and is non-fatal by design (D1); every failure that DOES abort
321
- # this step has already returned 1 above.
322
- log_elapsed_since credential-restore-step "${step_started_ms}" 0
323
- return 0
324
- }
325
-
326
- # Best-effort by design: /suspend must still drop the tunnel and /terminate must
327
- # still clean up, so a flush that cannot happen is loud but never fatal.
154
+ # Best-effort by design: the teardown hooks must still complete even when a
155
+ # credential flush cannot happen.
328
156
  sync_credentials() {
329
157
  load_state_config || return 0
330
158
  run_synchroniser sync-once claude || true
331
159
  run_synchroniser sync-once opencode || true
332
160
  }
333
161
 
334
- # --- Session DB restore (#812 WI-2) ----------------------------------------
335
- #
336
- # Restores opencode.db from S3 before opencode opens it — the read half only.
337
- # Writes NOTHING to S3 itself; replication back to S3 is started separately by
338
- # `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
339
- # `restore_session_db` are the MicroVM side of the same contract
340
- # packages/runner-image/entrypoint.sh's inlined restore already speaks, going
341
- # through the SAME runner-synchroniser CLI.
162
+ # --- boot pre-warms ----------------------------------------------------------
342
163
 
343
164
  # Reads the ~30 MB litestream binary into the page cache, in the background, so
344
165
  # the FIRST exec of it does not pay that read on the critical path.
345
166
  #
346
- # Why it is on the critical path at all: restore_session_db must finish before
347
- # start_opencode, and HOOK_SIGTERM_SECONDS is a POINT on the hook's SECONDS
348
- # clock, so a second spent here is a second taken from the rest of /run — and a
349
- # /run that overruns is SIGTERMed mid-boot and its VM destroyed. (Until #1172
350
- # the binding point was the tighter in-hook opencode readiness deadline; that
351
- # deadline is gone, the hook's own SIGTERM is not.)
352
- # Boot measurements put ~6.6-7.3s between the credential restore and the
353
- # litestream version line, of which only ~0.2s is accounted for by the two node
354
- # calls in between — the remainder is INFERRED to be this read, never measured.
167
+ # The pre-warm overlaps the CLI's node startup and auth round trip, so the first
168
+ # litestream operation does not also pay the binary's cold page-cache read.
169
+ # Boot measurements put ~6.6-7.3s before the litestream version line, of which
170
+ # only ~0.2s is accounted for by the two node calls in between — the remainder
171
+ # is INFERRED to be this read, never measured.
355
172
  # The `litestream-prewarm` and `litestream-version` timings are what settle it
356
173
  # on the next boot.
357
174
  #
@@ -378,246 +195,40 @@ prewarm_litestream() {
378
195
  log "pre-warming ${binary} in the background"
379
196
  }
380
197
 
381
- # Generates ${LITESTREAM_CONFIG_FILE} from runner-synchroniser's own renderer
382
- # the SAME config module the credential restore/sync already goes through,
383
- # so there is no second copy of the bucket/prefix/path logic to drift
384
- # (litestream-config.ts's own header makes the same point). Only when absent
385
- # or empty, so `/resume` (WI-4) restarting litestream after a suspend/resume
386
- # snapshot — /dev/shm intact — pays nothing to regenerate it.
387
- #
388
- # Returns 1 on failure — fatal to the CALLER, never to the hook: every caller
389
- # (restore_session_db here, start_litestream in WI-3) must still let opencode
390
- # and the tunnel start regardless.
391
- ensure_litestream_config() {
392
- if [ -s "${LITESTREAM_CONFIG_FILE}" ]; then
393
- return 0
394
- fi
395
-
396
- # Timed OUTSIDE the substitution, not with timed_synchroniser: this call's
397
- # stdout IS the config, so a timing line emitted inside `$(...)` would land in
398
- # litestream.yml.
399
- local rendered config_started_ms config_rc=0
400
- config_started_ms="$(now_ms)"
401
- rendered="$(run_synchroniser litestream-config)" || config_rc=$?
402
- log_elapsed_since litestream-config "${config_started_ms}" "${config_rc}"
403
- if [ "${config_rc}" -ne 0 ]; then
404
- error "could not generate ${LITESTREAM_CONFIG_FILE}: runner-synchroniser litestream-config failed (see the error above)"
405
- return 1
406
- fi
407
- printf '%s\n' "${rendered}" >"${LITESTREAM_CONFIG_FILE}" || {
408
- error "could not write ${LITESTREAM_CONFIG_FILE}"
409
- return 1
410
- }
411
-
412
- # Q7's diagnostic, for whoever can read this VM's CloudWatch log: which way
413
- # litestream resolved its AWS region, and whether it can even see the
414
- # names it would need to (never the VALUES of any OTHER AWS_* variable —
415
- # AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN may be present here).
416
- local region_empty="yes"
417
- if printf '%s\n' "${rendered}" | grep -Eq '^ *region: *[^[:space:]]'; then
418
- region_empty="no"
419
- fi
420
- # Hoisted out of the log line below so it can be timed: this is the FIRST exec
421
- # of the litestream binary on the boot, and the pre-warm /run fires (see
422
- # prewarm_litestream) is aimed squarely at what this number measures.
423
- local version_started_ms litestream_version
424
- version_started_ms="$(now_ms)"
425
- litestream_version="$(litestream version 2>/dev/null || echo 'unknown')"
426
- log_elapsed_since litestream-version "${version_started_ms}" 0
427
- log "litestream ${litestream_version}; AWS_REGION=${AWS_REGION:-<unset>} AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-<unset>}; rendered litestream.yml region empty: ${region_empty}"
428
- }
429
-
430
- # Writes the marker AND names the reason, in one call, so the marker can never
431
- # appear silently. Every give-up path in restore_session_db calls this EXCEPT
432
- # the classifier's 31 (#1106 — see that branch), and no success path ever does.
433
- # It is what a later `/resume`/`start_litestream` (WI-3) reads to skip
434
- # replicating this boot: a local DB that is only PARTIALLY restored must never
435
- # be allowed to overwrite a replica this boot never proved safe to write over.
436
- # A merely FRESH local DB is not that danger — litestream continues the txid
437
- # chain — which is precisely why 31 no longer belongs here.
438
- mark_no_replicate() {
439
- : >"${SESSION_DB_NO_REPLICATE_MARKER}"
440
- warn "SESSION-DB-NO-REPLICATE: $1"
441
- }
442
-
443
- # Removes local session-DB debris a give-up path leaves behind. Called on
444
- # EVERY give-up path and NO success path: skipping it on a give-up is a real
445
- # bug (a truncated file left in place makes opencode fail to open a malformed
446
- # database, which fails /run outright — worse than the history loss this
447
- # discards); calling it on a success path would delete a good restore. One
448
- # `rm -f` per path, not a single command, so a failure removing one does not
449
- # skip the other two.
450
- discard_session_db_debris() {
451
- rm -f "${OPENCODE_DB_PATH}" "${OPENCODE_DB_PATH}-wal" "${OPENCODE_DB_PATH}-shm"
452
- }
453
-
454
- # How long restore_session_db waits for ONE `litestream restore` before giving
455
- # up. From M4 (see the plan's §2): ~50 MB/s extrapolated from the x86_64 ECS
456
- # container's own live replica, so 7s covers roughly 300-350 MB — an
457
- # EXTRAPOLATION, not a MicroVM measurement (different arch, network path and
458
- # credential source); the boot line ensure_litestream_config logs, plus the
459
- # ${SECONDS} stamps in /run, are how it gets measured here for real.
198
+ # Reads the aws CLI v2 install tree into the page cache, in the background, so
199
+ # the CLI's first runner-secret fetch does not pay first-touch I/O. The read
200
+ # overlaps `evident run`'s node startup and auth round trip rather than a hook
201
+ # deadline.
460
202
  #
461
- # 7 rather than the 8 it was: hook-scripts.test.ts's pre-opencode ladder held an
462
- # allowance of 4s for the rest of this step, which live boots measured at
463
- # 6.7-7.4s, so the ladder passed while real boots blew the deadline. Correcting
464
- # that allowance is what takes this second — the remedy the ladder's own failure
465
- # message prescribes, and the one that leaves the SIGTERM worst case untouched.
466
- # The cost is real and unmeasured on this platform: ~50 MB less restorable
467
- # replica before a truncation that silently loses history. A fresh
468
- # state_prefix starts at zero, so this is generous for a long time, but
469
- # RoutingStrategy is 'per_user' only (routing-strategy.ts) and every prefix is
470
- # deterministic and permanent per (pool, routing key) — so a long-lived
471
- # runner's opencode.db grows monotonically across VM generations, exactly
472
- # like the ~1 GB ECS one M4 measured, and will eventually hit this ceiling.
473
- # Past it the restore is truncated (SESSION-DB-RESTORE-TRUNCATED, below) and
474
- # this boot skips replication — silent, permanent history loss for that
475
- # runner unless somebody reads the log. Not solved here (WI-0 Task 0.2).
476
- # The env override is for tests only, so they need not burn 7s of wall clock.
477
- SESSION_DB_RESTORE_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_RESTORE_DEADLINE_SECONDS:-7}"
478
- # The SIGKILL backstop `timeout -k` applies after its own SIGTERM: `timeout`
479
- # alone only SIGTERMs, and a litestream that ignored it would be unbounded
480
- # again. Hard ceiling on this step: DEADLINE + this = 9s.
481
- SESSION_DB_RESTORE_KILL_GRACE_SECONDS=2
482
-
483
- # The one bounded restore attempt, run before start_opencode (opencode opens
484
- # the DB the moment it starts, so this is the only place it can happen).
485
- # Every branch returns 0 — /run calls this bare under `set -euo pipefail`,
486
- # exactly like restore_credentials, and nothing about the session DB may ever
487
- # fail /run (Q3): a /run that exits 1 fails the whole lifecycle transition
488
- # and the user gets NO runner at all, which is worse than losing history.
489
- restore_session_db() {
490
- # Cleanup, not a decision, and unconditionally first: /dev/shm survives
491
- # suspend/resume and a /run retry, so a marker left by an earlier boot must
492
- # never silently disable replication for the rest of this VM's life.
493
- rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
494
-
495
- # Config resolved ONCE here, in the synchroniser (config.ts owns
496
- # OPENCODE_DB_PATH; no second copy of that path here), exactly as
497
- # entrypoint.sh's own line does. Unlike ECS's `|| die`, a failure here must
498
- # not abort the hook — `x="$(cmd)"` under `set -e` WOULD abort the whole
499
- # shell on a failing substitution, which is exactly why it is caught
500
- # explicitly rather than left to `set -e`.
501
- # Timed OUTSIDE the substitution for the same reason ensure_litestream_config
502
- # is: this call's stdout is `eval`'d, so a timing line emitted inside `$(...)`
503
- # would be evaluated as configuration.
504
- local synchroniser_env env_started_ms env_rc=0
505
- env_started_ms="$(now_ms)"
506
- synchroniser_env="$(run_synchroniser env)" || env_rc=$?
507
- log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
508
- if [ "${env_rc}" -ne 0 ]; then
509
- mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
510
- return 0
511
- fi
512
- # Guarded for the SAME reason as the substitution above, which is easy to miss:
513
- # `eval` returns the status of what it ran, so a malformed line from a skewed
514
- # bundle would abort /run right here under `set -e` — before start_opencode,
515
- # so the user gets NO runner at all. entrypoint.sh:110 leaves the identical
516
- # line bare because on ECS the blast radius is a crash-loop-and-replace; here
517
- # it is the whole lifecycle transition, so Q3 makes it a give-up instead.
518
- eval "${synchroniser_env}" || {
519
- mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
520
- return 0
521
- }
522
-
523
- # #931's exact lesson, one shell over (packages/runner-image/entrypoint.sh):
524
- # the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
525
- # 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
526
- # would otherwise abort right here under `set -u` the moment
527
- # `${OPENCODE_DB_PATH}` is dereferenced below. entrypoint.sh may `die` on
528
- # this (ECS just crash-loops and replaces the task); this hook cannot —
529
- # Q3 says nothing about the session DB may ever fail /run — so a missing
530
- # OPENCODE_DB_PATH is a give-up, not a silent default (a guessed path would
531
- # be actively wrong, not merely absent).
532
- if [ -z "${OPENCODE_DB_PATH+x}" ]; then
533
- mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
534
- return 0
535
- fi
536
-
537
- # #770's arm, checked FIRST of the decisions and with its OWN message —
538
- # never the SESSION-DB-REPLICA-UNUSABLE one below (#764's finding: an
539
- # operator grepping that marker must not hit the disabled case).
540
- # PERSISTENCE_BUCKET is empty exactly when LITESTREAM_BUCKET or
541
- # LITESTREAM_PREFIX is unset (config.ts's persistenceEnabled) — the
542
- # IDENTICAL predicate ECS's own litestream-launch gate uses
543
- # (entrypoint.sh), so the two images agree on what "disabled" means.
544
- if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
545
- warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated."
546
- return 0
547
- fi
548
-
549
- if ! ensure_litestream_config; then
550
- mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
203
+ # A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
204
+ # enough here: v2 ships as a real Python distribution (~7,500 files under the
205
+ # resolved binary's own directory), and a cold invocation demand-pages a
206
+ # scattered set of them (botocore's endpoints.json/partitions.json, service
207
+ # model JSON, shared libs) one boot measured #1997's fetch at 7,243ms cold
208
+ # vs ~400ms warm, an ~18x gap the litestream read-ahead trick alone cannot
209
+ # close. `find -exec cat` walks that whole tree instead of one file.
210
+ prewarm_aws_cli() {
211
+ local binary tree
212
+ binary="$(command -v aws 2>/dev/null || true)"
213
+ if [ -z "${binary}" ]; then
214
+ warn "aws CLI is not on PATH; skipping the boot pre-warm"
551
215
  return 0
552
216
  fi
217
+ tree="$(dirname "$(readlink -f "${binary}")")"
553
218
 
554
- local restore_rc=0 restore_started_ms
555
- restore_started_ms="$(now_ms)"
556
- timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_RESTORE_DEADLINE_SECONDS}" \
557
- litestream restore -config "${LITESTREAM_CONFIG_FILE}" \
558
- -if-db-not-exists -if-replica-exists "${OPENCODE_DB_PATH}" || restore_rc=$?
559
- log_elapsed_since session-db-restore "${restore_started_ms}" "${restore_rc}"
560
-
561
- # `timeout`'s OWN codes, handled BEFORE classifying: 124/137 are a
562
- # TRUNCATED restore, not a corrupt replica — handing them to
563
- # session-db-classify would mislabel a slow/large replica as one. 125-127
564
- # mean `timeout` (or litestream itself) is broken, not the replica.
565
- case "${restore_rc}" in
566
- 124 | 137)
567
- discard_session_db_debris
568
- mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
569
- return 0
570
- ;;
571
- 125 | 126 | 127)
572
- error "litestream restore could not even run (timeout exited ${restore_rc})"
573
- discard_session_db_debris
574
- mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
575
- return 0
576
- ;;
577
- esac
578
-
579
- local classify_rc=0
580
- run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave || classify_rc=$?
581
- case "${classify_rc}" in
582
- 0) ;; # restored, or no replica yet — the CLI already logged which
583
- 31)
584
- # The classifier already discarded the local debris (session-db.ts) — and
585
- # that is exactly why this is the ONE give-up path that does NOT
586
- # mark_no_replicate (#1106). Every other give-up here may be sitting on a
587
- # half-restored DB, so the marker stays; a 31 is structurally guaranteed
588
- # to be a FRESH one, and replicating it starts a new backup chain instead
589
- # of leaving this boot with a zero-width backup window. Same reasoning,
590
- # and same one-line change, as packages/runner-image/entrypoint.sh's `31)`.
591
- warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and starting a fresh backup chain this boot (see the WARNING above)"
592
- ;;
593
- 32)
594
- # The CLI's own contract for 32 is "re-run litestream restore and ask
595
- # again", but this hook has budget for exactly ONE attempt (Q4) — name
596
- # the deviation so nobody reads this as a bug.
597
- discard_session_db_debris
598
- mark_no_replicate "session-db-classify asked for another restore attempt (32), but this hook has budget for only one; treating it as a give-up rather than retrying"
599
- ;;
600
- 30)
601
- # The CLI already logged its own FATAL line above; Q3 still says boot
602
- # fresh rather than fail /run.
603
- discard_session_db_debris
604
- mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
605
- ;;
606
- *)
607
- # run_synchroniser already logged the "tool broke" ERROR for this.
608
- discard_session_db_debris
609
- mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
610
- ;;
611
- esac
612
-
613
- return 0
219
+ (
220
+ local started_ms rc=0
221
+ started_ms="$(now_ms)"
222
+ find "${tree}" -type f -exec cat {} + >/dev/null 2>&1 || rc=$?
223
+ log_elapsed_since aws-cli-prewarm "${started_ms}" "${rc}"
224
+ ) &
225
+ log "pre-warming ${tree} in the background"
614
226
  }
615
- # --- Session DB restore (end) -----------------------------------------------
616
227
 
617
228
  # `kill -0` answers "does this pid exist", which is not the question any caller
618
229
  # here is asking. A process that has exited but has not been reaped — a zombie —
619
230
  # still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
620
- # normal case for everything these hooks start: `start_tunnel`/`start_opencode`
231
+ # normal case for everything these hooks start: `start_tunnel` and its CLI child
621
232
  # background a process that outlives the hook, the hook shell must return so AWS
622
233
  # gets its 200, and PID 1 in this image is a bare node hook server with no init
623
234
  # (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
@@ -706,6 +317,7 @@ regenerate_machine_id() {
706
317
  tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
707
318
  opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
708
319
  litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
320
+ creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
709
321
 
710
322
  # `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
711
323
  # and an interpolation of a missing field is still a non-empty string, so a
@@ -719,24 +331,6 @@ payload_is_complete() {
719
331
  ' >/dev/null 2>&1
720
332
  }
721
333
 
722
- # `setsid` so opencode outlives this hook: the script must return so AWS gets its
723
- # 200, while opencode keeps serving. The guard makes /run idempotent — a retry
724
- # after a killed hook must not put a second instance on the same port.
725
- start_opencode() {
726
- if opencode_is_running; then
727
- warn "opencode already running (pid $(cat "${OPENCODE_PID_FILE}")); reusing it"
728
- return 0
729
- fi
730
-
731
- setsid opencode serve \
732
- --hostname 127.0.0.1 \
733
- --port "${OPENCODE_PORT}" \
734
- --print-logs &
735
-
736
- echo $! >"${OPENCODE_PID_FILE}"
737
- log "opencode starting (pid $(cat "${OPENCODE_PID_FILE}"))"
738
- }
739
-
740
334
  stop_opencode() {
741
335
  if ! opencode_is_running; then
742
336
  # Clear the file here too, for the same reason stop_tunnel does: opencode
@@ -801,69 +395,11 @@ stop_opencode_and_wait() {
801
395
  log "opencode stopped ${outcome}"
802
396
  }
803
397
 
804
- # --- litestream replicate (#812 WI-3) ---------------------------------------
805
- #
806
- # The write half of session-DB persistence. restore_session_db (above) is the
807
- # read half only and never starts this. Q8 chose one backgrounded, unsupervised
808
- # `litestream replicate` over a periodic flush or a `replicate -exec` wrapper
809
- # around opencode: SIGTERM is `replicate`'s own final-sync trigger, so
810
- # stop_litestream (below) doubles as the checked flush /suspend and /terminate
811
- # need, and there is no supervisor in this image to hand the process to (no
812
- # init, no `wait -n` shell that could stay alive after the hook returns).
813
-
814
398
  # The graceful wait before the SIGKILL backstop in stop_litestream, below.
815
399
  # litestream's own sync is normally sub-second, so 10s is generous headroom —
816
400
  # chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
817
401
  LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
818
402
 
819
- # Guards, in this exact order and no other — reordering any of them re-arms the
820
- # invariant it exists to protect:
821
- #
822
- # 1. persistence disabled (#770) — bucket AND prefix both non-empty — is its
823
- # OWN arm with its OWN message, checked FIRST, and must never log
824
- # SESSION-DB-REPLICA-UNUSABLE: an operator grepping that marker must not
825
- # hit the disabled case (#764).
826
- # 2. the no-replicate marker (restore_session_db above) — set when this boot's
827
- # local DB may be only PARTIALLY restored, so starting replicate here would
828
- # let it overwrite a replica this boot never proved safe to write over.
829
- # A classifier 31 no longer sets it (#1106): that DB is guaranteed fresh,
830
- # and skipping replicate for it was what made the history loss unbounded.
831
- # 3. no usable config — ensure_litestream_config (restore_session_db, above)
832
- # already logged why it is missing; this is not a place to retry it.
833
- # 4. already running — idempotence, mirroring start_tunnel's own guard.
834
- #
835
- # Only past all four does it actually spawn, backgrounded with `setsid` for the
836
- # same reason start_opencode/start_tunnel are: the hook must return so AWS gets
837
- # its 200, while replicate keeps running. litestream's OWN stderr is left to
838
- # reach CloudWatch — never redirected — because there is no supervisor in this
839
- # image, so that is the only channel that can say why a replicator died.
840
- start_litestream() {
841
- if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
842
- warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing is replicated."
843
- return 0
844
- fi
845
-
846
- if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
847
- log "skipping litestream replicate: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
848
- return 0
849
- fi
850
-
851
- if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
852
- error "no usable ${LITESTREAM_CONFIG_FILE}; not starting litestream replicate"
853
- return 0
854
- fi
855
-
856
- if litestream_is_running; then
857
- warn "litestream already running (pid $(cat "${LITESTREAM_PID_FILE}")); reusing it"
858
- return 0
859
- fi
860
-
861
- setsid litestream replicate -config "${LITESTREAM_CONFIG_FILE}" &
862
-
863
- echo $! >"${LITESTREAM_PID_FILE}"
864
- log "litestream replicate starting (pid $(cat "${LITESTREAM_PID_FILE}"))"
865
- }
866
-
867
403
  # The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
868
404
  # one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
869
405
  # would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
@@ -910,7 +446,7 @@ stop_litestream() {
910
446
  # stop_tunnel's own comment argues against an argument a future caller can get
911
447
  # wrong. litestream_is_running is the same O(1) check stop_litestream's own
912
448
  # no-op branch uses, not a poll — /run's cleanup trap fires this before
913
- # start_litestream is ever reached whenever an earlier step failed, and an
449
+ # the replicator start is ever reached whenever an earlier step failed, and an
914
450
  # operator reading that log must not be told a stop signal went to a process
915
451
  # that never started.
916
452
  kill_litestream() {
@@ -926,6 +462,149 @@ kill_litestream() {
926
462
  }
927
463
  # --- litestream replicate (end) ----------------------------------------------
928
464
 
465
+ # --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
466
+ #
467
+ # sync_credentials (above) covers the three boundary flushes /run's restore,
468
+ # /suspend and /terminate already call. What it does NOT cover is a VM that
469
+ # runs for a long time between those boundaries: a provider re-authenticated
470
+ # through the proxied UI hours into a run would sit unflushed until the next
471
+ # suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
472
+ # loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
473
+ # own sync_credentials_loop is the ECS side of the identical gap; this is the
474
+ # same fix, backgrounded like the other long-lived services so it
475
+ # outlives this hook process, `( … ) &` rather than `setsid`: a plain
476
+ # backgrounded subshell is reparented to init and keeps running once its
477
+ # parent hook script exits (verified: PPID=1, still alive, with no controlling
478
+ # terminal in this image to send it a stray SIGHUP), and it inherits every
479
+ # function this file defines, so it can call run_synchroniser directly with no
480
+ # re-exec.
481
+
482
+ # Bounded confirmation window `stop_credential_sync` polls after signalling the
483
+ # loop. The loop's current child is one fast `run_synchroniser sync-once` call,
484
+ # so this stays a short backstop rather than a graceful drain.
485
+ CREDS_SYNC_STOP_WAIT_SECONDS=2
486
+
487
+ # Best-effort per tick, exactly like sync_credentials above: a failed tick
488
+ # must never end the loop, or a single transient S3 error would silently
489
+ # disable sync for the rest of the VM's life.
490
+ start_credential_sync() {
491
+ if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
492
+ warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
493
+ return 0
494
+ fi
495
+
496
+ if creds_sync_is_running; then
497
+ warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
498
+ return 0
499
+ fi
500
+
501
+ # This is the same environment input runner-synchroniser validates. The
502
+ # variable is normally absent, so that ordinary case uses the documented
503
+ # default without a warning; a present invalid value is named and rejected.
504
+ local interval="${CREDS_SYNC_INTERVAL:-60}"
505
+ if [ -n "${CREDS_SYNC_INTERVAL+x}" ] && [[ ! "${CREDS_SYNC_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
506
+ warn "CREDS-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${CREDS_SYNC_INTERVAL}' is not a positive integer; using 60s"
507
+ interval=60
508
+ fi
509
+
510
+ rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
511
+
512
+ (
513
+ # Releases the fds this subshell inherited from the hook process before
514
+ # settling in for the VM's whole remaining life: nothing here writes to
515
+ # them (every synchroniser call already redirects its own), so there is
516
+ # no reason to keep holding the hook's original stdout/stderr open. A
517
+ # long-lived process that instead inherited a pipe's write end (a test
518
+ # harness reading the hook's own output, for one) would keep that pipe
519
+ # from ever reporting EOF — testing-guide.mdc's own lesson, and the same
520
+ # reason the long-lived services never inherit stdio either. That
521
+ # redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
522
+ # failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
523
+ # surfaced by stop_credential_sync, which DOES have live stdio.
524
+ exec >/dev/null 2>&1 </dev/null
525
+
526
+ # A TERM this subshell receives (from stop_credential_sync, below) only
527
+ # kills THIS wrapper by default — its currently-running child (`sleep`,
528
+ # or a `run_synchroniser sync-once` call) is a separate process that
529
+ # would otherwise be orphaned and keep running, free to upload STALE
530
+ # credentials to S3 after the boundary flush that /suspend and
531
+ # /terminate perform immediately following the stop. Tracking the
532
+ # current child explicitly and forwarding the signal closes that race.
533
+ creds_sync_child_pid=""
534
+ trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
535
+
536
+ while true; do
537
+ sleep "${interval}" &
538
+ creds_sync_child_pid=$!
539
+ wait "${creds_sync_child_pid}" 2>/dev/null
540
+ creds_sync_child_pid=""
541
+
542
+ run_synchroniser sync-once claude &
543
+ creds_sync_child_pid=$!
544
+ wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
545
+ creds_sync_child_pid=""
546
+
547
+ run_synchroniser sync-once opencode &
548
+ creds_sync_child_pid=$!
549
+ wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
550
+ creds_sync_child_pid=""
551
+ done
552
+ ) &
553
+
554
+ echo $! >"${CREDS_SYNC_PID_FILE}"
555
+ log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
556
+ }
557
+
558
+ # Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
559
+ # that it and its current child are actually gone before returning: /suspend
560
+ # and /terminate start their own boundary flush immediately after this call,
561
+ # and an orphaned in-flight sync-once surviving past that point can overwrite
562
+ # fresher credentials with stale ones. The TERM trap inside the loop (above)
563
+ # forwards the signal to its current child almost instantly — this poll is a
564
+ # defensive confirmation, not the primary mechanism, so it stays short; a
565
+ # SIGKILL backstop covers a child that ignores TERM entirely.
566
+ #
567
+ # The DIED branch is a liveness report, not a no-op: every recovery/no-op path
568
+ # must say what it found (development-workflow.mdc) — a stopped-before-called
569
+ # loop and a died-on-its-own loop are different facts an operator needs told
570
+ # apart, not the same "nothing to stop" line.
571
+ stop_credential_sync() {
572
+ if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
573
+ log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
574
+ return 0
575
+ fi
576
+
577
+ local pid
578
+ pid="$(cat "${CREDS_SYNC_PID_FILE}")"
579
+ if ! process_is_alive "${pid}"; then
580
+ rm -f "${CREDS_SYNC_PID_FILE}"
581
+ warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
582
+ return 0
583
+ fi
584
+
585
+ kill -TERM "${pid}" 2>/dev/null || true
586
+ rm -f "${CREDS_SYNC_PID_FILE}"
587
+
588
+ local waited_ms=0
589
+ while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
590
+ sleep 0.1
591
+ waited_ms=$((waited_ms + 100))
592
+ done
593
+
594
+ if process_is_alive "${pid}"; then
595
+ kill -KILL "${pid}" 2>/dev/null || true
596
+ warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
597
+ fi
598
+
599
+ if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
600
+ warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
601
+ rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
602
+ fi
603
+
604
+ log "CREDS-SYNC-STOPPED: pid=${pid}"
605
+ }
606
+ # --- credential sync loop (end) -----------------------------------------------
607
+
929
608
  # --- flush_session_db (#812 WI-4) -------------------------------------------
930
609
  #
931
610
  # The checked, synchronous flush /suspend and /terminate need before they
@@ -936,19 +615,16 @@ kill_litestream() {
936
615
  # Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
937
616
  # own sync is normally sub-second (M5, the plan's grounding); 10s is generous
938
617
  # headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
939
- # measured worst case. The SIGKILL backstop below reuses
940
- # SESSION_DB_RESTORE_KILL_GRACE_SECONDS rather than its own literal: it is the
941
- # identical concept (bare `timeout` only SIGTERMs, and a litestream that
942
- # ignored it would be unbounded again), and there is no reason for the restore
943
- # and flush paths to ever drift on how long a SIGTERM gets to land.
618
+ # measured worst case. The SIGKILL backstop gives the flush its own two-second
619
+ # grace after `timeout` sends SIGTERM.
944
620
  SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
621
+ SESSION_DB_FLUSH_KILL_GRACE_SECONDS=2
945
622
 
946
- # Guards mirror start_litestream's first three exactly (disabled -> marker ->
623
+ # Guards mirror the CLI replicator's first three exactly (disabled -> marker ->
947
624
  # config) — a flush must never attempt work those guards would have refused
948
625
  # to start in the first place. PERSISTENCE_BUCKET is the SAME variable
949
- # start_litestream reads, so both agree regardless of which earlier step in
950
- # THIS process set it (load_state_config, above, or restore_session_db's
951
- # eval, on /run).
626
+ # the CLI reads, so both agree regardless of which earlier step in
627
+ # THIS process set it (load_state_config above).
952
628
  #
953
629
  # Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
954
630
  # trigger — and only once it is confirmed stopped does the synchronous -once
@@ -988,7 +664,7 @@ flush_session_db() {
988
664
  # Only reached with the daemon confirmed stopped, or never running — never
989
665
  # concurrently with it (see above).
990
666
  local flush_rc=0
991
- timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
667
+ timeout -k "${SESSION_DB_FLUSH_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
992
668
  litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
993
669
 
994
670
  if [ "${flush_rc}" -eq 0 ]; then
@@ -1018,7 +694,7 @@ flush_session_db() {
1018
694
  # returned its 200, so it is deliberately outside that arithmetic.
1019
695
  #
1020
696
  # EVIDENT_IDLE_TIMEOUT_SECONDS, deliberately the SAME env var name
1021
- # packages/runner-image/entrypoint.sh uses for the ECS runner's own
697
+ # runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
1022
698
  # idle-timeout flag, so an operator who knows one knows the other. A
1023
699
  # non-numeric override must never silently DROP the flag — that degrades to
1024
700
  # an always-on VM burning ~$2.40/day, exactly the bug this closes — so it
@@ -1043,6 +719,26 @@ fi
1043
719
  # its 200, while the tunnel keeps serving.
1044
720
  start_tunnel() {
1045
721
  local runner_key="$1" api_url="$2" tunnel_url="$3"
722
+ local restore_runner_credentials="${4:-false}" restore_history="${5:-false}"
723
+ local -a credential_flags=()
724
+ local -a session_db_flags=()
725
+ local -a opencode_config_flags=()
726
+
727
+ # The explicit fourth argument is set only by /run; /resume uses the default
728
+ # so a resumed VM never restores credentials over stores it already has.
729
+ if [ "${restore_runner_credentials}" = true ]; then
730
+ credential_flags=(--restore-runner-credentials)
731
+ fi
732
+
733
+ # Only a fresh /run asks the CLI to restore session history. /resume keeps
734
+ # the database from the snapshot and must not restore over it.
735
+ if [ "${restore_history}" = true ]; then
736
+ session_db_flags=(--restore-session-db)
737
+ fi
738
+
739
+ if [ -n "${RUNNER_OPENCODE_CONFIG:-}" ]; then
740
+ opencode_config_flags=(--opencode-config-overlay "${RUNNER_OPENCODE_CONFIG}")
741
+ fi
1046
742
 
1047
743
  if tunnel_is_running; then
1048
744
  warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
@@ -1057,7 +753,7 @@ start_tunnel() {
1057
753
  # into a MicroVM, so it is the ONLY way to seed model credentials on a runner
1058
754
  # that is already up — the S3 credential store is read at /run and /resume,
1059
755
  # never mid-life. The allow-list stays a single directory, as on ECS
1060
- # (packages/runner-image/entrypoint.sh): the flag is repeatable, but every
756
+ # (runner/docker-images/fargate/entrypoint.sh): the flag is repeatable, but every
1061
757
  # extra entry widens what Evident can write into a VM that executes agent
1062
758
  # code. ${HOME} is set by the image (ENV HOME=/home/runner) and `set -u` makes
1063
759
  # an unset one abort rather than silently allow-list "/.claude".
@@ -1065,6 +761,13 @@ start_tunnel() {
1065
761
  --port "${OPENCODE_PORT}" \
1066
762
  --endpoint "${api_url}" \
1067
763
  --tunnel "${tunnel_url}" \
764
+ --opencode-pid-file "${OPENCODE_PID_FILE}" \
765
+ --litestream-config "${LITESTREAM_CONFIG_FILE}" \
766
+ --litestream-pid-file "${LITESTREAM_PID_FILE}" \
767
+ --session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
768
+ "${credential_flags[@]}" \
769
+ "${session_db_flags[@]}" \
770
+ "${opencode_config_flags[@]}" \
1068
771
  --idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
1069
772
  --enable-file-sync-to "${HOME}/.claude" &
1070
773
 
@@ -1104,7 +807,7 @@ CLI_SHUTDOWN_CEILING_SECONDS=32
1104
807
  # everything else in the hook. 10 s is ~4x the measured shutdown and gives a
1105
808
  # provable worst case of 10 + the 10 s `suspend` spends in sync_credentials
1106
809
  # before us = 20 s, inside the ~55 s the in-VM server allows this script
1107
- # (DEFAULT_TIMEOUT_SECONDS, packages/lambda-microvm-runtime/src/runtime.ts),
810
+ # (DEFAULT_TIMEOUT_SECONDS, aws/lambda-microvm-runtime/src/runtime.ts),
1108
811
  # itself inside AWS's 60 s (HOOK_TIMEOUT_SECONDS, src/constants.ts) —
1109
812
  # overrunning that fails the whole lifecycle transition, which is worse than the
1110
813
  # kill this replaces. The residual cost is a drain longer than 10 s being
@@ -1197,10 +900,9 @@ check_runner_key() {
1197
900
  local runner_key="$1" api_url="$2"
1198
901
 
1199
902
  # status.ts's own exit-code contract (its header comment) means this exits
1200
- # non-zero on EVERY branch except `ok` guarded exactly like
1201
- # ensure_litestream_config's `run_synchroniser litestream-config` call,
1202
- # above, so that expected non-zero does not abort this function under the
1203
- # caller's `set -e` before the case below ever runs. Only stdout is
903
+ # non-zero on EVERY branch except `ok`; guard the capture so an expected
904
+ # non-zero does not abort this function under the caller's `set -e` before
905
+ # the case below ever runs. Only stdout is
1204
906
  # captured: status.ts's own contract is one parseable JSON line and nothing
1205
907
  # else there, and — like litestream's stderr elsewhere in this file — its
1206
908
  # stderr is left to reach CloudWatch directly rather than being folded in,