@evident-ai/runner-cdk 3.4.1-dev.2f1b44b → 3.4.1-dev.38181ff
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/controller-lambda/handler.js +24 -13
- package/dist/evident-scale-to-zero-construct.d.ts +1 -1
- package/dist/evident-scale-to-zero-construct.js +2 -3
- package/dist/image-version-reporter-lambda/handler.js +129 -0
- package/dist/microvm/construct.d.ts +24 -0
- package/dist/microvm/construct.js +27 -0
- package/dist/microvm/controller/doorbell.d.ts +5 -0
- package/dist/microvm/controller/doorbell.js +11 -10
- package/dist/microvm/controller/handle-doorbell.js +17 -14
- package/dist/microvm/controller/microvm-client.d.ts +6 -0
- package/dist/microvm/image/stage-context.d.ts +14 -5
- package/dist/microvm/image/stage-context.js +46 -16
- package/dist/microvm/image-version-reporter/construct.d.ts +35 -0
- package/dist/microvm/image-version-reporter/construct.js +91 -0
- package/dist/microvm/image-version-reporter/handler.d.ts +26 -0
- package/dist/microvm/image-version-reporter/handler.js +104 -0
- package/dist/microvm-image-context/Dockerfile +31 -49
- package/dist/microvm-image-context/hooks/common.sh +209 -967
- package/dist/microvm-image-context/hooks/resume +14 -16
- package/dist/microvm-image-context/hooks/run +20 -57
- package/dist/microvm-image-context/hooks/suspend +4 -5
- package/dist/microvm-image-context/hooks/terminate +5 -6
- package/dist/waker/construct.js +1 -2
- package/package.json +3 -3
|
@@ -21,11 +21,15 @@ OPENCODE_PORT="${OPENCODE_PORT:-4096}"
|
|
|
21
21
|
# /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
|
|
22
22
|
# shellcheck disable=SC2034 # read by the scripts that source this file
|
|
23
23
|
CONTEXT_FILE="/dev/shm/evident-run-context"
|
|
24
|
+
|
|
25
|
+
# The runtime injects MICROVM_ID only into /run, never /resume. This tmpfs file
|
|
26
|
+
# survives suspend/resume so /resume can restore the id for every fresh CLI
|
|
27
|
+
# process to self-report and acknowledge a fulfilled recycle request (#1906).
|
|
28
|
+
# shellcheck disable=SC2034 # read by the scripts that source this file
|
|
29
|
+
MICROVM_ID_FILE="/dev/shm/evident-microvm-id"
|
|
24
30
|
TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
|
|
25
31
|
OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
|
|
26
32
|
LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
|
|
27
|
-
CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
|
|
28
|
-
CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
|
|
29
33
|
|
|
30
34
|
# Where the runner's credential store lives inside the durable-state bucket.
|
|
31
35
|
# The BUCKET is the same for every VM from an image version, so the stack bakes
|
|
@@ -39,92 +43,38 @@ STATE_PREFIX_FILE="/dev/shm/evident-state-prefix"
|
|
|
39
43
|
# The generated litestream.yml (#812). tmpfs for the same two reasons as the
|
|
40
44
|
# files above: it must never reach the block device (Q6, `docs`), and it
|
|
41
45
|
# survives suspend/resume, which is what lets `/resume` start litestream again
|
|
42
|
-
# with no regeneration cost.
|
|
43
|
-
# (`ensure_litestream_config`, below).
|
|
46
|
+
# with no regeneration cost. The CLI regenerates it only when absent or empty.
|
|
44
47
|
LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
|
|
45
48
|
|
|
46
|
-
# Set when
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
# so would let a partial/fresh local DB overwrite a replica this boot never
|
|
50
|
-
# proved is safe to write over. Cleared at the top of every `restore_session_db`
|
|
51
|
-
# call (i.e. every `/run`) before any other decision, so a marker left by an
|
|
52
|
-
# earlier boot cannot silently disable replication for the VM's whole life.
|
|
49
|
+
# Set by the CLI when session-DB restore or verification could not prove this
|
|
50
|
+
# boot safe to replicate. The teardown hooks read it before flushing, and
|
|
51
|
+
# `/terminate` removes it with the rest of the per-VM state.
|
|
53
52
|
SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
|
|
54
53
|
|
|
54
|
+
# Completion evidence for the CLI-owned credential flush. It lives in tmpfs,
|
|
55
|
+
# is removed before every handshake, and is also removed by /terminate.
|
|
56
|
+
CREDENTIAL_FLUSH_MARKER_FILE="/dev/shm/evident-credential-flush"
|
|
57
|
+
|
|
55
58
|
hook_name() { printf '%s' "${0##*/}"; }
|
|
56
59
|
log() { echo "[hook:$(hook_name)] $*"; }
|
|
57
60
|
warn() { echo "[hook:$(hook_name)] $*" >&2; }
|
|
58
61
|
error() { echo "[hook:$(hook_name)] ERROR: $*" >&2; }
|
|
59
62
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if [ -n "${path}" ]; then
|
|
65
|
-
printf '%s\n' "${path}"
|
|
66
|
-
return 0
|
|
67
|
-
fi
|
|
68
|
-
local home="${HOME:-}"
|
|
69
|
-
home="${home#"${home%%[![:space:]]*}"}"
|
|
70
|
-
home="${home%"${home##*[![:space:]]}"}"
|
|
71
|
-
[ -n "${home}" ] || return 1
|
|
72
|
-
printf '%s\n' "${home}/.local/state/evident/session-db-recovery.jsonl"
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
report_session_db_giveup() {
|
|
76
|
-
local reason="$1" exit_code="${2:-null}" outcome path at
|
|
77
|
-
case "${reason}" in
|
|
78
|
-
synchroniser_config_unresolved|synchroniser_config_unevaluable|synchroniser_config_incomplete|litestream_config_unavailable|classification_fatal) outcome="restore_misconfigured" ;;
|
|
79
|
-
restore_deadline_exceeded|restore_tool_unusable|classification_unrecognised) outcome="fresh_session_db" ;;
|
|
80
|
-
*) warn "SESSION-DB-RECOVERY-REPORT-UNKNOWN-REASON: ${reason}"; return 0 ;;
|
|
81
|
-
esac
|
|
82
|
-
if ! path="$(session_db_recovery_report_path)"; then
|
|
83
|
-
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
84
|
-
return 0
|
|
85
|
-
fi
|
|
86
|
-
at="${EPOCHREALTIME:-}"
|
|
87
|
-
[ -n "${at}" ] || warn "SESSION-DB-RECOVERY-REPORT-TIMESTAMP-UNAVAILABLE"
|
|
88
|
-
printf '{"v":1,"event":"session_db_recovery","at":"%s","stage":"restore","outcome":"%s","severity":"error","reason":"%s","litestream_exit_code":%s,"attempt":null,"replica_objects":null,"replica_bytes":null,"quarantine_destination":null,"quarantined_objects":null,"quarantine_failed_objects":null,"quarantined_bytes":null,"verified_restore_point":null,"restore_points_tried":null,"replication_suspended":true}\n' "${at}" "${outcome}" "${reason}" "${exit_code}" >>"${path}" || warn "SESSION-DB-RECOVERY-REPORT-APPEND-FAILED"
|
|
89
|
-
return 0
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
# Returns the CLI's own exit code. Domain outcomes (nothing persisted yet, a
|
|
93
|
-
# corrupt object) are LOGGED and exit 0, the predicates answer "no" with 10, and
|
|
63
|
+
# Returns the CLI's own exit code. `restore` logs domain outcomes and returns 0;
|
|
64
|
+
# a non-zero status means the tool itself failed. `sync-once` returns 40 for
|
|
65
|
+
# `failed`, `hashFailed`, and `localInvalid` (credentials not persisted), and 0
|
|
66
|
+
# for every other outcome. The predicates answer "no" with 10, and
|
|
94
67
|
# `session-db-classify`'s three typed answers are 30 (fatal)/31 (replica
|
|
95
68
|
# unusable)/32 (retry), extended by `session-db-verify`'s 33 (integrity
|
|
96
69
|
# exhausted, replica separated and local disposed) / 34 (could not prove
|
|
97
|
-
# separation or disposal)
|
|
98
|
-
#
|
|
99
|
-
# means the tool itself broke, which is the only case worth an ERROR here —
|
|
100
|
-
# EXCEPT 124/137 (a `timeout` deadline/SIGKILL) when the caller asked for one:
|
|
101
|
-
# that is an intentional bound firing, not a broken tool, so the caller
|
|
102
|
-
# classifies it instead (restore_credentials, #930). A trailing
|
|
103
|
-
# `--evident-deadline=N` (stripped below before forwarding to the CLI, and
|
|
104
|
-
# never produced by anything but timed_synchroniser) is how a caller opts in;
|
|
105
|
-
# every other caller — sync_credentials, restore_session_db's `env` /
|
|
106
|
-
# `litestream-config` / `session-db-classify`, and this file's own contract
|
|
107
|
-
# test — passes none, so 124/137 there still means a genuine external SIGKILL
|
|
108
|
-
# (e.g. an OOM kill) and must keep producing the ERROR below.
|
|
70
|
+
# separation or disposal). Any status outside a command's contractual answers
|
|
71
|
+
# means the tool itself broke, which is the only case worth an ERROR here.
|
|
109
72
|
run_synchroniser() {
|
|
110
|
-
local -a call_args=("$@")
|
|
111
|
-
local deadline=""
|
|
112
|
-
local last=$(( ${#call_args[@]} - 1 ))
|
|
113
|
-
if [ "${last}" -ge 0 ] && [[ "${call_args[last]}" == --evident-deadline=* ]]; then
|
|
114
|
-
deadline="${call_args[last]#--evident-deadline=}"
|
|
115
|
-
call_args=("${call_args[@]:0:${last}}")
|
|
116
|
-
fi
|
|
117
|
-
|
|
118
73
|
local rc=0
|
|
119
|
-
|
|
120
|
-
[ -n "${deadline}" ] && launcher=(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${deadline}")
|
|
121
|
-
"${launcher[@]}" "${SYNCHRONISER}" "${call_args[@]}" || rc=$?
|
|
74
|
+
"${SYNCHRONISER}" "$@" || rc=$?
|
|
122
75
|
case "${rc}" in
|
|
123
|
-
0 | 10 | 30 | 31 | 32 | 33 | 34) ;;
|
|
124
|
-
|
|
125
|
-
[ -n "${deadline}" ] || error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw"
|
|
126
|
-
;;
|
|
127
|
-
*) error "synchroniser '${call_args[*]}' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
|
|
76
|
+
0 | 10 | 30 | 31 | 32 | 33 | 34 | 40) ;;
|
|
77
|
+
*) error "synchroniser '$*' exited ${rc}; the '${SYNCHRONISER}' command is missing from PATH, corrupt, or it threw" ;;
|
|
128
78
|
esac
|
|
129
79
|
return "${rc}"
|
|
130
80
|
}
|
|
@@ -147,8 +97,8 @@ run_synchroniser() {
|
|
|
147
97
|
# Never fails its caller: a timing line is diagnostics, and hardening a
|
|
148
98
|
# currently-working path is worse than the gap it closes (#931). The `date`
|
|
149
99
|
# fallback covers a pre-5.0 bash; a 0 means "could not read the clock", which
|
|
150
|
-
#
|
|
151
|
-
# under `set -e` — and rather than a 0 that would read as a fast healthy call.
|
|
100
|
+
# `log_elapsed_since` turns into `elapsed_ms=unknown` rather than aborting a
|
|
101
|
+
# hook under `set -e` — and rather than a 0 that would read as a fast healthy call.
|
|
152
102
|
now_ms() {
|
|
153
103
|
local now="${EPOCHREALTIME:-}"
|
|
154
104
|
if [ -n "${now}" ]; then
|
|
@@ -159,16 +109,10 @@ now_ms() {
|
|
|
159
109
|
date +%s%3N 2>/dev/null || echo 0
|
|
160
110
|
}
|
|
161
111
|
|
|
162
|
-
# Reports what one
|
|
163
|
-
# `elapsed_ms=`, `rc=`) so "how long does this actually take in the
|
|
164
|
-
# log query rather than another spike. `at_s` is where the step
|
|
165
|
-
# hook's own SECONDS clock
|
|
166
|
-
# hook-scripts.test.ts is derived against.
|
|
167
|
-
#
|
|
168
|
-
# Callable directly, not only through timed_synchroniser below, because the
|
|
169
|
-
# steps that dominate the pre-opencode window CANNOT be wrapped: their stdout is
|
|
170
|
-
# captured (`x="$(run_synchroniser env)"`, later `eval`'d), so a `log` line
|
|
171
|
-
# emitted inside the substitution would be evaluated as configuration.
|
|
112
|
+
# Reports what one pre-warm COST, in a deliberately machine-greppable line
|
|
113
|
+
# (`op=`, `elapsed_ms=`, `rc=`) so "how long does this actually take in the
|
|
114
|
+
# fleet?" is a log query rather than another spike. `at_s` is where the step
|
|
115
|
+
# landed on the hook's own SECONDS clock.
|
|
172
116
|
#
|
|
173
117
|
# `unknown`, never a number, when either end failed to read the clock (now_ms's
|
|
174
118
|
# 0). Subtracting them would report `elapsed_ms=0` — indistinguishable from a
|
|
@@ -187,26 +131,6 @@ log_elapsed_since() {
|
|
|
187
131
|
log "SYNCHRONISER-TIMING op=${op} elapsed_ms=${elapsed_ms} rc=${rc} at_s=${SECONDS}"
|
|
188
132
|
}
|
|
189
133
|
|
|
190
|
-
# Runs ONE synchroniser call and reports what it COST, without changing what it
|
|
191
|
-
# MEANS: the call's own status is returned untouched, so every caller's
|
|
192
|
-
# `|| return 1` / `|| status=$?` keeps working exactly as before.
|
|
193
|
-
#
|
|
194
|
-
# `deadline` is an explicit parameter, forwarded to run_synchroniser as a
|
|
195
|
-
# trailing `--evident-deadline=N` (never a global or dynamically-scoped
|
|
196
|
-
# variable, #930) — every current caller (restore_credentials) always has a
|
|
197
|
-
# positive remaining-budget value by the time it calls this, having already
|
|
198
|
-
# decided to skip the call entirely otherwise.
|
|
199
|
-
timed_synchroniser() {
|
|
200
|
-
local op="$1" deadline="$2"
|
|
201
|
-
shift 2
|
|
202
|
-
|
|
203
|
-
local started_ms rc=0
|
|
204
|
-
started_ms="$(now_ms)"
|
|
205
|
-
run_synchroniser "$@" "--evident-deadline=${deadline}" || rc=$?
|
|
206
|
-
log_elapsed_since "${op}" "${started_ms}" "${rc}"
|
|
207
|
-
return "${rc}"
|
|
208
|
-
}
|
|
209
|
-
|
|
210
134
|
# Exports what `runner-synchroniser` resolves its object-store location from
|
|
211
135
|
# (runner/synchroniser/src/config.ts). It treats either being empty as
|
|
212
136
|
# "persistence disabled" and then reports every restore as a WARNING it still
|
|
@@ -228,269 +152,32 @@ load_state_config() {
|
|
|
228
152
|
|
|
229
153
|
# Mirrors runner-synchroniser's own persistenceEnabled predicate (config.ts):
|
|
230
154
|
# both non-empty, already checked above, so by this point PERSISTENCE_BUCKET
|
|
231
|
-
# is just LITESTREAM_BUCKET.
|
|
155
|
+
# is just LITESTREAM_BUCKET. The CLI's replicator and flush_session_db (#812
|
|
232
156
|
# WI-3/WI-4) read ONLY this var, never LITESTREAM_BUCKET/LITESTREAM_PREFIX
|
|
233
157
|
# directly, so every entry point agrees on one predicate regardless of which
|
|
234
|
-
# earlier step in THIS hook process set it
|
|
235
|
-
#
|
|
236
|
-
# /suspend and /terminate, none of which ever call the synchroniser before
|
|
237
|
-
# needing the answer.
|
|
158
|
+
# earlier step in THIS hook process set it; /resume, /suspend and /terminate
|
|
159
|
+
# all use this same function before any teardown decision.
|
|
238
160
|
export PERSISTENCE_BUCKET="${LITESTREAM_BUCKET}"
|
|
239
161
|
}
|
|
240
162
|
|
|
241
|
-
#
|
|
242
|
-
#
|
|
243
|
-
# depends on the model that turn's message asks for, so that question has no
|
|
244
|
-
# single boot-time answer; it is answered per message, not here. What IS still
|
|
245
|
-
# fatal is credential persistence being unavailable at all (an unset bucket/
|
|
246
|
-
# prefix, or a broken synchroniser bundle) — that would silently drop refreshed
|
|
247
|
-
# tokens on every suspend, so those two restores stay `|| return 1`.
|
|
248
|
-
# `model-auth-ready` therefore stays a boot-time diagnostic: its only output is
|
|
249
|
-
# a log line, since the runtime forwards no hook stderr to the doorbell caller.
|
|
250
|
-
|
|
251
|
-
# How long restore_credentials' shared step budget is, covering the runner-secret
|
|
252
|
-
# fetch plus its three synchroniser calls (restore claude, restore opencode,
|
|
253
|
-
# model-auth-ready) as one window rather than a timeout apiece — a fixed per-call cap at the same
|
|
254
|
-
# total would false-fire on the ordinary case of one slow call (a slow S3 GET),
|
|
255
|
-
# which is precisely the boot this exists to keep healthy. From #930's 3-boot
|
|
256
|
-
# sample: the step measured ~3-5s total (~1.2s per synchroniser call, three
|
|
257
|
-
# node cold starts of a 1.8 MB bundle) — a WEAK estimate this file's own
|
|
258
|
-
# SYNCHRONISER-TIMING lines are what will sharpen for real. Too tight and a
|
|
259
|
-
# routine slow call boots this VM with no model credentials until an operator
|
|
260
|
-
# reconnects it; too loose and a hung call burns more of the hook's own
|
|
261
|
-
# SIGTERM budget before the VM is destroyed mid-boot anyway. The env override
|
|
262
|
-
# is for tests only, so they need not burn wall clock.
|
|
263
|
-
CREDENTIAL_RESTORE_DEADLINE_SECONDS="${EVIDENT_CREDENTIAL_RESTORE_DEADLINE_SECONDS:-8}"
|
|
264
|
-
# The SIGKILL backstop `timeout -k` applies after its own SIGTERM, exactly like
|
|
265
|
-
# SESSION_DB_RESTORE_KILL_GRACE_SECONDS above (a bare `timeout` only SIGTERMs,
|
|
266
|
-
# and a call that ignored it would be unbounded again). Hard ceiling on the
|
|
267
|
-
# step: DEADLINE + this = 10s, once — not per call.
|
|
268
|
-
CREDENTIAL_RESTORE_KILL_GRACE_SECONDS=2
|
|
269
|
-
GITHUB_PROBE_DEADLINE_SECONDS="${EVIDENT_GITHUB_PROBE_DEADLINE_SECONDS:-10}"
|
|
270
|
-
|
|
271
|
-
# What is left of the shared step budget, in whole seconds, `step_started_s`
|
|
272
|
-
# seconds after it began. `SECONDS` (a bash builtin with no failure mode,
|
|
273
|
-
# unlike `now_ms`) truncates, so a step that began at true time `s0` can read
|
|
274
|
-
# less elapsed time than actually passed — the `- 1` restores the invariant
|
|
275
|
-
# that no call is granted more than DEADLINE + GRACE from step start.
|
|
276
|
-
remaining_credential_budget() {
|
|
277
|
-
local step_started_s="$1"
|
|
278
|
-
echo $(( CREDENTIAL_RESTORE_DEADLINE_SECONDS - (SECONDS - step_started_s) - 1 ))
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
# Runs one of the step's bounded calls: skips it with a named warn if the
|
|
282
|
-
# shared budget is already exhausted, treats a timeout (124/137) as the
|
|
283
|
-
# non-fatal warn-and-continue D1 approved, and keeps every other non-zero
|
|
284
|
-
# fatal exactly as restore_credentials always has (`|| return 1` aborts /run
|
|
285
|
-
# before opencode and the tunnel start).
|
|
286
|
-
bounded_restore_call() {
|
|
287
|
-
local op="$1" step_started_s="$2"
|
|
288
|
-
shift 2
|
|
289
|
-
|
|
290
|
-
local remaining
|
|
291
|
-
remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
292
|
-
if [ "${remaining}" -lt 1 ]; then
|
|
293
|
-
warn "CREDENTIAL-RESTORE-SKIPPED: ${op} skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
294
|
-
return 0
|
|
295
|
-
fi
|
|
296
|
-
|
|
297
|
-
local rc=0
|
|
298
|
-
timed_synchroniser "${op}" "${remaining}" "$@" || rc=$?
|
|
299
|
-
case "${rc}" in
|
|
300
|
-
0) ;;
|
|
301
|
-
124 | 137)
|
|
302
|
-
warn "CREDENTIAL-RESTORE-TIMEOUT: ${op} did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
303
|
-
;;
|
|
304
|
-
*) return 1 ;;
|
|
305
|
-
esac
|
|
306
|
-
return 0
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
fetch_runner_secret() {
|
|
310
|
-
local step_started_s="$1" remaining rc=0 payload stderr_file started_ms populated=0 skipped=0 key value
|
|
311
|
-
if [ -z "${RUNNER_SECRET_ARN:-}" ]; then
|
|
312
|
-
log "runner secret is not configured; continuing without GitHub and MCP credentials"
|
|
313
|
-
return 0
|
|
314
|
-
fi
|
|
315
|
-
remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
316
|
-
if [ "${remaining}" -lt 1 ]; then
|
|
317
|
-
warn "CREDENTIAL-RESTORE-SKIPPED: runner-secret skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
318
|
-
return 0
|
|
319
|
-
fi
|
|
320
|
-
if ! stderr_file="$(mktemp /dev/shm/runner-secret-stderr.XXXXXX)"; then
|
|
321
|
-
warn "RUNNER-SECRET-STDERR-UNAVAILABLE: could not allocate diagnostic storage; continuing without runner credentials"
|
|
322
|
-
return 0
|
|
323
|
-
fi
|
|
324
|
-
started_ms="$(now_ms)"
|
|
325
|
-
payload="$(timeout -k "${CREDENTIAL_RESTORE_KILL_GRACE_SECONDS}" "${remaining}" aws secretsmanager get-secret-value --secret-id "${RUNNER_SECRET_ARN}" --query SecretString --output text 2>"${stderr_file}")" || rc=$?
|
|
326
|
-
log_elapsed_since runner-secret-fetch "${started_ms}" "${rc}"
|
|
327
|
-
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then
|
|
328
|
-
warn "CREDENTIAL-RESTORE-TIMEOUT: runner-secret did not finish within its ${remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
329
|
-
rm -f "${stderr_file}"
|
|
330
|
-
return 0
|
|
331
|
-
fi
|
|
332
|
-
if [ "${rc}" -ne 0 ]; then
|
|
333
|
-
warn "RUNNER-SECRET-UNREADABLE: $(<"${stderr_file}")"
|
|
334
|
-
rm -f "${stderr_file}"
|
|
335
|
-
return 0
|
|
336
|
-
fi
|
|
337
|
-
rm -f "${stderr_file}"
|
|
338
|
-
if ! jq -e 'type == "object"' >/dev/null 2>&1 <<<"${payload}"; then
|
|
339
|
-
warn "RUNNER-SECRET-UNPARSEABLE: secret value is not a JSON object"
|
|
340
|
-
return 0
|
|
341
|
-
fi
|
|
342
|
-
while IFS= read -r -d '' key && IFS= read -r -d '' value; do
|
|
343
|
-
if [[ ! "${key}" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
|
|
344
|
-
warn "RUNNER-SECRET-KEY-SKIPPED: ${key@Q} is not a valid environment variable name"
|
|
345
|
-
skipped=$((skipped + 1))
|
|
346
|
-
continue
|
|
347
|
-
fi
|
|
348
|
-
export "${key}=${value}"
|
|
349
|
-
populated=$((populated + 1))
|
|
350
|
-
done < <(jq -j 'to_entries[] | select(.value | type == "string" and length > 0) | .key, "\u0000", .value, "\u0000"' <<<"${payload}")
|
|
351
|
-
if [ "${populated}" -eq 0 ]; then
|
|
352
|
-
warn "RUNNER-SECRET-UNPOPULATED: populate the runner secret as documented in infrastructure/evident-runner/MICROVM.md"
|
|
353
|
-
else
|
|
354
|
-
log "RUNNER-SECRET-OK: exported ${populated} secret values; skipped ${skipped} invalid environment variable names"
|
|
355
|
-
fi
|
|
356
|
-
return 0
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
restore_credentials() {
|
|
360
|
-
local step_started_ms step_started_s
|
|
361
|
-
step_started_ms="$(now_ms)"
|
|
362
|
-
step_started_s="${SECONDS}"
|
|
363
|
-
|
|
364
|
-
load_state_config || return 1
|
|
365
|
-
# This shares the existing bounded window so /run's worst-case duration does not grow.
|
|
366
|
-
fetch_runner_secret "${step_started_s}"
|
|
367
|
-
bounded_restore_call restore-claude "${step_started_s}" restore claude || return 1
|
|
368
|
-
bounded_restore_call restore-opencode "${step_started_s}" restore opencode || return 1
|
|
369
|
-
|
|
370
|
-
# `restore` exits 0 whether it restored, found nothing, or discarded a corrupt
|
|
371
|
-
# object, so the synchroniser's own predicate is the only reliable answer to
|
|
372
|
-
# "can this VM authenticate a model right now". Its status is captured, not
|
|
373
|
-
# used as a condition, so it can never become this function's return status —
|
|
374
|
-
# `run` calls this bare (no `||`) under `set -euo pipefail`, and any
|
|
375
|
-
# non-zero return here would abort the hook before opencode and the tunnel
|
|
376
|
-
# ever start.
|
|
377
|
-
local auth_status=0 auth_remaining
|
|
378
|
-
auth_remaining="$(remaining_credential_budget "${step_started_s}")"
|
|
379
|
-
if [ "${auth_remaining}" -lt 1 ]; then
|
|
380
|
-
warn "CREDENTIAL-RESTORE-SKIPPED: model-auth-ready skipped; the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore budget is exhausted"
|
|
381
|
-
else
|
|
382
|
-
timed_synchroniser model-auth-ready "${auth_remaining}" model-auth-ready || auth_status=$?
|
|
383
|
-
case "${auth_status}" in
|
|
384
|
-
0) ;; # some model auth is configured; the two restores above already logged what they found
|
|
385
|
-
10)
|
|
386
|
-
warn "no model credentials under s3://${LITESTREAM_BUCKET}/${LITESTREAM_PREFIX}/ \
|
|
387
|
-
(neither claude/credentials.json nor opencode/auth.json yielded valid JSON) and neither \
|
|
388
|
-
ANTHROPIC_API_KEY nor OPENAI_API_KEY is set. This VM boots and connects; a turn that needs \
|
|
389
|
-
a model provider fails until one is connected. See 'Seeding a credential store' in \
|
|
390
|
-
infrastructure/evident-runner/MICROVM.md."
|
|
391
|
-
;;
|
|
392
|
-
124 | 137)
|
|
393
|
-
warn "CREDENTIAL-RESTORE-TIMEOUT: model-auth-ready did not finish within its ${auth_remaining}s share of the ${CREDENTIAL_RESTORE_DEADLINE_SECONDS}s credential-restore deadline; continuing without it"
|
|
394
|
-
;;
|
|
395
|
-
*)
|
|
396
|
-
# run_synchroniser has already `error`ed the tool-broke line for a
|
|
397
|
-
# broken tool; any other unrecognised code lands here too. Do not
|
|
398
|
-
# claim there are no credentials — absent evidence is not contrary
|
|
399
|
-
# evidence (development-workflow.mdc).
|
|
400
|
-
warn "could not determine whether this VM has model credentials"
|
|
401
|
-
;;
|
|
402
|
-
esac
|
|
403
|
-
fi
|
|
404
|
-
# The step total, in the SAME greppable shape as the per-call lines above, so
|
|
405
|
-
# one query answers both "what does credential restore cost?" and "which of
|
|
406
|
-
# its four operations cost it". Bounded now by CREDENTIAL_RESTORE_DEADLINE_SECONDS
|
|
407
|
-
# + CREDENTIAL_RESTORE_KILL_GRACE_SECONDS as one shared window (see the
|
|
408
|
-
# comment above that constant) rather than per call. `rc=0` is deliberate
|
|
409
|
-
# here, not an absence of failure modes: a timed-out call already warned by
|
|
410
|
-
# name above and is non-fatal by design (D1); every failure that DOES abort
|
|
411
|
-
# this step has already returned 1 above.
|
|
412
|
-
log_elapsed_since credential-restore-step "${step_started_ms}" 0
|
|
413
|
-
return 0
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
apply_runner_opencode_config() {
|
|
417
|
-
if [ -z "${RUNNER_OPENCODE_CONFIG:-}" ]; then
|
|
418
|
-
log "runner OpenCode config is not configured; using the baked project config"
|
|
419
|
-
return 0
|
|
420
|
-
fi
|
|
421
|
-
local source="${RUNNER_OPENCODE_CONFIG}" target="opencode.json"
|
|
422
|
-
[[ "${source}" = /* ]] || source="${WORKSPACE}/${source}"
|
|
423
|
-
[ -f "${WORKSPACE}/opencode.jsonc" ] && target="opencode.jsonc"
|
|
424
|
-
if [ ! -f "${source}" ]; then
|
|
425
|
-
error "RUNNER-OPENCODE-CONFIG-MISSING: ${source} is not a file; headless turns will wedge on the first external-directory permission prompt (#563)"
|
|
426
|
-
return 0
|
|
427
|
-
fi
|
|
428
|
-
cp "${source}" "${WORKSPACE}/${target}"
|
|
429
|
-
git -C "${WORKSPACE}" update-index --skip-worktree "${target}" 2>/dev/null \
|
|
430
|
-
|| warn "could not mark ${target} skip-worktree; it may show as a local change"
|
|
431
|
-
log "Applied runner OpenCode config ${source} to ${WORKSPACE}/${target}"
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
configure_github_access() {
|
|
435
|
-
if [ -z "${GH_TOKEN:-}" ]; then
|
|
436
|
-
warn "GITHUB-CREDENTIALS-MISSING: GH_TOKEN is unavailable; see RUNNER-SECRET-* above"
|
|
437
|
-
return 0
|
|
438
|
-
fi
|
|
439
|
-
export GIT_CONFIG_GLOBAL=/tmp/gitconfig
|
|
440
|
-
if ! : >"${GIT_CONFIG_GLOBAL}" ||
|
|
441
|
-
! git config --global user.name "${GIT_USER_NAME:-evident-bot}" ||
|
|
442
|
-
! git config --global user.email "${GIT_USER_EMAIL:-evident-bot@users.noreply.github.com}" ||
|
|
443
|
-
! git config --global init.defaultBranch main ||
|
|
444
|
-
! printf '%s\n' '#!/usr/bin/env bash' '[ "$1" = get ] || exit 0' 'echo username=x-access-token' 'echo "password=${GH_TOKEN}"' >/tmp/git-credential-helper.sh ||
|
|
445
|
-
! chmod 0700 /tmp/git-credential-helper.sh ||
|
|
446
|
-
! git config --global credential."https://github.com".helper /tmp/git-credential-helper.sh; then
|
|
447
|
-
warn "GITHUB-SETUP-FAILED: could not configure local git credentials; continuing without GitHub access"
|
|
448
|
-
return 0
|
|
449
|
-
fi
|
|
450
|
-
(
|
|
451
|
-
local output rc=0 login repo_url repo
|
|
452
|
-
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api user --jq .login 2>&1)" || rc=$?
|
|
453
|
-
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: auth probe exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
454
|
-
if [ "${rc}" -ne 0 ]; then warn "GITHUB-AUTH-REJECTED: ${output}"; return; fi
|
|
455
|
-
log "GITHUB-AUTH-OK: ${output}"
|
|
456
|
-
repo_url="$(git -C "${WORKSPACE}" remote get-url origin 2>/dev/null || true)"
|
|
457
|
-
repo="$(printf '%s' "${repo_url}" | sed -E 's#(https://github.com/|git@github.com:)##; s#\.git$##')"
|
|
458
|
-
[ -n "${repo}" ] || return
|
|
459
|
-
output="$(timeout -k 1 "${GITHUB_PROBE_DEADLINE_SECONDS}" gh api "repos/${repo}" --jq .full_name 2>&1)" || rc=$?
|
|
460
|
-
if [ "${rc}" -eq 124 ] || [ "${rc}" -eq 137 ]; then warn "GITHUB-PROBE-TIMEOUT: repository probe for ${repo} exceeded ${GITHUB_PROBE_DEADLINE_SECONDS}s"; return; fi
|
|
461
|
-
[ "${rc}" -eq 0 ] || warn "GITHUB-REPO-INACCESSIBLE: ${repo}: ${output}"
|
|
462
|
-
) &
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
# Best-effort by design: /suspend must still drop the tunnel and /terminate must
|
|
466
|
-
# still clean up, so a flush that cannot happen is loud but never fatal.
|
|
163
|
+
# Best-effort by design: the teardown hooks must still complete even when a
|
|
164
|
+
# credential flush cannot happen.
|
|
467
165
|
sync_credentials() {
|
|
468
166
|
load_state_config || return 0
|
|
469
167
|
run_synchroniser sync-once claude || true
|
|
470
168
|
run_synchroniser sync-once opencode || true
|
|
471
169
|
}
|
|
472
170
|
|
|
473
|
-
# ---
|
|
474
|
-
#
|
|
475
|
-
# Restores opencode.db from S3 before opencode opens it — the read half only.
|
|
476
|
-
# Writes NOTHING to S3 itself; replication back to S3 is started separately by
|
|
477
|
-
# `start_litestream` (below) from the `run`/`resume` hooks. `ensure_litestream_config`/
|
|
478
|
-
# `restore_session_db` are the MicroVM side of the same contract
|
|
479
|
-
# runner/docker-images/fargate/entrypoint.sh's inlined restore already speaks, going
|
|
480
|
-
# through the SAME runner-synchroniser CLI.
|
|
171
|
+
# --- boot pre-warms ----------------------------------------------------------
|
|
481
172
|
|
|
482
173
|
# Reads the ~30 MB litestream binary into the page cache, in the background, so
|
|
483
174
|
# the FIRST exec of it does not pay that read on the critical path.
|
|
484
175
|
#
|
|
485
|
-
#
|
|
486
|
-
#
|
|
487
|
-
#
|
|
488
|
-
#
|
|
489
|
-
#
|
|
490
|
-
# deadline is gone, the hook's own SIGTERM is not.)
|
|
491
|
-
# Boot measurements put ~6.6-7.3s between the credential restore and the
|
|
492
|
-
# litestream version line, of which only ~0.2s is accounted for by the two node
|
|
493
|
-
# calls in between — the remainder is INFERRED to be this read, never measured.
|
|
176
|
+
# The pre-warm overlaps the CLI's node startup and auth round trip, so the first
|
|
177
|
+
# litestream operation does not also pay the binary's cold page-cache read.
|
|
178
|
+
# Boot measurements put ~6.6-7.3s before the litestream version line, of which
|
|
179
|
+
# only ~0.2s is accounted for by the two node calls in between — the remainder
|
|
180
|
+
# is INFERRED to be this read, never measured.
|
|
494
181
|
# The `litestream-prewarm` and `litestream-version` timings are what settle it
|
|
495
182
|
# on the next boot.
|
|
496
183
|
#
|
|
@@ -517,365 +204,40 @@ prewarm_litestream() {
|
|
|
517
204
|
log "pre-warming ${binary} in the background"
|
|
518
205
|
}
|
|
519
206
|
|
|
520
|
-
#
|
|
521
|
-
#
|
|
522
|
-
#
|
|
523
|
-
#
|
|
524
|
-
# or empty, so `/resume` (WI-4) restarting litestream after a suspend/resume
|
|
525
|
-
# snapshot — /dev/shm intact — pays nothing to regenerate it.
|
|
526
|
-
#
|
|
527
|
-
# Returns 1 on failure — fatal to the CALLER, never to the hook: every caller
|
|
528
|
-
# (restore_session_db here, start_litestream in WI-3) must still let opencode
|
|
529
|
-
# and the tunnel start regardless.
|
|
530
|
-
ensure_litestream_config() {
|
|
531
|
-
if [ -s "${LITESTREAM_CONFIG_FILE}" ]; then
|
|
532
|
-
return 0
|
|
533
|
-
fi
|
|
534
|
-
|
|
535
|
-
# Timed OUTSIDE the substitution, not with timed_synchroniser: this call's
|
|
536
|
-
# stdout IS the config, so a timing line emitted inside `$(...)` would land in
|
|
537
|
-
# litestream.yml.
|
|
538
|
-
local rendered config_started_ms config_rc=0
|
|
539
|
-
config_started_ms="$(now_ms)"
|
|
540
|
-
rendered="$(run_synchroniser litestream-config)" || config_rc=$?
|
|
541
|
-
log_elapsed_since litestream-config "${config_started_ms}" "${config_rc}"
|
|
542
|
-
if [ "${config_rc}" -ne 0 ]; then
|
|
543
|
-
error "could not generate ${LITESTREAM_CONFIG_FILE}: runner-synchroniser litestream-config failed (see the error above)"
|
|
544
|
-
return 1
|
|
545
|
-
fi
|
|
546
|
-
printf '%s\n' "${rendered}" >"${LITESTREAM_CONFIG_FILE}" || {
|
|
547
|
-
error "could not write ${LITESTREAM_CONFIG_FILE}"
|
|
548
|
-
return 1
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
# Q7's diagnostic, for whoever can read this VM's CloudWatch log: which way
|
|
552
|
-
# litestream resolved its AWS region, and whether it can even see the
|
|
553
|
-
# names it would need to (never the VALUES of any OTHER AWS_* variable —
|
|
554
|
-
# AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN may be present here).
|
|
555
|
-
local region_empty="yes"
|
|
556
|
-
if printf '%s\n' "${rendered}" | grep -Eq '^ *region: *[^[:space:]]'; then
|
|
557
|
-
region_empty="no"
|
|
558
|
-
fi
|
|
559
|
-
# Hoisted out of the log line below so it can be timed: this is the FIRST exec
|
|
560
|
-
# of the litestream binary on the boot, and the pre-warm /run fires (see
|
|
561
|
-
# prewarm_litestream) is aimed squarely at what this number measures.
|
|
562
|
-
local version_started_ms litestream_version
|
|
563
|
-
version_started_ms="$(now_ms)"
|
|
564
|
-
litestream_version="$(litestream version 2>/dev/null || echo 'unknown')"
|
|
565
|
-
log_elapsed_since litestream-version "${version_started_ms}" 0
|
|
566
|
-
log "litestream ${litestream_version}; AWS_REGION=${AWS_REGION:-<unset>} AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-<unset>}; rendered litestream.yml region empty: ${region_empty}"
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
# Writes the marker AND names the reason, in one call, so the marker can never
|
|
570
|
-
# appear silently. Every give-up path in restore_session_db calls this EXCEPT
|
|
571
|
-
# the classifier's 31 (#1106 — see that branch), and no success path ever does.
|
|
572
|
-
# It is what a later `/resume`/`start_litestream` (WI-3) reads to skip
|
|
573
|
-
# replicating this boot: a local DB that is only PARTIALLY restored must never
|
|
574
|
-
# be allowed to overwrite a replica this boot never proved safe to write over.
|
|
575
|
-
# A merely FRESH local DB is not that danger — litestream continues the txid
|
|
576
|
-
# chain — which is precisely why 31 no longer belongs here.
|
|
577
|
-
mark_no_replicate() {
|
|
578
|
-
: >"${SESSION_DB_NO_REPLICATE_MARKER}"
|
|
579
|
-
warn "SESSION-DB-NO-REPLICATE: $1"
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
# Removes local session-DB debris a give-up path leaves behind. Called on
|
|
583
|
-
# EVERY give-up path and NO success path: skipping it on a give-up is a real
|
|
584
|
-
# bug (a truncated file left in place makes opencode fail to open a malformed
|
|
585
|
-
# database, which fails /run outright — worse than the history loss this
|
|
586
|
-
# discards); calling it on a success path would delete a good restore. One
|
|
587
|
-
# `rm -f` per path, not a single command, so a failure removing one does not
|
|
588
|
-
# skip the other two.
|
|
589
|
-
discard_session_db_debris() {
|
|
590
|
-
rm -f "${OPENCODE_DB_PATH}" "${OPENCODE_DB_PATH}-wal" "${OPENCODE_DB_PATH}-shm"
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
# How long restore_session_db waits for ONE `litestream restore` before giving
|
|
594
|
-
# up. From M4 (see the plan's §2): ~50 MB/s extrapolated from the x86_64 ECS
|
|
595
|
-
# container's own live replica, so 7s covers roughly 300-350 MB — an
|
|
596
|
-
# EXTRAPOLATION, not a MicroVM measurement (different arch, network path and
|
|
597
|
-
# credential source); the boot line ensure_litestream_config logs, plus the
|
|
598
|
-
# ${SECONDS} stamps in /run, are how it gets measured here for real.
|
|
599
|
-
#
|
|
600
|
-
# 7 rather than the 8 it was: hook-scripts.test.ts's pre-opencode ladder held an
|
|
601
|
-
# allowance of 4s for the rest of this step, which live boots measured at
|
|
602
|
-
# 6.7-7.4s, so the ladder passed while real boots blew the deadline. Correcting
|
|
603
|
-
# that allowance is what takes this second — the remedy the ladder's own failure
|
|
604
|
-
# message prescribes, and the one that leaves the SIGTERM worst case untouched.
|
|
605
|
-
# The cost is real and unmeasured on this platform: ~50 MB less restorable
|
|
606
|
-
# replica before a truncation that silently loses history. A fresh
|
|
607
|
-
# state_prefix starts at zero, so this is generous for a long time, but
|
|
608
|
-
# RoutingStrategy is 'per_user' only (routing-strategy.ts) and every prefix is
|
|
609
|
-
# deterministic and permanent per (pool, routing key) — so a long-lived
|
|
610
|
-
# runner's opencode.db grows monotonically across VM generations, exactly
|
|
611
|
-
# like the ~1 GB ECS one M4 measured, and will eventually hit this ceiling.
|
|
612
|
-
# Past it the restore is truncated (SESSION-DB-RESTORE-TRUNCATED, below) and
|
|
613
|
-
# this boot skips replication — silent, permanent history loss for that
|
|
614
|
-
# runner unless somebody reads the log. Not solved here (WI-0 Task 0.2).
|
|
615
|
-
# The env override is for tests only, so they need not burn 7s of wall clock.
|
|
616
|
-
SESSION_DB_RESTORE_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_RESTORE_DEADLINE_SECONDS:-7}"
|
|
617
|
-
# The SIGKILL backstop `timeout -k` applies after its own SIGTERM: `timeout`
|
|
618
|
-
# alone only SIGTERMs, and a litestream that ignored it would be unbounded
|
|
619
|
-
# again. Hard ceiling on this step: DEADLINE + this = 9s.
|
|
620
|
-
SESSION_DB_RESTORE_KILL_GRACE_SECONDS=2
|
|
621
|
-
|
|
622
|
-
# The one bounded restore attempt, run before start_opencode (opencode opens
|
|
623
|
-
# the DB the moment it starts, so this is the only place it can happen).
|
|
624
|
-
# Every branch returns 0 — /run calls this bare under `set -euo pipefail`,
|
|
625
|
-
# exactly like restore_credentials, and nothing about the session DB may ever
|
|
626
|
-
# fail /run (Q3): a /run that exits 1 fails the whole lifecycle transition
|
|
627
|
-
# and the user gets NO runner at all, which is worse than losing history.
|
|
628
|
-
restore_session_db() {
|
|
629
|
-
# Cleanup, not a decision, and unconditionally first: /dev/shm survives
|
|
630
|
-
# suspend/resume and a /run retry, so a marker left by an earlier boot must
|
|
631
|
-
# never silently disable replication for the rest of this VM's life.
|
|
632
|
-
rm -f "${SESSION_DB_NO_REPLICATE_MARKER}"
|
|
633
|
-
local report_path
|
|
634
|
-
if report_path="$(session_db_recovery_report_path)"; then
|
|
635
|
-
# The default report directory is provisioned in the image, so this is a
|
|
636
|
-
# no-op there; a shared-path override (EVIDENT_SESSION_DB_RECOVERY_REPORT)
|
|
637
|
-
# can name a directory that isn't, and the typed synchroniser writer
|
|
638
|
-
# already creates it for that same override (session-db-recovery-report.ts).
|
|
639
|
-
# Best-effort only: report_session_db_giveup below still warns and
|
|
640
|
-
# continues if this didn't leave a writable path. `${path%/*}` leaves a
|
|
641
|
-
# bare relative filename (no `/`) unchanged rather than reducing to `.`
|
|
642
|
-
# like dirname(1) — without the explicit check, mkdir would create a
|
|
643
|
-
# directory AT the report path itself.
|
|
644
|
-
local report_dir="${report_path%/*}"
|
|
645
|
-
[ "${report_dir}" = "${report_path}" ] && report_dir="."
|
|
646
|
-
[ -d "${report_dir}" ] || mkdir -p "${report_dir}" 2>/dev/null || true
|
|
647
|
-
: >"${report_path}" || warn "SESSION-DB-RECOVERY-REPORT-TRUNCATE-FAILED"
|
|
648
|
-
else
|
|
649
|
-
warn "SESSION-DB-RECOVERY-REPORT-PATH-UNAVAILABLE"
|
|
650
|
-
fi
|
|
651
|
-
|
|
652
|
-
# Config resolved ONCE here, in the synchroniser (config.ts owns
|
|
653
|
-
# OPENCODE_DB_PATH; no second copy of that path here), exactly as
|
|
654
|
-
# entrypoint.sh's own line does. Unlike ECS's `|| die`, a failure here must
|
|
655
|
-
# not abort the hook — `x="$(cmd)"` under `set -e` WOULD abort the whole
|
|
656
|
-
# shell on a failing substitution, which is exactly why it is caught
|
|
657
|
-
# explicitly rather than left to `set -e`.
|
|
658
|
-
# Timed OUTSIDE the substitution for the same reason ensure_litestream_config
|
|
659
|
-
# is: this call's stdout is `eval`'d, so a timing line emitted inside `$(...)`
|
|
660
|
-
# would be evaluated as configuration.
|
|
661
|
-
local synchroniser_env env_started_ms env_rc=0
|
|
662
|
-
env_started_ms="$(now_ms)"
|
|
663
|
-
synchroniser_env="$(run_synchroniser env)" || env_rc=$?
|
|
664
|
-
log_elapsed_since session-db-env "${env_started_ms}" "${env_rc}"
|
|
665
|
-
if [ "${env_rc}" -ne 0 ]; then
|
|
666
|
-
mark_no_replicate "could not resolve the runner-synchroniser configuration (see the ERROR above)"
|
|
667
|
-
report_session_db_giveup synchroniser_config_unresolved
|
|
668
|
-
return 0
|
|
669
|
-
fi
|
|
670
|
-
# Guarded for the SAME reason as the substitution above, which is easy to miss:
|
|
671
|
-
# `eval` returns the status of what it ran, so a malformed line from a skewed
|
|
672
|
-
# bundle would abort /run right here under `set -e` — before start_opencode,
|
|
673
|
-
# so the user gets NO runner at all. entrypoint.sh:110 leaves the identical
|
|
674
|
-
# line bare because on ECS the blast radius is a crash-loop-and-replace; here
|
|
675
|
-
# it is the whole lifecycle transition, so Q3 makes it a give-up instead.
|
|
676
|
-
eval "${synchroniser_env}" || {
|
|
677
|
-
mark_no_replicate "the runner-synchroniser configuration could not be evaluated; the installed bundle likely does not match this hook"
|
|
678
|
-
report_session_db_giveup synchroniser_config_unevaluable
|
|
679
|
-
return 0
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
# #931's exact lesson, one shell over (runner/docker-images/fargate/entrypoint.sh):
|
|
683
|
-
# the `|| { ... }` above only catches a non-zero EXIT — an `env` that exits
|
|
684
|
-
# 0 with an INCOMPLETE contract (a runner-synchroniser version/build skew)
|
|
685
|
-
# would otherwise abort right here under `set -u` the moment
|
|
686
|
-
# `${OPENCODE_DB_PATH}` is dereferenced below. entrypoint.sh may `die` on
|
|
687
|
-
# this (ECS just crash-loops and replaces the task); this hook cannot —
|
|
688
|
-
# Q3 says nothing about the session DB may ever fail /run — so a missing
|
|
689
|
-
# OPENCODE_DB_PATH is a give-up, not a silent default (a guessed path would
|
|
690
|
-
# be actively wrong, not merely absent).
|
|
691
|
-
if [ -z "${OPENCODE_DB_PATH+x}" ]; then
|
|
692
|
-
mark_no_replicate "run_synchroniser env did not define OPENCODE_DB_PATH; the installed runner-synchroniser build likely does not match this hook"
|
|
693
|
-
report_session_db_giveup synchroniser_config_incomplete
|
|
694
|
-
return 0
|
|
695
|
-
fi
|
|
696
|
-
|
|
697
|
-
# #770's arm, checked FIRST of the decisions and with its OWN message —
|
|
698
|
-
# never the SESSION-DB-REPLICA-UNUSABLE one below (#764's finding: an
|
|
699
|
-
# operator grepping that marker must not hit the disabled case).
|
|
700
|
-
# PERSISTENCE_BUCKET is empty exactly when LITESTREAM_BUCKET or
|
|
701
|
-
# LITESTREAM_PREFIX is unset (config.ts's persistenceEnabled) — the
|
|
702
|
-
# IDENTICAL predicate ECS's own litestream-launch gate uses
|
|
703
|
-
# (entrypoint.sh), so the two images agree on what "disabled" means.
|
|
704
|
-
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
705
|
-
warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; opencode starts with a fresh session DB and nothing is replicated."
|
|
706
|
-
return 0
|
|
707
|
-
fi
|
|
708
|
-
|
|
709
|
-
if ! ensure_litestream_config; then
|
|
710
|
-
mark_no_replicate "could not generate ${LITESTREAM_CONFIG_FILE} (see the error above)"
|
|
711
|
-
report_session_db_giveup litestream_config_unavailable
|
|
712
|
-
return 0
|
|
713
|
-
fi
|
|
714
|
-
|
|
715
|
-
local restore_rc=0 restore_started_ms
|
|
716
|
-
restore_started_ms="$(now_ms)"
|
|
717
|
-
timeout -k "${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}" "${SESSION_DB_RESTORE_DEADLINE_SECONDS}" \
|
|
718
|
-
litestream restore -config "${LITESTREAM_CONFIG_FILE}" \
|
|
719
|
-
-if-db-not-exists -if-replica-exists "${OPENCODE_DB_PATH}" || restore_rc=$?
|
|
720
|
-
log_elapsed_since session-db-restore "${restore_started_ms}" "${restore_rc}"
|
|
721
|
-
|
|
722
|
-
# `timeout`'s OWN codes, handled BEFORE classifying: 124/137 are a
|
|
723
|
-
# TRUNCATED restore, not a corrupt replica — handing them to
|
|
724
|
-
# session-db-classify would mislabel a slow/large replica as one. 125-127
|
|
725
|
-
# mean `timeout` (or litestream itself) is broken, not the replica.
|
|
726
|
-
case "${restore_rc}" in
|
|
727
|
-
124 | 137)
|
|
728
|
-
discard_session_db_debris
|
|
729
|
-
mark_no_replicate "SESSION-DB-RESTORE-TRUNCATED: litestream restore did not finish within the ${SESSION_DB_RESTORE_DEADLINE_SECONDS}s deadline (+${SESSION_DB_RESTORE_KILL_GRACE_SECONDS}s kill grace), ${SECONDS}s into the hook; opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
730
|
-
report_session_db_giveup restore_deadline_exceeded "${restore_rc}"
|
|
731
|
-
return 0
|
|
732
|
-
;;
|
|
733
|
-
125 | 126 | 127)
|
|
734
|
-
error "litestream restore could not even run (timeout exited ${restore_rc})"
|
|
735
|
-
discard_session_db_debris
|
|
736
|
-
mark_no_replicate "restore tool is broken (timeout exited ${restore_rc}); opencode starts with a fresh session DB and nothing is replicated this boot"
|
|
737
|
-
report_session_db_giveup restore_tool_unusable "${restore_rc}"
|
|
738
|
-
return 0
|
|
739
|
-
;;
|
|
740
|
-
esac
|
|
741
|
-
|
|
742
|
-
local classify_rc=0
|
|
743
|
-
run_synchroniser session-db-classify "${restore_rc}" 1 --on-unusable-replica=leave --fresh-db-fallback || classify_rc=$?
|
|
744
|
-
case "${classify_rc}" in
|
|
745
|
-
0) ;; # restored, or no replica yet — the CLI already logged which
|
|
746
|
-
31)
|
|
747
|
-
# The classifier already discarded the local debris (session-db.ts) — and
|
|
748
|
-
# that is exactly why this is the ONE give-up path that does NOT
|
|
749
|
-
# mark_no_replicate (#1106). Every other give-up here may be sitting on a
|
|
750
|
-
# half-restored DB, so the marker stays; a 31 is structurally guaranteed
|
|
751
|
-
# to be a FRESH one, and replicating it starts a new backup chain instead
|
|
752
|
-
# of leaving this boot with a zero-width backup window. Same reasoning,
|
|
753
|
-
# and same one-line change, as runner/docker-images/fargate/entrypoint.sh's `31)`.
|
|
754
|
-
warn "SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot (see the WARNING above)"
|
|
755
|
-
;;
|
|
756
|
-
32)
|
|
757
|
-
# The CLI's own contract for 32 is "re-run litestream restore and ask
|
|
758
|
-
# again", but this hook has budget for exactly ONE attempt (Q4) — name
|
|
759
|
-
# the deviation so nobody reads this as a bug.
|
|
760
|
-
discard_session_db_debris
|
|
761
|
-
mark_no_replicate "session-db-classify asked for another restore attempt (32), but this hook has budget for only one; treating it as a give-up rather than retrying"
|
|
762
|
-
;;
|
|
763
|
-
30)
|
|
764
|
-
# The CLI already logged its own FATAL line above; Q3 still says boot
|
|
765
|
-
# fresh rather than fail /run.
|
|
766
|
-
discard_session_db_debris
|
|
767
|
-
mark_no_replicate "session-db-classify returned fatal (30); see the FATAL message above"
|
|
768
|
-
report_session_db_giveup classification_fatal
|
|
769
|
-
;;
|
|
770
|
-
*)
|
|
771
|
-
# run_synchroniser already logged the "tool broke" ERROR for this.
|
|
772
|
-
discard_session_db_debris
|
|
773
|
-
mark_no_replicate "session-db-classify exited ${classify_rc}, which is none of its documented answers"
|
|
774
|
-
report_session_db_giveup classification_unrecognised
|
|
775
|
-
;;
|
|
776
|
-
esac
|
|
777
|
-
|
|
778
|
-
return 0
|
|
779
|
-
}
|
|
780
|
-
# --- Session DB restore (end) -----------------------------------------------
|
|
781
|
-
|
|
782
|
-
# --- Session DB integrity verification (#1868 WI-4) --------------------------
|
|
783
|
-
#
|
|
784
|
-
# The restore loop above only proves litestream could WRITE a file; it never
|
|
785
|
-
# proves the file is intact (#1345: a leaf-page-corrupt-but-openable DB
|
|
786
|
-
# re-restores unchanged forever). `session-db-verify` runs a real `PRAGMA
|
|
787
|
-
# integrity_check` and, on failure, walks retained restore points back until
|
|
788
|
-
# one passes — the MicroVM side of the identical check
|
|
789
|
-
# runner/docker-images/fargate/entrypoint.sh already runs after ITS restore
|
|
790
|
-
# loop, through the same runner-synchroniser CLI.
|
|
791
|
-
|
|
792
|
-
# Bounded by `timed_synchroniser`, not an external `timeout` around
|
|
793
|
-
# `run_synchroniser` itself (a shell function, not an exported binary — that
|
|
794
|
-
# would fail with rc 127, silently mis-triggering the allowlist's "tool
|
|
795
|
-
# broke" branch). The grace period is `CREDENTIAL_RESTORE_KILL_GRACE_SECONDS`
|
|
796
|
-
# (2s): `run_synchroniser`'s `timeout -k` hardcodes that one constant for
|
|
797
|
-
# every `timed_synchroniser` caller, not a value unique to this step, so the
|
|
798
|
-
# real worst case is DEADLINE + 2s, once — accounted for in
|
|
799
|
-
# hook-scripts.test.ts's budget-ladder test.
|
|
207
|
+
# Reads the aws CLI v2 install tree into the page cache, in the background, so
|
|
208
|
+
# the CLI's first runner-secret fetch does not pay first-touch I/O. The read
|
|
209
|
+
# overlaps `evident run`'s node startup and auth round trip rather than a hook
|
|
210
|
+
# deadline.
|
|
800
211
|
#
|
|
801
|
-
#
|
|
802
|
-
#
|
|
803
|
-
#
|
|
804
|
-
#
|
|
805
|
-
#
|
|
806
|
-
#
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
# proceeds, and would spend budget only to report on a moot outcome.
|
|
814
|
-
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
815
|
-
log "skipping session-DB verification: persistence is disabled"
|
|
816
|
-
return 0
|
|
817
|
-
fi
|
|
818
|
-
|
|
819
|
-
if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
|
|
820
|
-
log "skipping session-DB verification: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
|
|
821
|
-
return 0
|
|
822
|
-
fi
|
|
823
|
-
|
|
824
|
-
if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
|
|
825
|
-
log "skipping session-DB verification: no usable ${LITESTREAM_CONFIG_FILE}"
|
|
212
|
+
# A single `cat` of the `aws` entrypoint (litestream's pattern above) is NOT
|
|
213
|
+
# enough here: v2 ships as a real Python distribution (~7,500 files under the
|
|
214
|
+
# resolved binary's own directory), and a cold invocation demand-pages a
|
|
215
|
+
# scattered set of them (botocore's endpoints.json/partitions.json, service
|
|
216
|
+
# model JSON, shared libs) — one boot measured #1997's fetch at 7,243ms cold
|
|
217
|
+
# vs ~400ms warm, an ~18x gap the litestream read-ahead trick alone cannot
|
|
218
|
+
# close. `find -exec cat` walks that whole tree instead of one file.
|
|
219
|
+
prewarm_aws_cli() {
|
|
220
|
+
local binary tree
|
|
221
|
+
binary="$(command -v aws 2>/dev/null || true)"
|
|
222
|
+
if [ -z "${binary}" ]; then
|
|
223
|
+
warn "aws CLI is not on PATH; skipping the boot pre-warm"
|
|
826
224
|
return 0
|
|
827
225
|
fi
|
|
226
|
+
tree="$(dirname "$(readlink -f "${binary}")")"
|
|
828
227
|
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
session-db-verify "${LITESTREAM_CONFIG_FILE}" || verify_rc=$?
|
|
837
|
-
|
|
838
|
-
case "${verify_rc}" in
|
|
839
|
-
0) ;; # verified intact, or nothing to verify yet — the CLI already logged which
|
|
840
|
-
33)
|
|
841
|
-
# Integrity exhausted, but the classifier already proved the corrupt
|
|
842
|
-
# replica separated and the local copy disposed of — booting with a
|
|
843
|
-
# fresh DB and a new backup chain is safe, exactly like ECS's own `33)`.
|
|
844
|
-
log "SESSION-DB-INTEGRITY-EXHAUSTED: booting continues and litestream still replicates, starting an empty backup chain after the corrupt replica was separated."
|
|
845
|
-
;;
|
|
846
|
-
34)
|
|
847
|
-
# The ONE deliberate exception to "nothing about the session DB may
|
|
848
|
-
# ever fail /run" (Q3): a 34 means separation/disposal could NOT be
|
|
849
|
-
# proven, so continuing would hand opencode a DB it may not be safe to
|
|
850
|
-
# open or write — the same evidence-quality bar `check_runner_key`
|
|
851
|
-
# already applies one step earlier in this hook (contrary evidence,
|
|
852
|
-
# not absent evidence, is what's fatal). The CLI already logged
|
|
853
|
-
# SESSION-DB-REPLICA-SEPARATION-UNVERIFIED / SESSION-DB-LOCAL-DISCARD-FAILED.
|
|
854
|
-
error "SESSION-DB-INTEGRITY-EXHAUSTED: the corrupt session DB could not be proven separated from the active backup prefix or removed from disk, so nothing will be started (see the ERROR above)."
|
|
855
|
-
return 1
|
|
856
|
-
;;
|
|
857
|
-
124 | 137)
|
|
858
|
-
# The designed-for outcome on a large DB (see the deadline comment
|
|
859
|
-
# above), not a broken tool: continue with the restored DB exactly as
|
|
860
|
-
# ECS's own `*)` branch does for an unexpected code.
|
|
861
|
-
warn "SESSION-DB-VERIFY-TIMEOUT: verification did not finish within its ${SESSION_DB_VERIFY_DEADLINE_SECONDS}s deadline; continuing with the restored opencode.db as-is, unverified"
|
|
862
|
-
;;
|
|
863
|
-
*)
|
|
864
|
-
# run_synchroniser already logged the "tool broke" ERROR for this. A
|
|
865
|
-
# broken verifier must not turn a boot that works today into a
|
|
866
|
-
# crash-loop.
|
|
867
|
-
warn "SESSION-DB-VERIFY-UNKNOWN: session-db-verify exited ${verify_rc}, which is none of its documented answers; continuing with the restored opencode.db as-is"
|
|
868
|
-
;;
|
|
869
|
-
esac
|
|
870
|
-
|
|
871
|
-
return 0
|
|
228
|
+
(
|
|
229
|
+
local started_ms rc=0
|
|
230
|
+
started_ms="$(now_ms)"
|
|
231
|
+
find "${tree}" -type f -exec cat {} + >/dev/null 2>&1 || rc=$?
|
|
232
|
+
log_elapsed_since aws-cli-prewarm "${started_ms}" "${rc}"
|
|
233
|
+
) &
|
|
234
|
+
log "pre-warming ${tree} in the background"
|
|
872
235
|
}
|
|
873
|
-
# --- Session DB integrity verification (end) ---------------------------------
|
|
874
236
|
|
|
875
237
|
# `kill -0` answers "does this pid exist", which is not the question any caller
|
|
876
238
|
# here is asking. A process that has exited but has not been reaped — a zombie —
|
|
877
239
|
# still exists, so `kill -0` reports a corpse as ALIVE. That condition is the
|
|
878
|
-
# normal case for everything these hooks start: `start_tunnel
|
|
240
|
+
# normal case for everything these hooks start: `start_tunnel` and its CLI child
|
|
879
241
|
# background a process that outlives the hook, the hook shell must return so AWS
|
|
880
242
|
# gets its 200, and PID 1 in this image is a bare node hook server with no init
|
|
881
243
|
# (Dockerfile) — so nothing ever wait()s for the orphan. It stayed a zombie for
|
|
@@ -964,7 +326,6 @@ regenerate_machine_id() {
|
|
|
964
326
|
tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
|
|
965
327
|
opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
|
|
966
328
|
litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
|
|
967
|
-
creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
|
|
968
329
|
|
|
969
330
|
# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
|
|
970
331
|
# and an interpolation of a missing field is still a non-empty string, so a
|
|
@@ -978,24 +339,6 @@ payload_is_complete() {
|
|
|
978
339
|
' >/dev/null 2>&1
|
|
979
340
|
}
|
|
980
341
|
|
|
981
|
-
# `setsid` so opencode outlives this hook: the script must return so AWS gets its
|
|
982
|
-
# 200, while opencode keeps serving. The guard makes /run idempotent — a retry
|
|
983
|
-
# after a killed hook must not put a second instance on the same port.
|
|
984
|
-
start_opencode() {
|
|
985
|
-
if opencode_is_running; then
|
|
986
|
-
warn "opencode already running (pid $(cat "${OPENCODE_PID_FILE}")); reusing it"
|
|
987
|
-
return 0
|
|
988
|
-
fi
|
|
989
|
-
|
|
990
|
-
setsid opencode serve \
|
|
991
|
-
--hostname 127.0.0.1 \
|
|
992
|
-
--port "${OPENCODE_PORT}" \
|
|
993
|
-
--print-logs &
|
|
994
|
-
|
|
995
|
-
echo $! >"${OPENCODE_PID_FILE}"
|
|
996
|
-
log "opencode starting (pid $(cat "${OPENCODE_PID_FILE}"))"
|
|
997
|
-
}
|
|
998
|
-
|
|
999
342
|
stop_opencode() {
|
|
1000
343
|
if ! opencode_is_running; then
|
|
1001
344
|
# Clear the file here too, for the same reason stop_tunnel does: opencode
|
|
@@ -1060,69 +403,11 @@ stop_opencode_and_wait() {
|
|
|
1060
403
|
log "opencode stopped ${outcome}"
|
|
1061
404
|
}
|
|
1062
405
|
|
|
1063
|
-
# --- litestream replicate (#812 WI-3) ---------------------------------------
|
|
1064
|
-
#
|
|
1065
|
-
# The write half of session-DB persistence. restore_session_db (above) is the
|
|
1066
|
-
# read half only and never starts this. Q8 chose one backgrounded, unsupervised
|
|
1067
|
-
# `litestream replicate` over a periodic flush or a `replicate -exec` wrapper
|
|
1068
|
-
# around opencode: SIGTERM is `replicate`'s own final-sync trigger, so
|
|
1069
|
-
# stop_litestream (below) doubles as the checked flush /suspend and /terminate
|
|
1070
|
-
# need, and there is no supervisor in this image to hand the process to (no
|
|
1071
|
-
# init, no `wait -n` shell that could stay alive after the hook returns).
|
|
1072
|
-
|
|
1073
406
|
# The graceful wait before the SIGKILL backstop in stop_litestream, below.
|
|
1074
407
|
# litestream's own sync is normally sub-second, so 10s is generous headroom —
|
|
1075
408
|
# chosen the same way TUNNEL_STOP_WAIT_SECONDS was, not a measured worst case.
|
|
1076
409
|
LITESTREAM_STOP_WAIT_SECONDS="${EVIDENT_LITESTREAM_STOP_WAIT_SECONDS:-10}"
|
|
1077
410
|
|
|
1078
|
-
# Guards, in this exact order and no other — reordering any of them re-arms the
|
|
1079
|
-
# invariant it exists to protect:
|
|
1080
|
-
#
|
|
1081
|
-
# 1. persistence disabled (#770) — bucket AND prefix both non-empty — is its
|
|
1082
|
-
# OWN arm with its OWN message, checked FIRST, and must never log
|
|
1083
|
-
# SESSION-DB-REPLICA-UNUSABLE: an operator grepping that marker must not
|
|
1084
|
-
# hit the disabled case (#764).
|
|
1085
|
-
# 2. the no-replicate marker (restore_session_db above) — set when this boot's
|
|
1086
|
-
# local DB may be only PARTIALLY restored, so starting replicate here would
|
|
1087
|
-
# let it overwrite a replica this boot never proved safe to write over.
|
|
1088
|
-
# A classifier 31 no longer sets it (#1106): that DB is guaranteed fresh,
|
|
1089
|
-
# and skipping replicate for it was what made the history loss unbounded.
|
|
1090
|
-
# 3. no usable config — ensure_litestream_config (restore_session_db, above)
|
|
1091
|
-
# already logged why it is missing; this is not a place to retry it.
|
|
1092
|
-
# 4. already running — idempotence, mirroring start_tunnel's own guard.
|
|
1093
|
-
#
|
|
1094
|
-
# Only past all four does it actually spawn, backgrounded with `setsid` for the
|
|
1095
|
-
# same reason start_opencode/start_tunnel are: the hook must return so AWS gets
|
|
1096
|
-
# its 200, while replicate keeps running. litestream's OWN stderr is left to
|
|
1097
|
-
# reach CloudWatch — never redirected — because there is no supervisor in this
|
|
1098
|
-
# image, so that is the only channel that can say why a replicator died.
|
|
1099
|
-
start_litestream() {
|
|
1100
|
-
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
1101
|
-
warn "SESSION-DB-PERSISTENCE-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; nothing is replicated."
|
|
1102
|
-
return 0
|
|
1103
|
-
fi
|
|
1104
|
-
|
|
1105
|
-
if [ -e "${SESSION_DB_NO_REPLICATE_MARKER}" ]; then
|
|
1106
|
-
log "skipping litestream replicate: this boot's session DB was not proven safe to replicate (see the SESSION-DB-* warning above)"
|
|
1107
|
-
return 0
|
|
1108
|
-
fi
|
|
1109
|
-
|
|
1110
|
-
if [ ! -s "${LITESTREAM_CONFIG_FILE}" ]; then
|
|
1111
|
-
error "no usable ${LITESTREAM_CONFIG_FILE}; not starting litestream replicate"
|
|
1112
|
-
return 0
|
|
1113
|
-
fi
|
|
1114
|
-
|
|
1115
|
-
if litestream_is_running; then
|
|
1116
|
-
warn "litestream already running (pid $(cat "${LITESTREAM_PID_FILE}")); reusing it"
|
|
1117
|
-
return 0
|
|
1118
|
-
fi
|
|
1119
|
-
|
|
1120
|
-
setsid litestream replicate -config "${LITESTREAM_CONFIG_FILE}" &
|
|
1121
|
-
|
|
1122
|
-
echo $! >"${LITESTREAM_PID_FILE}"
|
|
1123
|
-
log "litestream replicate starting (pid $(cat "${LITESTREAM_PID_FILE}"))"
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
411
|
# The graceful stop: mirrors stop_tunnel exactly, including PROC_DEAD_REASON's
|
|
1127
412
|
# one deciding read AFTER the poll loop (never inside it, which polls 10x/s and
|
|
1128
413
|
# would bury the log) — gone / zombie / SIGKILLed are the three outcomes an
|
|
@@ -1169,7 +454,7 @@ stop_litestream() {
|
|
|
1169
454
|
# stop_tunnel's own comment argues against an argument a future caller can get
|
|
1170
455
|
# wrong. litestream_is_running is the same O(1) check stop_litestream's own
|
|
1171
456
|
# no-op branch uses, not a poll — /run's cleanup trap fires this before
|
|
1172
|
-
#
|
|
457
|
+
# the replicator start is ever reached whenever an earlier step failed, and an
|
|
1173
458
|
# operator reading that log must not be told a stop signal went to a process
|
|
1174
459
|
# that never started.
|
|
1175
460
|
kill_litestream() {
|
|
@@ -1185,155 +470,6 @@ kill_litestream() {
|
|
|
1185
470
|
}
|
|
1186
471
|
# --- litestream replicate (end) ----------------------------------------------
|
|
1187
472
|
|
|
1188
|
-
# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
|
|
1189
|
-
#
|
|
1190
|
-
# sync_credentials (above) covers the three boundary flushes /run's restore,
|
|
1191
|
-
# /suspend and /terminate already call. What it does NOT cover is a VM that
|
|
1192
|
-
# runs for a long time between those boundaries: a provider re-authenticated
|
|
1193
|
-
# through the proxied UI hours into a run would sit unflushed until the next
|
|
1194
|
-
# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
|
|
1195
|
-
# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
|
|
1196
|
-
# own sync_credentials_loop is the ECS side of the identical gap; this is the
|
|
1197
|
-
# same fix, backgrounded the same way as start_opencode/start_litestream so it
|
|
1198
|
-
# outlives this hook process, `( … ) &` rather than `setsid`: a plain
|
|
1199
|
-
# backgrounded subshell is reparented to init and keeps running once its
|
|
1200
|
-
# parent hook script exits (verified: PPID=1, still alive, with no controlling
|
|
1201
|
-
# terminal in this image to send it a stray SIGHUP), and it inherits every
|
|
1202
|
-
# function this file defines, so it can call run_synchroniser directly with no
|
|
1203
|
-
# re-exec.
|
|
1204
|
-
|
|
1205
|
-
# Bounded confirmation window `stop_credential_sync` polls after signalling the
|
|
1206
|
-
# loop, sized against the SIGTERM budget ladder (#812 WI-4's
|
|
1207
|
-
# hook-scripts.test.ts): /terminate's own steps already use 49 of the 55s
|
|
1208
|
-
# ceiling, leaving 6s of headroom — this matches the *_KILL_GRACE_SECONDS
|
|
1209
|
-
# convention (CREDENTIAL_RESTORE_KILL_GRACE_SECONDS,
|
|
1210
|
-
# SESSION_DB_RESTORE_KILL_GRACE_SECONDS, both 2s) rather than a longer
|
|
1211
|
-
# drain-style wait, since the loop's current child is one fast
|
|
1212
|
-
# `run_synchroniser sync-once` call (#930: ~1.2s measured), not a writer
|
|
1213
|
-
# needing a graceful drain.
|
|
1214
|
-
CREDS_SYNC_STOP_WAIT_SECONDS=2
|
|
1215
|
-
|
|
1216
|
-
# Best-effort per tick, exactly like sync_credentials above: a failed tick
|
|
1217
|
-
# must never end the loop, or a single transient S3 error would silently
|
|
1218
|
-
# disable sync for the rest of the VM's life.
|
|
1219
|
-
start_credential_sync() {
|
|
1220
|
-
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
1221
|
-
warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
|
|
1222
|
-
return 0
|
|
1223
|
-
fi
|
|
1224
|
-
|
|
1225
|
-
if creds_sync_is_running; then
|
|
1226
|
-
warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
|
|
1227
|
-
return 0
|
|
1228
|
-
fi
|
|
1229
|
-
|
|
1230
|
-
# CREDS_SYNC_INTERVAL is exported by restore_session_db's `eval "$(run_synchroniser env)"`
|
|
1231
|
-
# on /run (config.ts's own default is 60s), but that eval can fail or be skipped by an
|
|
1232
|
-
# earlier give-up — never leave the loop unbound under set -u for a value with a safe,
|
|
1233
|
-
# named fallback (unlike OPENCODE_DB_PATH, a guessed sync cadence is not actively wrong).
|
|
1234
|
-
local interval="${CREDS_SYNC_INTERVAL:-60}"
|
|
1235
|
-
if [ -z "${CREDS_SYNC_INTERVAL:-}" ]; then
|
|
1236
|
-
warn "CREDS-SYNC-INTERVAL-DEFAULTED: CREDS_SYNC_INTERVAL was not set by run_synchroniser env; using ${interval}s"
|
|
1237
|
-
fi
|
|
1238
|
-
|
|
1239
|
-
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1240
|
-
|
|
1241
|
-
(
|
|
1242
|
-
# Releases the fds this subshell inherited from the hook process before
|
|
1243
|
-
# settling in for the VM's whole remaining life: nothing here writes to
|
|
1244
|
-
# them (every synchroniser call already redirects its own), so there is
|
|
1245
|
-
# no reason to keep holding the hook's original stdout/stderr open. A
|
|
1246
|
-
# long-lived process that instead inherited a pipe's write end (a test
|
|
1247
|
-
# harness reading the hook's own output, for one) would keep that pipe
|
|
1248
|
-
# from ever reporting EOF — testing-guide.mdc's own lesson, and the same
|
|
1249
|
-
# reason start_opencode/start_litestream never inherit stdio either. That
|
|
1250
|
-
# redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
|
|
1251
|
-
# failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
|
|
1252
|
-
# surfaced by stop_credential_sync, which DOES have live stdio.
|
|
1253
|
-
exec >/dev/null 2>&1 </dev/null
|
|
1254
|
-
|
|
1255
|
-
# A TERM this subshell receives (from stop_credential_sync, below) only
|
|
1256
|
-
# kills THIS wrapper by default — its currently-running child (`sleep`,
|
|
1257
|
-
# or a `run_synchroniser sync-once` call) is a separate process that
|
|
1258
|
-
# would otherwise be orphaned and keep running, free to upload STALE
|
|
1259
|
-
# credentials to S3 after the boundary flush that /suspend and
|
|
1260
|
-
# /terminate perform immediately following the stop. Tracking the
|
|
1261
|
-
# current child explicitly and forwarding the signal closes that race.
|
|
1262
|
-
creds_sync_child_pid=""
|
|
1263
|
-
trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
|
|
1264
|
-
|
|
1265
|
-
while true; do
|
|
1266
|
-
sleep "${interval}" &
|
|
1267
|
-
creds_sync_child_pid=$!
|
|
1268
|
-
wait "${creds_sync_child_pid}" 2>/dev/null
|
|
1269
|
-
creds_sync_child_pid=""
|
|
1270
|
-
|
|
1271
|
-
run_synchroniser sync-once claude &
|
|
1272
|
-
creds_sync_child_pid=$!
|
|
1273
|
-
wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1274
|
-
creds_sync_child_pid=""
|
|
1275
|
-
|
|
1276
|
-
run_synchroniser sync-once opencode &
|
|
1277
|
-
creds_sync_child_pid=$!
|
|
1278
|
-
wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1279
|
-
creds_sync_child_pid=""
|
|
1280
|
-
done
|
|
1281
|
-
) &
|
|
1282
|
-
|
|
1283
|
-
echo $! >"${CREDS_SYNC_PID_FILE}"
|
|
1284
|
-
log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
|
|
1285
|
-
}
|
|
1286
|
-
|
|
1287
|
-
# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
|
|
1288
|
-
# that it and its current child are actually gone before returning: /suspend
|
|
1289
|
-
# and /terminate start their own boundary flush immediately after this call,
|
|
1290
|
-
# and an orphaned in-flight sync-once surviving past that point can overwrite
|
|
1291
|
-
# fresher credentials with stale ones. The TERM trap inside the loop (above)
|
|
1292
|
-
# forwards the signal to its current child almost instantly — this poll is a
|
|
1293
|
-
# defensive confirmation, not the primary mechanism, so it stays short; a
|
|
1294
|
-
# SIGKILL backstop covers a child that ignores TERM entirely.
|
|
1295
|
-
#
|
|
1296
|
-
# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
|
|
1297
|
-
# must say what it found (development-workflow.mdc) — a stopped-before-called
|
|
1298
|
-
# loop and a died-on-its-own loop are different facts an operator needs told
|
|
1299
|
-
# apart, not the same "nothing to stop" line.
|
|
1300
|
-
stop_credential_sync() {
|
|
1301
|
-
if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
|
|
1302
|
-
log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
|
|
1303
|
-
return 0
|
|
1304
|
-
fi
|
|
1305
|
-
|
|
1306
|
-
local pid
|
|
1307
|
-
pid="$(cat "${CREDS_SYNC_PID_FILE}")"
|
|
1308
|
-
if ! process_is_alive "${pid}"; then
|
|
1309
|
-
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
1310
|
-
warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
|
|
1311
|
-
return 0
|
|
1312
|
-
fi
|
|
1313
|
-
|
|
1314
|
-
kill -TERM "${pid}" 2>/dev/null || true
|
|
1315
|
-
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
1316
|
-
|
|
1317
|
-
local waited_ms=0
|
|
1318
|
-
while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
|
|
1319
|
-
sleep 0.1
|
|
1320
|
-
waited_ms=$((waited_ms + 100))
|
|
1321
|
-
done
|
|
1322
|
-
|
|
1323
|
-
if process_is_alive "${pid}"; then
|
|
1324
|
-
kill -KILL "${pid}" 2>/dev/null || true
|
|
1325
|
-
warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
|
|
1326
|
-
fi
|
|
1327
|
-
|
|
1328
|
-
if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
|
|
1329
|
-
warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
|
|
1330
|
-
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
1331
|
-
fi
|
|
1332
|
-
|
|
1333
|
-
log "CREDS-SYNC-STOPPED: pid=${pid}"
|
|
1334
|
-
}
|
|
1335
|
-
# --- credential sync loop (end) -----------------------------------------------
|
|
1336
|
-
|
|
1337
473
|
# --- flush_session_db (#812 WI-4) -------------------------------------------
|
|
1338
474
|
#
|
|
1339
475
|
# The checked, synchronous flush /suspend and /terminate need before they
|
|
@@ -1344,19 +480,16 @@ stop_credential_sync() {
|
|
|
1344
480
|
# Ceiling for the one-shot `litestream replicate -once` flush below. litestream's
|
|
1345
481
|
# own sync is normally sub-second (M5, the plan's grounding); 10s is generous
|
|
1346
482
|
# headroom, chosen the same way LITESTREAM_STOP_WAIT_SECONDS was above — not a
|
|
1347
|
-
# measured worst case. The SIGKILL backstop
|
|
1348
|
-
#
|
|
1349
|
-
# identical concept (bare `timeout` only SIGTERMs, and a litestream that
|
|
1350
|
-
# ignored it would be unbounded again), and there is no reason for the restore
|
|
1351
|
-
# and flush paths to ever drift on how long a SIGTERM gets to land.
|
|
483
|
+
# measured worst case. The SIGKILL backstop gives the flush its own two-second
|
|
484
|
+
# grace after `timeout` sends SIGTERM.
|
|
1352
485
|
SESSION_DB_FLUSH_DEADLINE_SECONDS="${EVIDENT_SESSION_DB_FLUSH_DEADLINE_SECONDS:-10}"
|
|
486
|
+
SESSION_DB_FLUSH_KILL_GRACE_SECONDS=2
|
|
1353
487
|
|
|
1354
|
-
# Guards mirror
|
|
488
|
+
# Guards mirror the CLI replicator's first three exactly (disabled -> marker ->
|
|
1355
489
|
# config) — a flush must never attempt work those guards would have refused
|
|
1356
490
|
# to start in the first place. PERSISTENCE_BUCKET is the SAME variable
|
|
1357
|
-
#
|
|
1358
|
-
# THIS process set it (load_state_config
|
|
1359
|
-
# eval, on /run).
|
|
491
|
+
# the CLI reads, so both agree regardless of which earlier step in
|
|
492
|
+
# THIS process set it (load_state_config above).
|
|
1360
493
|
#
|
|
1361
494
|
# Then: stop the daemon FIRST — its SIGTERM IS litestream's own final-sync
|
|
1362
495
|
# trigger — and only once it is confirmed stopped does the synchronous -once
|
|
@@ -1396,7 +529,7 @@ flush_session_db() {
|
|
|
1396
529
|
# Only reached with the daemon confirmed stopped, or never running — never
|
|
1397
530
|
# concurrently with it (see above).
|
|
1398
531
|
local flush_rc=0
|
|
1399
|
-
timeout -k "${
|
|
532
|
+
timeout -k "${SESSION_DB_FLUSH_KILL_GRACE_SECONDS}" "${SESSION_DB_FLUSH_DEADLINE_SECONDS}" \
|
|
1400
533
|
litestream replicate -once -config "${LITESTREAM_CONFIG_FILE}" || flush_rc=$?
|
|
1401
534
|
|
|
1402
535
|
if [ "${flush_rc}" -eq 0 ]; then
|
|
@@ -1408,16 +541,19 @@ flush_session_db() {
|
|
|
1408
541
|
# --- flush_session_db (end) --------------------------------------------------
|
|
1409
542
|
|
|
1410
543
|
# How long the guest CLI runs with no activity before it exits itself
|
|
1411
|
-
# (`evident run --idle-timeout
|
|
544
|
+
# (`evident run --idle-timeout`), which is what
|
|
1412
545
|
# turns a truly-abandoned VM into the clean-offline POST that lets Evident
|
|
1413
546
|
# suspend it (#732). Sized from the measured cost of guessing wrong rather than
|
|
1414
547
|
# the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
|
|
1415
548
|
# ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
|
|
1416
|
-
# napping a VM whose user comes straight back costs ~10 s — against
|
|
1417
|
-
#
|
|
1418
|
-
#
|
|
1419
|
-
#
|
|
1420
|
-
#
|
|
549
|
+
# napping a VM whose user comes straight back costs ~10 s — against the
|
|
550
|
+
# legacy conservative baseline input of ~$0.30/h, AWS can burst a loaded VM
|
|
551
|
+
# to a 16 GB / 8 vCPU peak (~$1.06/h at sustained full load—a worst-case
|
|
552
|
+
# ceiling, not an expectation); that is cheap enough that the balance
|
|
553
|
+
# sits far nearer the floor than the ceiling. Not AT the floor, though: the
|
|
554
|
+
# CLI's idle detector needs 2 clear poll cycles (≥4 s of real time), so a value
|
|
555
|
+
# near that would spend more time
|
|
556
|
+
# suspending/resuming than idle.
|
|
1421
557
|
# ECS's waker uses 900 s instead only because *its* cold start is far slower
|
|
1422
558
|
# than this VM's ~2 s resume — not evidence this default should match it.
|
|
1423
559
|
#
|
|
@@ -1429,7 +565,9 @@ flush_session_db() {
|
|
|
1429
565
|
# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
|
|
1430
566
|
# idle-timeout flag, so an operator who knows one knows the other. A
|
|
1431
567
|
# non-numeric override must never silently DROP the flag — that degrades to
|
|
1432
|
-
# an always-on VM burning ~$
|
|
568
|
+
# an always-on VM burning ~$7.17/day at baseline, or up to ~$25.49/day at
|
|
569
|
+
# sustained full peak as load increases (a worst-case ceiling, not an
|
|
570
|
+
# expectation), exactly the bug this closes — so it
|
|
1433
571
|
# warns and falls back to the default instead.
|
|
1434
572
|
#
|
|
1435
573
|
# Deliberately NOT in the /run payload yet: doing so would touch the doorbell
|
|
@@ -1451,6 +589,26 @@ fi
|
|
|
1451
589
|
# its 200, while the tunnel keeps serving.
|
|
1452
590
|
start_tunnel() {
|
|
1453
591
|
local runner_key="$1" api_url="$2" tunnel_url="$3"
|
|
592
|
+
local restore_runner_credentials="${4:-false}" restore_history="${5:-false}"
|
|
593
|
+
local -a credential_flags=()
|
|
594
|
+
local -a session_db_flags=()
|
|
595
|
+
local -a opencode_config_flags=()
|
|
596
|
+
|
|
597
|
+
# The explicit fourth argument is set only by /run; /resume uses the default
|
|
598
|
+
# so a resumed VM never restores credentials over stores it already has.
|
|
599
|
+
if [ "${restore_runner_credentials}" = true ]; then
|
|
600
|
+
credential_flags=(--restore-runner-credentials)
|
|
601
|
+
fi
|
|
602
|
+
|
|
603
|
+
# Only a fresh /run asks the CLI to restore session history. /resume keeps
|
|
604
|
+
# the database from the snapshot and must not restore over it.
|
|
605
|
+
if [ "${restore_history}" = true ]; then
|
|
606
|
+
session_db_flags=(--restore-session-db)
|
|
607
|
+
fi
|
|
608
|
+
|
|
609
|
+
if [ -n "${RUNNER_OPENCODE_CONFIG:-}" ]; then
|
|
610
|
+
opencode_config_flags=(--opencode-config-overlay "${RUNNER_OPENCODE_CONFIG}")
|
|
611
|
+
fi
|
|
1454
612
|
|
|
1455
613
|
if tunnel_is_running; then
|
|
1456
614
|
warn "tunnel already running (pid $(cat "${TUNNEL_PID_FILE}")); not starting a second one"
|
|
@@ -1473,6 +631,14 @@ start_tunnel() {
|
|
|
1473
631
|
--port "${OPENCODE_PORT}" \
|
|
1474
632
|
--endpoint "${api_url}" \
|
|
1475
633
|
--tunnel "${tunnel_url}" \
|
|
634
|
+
--opencode-pid-file "${OPENCODE_PID_FILE}" \
|
|
635
|
+
--litestream-config "${LITESTREAM_CONFIG_FILE}" \
|
|
636
|
+
--litestream-pid-file "${LITESTREAM_PID_FILE}" \
|
|
637
|
+
--session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
|
|
638
|
+
--credential-sync-marker "${CREDENTIAL_FLUSH_MARKER_FILE}" \
|
|
639
|
+
"${credential_flags[@]}" \
|
|
640
|
+
"${session_db_flags[@]}" \
|
|
641
|
+
"${opencode_config_flags[@]}" \
|
|
1476
642
|
--idle-timeout "${IDLE_TIMEOUT_SECONDS}" \
|
|
1477
643
|
--enable-file-sync-to "${HOME}/.claude" &
|
|
1478
644
|
|
|
@@ -1481,22 +647,20 @@ start_tunnel() {
|
|
|
1481
647
|
}
|
|
1482
648
|
|
|
1483
649
|
# The worst case `evident run` can take to shut down gracefully on SIGTERM, in
|
|
1484
|
-
# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS
|
|
1485
|
-
#
|
|
1486
|
-
#
|
|
1487
|
-
#
|
|
1488
|
-
#
|
|
1489
|
-
#
|
|
1490
|
-
#
|
|
1491
|
-
#
|
|
1492
|
-
#
|
|
1493
|
-
#
|
|
1494
|
-
#
|
|
1495
|
-
#
|
|
1496
|
-
# cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
|
|
1497
|
-
# three bounds cited above, so if one of them moves, update it here.
|
|
650
|
+
# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS) + 2 s offline
|
|
651
|
+
# POST (notifyAgentDisconnected) + 5 s telemetry flush
|
|
652
|
+
# (TELEMETRY_SHUTDOWN_TIMEOUT_MS) + 8.5 s pre-drain credential flush
|
|
653
|
+
# + 8.5 s post-drain credential flush. Each phase is bounded there, so this is
|
|
654
|
+
# a ceiling rather than a typical cost — an idle suspend finishes in a couple
|
|
655
|
+
# of seconds. This is the ONE place the hand-maintained budget is written down;
|
|
656
|
+
# a guard in the source repository keeps this number in step with the CLI's
|
|
657
|
+
# declared bounds.
|
|
658
|
+
#
|
|
659
|
+
# DOCUMENTATION ONLY — nothing is derived from this value. It is the hand-maintained
|
|
660
|
+
# sum of the five bounded phases above (25 + 2 + 5 + 8.5 + 8.5); update it here if
|
|
661
|
+
# any of those bounds changes.
|
|
1498
662
|
# shellcheck disable=SC2034 # documentation; deliberately read by nothing
|
|
1499
|
-
CLI_SHUTDOWN_CEILING_SECONDS=
|
|
663
|
+
CLI_SHUTDOWN_CEILING_SECONDS=49
|
|
1500
664
|
|
|
1501
665
|
# How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
|
|
1502
666
|
# Since #718 this binds ONLY for a CLI that is still draining: one that has
|
|
@@ -1572,12 +736,91 @@ stop_tunnel() {
|
|
|
1572
736
|
log "tunnel stopped ${outcome}"
|
|
1573
737
|
}
|
|
1574
738
|
|
|
739
|
+
# The CLI owns the interval loop and its boundary flush. Two seconds of slack
|
|
740
|
+
# over the CLI's 8s flush deadline keeps this marker handshake inside the 55s
|
|
741
|
+
# hook ceiling while leaving the fallback flush and tunnel stop budget intact.
|
|
742
|
+
#
|
|
743
|
+
# This bounds only the CLI's pre-drain flush, which always runs first and
|
|
744
|
+
# unconditionally (run.ts's cleanup(), before the channel-work drain) — not
|
|
745
|
+
# the CLI's post-drain second pass, which can take up to
|
|
746
|
+
# SHUTDOWN_DRAIN_TIMEOUT_MS longer than this wait covers. A drain that
|
|
747
|
+
# consumes the whole window loses the SECOND pass, not the credential
|
|
748
|
+
# guarantee itself: the pre-drain flush already persisted everything on disk
|
|
749
|
+
# at signal time, exactly what the old bash `sync_credentials` guaranteed in
|
|
750
|
+
# one synchronous call — so the worst case here is no worse than before this
|
|
751
|
+
# handshake existed, never a fresh data-loss window. See
|
|
752
|
+
# docs/decisions/0063-microvm-boot-orchestration-in-cli.md's two-phase-flush
|
|
753
|
+
# section for the full reasoning.
|
|
754
|
+
CREDENTIAL_FLUSH_WAIT_SECONDS="${EVIDENT_CREDENTIAL_FLUSH_WAIT_SECONDS:-10}"
|
|
755
|
+
|
|
756
|
+
# Remove the previous answer, signal the same CLI that stop_tunnel handles, and
|
|
757
|
+
# wait for either its marker or its death. A fallback sync runs only after the
|
|
758
|
+
# CLI is known to be absent, never alongside a live CLI that may still write.
|
|
759
|
+
stop_runner_and_flush_credentials() {
|
|
760
|
+
rm -f "${CREDENTIAL_FLUSH_MARKER_FILE}"
|
|
761
|
+
|
|
762
|
+
if ! tunnel_is_running; then
|
|
763
|
+
warn "CREDS-FLUSH-NO-RUNNER: no live CLI at handshake entry"
|
|
764
|
+
stop_tunnel
|
|
765
|
+
sync_credentials
|
|
766
|
+
return 0
|
|
767
|
+
fi
|
|
768
|
+
|
|
769
|
+
local pid
|
|
770
|
+
pid="$(cat "${TUNNEL_PID_FILE}")"
|
|
771
|
+
kill -TERM "${pid}" 2>/dev/null || true
|
|
772
|
+
|
|
773
|
+
local waited_ms=0 marker_found=false runner_exited=false
|
|
774
|
+
while [ "${waited_ms}" -lt $((CREDENTIAL_FLUSH_WAIT_SECONDS * 1000)) ]; do
|
|
775
|
+
if [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
|
|
776
|
+
marker_found=true
|
|
777
|
+
break
|
|
778
|
+
fi
|
|
779
|
+
if ! process_is_alive "${pid}"; then
|
|
780
|
+
runner_exited=true
|
|
781
|
+
break
|
|
782
|
+
fi
|
|
783
|
+
sleep 0.1
|
|
784
|
+
waited_ms=$((waited_ms + 100))
|
|
785
|
+
done
|
|
786
|
+
|
|
787
|
+
# The CLI can publish the marker and exit inside one poll tick. A final
|
|
788
|
+
# marker check after a death break preserves that answer instead of falling
|
|
789
|
+
# through to the fallback.
|
|
790
|
+
if [ "${runner_exited}" = true ] && [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
|
|
791
|
+
marker_found=true
|
|
792
|
+
runner_exited=false
|
|
793
|
+
fi
|
|
794
|
+
|
|
795
|
+
if [ "${marker_found}" = true ]; then
|
|
796
|
+
local failures
|
|
797
|
+
failures="$(grep -Ev '^(claude|opencode)=ok$' "${CREDENTIAL_FLUSH_MARKER_FILE}" || true)"
|
|
798
|
+
if [ -n "${failures}" ]; then
|
|
799
|
+
warn "CREDS-FLUSH-HAD-FAILURES: ${failures//$'\n'/ }"
|
|
800
|
+
else
|
|
801
|
+
log "CREDS-FLUSH-OK"
|
|
802
|
+
fi
|
|
803
|
+
stop_tunnel
|
|
804
|
+
return 0
|
|
805
|
+
fi
|
|
806
|
+
|
|
807
|
+
if [ "${runner_exited}" = true ]; then
|
|
808
|
+
warn "CREDS-FLUSH-RUNNER-EXITED: CLI exited without a completion marker"
|
|
809
|
+
stop_tunnel
|
|
810
|
+
sync_credentials
|
|
811
|
+
return 0
|
|
812
|
+
fi
|
|
813
|
+
|
|
814
|
+
warn "CREDS-FLUSH-TIMEOUT: live CLI did not write a completion marker within ${CREDENTIAL_FLUSH_WAIT_SECONDS}s"
|
|
815
|
+
stop_tunnel
|
|
816
|
+
}
|
|
817
|
+
|
|
1575
818
|
# --- check_runner_key (#1172) ------------------------------------------------
|
|
1576
819
|
#
|
|
1577
820
|
# Answers exactly one question before opencode/the tunnel start spending this
|
|
1578
821
|
# boot's SIGTERM budget on a key that cannot work: "does the runner key in this
|
|
1579
822
|
# payload authenticate against Evident?" Delegates entirely to the CLI's own
|
|
1580
|
-
# `evident status --json`
|
|
823
|
+
# `evident status --json` rather than
|
|
1581
824
|
# reimplementing its auth logic here — that command's `reason` field is the
|
|
1582
825
|
# published contract this function reads, and its own header states the
|
|
1583
826
|
# absent-vs-contrary distinction this function must honour.
|
|
@@ -1605,10 +848,9 @@ check_runner_key() {
|
|
|
1605
848
|
local runner_key="$1" api_url="$2"
|
|
1606
849
|
|
|
1607
850
|
# status.ts's own exit-code contract (its header comment) means this exits
|
|
1608
|
-
# non-zero on EVERY branch except `ok
|
|
1609
|
-
#
|
|
1610
|
-
#
|
|
1611
|
-
# caller's `set -e` before the case below ever runs. Only stdout is
|
|
851
|
+
# non-zero on EVERY branch except `ok`; guard the capture so an expected
|
|
852
|
+
# non-zero does not abort this function under the caller's `set -e` before
|
|
853
|
+
# the case below ever runs. Only stdout is
|
|
1612
854
|
# captured: status.ts's own contract is one parseable JSON line and nothing
|
|
1613
855
|
# else there, and — like litestream's stderr elsewhere in this file — its
|
|
1614
856
|
# stderr is left to reach CloudWatch directly rather than being folded in,
|