@evident-ai/runner-cdk 3.4.1-dev.cb56b3f → 3.4.1-dev.f80aa1c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/microvm-image-context/Dockerfile +10 -0
- package/dist/microvm-image-context/hooks/common.sh +110 -163
- package/dist/microvm-image-context/hooks/resume +9 -5
- package/dist/microvm-image-context/hooks/run +6 -4
- package/dist/microvm-image-context/hooks/suspend +4 -5
- package/dist/microvm-image-context/hooks/terminate +5 -6
- package/package.json +1 -1
|
@@ -95,13 +95,23 @@ ARG OPENCODE_VERSION=1.18.3
|
|
|
95
95
|
ARG EVIDENT_CLI_VERSION=dev
|
|
96
96
|
ARG CLAUDE_CODE_VERSION=latest
|
|
97
97
|
ARG RUNNER_SYNCHRONISER_VERSION=dev
|
|
98
|
+
#
|
|
99
|
+
# @brave/brave-search-mcp-server is pinned (not a build arg — a single caller,
|
|
100
|
+
# opencode.evident.jsonc, needs one version) and baked here for the same reason
|
|
101
|
+
# as the ECS runner image: a cold `npx -y` install of it took ~38s in a fresh
|
|
102
|
+
# MicroVM, over OpenCode's ~30s MCP connect timeout, so the server silently
|
|
103
|
+
# failed to connect on every first turn after a boot. Baking it lets
|
|
104
|
+
# opencode.evident.jsonc invoke the installed `brave-search-mcp-server` binary
|
|
105
|
+
# directly instead of through `npx`.
|
|
98
106
|
RUN npm install -g \
|
|
99
107
|
"opencode-ai@${OPENCODE_VERSION}" \
|
|
100
108
|
"@evident-ai/cli@${EVIDENT_CLI_VERSION}" \
|
|
101
109
|
"@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \
|
|
102
110
|
"@evident-ai/runner-synchroniser@${RUNNER_SYNCHRONISER_VERSION}" \
|
|
111
|
+
"@brave/brave-search-mcp-server@2.1.0" \
|
|
103
112
|
&& opencode --version \
|
|
104
113
|
&& evident --version \
|
|
114
|
+
&& brave-search-mcp-server --help >/dev/null \
|
|
105
115
|
&& npm cache clean --force
|
|
106
116
|
|
|
107
117
|
# Playwright chromium + OS deps for E2E, baked at build time to avoid a slow
|
|
@@ -21,11 +21,15 @@ OPENCODE_PORT="${OPENCODE_PORT:-4096}"
|
|
|
21
21
|
# /proc/<pid>/environ, which runs as that same uid. /terminate removes it.
|
|
22
22
|
# shellcheck disable=SC2034 # read by the scripts that source this file
|
|
23
23
|
CONTEXT_FILE="/dev/shm/evident-run-context"
|
|
24
|
+
|
|
25
|
+
# The runtime injects MICROVM_ID only into /run, never /resume. This tmpfs file
|
|
26
|
+
# survives suspend/resume so /resume can restore the id for every fresh CLI
|
|
27
|
+
# process to self-report and acknowledge a fulfilled recycle request (#1906).
|
|
28
|
+
# shellcheck disable=SC2034 # read by the scripts that source this file
|
|
29
|
+
MICROVM_ID_FILE="/dev/shm/evident-microvm-id"
|
|
24
30
|
TUNNEL_PID_FILE="/dev/shm/evident-tunnel.pid"
|
|
25
31
|
OPENCODE_PID_FILE="/dev/shm/evident-opencode.pid"
|
|
26
32
|
LITESTREAM_PID_FILE="/dev/shm/evident-litestream.pid"
|
|
27
|
-
CREDS_SYNC_PID_FILE="/dev/shm/evident-creds-sync.pid"
|
|
28
|
-
CREDS_SYNC_LAST_ERROR_FILE="/dev/shm/evident-creds-sync.last-error"
|
|
29
33
|
|
|
30
34
|
# Where the runner's credential store lives inside the durable-state bucket.
|
|
31
35
|
# The BUCKET is the same for every VM from an image version, so the stack bakes
|
|
@@ -47,6 +51,10 @@ LITESTREAM_CONFIG_FILE="/dev/shm/evident-litestream.yml"
|
|
|
47
51
|
# `/terminate` removes it with the rest of the per-VM state.
|
|
48
52
|
SESSION_DB_NO_REPLICATE_MARKER="/dev/shm/evident-session-db-no-replicate"
|
|
49
53
|
|
|
54
|
+
# Completion evidence for the CLI-owned credential flush. It lives in tmpfs,
|
|
55
|
+
# is removed before every handshake, and is also removed by /terminate.
|
|
56
|
+
CREDENTIAL_FLUSH_MARKER_FILE="/dev/shm/evident-credential-flush"
|
|
57
|
+
|
|
50
58
|
hook_name() { printf '%s' "${0##*/}"; }
|
|
51
59
|
log() { echo "[hook:$(hook_name)] $*"; }
|
|
52
60
|
warn() { echo "[hook:$(hook_name)] $*" >&2; }
|
|
@@ -318,7 +326,6 @@ regenerate_machine_id() {
|
|
|
318
326
|
tunnel_is_running() { is_running "${TUNNEL_PID_FILE}"; }
|
|
319
327
|
opencode_is_running() { is_running "${OPENCODE_PID_FILE}"; }
|
|
320
328
|
litestream_is_running() { is_running "${LITESTREAM_PID_FILE}"; }
|
|
321
|
-
creds_sync_is_running() { is_running "${CREDS_SYNC_PID_FILE}"; }
|
|
322
329
|
|
|
323
330
|
# `jq -e` alone is not enough: its exit status reflects the LAST OUTPUT VALUE,
|
|
324
331
|
# and an interpolation of a missing field is still a non-empty string, so a
|
|
@@ -463,149 +470,6 @@ kill_litestream() {
|
|
|
463
470
|
}
|
|
464
471
|
# --- litestream replicate (end) ----------------------------------------------
|
|
465
472
|
|
|
466
|
-
# --- credential sync loop (#1868 WI-3, ECS parity) ---------------------------
|
|
467
|
-
#
|
|
468
|
-
# sync_credentials (above) covers the three boundary flushes /run's restore,
|
|
469
|
-
# /suspend and /terminate already call. What it does NOT cover is a VM that
|
|
470
|
-
# runs for a long time between those boundaries: a provider re-authenticated
|
|
471
|
-
# through the proxied UI hours into a run would sit unflushed until the next
|
|
472
|
-
# suspend/terminate, and a VM that dies without one (a crash, an OOM kill)
|
|
473
|
-
# loses everything since boot. runner/docker-images/fargate/entrypoint.sh's
|
|
474
|
-
# own sync_credentials_loop is the ECS side of the identical gap; this is the
|
|
475
|
-
# same fix, backgrounded like the other long-lived services so it
|
|
476
|
-
# outlives this hook process, `( … ) &` rather than `setsid`: a plain
|
|
477
|
-
# backgrounded subshell is reparented to init and keeps running once its
|
|
478
|
-
# parent hook script exits (verified: PPID=1, still alive, with no controlling
|
|
479
|
-
# terminal in this image to send it a stray SIGHUP), and it inherits every
|
|
480
|
-
# function this file defines, so it can call run_synchroniser directly with no
|
|
481
|
-
# re-exec.
|
|
482
|
-
|
|
483
|
-
# Bounded confirmation window `stop_credential_sync` polls after signalling the
|
|
484
|
-
# loop. The loop's current child is one fast `run_synchroniser sync-once` call,
|
|
485
|
-
# so this stays a short backstop rather than a graceful drain.
|
|
486
|
-
CREDS_SYNC_STOP_WAIT_SECONDS=2
|
|
487
|
-
|
|
488
|
-
# Best-effort per tick, exactly like sync_credentials above: a failed tick
|
|
489
|
-
# must never end the loop, or a single transient S3 error would silently
|
|
490
|
-
# disable sync for the rest of the VM's life.
|
|
491
|
-
start_credential_sync() {
|
|
492
|
-
if [ -z "${PERSISTENCE_BUCKET:-}" ]; then
|
|
493
|
-
warn "CREDS-SYNC-DISABLED: LITESTREAM_BUCKET/LITESTREAM_PREFIX are not both set; no interval credential sync this boot."
|
|
494
|
-
return 0
|
|
495
|
-
fi
|
|
496
|
-
|
|
497
|
-
if creds_sync_is_running; then
|
|
498
|
-
warn "credential sync loop already running (pid $(cat "${CREDS_SYNC_PID_FILE}")); reusing it"
|
|
499
|
-
return 0
|
|
500
|
-
fi
|
|
501
|
-
|
|
502
|
-
# This is the same environment input runner-synchroniser validates. The
|
|
503
|
-
# variable is normally absent, so that ordinary case uses the documented
|
|
504
|
-
# default without a warning; a present invalid value is named and rejected.
|
|
505
|
-
local interval="${CREDS_SYNC_INTERVAL:-60}"
|
|
506
|
-
if [ -n "${CREDS_SYNC_INTERVAL+x}" ] && [[ ! "${CREDS_SYNC_INTERVAL}" =~ ^[1-9][0-9]*$ ]]; then
|
|
507
|
-
warn "CREDS-SYNC-INTERVAL-INVALID: CREDS_SYNC_INTERVAL='${CREDS_SYNC_INTERVAL}' is not a positive integer; using 60s"
|
|
508
|
-
interval=60
|
|
509
|
-
fi
|
|
510
|
-
|
|
511
|
-
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
512
|
-
|
|
513
|
-
(
|
|
514
|
-
# Releases the fds this subshell inherited from the hook process before
|
|
515
|
-
# settling in for the VM's whole remaining life: nothing here writes to
|
|
516
|
-
# them (every synchroniser call already redirects its own), so there is
|
|
517
|
-
# no reason to keep holding the hook's original stdout/stderr open. A
|
|
518
|
-
# long-lived process that instead inherited a pipe's write end (a test
|
|
519
|
-
# harness reading the hook's own output, for one) would keep that pipe
|
|
520
|
-
# from ever reporting EOF — testing-guide.mdc's own lesson, and the same
|
|
521
|
-
# reason the long-lived services never inherit stdio either. That
|
|
522
|
-
# redirect also means `warn`/`log`/`error` calls in here go nowhere, so a
|
|
523
|
-
# failed sync-once is instead recorded to CREDS_SYNC_LAST_ERROR_FILE and
|
|
524
|
-
# surfaced by stop_credential_sync, which DOES have live stdio.
|
|
525
|
-
exec >/dev/null 2>&1 </dev/null
|
|
526
|
-
|
|
527
|
-
# A TERM this subshell receives (from stop_credential_sync, below) only
|
|
528
|
-
# kills THIS wrapper by default — its currently-running child (`sleep`,
|
|
529
|
-
# or a `run_synchroniser sync-once` call) is a separate process that
|
|
530
|
-
# would otherwise be orphaned and keep running, free to upload STALE
|
|
531
|
-
# credentials to S3 after the boundary flush that /suspend and
|
|
532
|
-
# /terminate perform immediately following the stop. Tracking the
|
|
533
|
-
# current child explicitly and forwarding the signal closes that race.
|
|
534
|
-
creds_sync_child_pid=""
|
|
535
|
-
trap 'trap - TERM; [ -n "${creds_sync_child_pid}" ] && kill -TERM "${creds_sync_child_pid}" 2>/dev/null; exit 0' TERM
|
|
536
|
-
|
|
537
|
-
while true; do
|
|
538
|
-
sleep "${interval}" &
|
|
539
|
-
creds_sync_child_pid=$!
|
|
540
|
-
wait "${creds_sync_child_pid}" 2>/dev/null
|
|
541
|
-
creds_sync_child_pid=""
|
|
542
|
-
|
|
543
|
-
run_synchroniser sync-once claude &
|
|
544
|
-
creds_sync_child_pid=$!
|
|
545
|
-
wait "${creds_sync_child_pid}" 2>/dev/null || echo "claude" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
546
|
-
creds_sync_child_pid=""
|
|
547
|
-
|
|
548
|
-
run_synchroniser sync-once opencode &
|
|
549
|
-
creds_sync_child_pid=$!
|
|
550
|
-
wait "${creds_sync_child_pid}" 2>/dev/null || echo "opencode" >"${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
551
|
-
creds_sync_child_pid=""
|
|
552
|
-
done
|
|
553
|
-
) &
|
|
554
|
-
|
|
555
|
-
echo $! >"${CREDS_SYNC_PID_FILE}"
|
|
556
|
-
log "CREDS-SYNC-STARTED: pid=$! interval=${interval}s"
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
# Signals the loop, then confirms (bounded — see CREDS_SYNC_STOP_WAIT_SECONDS)
|
|
560
|
-
# that it and its current child are actually gone before returning: /suspend
|
|
561
|
-
# and /terminate start their own boundary flush immediately after this call,
|
|
562
|
-
# and an orphaned in-flight sync-once surviving past that point can overwrite
|
|
563
|
-
# fresher credentials with stale ones. The TERM trap inside the loop (above)
|
|
564
|
-
# forwards the signal to its current child almost instantly — this poll is a
|
|
565
|
-
# defensive confirmation, not the primary mechanism, so it stays short; a
|
|
566
|
-
# SIGKILL backstop covers a child that ignores TERM entirely.
|
|
567
|
-
#
|
|
568
|
-
# The DIED branch is a liveness report, not a no-op: every recovery/no-op path
|
|
569
|
-
# must say what it found (development-workflow.mdc) — a stopped-before-called
|
|
570
|
-
# loop and a died-on-its-own loop are different facts an operator needs told
|
|
571
|
-
# apart, not the same "nothing to stop" line.
|
|
572
|
-
stop_credential_sync() {
|
|
573
|
-
if [ ! -s "${CREDS_SYNC_PID_FILE}" ]; then
|
|
574
|
-
log "CREDS-SYNC-NOT-RUNNING: no credential sync loop to stop"
|
|
575
|
-
return 0
|
|
576
|
-
fi
|
|
577
|
-
|
|
578
|
-
local pid
|
|
579
|
-
pid="$(cat "${CREDS_SYNC_PID_FILE}")"
|
|
580
|
-
if ! process_is_alive "${pid}"; then
|
|
581
|
-
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
582
|
-
warn "CREDS-SYNC-DIED: credential sync loop (pid=${pid}) had already exited before this stop"
|
|
583
|
-
return 0
|
|
584
|
-
fi
|
|
585
|
-
|
|
586
|
-
kill -TERM "${pid}" 2>/dev/null || true
|
|
587
|
-
rm -f "${CREDS_SYNC_PID_FILE}"
|
|
588
|
-
|
|
589
|
-
local waited_ms=0
|
|
590
|
-
while process_is_alive "${pid}" && [ "${waited_ms}" -lt $((CREDS_SYNC_STOP_WAIT_SECONDS * 1000)) ]; do
|
|
591
|
-
sleep 0.1
|
|
592
|
-
waited_ms=$((waited_ms + 100))
|
|
593
|
-
done
|
|
594
|
-
|
|
595
|
-
if process_is_alive "${pid}"; then
|
|
596
|
-
kill -KILL "${pid}" 2>/dev/null || true
|
|
597
|
-
warn "CREDS-SYNC-STOP-TIMEOUT: pid=${pid} still alive after ${CREDS_SYNC_STOP_WAIT_SECONDS}s; sent SIGKILL"
|
|
598
|
-
fi
|
|
599
|
-
|
|
600
|
-
if [ -s "${CREDS_SYNC_LAST_ERROR_FILE}" ]; then
|
|
601
|
-
warn "CREDS-SYNC-HAD-FAILURES: sync-once failed at least once for: $(tr '\n' ' ' <"${CREDS_SYNC_LAST_ERROR_FILE}")"
|
|
602
|
-
rm -f "${CREDS_SYNC_LAST_ERROR_FILE}"
|
|
603
|
-
fi
|
|
604
|
-
|
|
605
|
-
log "CREDS-SYNC-STOPPED: pid=${pid}"
|
|
606
|
-
}
|
|
607
|
-
# --- credential sync loop (end) -----------------------------------------------
|
|
608
|
-
|
|
609
473
|
# --- flush_session_db (#812 WI-4) -------------------------------------------
|
|
610
474
|
#
|
|
611
475
|
# The checked, synchronous flush /suspend and /terminate need before they
|
|
@@ -682,11 +546,14 @@ flush_session_db() {
|
|
|
682
546
|
# suspend it (#732). Sized from the measured cost of guessing wrong rather than
|
|
683
547
|
# the saving: suspend reaches SUSPENDED in ~7 s and a resume is RUNNING in
|
|
684
548
|
# ~0.6 s with the tunnel back ~2 s later (README, "Measured, end to end"), so
|
|
685
|
-
# napping a VM whose user comes straight back costs ~10 s — against
|
|
686
|
-
#
|
|
687
|
-
#
|
|
688
|
-
#
|
|
689
|
-
#
|
|
549
|
+
# napping a VM whose user comes straight back costs ~10 s — against the
|
|
550
|
+
# legacy conservative baseline input of ~$0.30/h, AWS can burst a loaded VM
|
|
551
|
+
# to a 16 GB / 8 vCPU peak (~$1.06/h at sustained full load—a worst-case
|
|
552
|
+
# ceiling, not an expectation); that is cheap enough that the balance
|
|
553
|
+
# sits far nearer the floor than the ceiling. Not AT the floor, though: the
|
|
554
|
+
# CLI's idle detector needs 2 clear poll cycles (`run.ts`'s `idlePolls >= 2`,
|
|
555
|
+
# ≥4 s of real time), so a value near that would spend more time
|
|
556
|
+
# suspending/resuming than idle.
|
|
690
557
|
# ECS's waker uses 900 s instead only because *its* cold start is far slower
|
|
691
558
|
# than this VM's ~2 s resume — not evidence this default should match it.
|
|
692
559
|
#
|
|
@@ -698,7 +565,9 @@ flush_session_db() {
|
|
|
698
565
|
# runner/docker-images/fargate/entrypoint.sh uses for the ECS runner's own
|
|
699
566
|
# idle-timeout flag, so an operator who knows one knows the other. A
|
|
700
567
|
# non-numeric override must never silently DROP the flag — that degrades to
|
|
701
|
-
# an always-on VM burning ~$
|
|
568
|
+
# an always-on VM burning ~$7.17/day at baseline, or up to ~$25.49/day at
|
|
569
|
+
# sustained full peak as load increases (a worst-case ceiling, not an
|
|
570
|
+
# expectation), exactly the bug this closes — so it
|
|
702
571
|
# warns and falls back to the default instead.
|
|
703
572
|
#
|
|
704
573
|
# Deliberately NOT in the /run payload yet: doing so would touch the doorbell
|
|
@@ -766,6 +635,7 @@ start_tunnel() {
|
|
|
766
635
|
--litestream-config "${LITESTREAM_CONFIG_FILE}" \
|
|
767
636
|
--litestream-pid-file "${LITESTREAM_PID_FILE}" \
|
|
768
637
|
--session-db-no-replicate-marker "${SESSION_DB_NO_REPLICATE_MARKER}" \
|
|
638
|
+
--credential-sync-marker "${CREDENTIAL_FLUSH_MARKER_FILE}" \
|
|
769
639
|
"${credential_flags[@]}" \
|
|
770
640
|
"${session_db_flags[@]}" \
|
|
771
641
|
"${opencode_config_flags[@]}" \
|
|
@@ -780,19 +650,17 @@ start_tunnel() {
|
|
|
780
650
|
# whole seconds: 25 s in-flight drain (SHUTDOWN_DRAIN_TIMEOUT_MS,
|
|
781
651
|
# apps/cli/src/commands/run.ts) + 2 s offline POST (notifyAgentDisconnected,
|
|
782
652
|
# apps/cli/src/commands/agent-lookup.ts) + 5 s telemetry flush
|
|
783
|
-
# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts)
|
|
784
|
-
#
|
|
785
|
-
#
|
|
786
|
-
#
|
|
653
|
+
# (TELEMETRY_SHUTDOWN_TIMEOUT_MS, run.ts) + 8.5 s pre-drain credential flush
|
|
654
|
+
# + 8.5 s post-drain credential flush. Each phase is bounded there, so this is
|
|
655
|
+
# a ceiling rather than a typical cost — an idle suspend finishes in a couple
|
|
656
|
+
# of seconds. This is the ONE place the hand-maintained budget is written down;
|
|
657
|
+
# the CLI points back here when its bounds change.
|
|
787
658
|
#
|
|
788
|
-
# DOCUMENTATION ONLY — nothing is derived from this
|
|
789
|
-
#
|
|
790
|
-
#
|
|
791
|
-
# backstop chosen on its own merits and the two numbers are unrelated. Nothing
|
|
792
|
-
# cross-checks the 32 against apps/cli either: it is a hand-maintained sum of the
|
|
793
|
-
# three bounds cited above, so if one of them moves, update it here.
|
|
659
|
+
# DOCUMENTATION ONLY — nothing is derived from this value. It is the hand-maintained
|
|
660
|
+
# sum of the five bounded phases above (25 + 2 + 5 + 8.5 + 8.5); update it here if
|
|
661
|
+
# any of those bounds changes.
|
|
794
662
|
# shellcheck disable=SC2034 # documentation; deliberately read by nothing
|
|
795
|
-
CLI_SHUTDOWN_CEILING_SECONDS=
|
|
663
|
+
CLI_SHUTDOWN_CEILING_SECONDS=49
|
|
796
664
|
|
|
797
665
|
# How long stop_tunnel waits for that shutdown before the SIGKILL backstop.
|
|
798
666
|
# Since #718 this binds ONLY for a CLI that is still draining: one that has
|
|
@@ -868,6 +736,85 @@ stop_tunnel() {
|
|
|
868
736
|
log "tunnel stopped ${outcome}"
|
|
869
737
|
}
|
|
870
738
|
|
|
739
|
+
# The CLI owns the interval loop and its boundary flush. Two seconds of slack
|
|
740
|
+
# over the CLI's 8s flush deadline keeps this marker handshake inside the 55s
|
|
741
|
+
# hook ceiling while leaving the fallback flush and tunnel stop budget intact.
|
|
742
|
+
#
|
|
743
|
+
# This bounds only the CLI's pre-drain flush, which always runs first and
|
|
744
|
+
# unconditionally (run.ts's cleanup(), before the channel-work drain) — not
|
|
745
|
+
# the CLI's post-drain second pass, which can take up to
|
|
746
|
+
# SHUTDOWN_DRAIN_TIMEOUT_MS longer than this wait covers. A drain that
|
|
747
|
+
# consumes the whole window loses the SECOND pass, not the credential
|
|
748
|
+
# guarantee itself: the pre-drain flush already persisted everything on disk
|
|
749
|
+
# at signal time, exactly what the old bash `sync_credentials` guaranteed in
|
|
750
|
+
# one synchronous call — so the worst case here is no worse than before this
|
|
751
|
+
# handshake existed, never a fresh data-loss window. See
|
|
752
|
+
# docs/decisions/0063-microvm-boot-orchestration-in-cli.md's two-phase-flush
|
|
753
|
+
# section for the full reasoning.
|
|
754
|
+
CREDENTIAL_FLUSH_WAIT_SECONDS="${EVIDENT_CREDENTIAL_FLUSH_WAIT_SECONDS:-10}"
|
|
755
|
+
|
|
756
|
+
# Remove the previous answer, signal the same CLI that stop_tunnel handles, and
|
|
757
|
+
# wait for either its marker or its death. A fallback sync runs only after the
|
|
758
|
+
# CLI is known to be absent, never alongside a live CLI that may still write.
|
|
759
|
+
stop_runner_and_flush_credentials() {
|
|
760
|
+
rm -f "${CREDENTIAL_FLUSH_MARKER_FILE}"
|
|
761
|
+
|
|
762
|
+
if ! tunnel_is_running; then
|
|
763
|
+
warn "CREDS-FLUSH-NO-RUNNER: no live CLI at handshake entry"
|
|
764
|
+
stop_tunnel
|
|
765
|
+
sync_credentials
|
|
766
|
+
return 0
|
|
767
|
+
fi
|
|
768
|
+
|
|
769
|
+
local pid
|
|
770
|
+
pid="$(cat "${TUNNEL_PID_FILE}")"
|
|
771
|
+
kill -TERM "${pid}" 2>/dev/null || true
|
|
772
|
+
|
|
773
|
+
local waited_ms=0 marker_found=false runner_exited=false
|
|
774
|
+
while [ "${waited_ms}" -lt $((CREDENTIAL_FLUSH_WAIT_SECONDS * 1000)) ]; do
|
|
775
|
+
if [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
|
|
776
|
+
marker_found=true
|
|
777
|
+
break
|
|
778
|
+
fi
|
|
779
|
+
if ! process_is_alive "${pid}"; then
|
|
780
|
+
runner_exited=true
|
|
781
|
+
break
|
|
782
|
+
fi
|
|
783
|
+
sleep 0.1
|
|
784
|
+
waited_ms=$((waited_ms + 100))
|
|
785
|
+
done
|
|
786
|
+
|
|
787
|
+
# The CLI can publish the marker and exit inside one poll tick. A final
|
|
788
|
+
# marker check after a death break preserves that answer instead of falling
|
|
789
|
+
# through to the fallback.
|
|
790
|
+
if [ "${runner_exited}" = true ] && [ -s "${CREDENTIAL_FLUSH_MARKER_FILE}" ]; then
|
|
791
|
+
marker_found=true
|
|
792
|
+
runner_exited=false
|
|
793
|
+
fi
|
|
794
|
+
|
|
795
|
+
if [ "${marker_found}" = true ]; then
|
|
796
|
+
local failures
|
|
797
|
+
failures="$(grep -Ev '^(claude|opencode)=ok$' "${CREDENTIAL_FLUSH_MARKER_FILE}" || true)"
|
|
798
|
+
if [ -n "${failures}" ]; then
|
|
799
|
+
warn "CREDS-FLUSH-HAD-FAILURES: ${failures//$'\n'/ }"
|
|
800
|
+
else
|
|
801
|
+
log "CREDS-FLUSH-OK"
|
|
802
|
+
fi
|
|
803
|
+
stop_tunnel
|
|
804
|
+
return 0
|
|
805
|
+
fi
|
|
806
|
+
|
|
807
|
+
if [ "${runner_exited}" = true ]; then
|
|
808
|
+
warn "CREDS-FLUSH-RUNNER-EXITED: CLI exited without a completion marker"
|
|
809
|
+
stop_tunnel
|
|
810
|
+
sync_credentials
|
|
811
|
+
return 0
|
|
812
|
+
fi
|
|
813
|
+
|
|
814
|
+
warn "CREDS-FLUSH-TIMEOUT: live CLI did not write a completion marker within ${CREDENTIAL_FLUSH_WAIT_SECONDS}s"
|
|
815
|
+
stop_tunnel
|
|
816
|
+
}
|
|
817
|
+
|
|
871
818
|
# --- check_runner_key (#1172) ------------------------------------------------
|
|
872
819
|
#
|
|
873
820
|
# Answers exactly one question before opencode/the tunnel start spending this
|
|
@@ -2,10 +2,8 @@
|
|
|
2
2
|
#
|
|
3
3
|
# Re-dial with the identity /run left behind. There is no step that could fetch
|
|
4
4
|
# a fresh runner key, so the one from /run is what resumes. The CLI delegated by
|
|
5
|
-
# start_tunnel owns the session-DB replicator's
|
|
6
|
-
#
|
|
7
|
-
# a resumed VM that never restarted it here would never sync credentials again
|
|
8
|
-
# for the rest of its life.
|
|
5
|
+
# start_tunnel owns the credential loop and the session-DB replicator's
|
|
6
|
+
# post-resume start.
|
|
9
7
|
set -euo pipefail
|
|
10
8
|
|
|
11
9
|
# shellcheck source=./common.sh
|
|
@@ -39,12 +37,18 @@ fi
|
|
|
39
37
|
read -r runner_key
|
|
40
38
|
} <"${CONTEXT_FILE}"
|
|
41
39
|
|
|
40
|
+
# Restore the id so the resumed CLI can self-report and acknowledge any
|
|
41
|
+
# outstanding recycle request (#1906). Older images may not have this file.
|
|
42
|
+
if [ -s "${MICROVM_ID_FILE}" ]; then
|
|
43
|
+
MICROVM_ID="$(cat "${MICROVM_ID_FILE}")"
|
|
44
|
+
export MICROVM_ID
|
|
45
|
+
fi
|
|
46
|
+
|
|
42
47
|
# Tolerant, never fatal: a resume that fails costs the user their whole
|
|
43
48
|
# session, so a broken durable-state config degrades to "no session-DB
|
|
44
49
|
# replication" rather than a failed resume. The CLI's own guards handle the
|
|
45
50
|
# rest — this needs no logic of its own.
|
|
46
51
|
load_state_config || warn "could not resolve durable-state config; the session DB will not resume replicating"
|
|
47
|
-
start_credential_sync
|
|
48
52
|
|
|
49
53
|
# Diagnostic-only, unlike /run's gate: a failed resume costs the user their
|
|
50
54
|
# whole session, so this never exits — it only converts a silent "resumed but
|
|
@@ -28,7 +28,6 @@ cleanup() {
|
|
|
28
28
|
stop_tunnel || warn "stop_tunnel failed while cleaning up"
|
|
29
29
|
stop_opencode || warn "stop_opencode failed while cleaning up"
|
|
30
30
|
kill_litestream || warn "kill_litestream failed while cleaning up"
|
|
31
|
-
stop_credential_sync || warn "stop_credential_sync failed while cleaning up"
|
|
32
31
|
}
|
|
33
32
|
trap cleanup EXIT
|
|
34
33
|
|
|
@@ -73,9 +72,12 @@ load_state_config || exit 1
|
|
|
73
72
|
# it warns and this VM still boots.
|
|
74
73
|
check_runner_key "${runner_key}" "${api_url}" || exit 1
|
|
75
74
|
|
|
76
|
-
# 6 — the
|
|
77
|
-
#
|
|
78
|
-
|
|
75
|
+
# 6 — preserve the VM id across suspend/resume. The runtime always supplies it
|
|
76
|
+
# for a /run hook, and the subshell keeps the persisted value protected.
|
|
77
|
+
(
|
|
78
|
+
umask 077
|
|
79
|
+
printf '%s\n' "${MICROVM_ID}" >"${MICROVM_ID_FILE}"
|
|
80
|
+
)
|
|
79
81
|
|
|
80
82
|
# 7 — the first per-VM identity on the wire. The subshell's umask makes the file
|
|
81
83
|
# unreadable to anyone else from the moment it exists, before the key is in it.
|
|
@@ -8,11 +8,10 @@ set -euo pipefail
|
|
|
8
8
|
# shellcheck source=./common.sh
|
|
9
9
|
source "$(dirname "$0")/common.sh"
|
|
10
10
|
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
stop_tunnel
|
|
11
|
+
# load_state_config exports PERSISTENCE_BUCKET, which flush_session_db below
|
|
12
|
+
# requires; the CLI flush completes, or is proven absent, before teardown continues.
|
|
13
|
+
load_state_config || warn "could not resolve durable-state config; neither the credential flush nor the session-DB flush below can run"
|
|
14
|
+
stop_runner_and_flush_credentials
|
|
16
15
|
|
|
17
16
|
# opencode is deliberately NOT stopped here (#812 WI-4) — it is inside the
|
|
18
17
|
# snapshot and must resume with it — so a turn still in flight may not be
|
|
@@ -8,11 +8,10 @@ set -uo pipefail
|
|
|
8
8
|
# shellcheck source=./common.sh
|
|
9
9
|
source "$(dirname "$0")/common.sh"
|
|
10
10
|
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
stop_tunnel
|
|
11
|
+
# load_state_config exports PERSISTENCE_BUCKET, which flush_session_db below
|
|
12
|
+
# requires; the CLI flush completes, or is proven absent, before teardown continues.
|
|
13
|
+
load_state_config || warn "could not resolve durable-state config; neither the credential flush nor the session-DB flush below can run"
|
|
14
|
+
stop_runner_and_flush_credentials
|
|
16
15
|
|
|
17
16
|
# Writers stopped BEFORE litestream's final sync (#812 WI-4, the
|
|
18
17
|
# ordered-shutdown invariant, plan §6): otherwise litestream could snapshot
|
|
@@ -32,6 +31,6 @@ flush_session_db
|
|
|
32
31
|
# this hook runs under `set -u` (above), so an unbound one aborts the script
|
|
33
32
|
# AT THIS LINE, leaving the runner key sitting in CONTEXT_FILE on a VM that is
|
|
34
33
|
# being torn down. That is the one thing this line exists to prevent.
|
|
35
|
-
rm -f "${CONTEXT_FILE}" "${SESSION_DB_NO_REPLICATE_MARKER}"
|
|
34
|
+
rm -f "${CONTEXT_FILE}" "${SESSION_DB_NO_REPLICATE_MARKER}" "${CREDENTIAL_FLUSH_MARKER_FILE}"
|
|
36
35
|
|
|
37
36
|
exit 0
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evident-ai/runner-cdk",
|
|
3
|
-
"version": "3.4.1-dev.
|
|
3
|
+
"version": "3.4.1-dev.f80aa1c",
|
|
4
4
|
"description": "Reusable CDK constructs for an Evident agent runner: a single scale-to-zero Fargate runner (task + service + per-agent self-stop role + waker Lambda), or a per-session AWS Lambda MicroVM that boots on demand and suspends between messages. Instantiate once per agent from your own stack.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|