@elizaos/app-core 2.0.0-beta.3 → 2.0.3-beta.3

package/scripts/docker-ci-smoke.sh

@@ -7,29 +7,109 @@ set -Eeuo pipefail
 #   1. Installs deps with bun using the committed lockfile
 #   2. Builds required runtime/UI artifacts for Dockerfile.ci
 #   3. Builds the production image locally
-#   4. Optionally boots the container and probes /api/health or /api/status
+#   4. Boots the container running the REAL agent entrypoint and probes
+#      /api/health or /api/status, failing if the agent crashes on startup
+#
+# The boot probe runs the image's default command (APP_CMD_START =
+# the absolute tsx loader plus `packages/agent/dist/bin.js start`), i.e. the
+# actual production entrypoint. It does NOT substitute a stub health server, so a
+# broken runtime dependency closure (missing zod / @elizaos/core / chalk /
+# etc.) surfaces as a RED build instead of shipping a container that crashes
+# the moment it boots in production.
 #
 # Usage:
 #   bash packages/app-core/scripts/docker-ci-smoke.sh [--tag TAG] [--version VERSION] [--skip-smoke]
+#   # Verify an already-built/pulled image without rebuilding:
+#   DOCKER_IMAGE=ghcr.io/... bash packages/app-core/scripts/docker-ci-smoke.sh --boot-verify-only
 #
 # Environment:
 #   BUN_VERSION          Bun version to install/use in CI (default: 1.3.9)
 #   SMOKE_PORT           Host port to bind for smoke boot (default: 32138)
 #   SMOKE_TIMEOUT_SEC    Max wait for boot probe (default: 420)
 #   DOCKER_IMAGE         Override image tag completely
+#   BOOT_VERIFY_ONLY     If "true", skip the build and only boot-verify
+#                        DOCKER_IMAGE (must be set). Equivalent to
+#                        --boot-verify-only.
+#   SMOKE_CHARACTER_JSON Optional minimal character JSON injected via
+#                        ELIZA_AGENT_CHARACTER_JSON for the boot.
+#   BOOT_KPI_ENFORCE     If "1", a cold-start readyMs budget breach (measured
+#                        boot readyMs > boot.coldReadyMs from
+#                        packages/benchmarks/loadperf/budgets.json) FAILS the
+#                        smoke. Default (unset/!=1) is WARN-FIRST: a breach is
+#                        reported via a ::warning:: line but does NOT fail. Flip
+#                        this to "1" in the workflow once the baseline is
+#                        confirmed green in CI. The KPI measurement is additive
+#                        and only runs in --boot-verify-only mode; a measurement
+#                        failure (missing jq/budget) never fails a smoke that
+#                        otherwise passed.
 
 BUN_VERSION="${BUN_VERSION:-1.3.10}"
 SMOKE_PORT="${SMOKE_PORT:-32138}"
 CONTAINER_PORT="${CONTAINER_PORT:-42138}"
 SMOKE_TIMEOUT_SEC="${SMOKE_TIMEOUT_SEC:-420}"
 SKIP_SMOKE=false
+BOOT_VERIFY_ONLY="${BOOT_VERIFY_ONLY:-false}"
 TAG="docker-smoke"
 VERSION=""
 
+# Log signatures that mean the agent crashed during startup. If any of these
+# show up in the container logs we fail immediately rather than waiting out the
+# full timeout. These are the exact failure modes (missing runtime deps,
+# entrypoint blowups) that previously shipped green because the smoke step
+# booted a stub health server instead of the real entrypoint.
+#
+# Kept deliberately specific to module-resolution failures and explicit
+# fatal-startup markers so a benign warning that merely contains a generic word
+# (FATAL, Cannot read properties of undefined, ...) does not flap the gate. The
+# container-exit and health-timeout checks below are the backstops for any crash
+# that does not print one of these.
+BOOT_CRASH_PATTERNS=(
+  'Cannot find package'
+  'Cannot find module'
+  'ERR_MODULE_NOT_FOUND'
+  'ERR_REQUIRE_ESM'
+  'MODULE_NOT_FOUND'
+  # Defense-in-depth: the agent boots under the tsx loader (see APP_CMD_START),
+  # so .ts extensions resolve at runtime today. But if a workspace package ever
+  # shipped without a built dist, link-docker-local-app-packages.mjs rewrites its
+  # exports to ./src/*.ts, and importing that under a tsx-less or mislinked boot
+  # would throw ERR_UNKNOWN_FILE_EXTENSION on the core boot path. Fail fast on that
+  # signature instead of waiting out the full health timeout.
+  'ERR_UNKNOWN_FILE_EXTENSION'
+  'Unknown file extension'
+  'Failed to start'
+  'crashed during init'
+)
+
 log() {
   printf '[docker-ci-smoke] %s\n' "$*"
 }
 
+# Portable millisecond epoch timer used by the boot-KPI cold-start measurement.
+# Prefers bash 5's EPOCHREALTIME (no subprocess), falls back to GNU date
+# (%s%3N), then python3, then whole-second `date` * 1000. Always prints an
+# integer count of milliseconds. Never used for liveness decisions, only for
+# the additive boot-KPI readyMs value, so a coarse fallback is harmless.
+now_ms() {
+  if [[ -n "${EPOCHREALTIME:-}" ]]; then
+    # EPOCHREALTIME is "<seconds>.<microseconds>"; strip the dot and trim to ms.
+    local er="${EPOCHREALTIME/[.,]/}"
+    printf '%s\n' "${er:0:${#er}-3}"
+    return 0
+  fi
+  local d
+  d="$(date +%s%3N 2>/dev/null || true)"
+  if [[ "$d" =~ ^[0-9]+$ && "$d" != *N ]]; then
+    printf '%s\n' "$d"
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    python3 -c 'import time; print(int(time.time()*1000))'
+    return 0
+  fi
+  printf '%s\n' "$(( $(date +%s) * 1000 ))"
+}
+
 on_error() {
   local status=$?
   local line="${BASH_LINENO[0]:-0}"
@@ -86,6 +166,10 @@ while [[ $# -gt 0 ]]; do
       SKIP_SMOKE=true
       shift
       ;;
+    --boot-verify-only)
+      BOOT_VERIFY_ONLY=true
+      shift
+      ;;
     -h|--help)
       sed -n '1,24p' "$0" | sed 's/^# \{0,1\}//'
       exit 0
@@ -140,8 +224,8 @@ load_env_file "$APP_CORE_DIR/deploy/deploy.defaults.env"
 load_env_file "deploy/deploy.env"
 
 APP_IMAGE="${APP_IMAGE:-eliza/agent}"
-APP_ENTRYPOINT="${APP_ENTRYPOINT:-$AGENT_DIR/dist/packages/agent/src/bin.js}"
-APP_CMD_START="${APP_CMD_START:-node --import tsx ${APP_ENTRYPOINT} start}"
+APP_ENTRYPOINT="${APP_ENTRYPOINT:-$AGENT_DIR/dist/bin.js}"
+APP_CMD_START="${APP_CMD_START:-node --import /opt/tsx/node_modules/tsx/dist/loader.mjs ${APP_ENTRYPOINT} start}"
 APP_PORT="${APP_PORT:-2138}"
 APP_API_BIND="${APP_API_BIND:-127.0.0.1}"
 OCI_SOURCE="${OCI_SOURCE:-}"
@@ -166,9 +250,13 @@ log "Smoke port: $SMOKE_PORT"
 log "Container port override: $CONTAINER_PORT"
 log "Artifact dir: $SMOKE_ARTIFACT_DIR"
 
-command -v node >/dev/null 2>&1 || fail "node is required"
-command -v bun >/dev/null 2>&1 || fail "bun is required"
-BUN_BIN="$(command -v bun)"
+# bun/node are only needed for the build path. In boot-verify-only mode we
+# just run an already-built image, so don't hard-require them.
+if [[ "$BOOT_VERIFY_ONLY" != "true" ]]; then
+  command -v node >/dev/null 2>&1 || fail "node is required"
+  command -v bun >/dev/null 2>&1 || fail "bun is required"
+  BUN_BIN="$(command -v bun)"
+fi
 
 DOCKER_BIN="$(find_docker_bin)" || fail "docker is required"
 
@@ -204,6 +292,331 @@ cleanup() {
 }
 trap cleanup EXIT
 
+# boot_verify boots the just-built (or pre-supplied) image running the REAL
+# agent entrypoint and fails the script if the agent crashes on startup or
+# never serves health. This is the gate that stops a container that crashes on
+# boot (missing runtime deps, broken entrypoint) from shipping green.
+boot_verify() {
+  log "Starting container boot verification (real agent entrypoint)"
+  log "Command under test: ${APP_CMD_START}"
+
+  # A tiny, valid character so the agent has a concrete identity to boot with.
+  # The runtime falls back to a bundled default if this is unset, but pinning
+  # one keeps the boot deterministic across branches.
+  local character_json
+  character_json="${SMOKE_CHARACTER_JSON:-{\"name\":\"CiSmoke\",\"bio\":[\"CI boot verification agent.\"],\"system\":\"You are a CI boot probe.\"}}"
+
+  # Unique pglite data dir per run avoids the data-dir lock that makes a
+  # second container on the same host fail to acquire the store.
+  local pglite_dir="/tmp/ci-db-${RANDOM}-${RANDOM}"
+
+  "$DOCKER_BIN" rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
+  # Boot-KPI cold-start: stamp the instant just before the container starts.
+  # The elapsed time from here to the FIRST /api/health (or /api/status) ready
+  # is the cold readyMs we compare against boot.coldReadyMs. Only meaningful in
+  # --boot-verify-only mode (the workflow's gate); see emit_boot_kpi below.
+  BOOT_KPI_START_MS=""
+  BOOT_KPI_READY_MS=""
+  if [[ "$BOOT_VERIFY_ONLY" == "true" ]]; then
+    BOOT_KPI_START_MS="$(now_ms 2>/dev/null || true)"
+  fi
+  # NOTE: no command override here. The container runs its default CMD
+  # (APP_CMD_START = the real absolute tsx loader plus agent dist start).
+  # Substituting a stub health server is exactly what let broken
+  # images ship green before; do not reintroduce it.
+  "$DOCKER_BIN" run -d \
+    --name "$CONTAINER_NAME" \
+    -e PORT="$CONTAINER_PORT" \
+    -e APP_PORT="$CONTAINER_PORT" \
+    -e ELIZA_PORT="$CONTAINER_PORT" \
+    -e ELIZA_API_PORT="$CONTAINER_PORT" \
+    -e ELIZA_AGENT_PORT="$CONTAINER_PORT" \
+    -e APP_API_BIND=0.0.0.0 \
+    -e ELIZA_API_BIND=0.0.0.0 \
+    -e AGENT_API_BIND=0.0.0.0 \
+    -e ELIZA_AGENT_CHARACTER_JSON="$character_json" \
+    -e ELIZA_STATE_DIR=/tmp/eliza-smoke/state \
+    -e ELIZA_CONFIG_DIR=/tmp/eliza-smoke/config \
+    -e ELIZA_WORKSPACE_DIR=/tmp/eliza-smoke/workspace \
+    -e ELIZA_VAULT_PASSPHRASE=docker-smoke-vault-passphrase \
+    -e PGLITE_DATA_DIR="$pglite_dir" \
+    -e ELIZA_DISABLE_LOCAL_EMBEDDINGS=1 \
+    -p "${SMOKE_PORT}:${CONTAINER_PORT}" \
+    "$DOCKER_IMAGE" >/dev/null
+
+  local status_url="http://127.0.0.1:${SMOKE_PORT}/api/status"
+  local health_url="http://127.0.0.1:${SMOKE_PORT}/api/health"
+
+  # Boot-KPI: record the FIRST instant any probe reports ready. Called from the
+  # 200/401 success paths below so the readyMs reflects the very first health
+  # success, not the end of confirm_ok's confirmation re-probes. Stamps once and
+  # only in --boot-verify-only mode; never affects the liveness return value.
+  mark_first_ready() {
+    if [[ "$BOOT_VERIFY_ONLY" == "true" && -z "${BOOT_KPI_READY_MS:-}" ]]; then
+      BOOT_KPI_READY_MS="$(now_ms 2>/dev/null || true)"
+    fi
+  }
+
+  probe_ok() {
+    local url="$1"
+    local out="$2"
+    local code
+    code="$(curl -sS --connect-timeout 1 --max-time 3 -o "$out" -w '%{http_code}' "$url" || true)"
+    case "$code" in
+      200)
+        mark_first_ready
+        return 0
+        ;;
+      401)
+        if grep -q 'Unauthorized' "$out" 2>/dev/null; then
+          mark_first_ready
+          return 0
+        fi
+        ;;
+    esac
+    return 1
+  }
+
+  # Scan container logs for known crash signatures. Returns 0 (match) when the
+  # agent has crashed on startup so we can fail fast with a clear message
+  # instead of waiting out the whole timeout.
+  scan_crash() {
+    local logs_file="$1"
+    local pattern
+    for pattern in "${BOOT_CRASH_PATTERNS[@]}"; do
+      if grep -qiF "$pattern" "$logs_file" 2>/dev/null; then
+        printf '%s' "$pattern"
+        return 0
+      fi
+    done
+    return 1
+  }
+
+  # Confirm health by probing a few times in a row before declaring success,
+  # so a single transient curl failure does not flap the gate.
+  confirm_ok() {
+    local url="$1"
+    local out="$2"
+    local i
+    for i in 1 2 3; do
+      probe_ok "$url" "$out" || return 1
+      [[ "$i" -lt 3 ]] && sleep 1
+    done
+    return 0
+  }
+
+  # Post-health observation window: after health first comes up, keep watching
+  # the container for a short window so a crash that happens just AFTER the
+  # server binds (async plugin/runtime init blowing up post-listen) still turns
+  # the gate RED instead of slipping through. Only scans logs emitted AFTER the
+  # passed-in checkpoint ("$1", an RFC3339 UTC timestamp captured when health
+  # first succeeded) so a benign pre-health crash signature can't false-positive.
+  # Returns non-zero if the container dies or logs a crash signature in-window.
+  confirm_stable() {
+    local since="$1"
+    local window="${BOOT_STABLE_WINDOW_SEC:-15}"
+    local stable_logs="$SMOKE_ARTIFACT_DIR/post-health.log"
+    local stable_deadline=$((SECONDS + window))
+    while (( SECONDS < stable_deadline )); do
+      timeout 30 "$DOCKER_BIN" logs --since "$since" "$CONTAINER_NAME" >"$stable_logs" 2>&1 || true
+      local m
+      if m="$(scan_crash "$stable_logs")"; then
+        log "Crash signature appeared after health came up: '${m}'"
+        timeout 30 "$DOCKER_BIN" logs --tail 120 "$CONTAINER_NAME" || true
+        return 1
+      fi
+      if ! timeout 10 "$DOCKER_BIN" ps --format '{{.Names}}' 2>/dev/null | grep -Fxq "$CONTAINER_NAME"; then
+        local code
+        code="$(timeout 10 "$DOCKER_BIN" inspect -f '{{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo '?')"
+        log "Container exited (exit code ${code}) during the post-health window"
+        timeout 30 "$DOCKER_BIN" logs --tail 120 "$CONTAINER_NAME" || true
+        return 1
+      fi
+      sleep 3
+    done
+    return 0
+  }
+
+  # Boot-KPI cold-start budget (item 5 of #8812). Computes cold readyMs from the
+  # pre-`docker run` checkpoint to the FIRST health/status success, reads the
+  # boot.coldReadyMs budget from packages/benchmarks/loadperf/budgets.json, and
+  # reports the comparison. WARN-FIRST: a breach prints a ::warning:: line and
+  # returns 0 (does not fail) UNLESS BOOT_KPI_ENFORCE=1, in which case it returns
+  # 1 so the caller can fail the smoke. Every measurement step is guarded so a
+  # missing jq / budget / timer NEVER turns a liveness-passing boot RED.
+  # peakRss is intentionally NOT measured: `docker stats` reports an inst
+  # sample, not the boot peak, so a number here would be misleading.
+  emit_boot_kpi() {
+    if [[ "$BOOT_VERIFY_ONLY" != "true" ]]; then
+      return 0
+    fi
+    if [[ -z "${BOOT_KPI_START_MS:-}" || -z "${BOOT_KPI_READY_MS:-}" ]]; then
+      log "[boot-kpi] cold readyMs: not measured (missing start/ready timestamp)"
+      return 0
+    fi
+
+    local ready_ms
+    ready_ms=$(( BOOT_KPI_READY_MS - BOOT_KPI_START_MS ))
+    if (( ready_ms < 0 )); then
+      log "[boot-kpi] cold readyMs: not measured (non-monotonic clock)"
+      return 0
+    fi
+
+    # Default budget mirrors packages/benchmarks/loadperf/budgets.json boot.coldReadyMs.
+    local budget_default=25000
+    local budget="$budget_default"
+    local budgets_file="$REPO_ROOT/packages/benchmarks/loadperf/budgets.json"
+    if command -v jq >/dev/null 2>&1 && [[ -f "$budgets_file" ]]; then
+      local parsed
+      parsed="$(jq -r '.boot.coldReadyMs // empty' "$budgets_file" 2>/dev/null || true)"
+      if [[ "$parsed" =~ ^[0-9]+$ ]]; then
+        budget="$parsed"
+      else
+        log "[boot-kpi] WARNING: could not read boot.coldReadyMs from $budgets_file; using default ${budget_default}ms"
+      fi
+    else
+      log "[boot-kpi] WARNING: jq or budgets.json unavailable; using default coldReadyMs budget ${budget_default}ms"
+    fi
+
+    # Record runner contention alongside readyMs (#8812 item 5). Boot is
+    # single-threaded and import-bound, so a heavily loaded runner inflates
+    # readyMs; we log loadavg/cpu so a breach can be triaged, and (below) refuse
+    # to FAIL on a breach when the runner is clearly overloaded, so enforcement
+    # only catches a genuine boot-time regression — not transient runner load.
+    local load1="" cpus="" contended="false"
+    if [[ -r /proc/loadavg ]]; then
+      load1="$(cut -d' ' -f1 /proc/loadavg 2>/dev/null || true)"
+    fi
+    cpus="$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)"
+    log "[boot-kpi] runner loadavg(1m)=${load1:-?} cpus=${cpus}"
+    # Heavy contention = loadavg(1m) above 2x the cpu count. Float compare via
+    # awk (no bc dependency); guarded so a missing loadavg never trips it.
+    if [[ -n "$load1" && "$cpus" =~ ^[0-9]+$ ]]; then
+      if awk -v l="$load1" -v c="$cpus" 'BEGIN{exit !(l > 2*c)}' 2>/dev/null; then
+        contended="true"
+      fi
+    fi
+
+    log "[boot-kpi] cold readyMs=${ready_ms} budget=${budget}"
+    log "[boot-kpi] peakRss: not measured (docker stats samples instantaneously, not boot peak)"
+
+    if (( ready_ms > budget )); then
+      local msg="boot-kpi cold readyMs=${ready_ms} exceeded budget=${budget} (boot.coldReadyMs)"
+      if [[ "${BOOT_KPI_ENFORCE:-}" == "1" && "$contended" == "true" ]]; then
+        if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+          printf '::warning::%s — runner contended (loadavg %s over %s cpus), not failing\n' "$msg" "$load1" "$cpus" >&2
+        fi
+        log "[boot-kpi] WARNING (runner contended, loadavg ${load1} over ${cpus} cpus — not failing): $msg"
+        return 0
+      fi
+      if [[ "${BOOT_KPI_ENFORCE:-}" == "1" ]]; then
+        if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+          printf '::error::%s\n' "$msg" >&2
+        fi
+        log "[boot-kpi] ENFORCING: $msg"
+        return 1
+      fi
+      if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+        printf '::warning::%s (warn-first; set BOOT_KPI_ENFORCE=1 to gate)\n' "$msg" >&2
+      fi
+      log "[boot-kpi] WARNING (warn-first, not failing): $msg"
+      log "[boot-kpi] Set BOOT_KPI_ENFORCE=1 to make this breach fail CI."
+      return 0
+    fi
+
+    log "[boot-kpi] within budget (readyMs ${ready_ms} <= ${budget})"
+    return 0
+  }
+
+  local logs_file="$SMOKE_ARTIFACT_DIR/boot.log"
+  local deadline=$((SECONDS + SMOKE_TIMEOUT_SEC))
+  local last_log_dump=0
+  while (( SECONDS < deadline )); do
+    timeout 30 "$DOCKER_BIN" logs "$CONTAINER_NAME" >"$logs_file" 2>&1 || true
+
+    # Fail fast on a known crash signature in the logs.
+    local matched
+    if matched="$(scan_crash "$logs_file")"; then
+      log "Detected startup crash signature in container logs: '${matched}'"
+      timeout 30 "$DOCKER_BIN" logs --tail 120 "$CONTAINER_NAME" || true
+      log "Preserved failure artifacts in $SMOKE_ARTIFACT_DIR"
+      fail "Agent crashed on startup (matched '${matched}'); image is NOT bootable"
+    fi
+
+    local running_containers_file
+    running_containers_file="$(mktemp 2>/dev/null || printf '%s\n' "$SMOKE_ARTIFACT_DIR/docker-running-containers.txt")"
+    timeout 10 "$DOCKER_BIN" ps --format '{{.Names}}' >"$running_containers_file" 2>&1 || true
+    if ! grep -Fxq "$CONTAINER_NAME" "$running_containers_file" 2>/dev/null; then
+      rm -f "$running_containers_file" >/dev/null 2>&1 || true
+      local exit_code
+      exit_code="$(timeout 10 "$DOCKER_BIN" inspect -f '{{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo '?')"
+      timeout 30 "$DOCKER_BIN" logs --tail 120 "$CONTAINER_NAME" || true
+      log "Preserved failure artifacts in $SMOKE_ARTIFACT_DIR"
+      fail "Container exited (exit code ${exit_code}) before health came up; image is NOT bootable"
+    fi
+    rm -f "$running_containers_file" >/dev/null 2>&1 || true
+
+    if (( SECONDS - last_log_dump >= 30 )); then
+      last_log_dump=$SECONDS
+      log "Container still booting; recent logs follow"
+      timeout 10 "$DOCKER_BIN" logs --tail 80 "$CONTAINER_NAME" || true
+    fi
+
+    if confirm_ok "$health_url" /tmp/eliza-docker-health.txt; then
+      # Boot-KPI readyMs was already stamped by probe_ok (mark_first_ready) at
+      # the FIRST health success, BEFORE the post-health stability window.
+      log "Health probe succeeded against the real entrypoint: $health_url"
+      cat /tmp/eliza-docker-health.txt
+      local health_ts
+      health_ts="$(date -u +%Y-%m-%dT%H:%M:%S 2>/dev/null || echo '')"
+      if confirm_stable "$health_ts"; then
+        log "Boot verified: agent stayed up after health came up"
+        emit_boot_kpi || fail "boot-kpi cold readyMs budget exceeded (BOOT_KPI_ENFORCE=1)"
+        return 0
+      fi
+      fail "Agent served health then crashed during the post-health window; image is NOT bootable"
+    fi
+
+    if confirm_ok "$status_url" /tmp/eliza-docker-status.txt; then
+      # Boot-KPI readyMs was already stamped by probe_ok (mark_first_ready) at
+      # the FIRST status success.
+      log "Status probe succeeded against the real entrypoint: $status_url"
+      cat /tmp/eliza-docker-status.txt
+      local status_ts
+      status_ts="$(date -u +%Y-%m-%dT%H:%M:%S 2>/dev/null || echo '')"
+      if confirm_stable "$status_ts"; then
+        log "Boot verified: agent stayed up after status came up"
+        emit_boot_kpi || fail "boot-kpi cold readyMs budget exceeded (BOOT_KPI_ENFORCE=1)"
+        return 0
+      fi
+      fail "Agent served status then crashed during the post-health window; image is NOT bootable"
+    fi
+
+    sleep 5
+  done
+
+  timeout 30 "$DOCKER_BIN" logs --tail 120 "$CONTAINER_NAME" || true
+  log "Preserved timeout artifacts in $SMOKE_ARTIFACT_DIR"
+  fail "Timed out waiting for the real agent to serve health (${SMOKE_TIMEOUT_SEC}s); image is NOT bootable"
+}
+
+# Boot-verify-only mode: skip the entire build and just verify a pre-built
+# image (DOCKER_IMAGE must be set/pullable). Used by the build workflow to
+# verify the image it just built locally before pushing.
+if [[ "$BOOT_VERIFY_ONLY" == "true" ]]; then
+  log "Boot-verify-only mode: skipping build, verifying $DOCKER_IMAGE"
+  if ! "$DOCKER_BIN" image inspect "$DOCKER_IMAGE" >/dev/null 2>&1; then
+    log "Image $DOCKER_IMAGE not present locally; attempting pull"
+    "$DOCKER_BIN" pull "$DOCKER_IMAGE" >/dev/null 2>&1 || fail "Image $DOCKER_IMAGE not available locally and pull failed"
+  fi
+  if $SKIP_SMOKE; then
+    log "Skipping runtime boot (--skip-smoke)"
+    exit 0
+  fi
+  boot_verify
+  exit 0
+fi
+
 log "Installing dependencies"
 node "$APP_CORE_SCRIPTS_DIR/init-submodules.mjs"
 ELIZA_SKIP_LOCAL_UPSTREAMS=1 ELIZA_SKIP_LOCAL_UPSTREAMS=1 node "$APP_CORE_SCRIPTS_DIR/disable-local-eliza-workspace.mjs"
@@ -237,33 +650,61 @@ else
 fi
 
 log "Running repository postinstall"
-if [[ -f scripts/setup-upstreams.mjs ]]; then
+if [[ -f packages/scripts/setup-upstreams.mjs ]]; then
   SKIP_AVATAR_CLONE=1 ELIZA_NO_VISION_DEPS=1 node "$APP_CORE_SCRIPTS_DIR/run-repo-setup.mjs"
 else
-  node scripts/patch-nested-core-dist.mjs || true
+  node packages/scripts/patch-nested-core-dist.mjs || true
   node "$APP_CORE_SCRIPTS_DIR/ensure-shared-i18n-data.mjs"
   node "$APP_CORE_SCRIPTS_DIR/patch-deps.mjs" || true
   node "$APP_CORE_SCRIPTS_DIR/ensure-type-package-aliases.mjs" || true
 fi
-node scripts/patch-tsup-dts.mjs || true
+# @elizaos/contracts must be built BEFORE @elizaos/core: core's
+# tsconfig.declarations.json maps `@elizaos/contracts` to
+# `../contracts/dist/index.d.ts`, so the declarations build aborts with
+# TS2307 if dist/ doesn't exist yet.
+if [[ -f packages/contracts/package.json ]] && jq -e '.scripts.build' packages/contracts/package.json >/dev/null; then
+  log "Building @elizaos/contracts (required by core declarations)"
+  pushd packages/contracts >/dev/null
+  "$BUN_BIN" run build
+  popd >/dev/null
+  mkdir -p node_modules/@elizaos
+  rm -rf node_modules/@elizaos/contracts
+  ln -s ../../packages/contracts node_modules/@elizaos/contracts
+fi
+
+# @elizaos/logger must also be built BEFORE @elizaos/core: core's
+# tsconfig.declarations.json maps `@elizaos/logger` to
+# `../logger/dist/index.d.ts`, so the declarations build aborts with TS2307
+# (src/logger.ts) if dist/ doesn't exist yet.
+if [[ -f packages/logger/package.json ]] && jq -e '.scripts.build' packages/logger/package.json >/dev/null; then
+  log "Building @elizaos/logger (required by core declarations)"
+  pushd packages/logger >/dev/null
+  "$BUN_BIN" run build
+  popd >/dev/null
+  mkdir -p node_modules/@elizaos
+  rm -rf node_modules/@elizaos/logger
+  ln -s ../../packages/logger node_modules/@elizaos/logger
+fi
 
 if [[ -f "$TYPESCRIPT_DIR/package.json" ]]; then
   log "Building @elizaos/core source artifacts"
   pushd "$TYPESCRIPT_DIR" >/dev/null
   "$BUN_BIN" run build.ts --node-only
   popd >/dev/null
-  node scripts/prepare-package-dist.mjs "$TYPESCRIPT_DIR"
+  node packages/scripts/prepare-package-dist.mjs "$TYPESCRIPT_DIR"
   CORE_NODE_MODULE="node_modules/@elizaos/core"
   rm -rf "$CORE_NODE_MODULE"
   mkdir -p "$(dirname "$CORE_NODE_MODULE")"
   ln -s "../../$TYPESCRIPT_DIR" "$CORE_NODE_MODULE"
-  node scripts/patch-nested-core-dist.mjs || true
+  node packages/scripts/patch-nested-core-dist.mjs || true
 else
   log "No local @elizaos/core source package found at $TYPESCRIPT_DIR; using installed package"
 fi
 
 log "Building shared/cloud package artifacts"
-for package_dir in packages/shared cloud/packages/sdk packages/cloud-routing; do
+# plugin-streaming imports @elizaos/cloud-routing during its declarations build,
+# so Docker smoke needs the local workspace package built and linked.
+for package_dir in packages/shared packages/cloud-sdk packages/cloud-routing packages/skills; do
   if [[ -f "$package_dir/package.json" ]] && jq -e '.scripts.build' "$package_dir/package.json" >/dev/null; then
     log "Building $(node -p "require('./$package_dir/package.json').name") workspace artifacts"
     pushd "$package_dir" >/dev/null
@@ -272,10 +713,11 @@ for package_dir in packages/shared cloud/packages/sdk packages/cloud-routing; do
   fi
 done
 mkdir -p node_modules/@elizaos
-rm -rf node_modules/@elizaos/shared node_modules/@elizaos/cloud-sdk node_modules/@elizaos/cloud-routing
+rm -rf node_modules/@elizaos/shared node_modules/@elizaos/cloud-sdk node_modules/@elizaos/cloud-routing node_modules/@elizaos/skills
 ln -s ../../packages/shared node_modules/@elizaos/shared
-ln -s ../../cloud/packages/sdk node_modules/@elizaos/cloud-sdk
+ln -s ../../packages/cloud-sdk node_modules/@elizaos/cloud-sdk
 ln -s ../../packages/cloud-routing node_modules/@elizaos/cloud-routing
+ln -s ../../packages/skills node_modules/@elizaos/skills
 
 log "Building Capacitor plugins"
 "$BUN_BIN" packages/app-core/scripts/build-native-plugins.mjs
@@ -297,10 +739,12 @@ for plugin in \
   plugin-sql \
   plugin-video \
   plugin-agent-skills \
+  plugin-app-manager \
   plugin-pdf \
   plugin-browser \
   plugin-capacitor-bridge \
   plugin-coding-tools \
+  plugin-commands \
   plugin-computeruse \
   plugin-discord \
   plugin-elizacloud \
@@ -334,7 +778,7 @@ log "Building all @elizaos/app workspace deps (turbo, --force to bypass cache)"
 # turbo build of @elizaos/app's dep graph (build:core covers a subset
 # but misses @elizaos/ui and the @elizaos/app-* surface packages).
 # --force forces fresh builds, sidestepping any poisoned remote cache
-# that contains incomplete dist artifacts.
+# that contains partial dist artifacts.
 "$BUN_BIN" run build:client -- --force
 
 log "Building agent workspace"
@@ -392,6 +836,11 @@ log "Building Docker image"
 log "Docker build disk usage"
 df -h "$REPO_ROOT" || true
 "$DOCKER_BIN" system df || true
+available_kb="$(df -Pk "$REPO_ROOT" | awk 'NR == 2 { print $4 }')"
+minimum_kb=$((18 * 1024 * 1024))
+if [[ -n "$available_kb" && "$available_kb" -lt "$minimum_kb" ]]; then
+  fail "Insufficient disk for Docker smoke build: available=$((available_kb / 1024 / 1024))GiB required>=18GiB"
+fi
 "$DOCKER_BIN" build \
   --file "$APP_CORE_DIR/deploy/Dockerfile.ci" \
   --tag "$DOCKER_IMAGE" \
@@ -417,84 +866,4 @@ if $SKIP_SMOKE; then
   exit 0
 fi
 
-log "Starting container smoke boot"
-"$DOCKER_BIN" rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
-"$DOCKER_BIN" run -d \
-  --name "$CONTAINER_NAME" \
-  -e PORT="$CONTAINER_PORT" \
-  -e APP_PORT="$CONTAINER_PORT" \
-  -e ELIZA_API_PORT="$CONTAINER_PORT" \
-  -e ELIZA_API_PORT="$CONTAINER_PORT" \
-  -e ELIZA_PORT="$CONTAINER_PORT" \
-  -e APP_API_BIND=0.0.0.0 \
-  -e ELIZA_STATE_DIR=/tmp/eliza-smoke/state \
-  -e ELIZA_STATE_DIR=/tmp/eliza-smoke/state \
-  -e ELIZA_CONFIG_DIR=/tmp/eliza-smoke/config \
-  -e ELIZA_CONFIG_DIR=/tmp/eliza-smoke/config \
-  -e ELIZA_WORKSPACE_DIR=/tmp/eliza-smoke/workspace \
-  -e ELIZA_WORKSPACE_DIR=/tmp/eliza-smoke/workspace \
-  -e ELIZA_VAULT_PASSPHRASE=docker-smoke-vault-passphrase \
-  -e PGLITE_DATA_DIR=/tmp/eliza-smoke/pglite \
-  -e ELIZA_DISABLE_LOCAL_EMBEDDINGS=1 \
-  -e ELIZA_API_BIND=0.0.0.0 \
-  -p "${SMOKE_PORT}:${CONTAINER_PORT}" \
-  "$DOCKER_IMAGE" >/dev/null
-
-status_url="http://127.0.0.1:${SMOKE_PORT}/api/status"
-health_url="http://127.0.0.1:${SMOKE_PORT}/api/health"
-
-probe_ok() {
-  local url="$1"
-  local out="$2"
-  local code
-  code="$(curl -sS --connect-timeout 1 --max-time 3 -o "$out" -w '%{http_code}' "$url" || true)"
-  case "$code" in
-    200)
-      return 0
-      ;;
-    401)
-      if grep -q 'Unauthorized' "$out" 2>/dev/null; then
-        return 0
-      fi
-      ;;
-  esac
-  return 1
-}
-
-deadline=$((SECONDS + SMOKE_TIMEOUT_SEC))
-last_log_dump=0
-while (( SECONDS < deadline )); do
-  running_containers_file="$(mktemp 2>/dev/null || printf '%s\n' "$SMOKE_ARTIFACT_DIR/docker-running-containers.txt")"
-  timeout 10 "$DOCKER_BIN" ps --format '{{.Names}}' >"$running_containers_file" 2>&1 || true
-  if ! grep -Fxq "$CONTAINER_NAME" "$running_containers_file" 2>/dev/null; then
-    rm -f "$running_containers_file" >/dev/null 2>&1 || true
-    timeout 30 "$DOCKER_BIN" logs "$CONTAINER_NAME" || true
-    log "Preserved failure artifacts in $SMOKE_ARTIFACT_DIR"
-    fail "Container exited before smoke probe succeeded"
-  fi
-  rm -f "$running_containers_file" >/dev/null 2>&1 || true
-
-  if (( SECONDS - last_log_dump >= 30 )); then
-    last_log_dump=$SECONDS
-    log "Container still booting; recent logs follow"
-    timeout 10 "$DOCKER_BIN" logs --tail 80 "$CONTAINER_NAME" || true
-  fi
-
-  if probe_ok "$health_url" /tmp/eliza-docker-health.txt; then
-    log "Health probe succeeded: $health_url"
-    cat /tmp/eliza-docker-health.txt
-    exit 0
-  fi
-
-  if probe_ok "$status_url" /tmp/eliza-docker-status.txt; then
-    log "Status probe succeeded: $status_url"
-    cat /tmp/eliza-docker-status.txt
-    exit 0
-  fi
-
-  sleep 5
-done
-
-timeout 30 "$DOCKER_BIN" logs "$CONTAINER_NAME" || true
-log "Preserved timeout artifacts in $SMOKE_ARTIFACT_DIR"
-fail "Timed out waiting for container smoke probe (${SMOKE_TIMEOUT_SEC}s)"
+boot_verify