@askexenow/exe-os 0.9.302 → 0.9.310

package/deploy/compose/.env.example

@@ -123,8 +123,14 @@ EXE_GATEWAY_WHATSAPP_VERIFY_TOKEN=CHANGEME_EXE_GATEWAY_WHATSAPP_VERIFY_TOKEN
 WHATSAPP_ACCESS_TOKEN=CHANGEME_WHATSAPP_ACCESS_TOKEN
 API_ROUTER_URL=https://gateway.askexe.com
 API_ROUTER_KEY=CHANGEME_API_ROUTER_KEY
-# BYOK: to use your own API keys instead of the Exe API Router,
-# set BYOK_ENABLED=true and provide ANTHROPIC_API_KEY below.
+# Billing enforcement contract (bug df8823d0). These are the vars the gateway
+# ACTUALLY reads to decide metered vs BYOK. Default: metered (fail-closed) —
+# direct provider keys are rejected and all AI traffic is billed via the router.
+EXE_BILLING_MODE=metered
+EXE_ALLOW_DIRECT_PROVIDER_KEYS=false
+# BYOK: to use your own API keys instead of the Exe API Router, set
+# BYOK_ENABLED=true (the customer-facing alias for EXE_BILLING_MODE=byok +
+# EXE_ALLOW_DIRECT_PROVIDER_KEYS=true) and provide ANTHROPIC_API_KEY below.
 # BYOK_ENABLED=false
 # ANTHROPIC_API_KEY=CHANGEME_ANTHROPIC_API_KEY
 GATEWAY_HTTP_HOST_PORT=3100

package/deploy/compose/README.md

@@ -6,9 +6,13 @@ Customer VPS deployments (Hygo/High/etc.) must start from `.env.customer.example
 
 Full production stack for a single client VPS: CRM + wiki + gateway + exed
 backed by Postgres, ClickHouse, and Redis. Pinned image tags, healthchecks on
-every service, named volumes for persistence, and host-nginx-friendly port
+all core services, named volumes for persistence, and host-nginx-friendly port
 publishing on `127.0.0.1`.
 
+> **Note:** `exe-otel-collector` does not have a container-level healthcheck
+> (scratch image with no shell) — it exposes a health extension on `:13133`
+> which is monitored externally.
+
 This is the **Lane A-2** deliverable from the v1.6 execution plan
 (`exe/output/v16-execution-plan.md`). Pairs with **Lane A-1** Ansible roles
 which provision the host and install nginx-tls; the `nginx-tls` role includes
@@ -56,8 +60,8 @@ nginx (Lane A-1 `nginx-tls`) can reverse-proxy them. Override the host port via
 
 `exe-monitor-agent` reports host/container health to the monitoring hub, but it
 needs **sensitive read-only host access** to do so: `/var/run/docker.sock`
-(root-equivalent — can inspect every container), `/proc`, `/sys`, and
-`/etc/os-release`. Because that access is effectively root on the box, the agent
+(root-equivalent — can inspect every container) and `/etc/os-release`.
+Because that access is effectively root on the box, the agent
 is **gated behind the `monitor-agent` compose profile** and does **not** start by
 default.
 

package/deploy/compose/add-routes.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# exe-os overlay-apply / post-deploy hook (add-routes.sh)
+#
+# Re-applies CUSTOMER OVERLAY files into the exe-gateway container after every
+# image upgrade / container recreate, then reloads the gateway so the overlays
+# take effect.
+#
+# ─────────────────────────────────────────────────────────────────────────────
+# BUG HISTORY (02bc1bb8, P1)
+# ─────────────────────────────────────────────────────────────────────────────
+# On a stack image upgrade the gateway container is RECREATED. Anything that lived
+# only inside the old container's writable layer (a customer overlay such as
+# po-intake.js, copied in by hand) is DROPPED with no warning. HYGO's working
+# po-intake.js overlay — 37 orders processed — was killed on v0.9.17 → v0.9.22.
+# The intended post-deploy hook (this file) shipped as a 0-BYTE empty file, and
+# /opt/exe-stack/overlay/ existed on the host but was never mounted into the
+# container, so nothing re-applied it.
+#
+# This script + the compose bind-mount (`./overlay:/data/overlay:ro`) close the
+# gap two ways:
+#   (a) overlays survive recreate via the persistent host bind-mount, AND
+#   (b) this hook re-applies them into the container's app dir + reloads after
+#       every recreate. Idempotent + safe: a no-op when no overlays exist.
+#
+# Mechanism:
+#   Host overlay dir  : ${OVERLAY_DIR:-/opt/exe-stack/overlay}
+#   In-container view : /data/overlay  (read-only bind-mount, always present)
+#   Applied into      : ${OVERLAY_TARGET:-/app/overlay} inside the container
+#
+# Overlays are loaded by the gateway from EXE_GATEWAY_OVERLAY_DIR (defaults to
+# /data/overlay — the bind-mount — so on a modern gateway no copy is even needed;
+# the copy below is belt-and-suspenders for gateway builds that load from /app).
+
+set -uo pipefail
+
+STACK_DIR="${EXE_STACK_DIR:-/opt/exe-stack}"
+OVERLAY_DIR="${OVERLAY_DIR:-${STACK_DIR}/overlay}"
+CONTAINER="${EXE_GATEWAY_CONTAINER:-exe-gateway}"
+# Where the gateway image expects overlays if it does NOT read the bind-mount directly.
+OVERLAY_TARGET="${OVERLAY_TARGET:-/app/overlay}"
+
+ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+log() { echo "[$(ts)] add-routes: $*"; }
+
+# Nothing to do if there is no overlay dir or it is empty. This is the common
+# (no customer overlay) case — exit cleanly so the post-deploy step is a no-op.
+if [[ ! -d "$OVERLAY_DIR" ]]; then
+  log "no overlay dir at ${OVERLAY_DIR} — nothing to apply."
+  exit 0
+fi
+
+shopt -s nullglob dotglob
+overlay_files=("$OVERLAY_DIR"/*)
+shopt -u nullglob dotglob
+if [[ ${#overlay_files[@]} -eq 0 ]]; then
+  log "overlay dir ${OVERLAY_DIR} is empty — nothing to apply."
+  exit 0
+fi
+
+# Gateway must be running to receive the overlay copy + reload.
+running=$(docker inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo "false")
+if [[ "$running" != "true" ]]; then
+  log "WARN gateway container ${CONTAINER} is not running — cannot apply overlays now. They remain at ${OVERLAY_DIR} and the read-only bind-mount (/data/overlay) still exposes them once the gateway is up."
+  exit 0
+fi
+
+log "applying ${#overlay_files[@]} overlay file(s) from ${OVERLAY_DIR} into ${CONTAINER}:${OVERLAY_TARGET}"
+
+# Ensure the in-container target dir exists, then copy each overlay file in.
+# `docker cp` works even on a read-only-bind-mounted source because we copy from
+# the HOST path, not from the (ro) /data/overlay mount.
+docker exec "$CONTAINER" mkdir -p "$OVERLAY_TARGET" >/dev/null 2>&1 || {
+  log "ERROR could not create ${OVERLAY_TARGET} inside ${CONTAINER}."
+  exit 1
+}
+
+applied=0
+for f in "${overlay_files[@]}"; do
+  [[ -f "$f" ]] || continue   # skip sub-directories for the flat copy
+  base=$(basename "$f")
+  if docker cp "$f" "${CONTAINER}:${OVERLAY_TARGET}/${base}" >/dev/null 2>&1; then
+    log "  applied ${base}"
+    applied=$((applied + 1))
+  else
+    log "  ERROR failed to apply ${base}"
+  fi
+done
+
+if [[ "$applied" -eq 0 ]]; then
+  log "no overlay files applied."
+  exit 0
+fi
+
+# Reload the gateway so the overlays take effect. A graceful in-container reload is
+# preferred; fall back to a container restart if the image exposes no reload hook.
+# (The restart here is INTENTIONAL + one-shot after a deploy — not the watchdog
+# loop — and only happens when overlays were actually applied.)
+if docker exec "$CONTAINER" sh -c 'test -x /app/reload.sh' >/dev/null 2>&1; then
+  log "reloading gateway via /app/reload.sh"
+  docker exec "$CONTAINER" /app/reload.sh >/dev/null 2>&1 || log "WARN /app/reload.sh exited non-zero"
+else
+  log "no in-container reload hook; restarting ${CONTAINER} once to load overlays"
+  docker restart "$CONTAINER" >/dev/null 2>&1 || log "WARN docker restart ${CONTAINER} failed"
+fi
+
+log "applied ${applied} overlay file(s)."
+exit 0

package/deploy/compose/docker-compose.yml

@@ -3,10 +3,10 @@
 # Services: exe-db (postgres) + clickhouse + redis + exe-crm + exe-wiki + exe-os + exe-gateway + exe-erp + otel-collector
 # ONE postgres (exe-db) — all services connect to it via DATABASE_URL.
 # Optional: exe-monitor-agent reports fleet health to the monitoring hub. It needs
-#   sensitive host access (docker.sock, /proc, /sys) so it is gated behind the
+#   sensitive host access (docker.sock, /etc/os-release) so it is gated behind the
 #   "monitor-agent" compose profile and stays OFF until the operator consents.
 #   See deploy/monitor/HOST-ACCESS-CONSENT.md.
-# All image tags pinned per-client via .env (no :latest). Healthchecks on every service.
+# All image tags pinned per-client via .env (no :latest). Healthchecks on all core services.
 # Named volumes for state; explicit subnets; depends_on with service_healthy gates.
 #
 # Validate without secrets:
@@ -26,7 +26,10 @@ services:
   # ------------------------------------------------------------------
 
   exe-db:
-    image: ${EXE_DB_IMAGE:-pgvector/pgvector:pg16}
+    # bug ee16e7ef: pin a versioned tag, not the mutable :pg16 rolling tag.
+    # pgvector/pgvector:pg16 can change upstream without notice; 0.8.0-pg16
+    # is the tested stable release. Override via EXE_DB_IMAGE in .env.
+    image: ${EXE_DB_IMAGE:-pgvector/pgvector:0.8.0-pg16@sha256:a132765ec351c65111b5b675928a3a0515a466a40f97277329db8b8209ad8bc9}
     container_name: exe-db
     restart: unless-stopped
     # SECURITY (bug 67d62490): no blanket `env_file: .env`. Each service
@@ -78,7 +81,7 @@ services:
       options: { max-size: "10m", max-file: "3" }
 
   clickhouse:
-    image: clickhouse/clickhouse-server:24.8.4.13-alpine
+    image: clickhouse/clickhouse-server:24.8.4.13-alpine@sha256:88a45f9e328549b2579256c46ee38e5c0e25ae58303d9eb6d9c7ed8d6d2bbf3c
     container_name: clickhouse
     restart: unless-stopped
     # SECURITY (bug 67d62490): no blanket `env_file: .env`. Each service
@@ -119,7 +122,7 @@ services:
       options: { max-size: "10m", max-file: "3" }
 
   redis:
-    image: redis:7.4-alpine
+    image: redis:7.4-alpine@sha256:6ab0b6e7381779332f97b8ca76193e45b0756f38d4c0dcda72dbb3c32061ab99
     container_name: redis
     restart: unless-stopped
     # SECURITY (bug 67d62490): no blanket `env_file: .env`. Each service
@@ -236,17 +239,21 @@ services:
     image: ${MONITOR_HUB_IMAGE_TAG:-ghcr.io/askexe/exe-monitor-hub:v0.9.4}
     container_name: exe-monitor-hub
     restart: unless-stopped
-    # bug d3d49101: PocketBase's data dir for this hub is a RELATIVE path
-    # (DefaultDataDir="exe-monitor_data", see exe-monitor internal/cmd/hub).
-    # The scratch image runs the binary from / with no WORKDIR, so it would
-    # persist to /exe-monitor_data — NOT /app/pb_data and NOT /beszel_data.
-    # Pin it explicitly with --dir=/beszel_data so the binary writes to the same
-    # path the named volume is mounted at (and the same path the image VOLUME
-    # declares), so data survives container recreation.
+    # bug d3d49101 / 1fdb8be9: PocketBase's data dir for this hub is a RELATIVE
+    # path (DefaultDataDir="exe-monitor_data", see exe-monitor internal/cmd/hub).
+    # The image's WORKDIR is /app, so DefaultDataDir resolves to /app/exe-monitor_data.
+    # Pin it explicitly with --dir=/app/exe-monitor_data so the binary writes to
+    # the same path the named volume is mounted at, so data survives container
+    # recreation. Previous bug: mount was at /beszel_data but data went to
+    # /app/exe-monitor_data → writable layer → wiped on every recreate.
     # bug 182c6da3 / a125785d: EXE_MONITOR_KEY enables /api/exe-monitor/errors
     # ingestion (fail-closed in the hub when unset) and GATEWAY_ALERT_URL enables
     # error-spike alerts (silently disabled in the hub when unset).
-    command: ["serve", "--http=0.0.0.0:8090", "--dir=/beszel_data"]
+    # bug 1fdb8be9: the custom hub binary resolves DefaultDataDir="exe-monitor_data"
+    # relative to the WORKDIR (/app), so actual data lives at /app/exe-monitor_data.
+    # --dir must point there (NOT /beszel_data) and the volume must mount there too,
+    # otherwise container recreates wipe all monitor history.
+    command: ["serve", "--http=0.0.0.0:8090", "--dir=/app/exe-monitor_data"]
     environment:
       EXE_MONITOR_ADMIN_TOKEN: ${EXE_MONITOR_ADMIN_TOKEN:-}
       # bug a125785d: shared secret for POST /api/exe-monitor/errors. The hub
@@ -266,7 +273,11 @@ services:
     ports:
       - "127.0.0.1:${MONITOR_HUB_PORT:-8090}:8090"
     volumes:
-      - monitor_hub_data:/beszel_data
+      # bug 1fdb8be9: mount at the ACTUAL data dir the hub binary writes to.
+      # Was /beszel_data (stock Beszel path), but the exe-monitor fork uses
+      # /app/exe-monitor_data. Mismatch meant data lived in the writable layer
+      # and was lost on every container recreate / stack-update.
+      - monitor_hub_data:/app/exe-monitor_data
     networks:
       - backend
     healthcheck:
@@ -329,6 +340,11 @@ services:
       NODE_PORT: "3000"
       EXE_LICENSE_KEY: ${EXE_LICENSE_KEY:?EXE_LICENSE_KEY is required — purchase at https://askexe.com}
       SERVER_URL: ${CRM_SERVER_URL:-https://crm.askexe.com}
+      # bug b99d3762 / 83192765: the CRM error-forwarder POSTs to the monitor hub and must
+      # send X-Monitor-Key once the hub requires it on every path (mirrors gateway/erp).
+      ERROR_REPORTING_ENABLED: "true"
+      MONITOR_ERROR_URL: http://exe-monitor-hub:8090/api/exe-monitor/errors
+      MONITOR_API_KEY: ${MONITOR_API_KEY:-${EXE_MONITOR_KEY:-}}
       APP_SECRET: ${CRM_APP_SECRET:?CRM_APP_SECRET is required}
       EXE_CRM_ADMIN_TOKEN: ${EXE_CRM_ADMIN_TOKEN:-}
       EXE_CRM_ADMIN_EMAIL: ${EXE_CRM_ADMIN_EMAIL:-}
@@ -444,6 +460,11 @@ services:
       SERVER_PORT: "3001"
       EXE_LICENSE_KEY: ${EXE_LICENSE_KEY:?EXE_LICENSE_KEY is required — purchase at https://askexe.com}
       STORAGE_DIR: /app/server/storage
+      # bug b99d3762 / 83192765: the wiki error-reporter POSTs to the monitor hub and must
+      # send X-Monitor-Key once the hub requires it on every path (mirrors gateway/erp).
+      ERROR_REPORTING_ENABLED: "true"
+      MONITOR_ERROR_URL: http://exe-monitor-hub:8090/api/exe-monitor/errors
+      MONITOR_API_KEY: ${MONITOR_API_KEY:-${EXE_MONITOR_KEY:-}}
       DATABASE_URL: postgres://${POSTGRES_USER:-exe}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@exe-db:5432/${POSTGRES_DB:-exedb}?schema=${WIKI_DB_SCHEMA:-wiki}&sslmode=disable
       AUTH_TOKEN: ${WIKI_AUTH_TOKEN:?WIKI_AUTH_TOKEN is required}
       EXE_WIKI_ADMIN_TOKEN: ${EXE_WIKI_ADMIN_TOKEN:-}
@@ -597,6 +618,14 @@ services:
       - exe_os_data:/home/exed/.exe-os
     networks:
       - backend
+    healthcheck:
+      # The worker is a long-running node process (embed-worker.js). Verify the
+      # PID-1 node process is alive via /proc/1/cmdline.
+      test: ["CMD-SHELL", "test -f /proc/1/cmdline"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -637,6 +666,13 @@ services:
       NODE_ENV: production
       EXE_GATEWAY_HOME: /data
       EXE_GATEWAY_CONFIG: /data/gateway.json
+      # bug 02bc1bb8: customer overlay dir. The host ./overlay is bind-mounted
+      # read-only at /data/overlay (see volumes below) so customer overlays
+      # (e.g. po-intake.js) SURVIVE every image upgrade / container recreate
+      # instead of being silently dropped from the old container layer. The
+      # gateway loads overlays from this dir; the post-deploy add-routes.sh hook
+      # additionally re-applies them for gateway builds that load from /app.
+      EXE_GATEWAY_OVERLAY_DIR: /data/overlay
       DATABASE_URL: postgres://${POSTGRES_USER:-exe}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@exe-db:5432/${POSTGRES_DB:-exedb}?sslmode=disable
       EXE_GATEWAY_PORT: "3100"
       EXE_GATEWAY_HOST: "0.0.0.0"
@@ -665,6 +701,13 @@ services:
       # one secret without extra wiring.
       MONITOR_API_KEY: ${MONITOR_API_KEY:-${EXE_MONITOR_KEY:-}}
       EXE_DAEMON_TOKEN: ${EXED_MCP_TOKEN:?EXED_MCP_TOKEN is required}
+      # Billing enforcement contract (bug df8823d0). The gateway reads THESE to
+      # decide metered vs BYOK. Default metered + direct keys disallowed (fail
+      # closed). BYOK_ENABLED is the customer-facing alias: =true implies
+      # EXE_BILLING_MODE=byok + EXE_ALLOW_DIRECT_PROVIDER_KEYS=true. A mismatch
+      # (BYOK_ENABLED=true but EXE_BILLING_MODE=metered) resolves to metered.
+      EXE_BILLING_MODE: ${EXE_BILLING_MODE:-metered}
+      EXE_ALLOW_DIRECT_PROVIDER_KEYS: ${EXE_ALLOW_DIRECT_PROVIDER_KEYS:-false}
       BYOK_ENABLED: ${BYOK_ENABLED:-false}
       ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
       EXE_LICENSE_KEY: ${EXE_LICENSE_KEY:?EXE_LICENSE_KEY is required — purchase at https://askexe.com}
@@ -675,6 +718,14 @@ services:
       - gateway_data:/data
       - gateway_whatsapp_auth:/app/auth_info
       - ./gateway.json:/data/gateway.json:ro
+      # bug 02bc1bb8: PERSISTENT customer-overlay bind-mount. Anything the
+      # customer drops in the host ./overlay dir (e.g. po-intake.js) is exposed
+      # read-only inside the container at /data/overlay and therefore SURVIVES
+      # image upgrades / `docker compose up` recreate — unlike files that only
+      # ever lived in the container's writable layer (which a recreate drops).
+      # bootstrapStackHost ensures ./overlay exists so docker never auto-creates
+      # the bind path as a stray file. Empty for customers with no overlay.
+      - ./overlay:/data/overlay:ro
     networks:
       - backend
       - frontend
@@ -718,8 +769,9 @@ services:
       exe-monitor-hub:
         condition: service_healthy
     environment:
-      # Reports to the local hub by default. stack-update auto-pairs this agent
-      # with the hub via /api/register-agent and writes TOKEN + KEY back to .env.
+      # Reports to the local hub by default. The agent must be paired manually
+      # from the hub UI (Settings → Add System) — the hub does not expose a
+      # REST registration endpoint. Write the generated TOKEN + KEY into .env.
       HUB_URL: ${MONITOR_HUB_URL:-http://exe-monitor-hub:8090}
       # TOKEN and KEY are written automatically by `exe-os stack-update` during
       # bootstrap. They start empty; the agent retries connecting until the hub
@@ -838,7 +890,7 @@ services:
   # ------------------------------------------------------------------
 
   exe-sso-edge:
-    image: nginx:alpine
+    image: nginx:alpine@sha256:20316569d8f81a160065d7d2a5eeffc7ca97d79022462ee255fd23fa103a6b5c
     container_name: exe-sso-edge
     restart: unless-stopped
     depends_on:
@@ -1069,7 +1121,7 @@ services:
   # API to gunicorn, WebSocket to socketio. Without this, gunicorn returns
   # 404 for all /assets/* requests (bug 6776edf0).
   exe-erp-nginx:
-    image: nginx:alpine
+    image: nginx:alpine@sha256:20316569d8f81a160065d7d2a5eeffc7ca97d79022462ee255fd23fa103a6b5c
     container_name: exe-erp-nginx
     restart: unless-stopped
     depends_on:
@@ -1154,6 +1206,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the socketio node process is listening on port 9000.
+      test: ["CMD-SHELL", "node -e \"const s=require('net').connect(9000,'127.0.0.1',()=>{s.destroy();process.exit(0)});s.setTimeout(4000,()=>{s.destroy();process.exit(1)});s.on('error',()=>process.exit(1))\""]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -1193,6 +1252,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the bench worker process is alive and Redis queue is reachable.
+      test: ["CMD-SHELL", "pgrep -f 'rq worker' > /dev/null"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -1232,6 +1298,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the bench schedule process is alive.
+      test: ["CMD-SHELL", "pgrep -f 'bench schedule' > /dev/null"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:

package/deploy/compose/gateway-watchdog.sh

@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+# exe-os gateway watchdog — verifies the exe-gateway WhatsApp adapter is actually
+# CONNECTED (not just that the HTTP server answers) and restarts the gateway only
+# when it is genuinely wedged.
+#
+# Install (cron, every 2 min):
+#   chmod +x /opt/exe-stack/gateway-watchdog.sh
+#   crontab -l | { cat; echo "*/2 * * * * /opt/exe-stack/gateway-watchdog.sh 2>&1 | logger -t exe-gw-watchdog"; } | crontab -
+#
+# ─────────────────────────────────────────────────────────────────────────────
+# BUG HISTORY (facd5078, P0)
+# ─────────────────────────────────────────────────────────────────────────────
+# The previous watchdog curled the UNAUTHENTICATED http://localhost:3100/health
+# and grepped for `"connected":true`. But the unauth /health only returns
+# {status, uptime, adapterCount} — it carries NO adapter connection state — so the
+# grep ALWAYS failed → 3 strikes every ~6 min → `docker restart exe-gateway`
+# 24/7 (1,668 restarts in 72h on HYGO). `docker restart` does NOT increment the
+# container RestartCount, which masked the loop entirely.
+#
+# This rewrite makes the check ACCURATE and FAIL-SAFE:
+#   1. AUTHENTICATED probe of /admin/webhooks/health (Bearer EXE_GATEWAY_AUTH_TOKEN),
+#      which reports real per-adapter state. Preferred — correct regardless of
+#      gateway version.
+#   2. If the running gateway already exposes `adapters_connected` on the UNAUTH
+#      /health (exe-gateway bug 7e9fc713), that is honored too — but we do NOT
+#      hard-depend on it shipping first.
+#   3. FAIL SAFE: if EXE_GATEWAY_AUTH_TOKEN is unset, OR the health body cannot be
+#      parsed into a definite connected/disconnected verdict, the result is
+#      INCONCLUSIVE → we LOG and do NOT restart. A watchdog must never restart on
+#      an unknown.
+#   4. STARTUP GRACE: the gateway is given GRACE_SECS of container uptime before
+#      any strike is counted, so a gateway that is merely (re)starting / pairing
+#      WhatsApp is never restart-looped.
+#   5. BACKOFF: after a restart we refuse to restart again for COOLDOWN_SECS, so we
+#      never hammer a gateway that needs time to reconnect.
+
+set -uo pipefail
+
+# ── Config ───────────────────────────────────────────────────────────────────
+STACK_DIR="${EXE_STACK_DIR:-/opt/exe-stack}"
+ENV_FILE="${EXE_STACK_ENV_FILE:-${STACK_DIR}/.env}"
+CONTAINER="${EXE_GATEWAY_CONTAINER:-exe-gateway}"
+PORT="${EXE_GATEWAY_HTTP_PORT:-3100}"
+HOST="${EXE_GATEWAY_HOST:-127.0.0.1}"
+TIMEOUT="${EXE_GATEWAY_WATCHDOG_TIMEOUT:-5}"
+
+# Strikes before a restart. Each cron tick = one probe.
+MAX_STRIKES="${EXE_GATEWAY_WATCHDOG_STRIKES:-3}"
+# Don't count strikes until the container has been up at least this long.
+GRACE_SECS="${EXE_GATEWAY_WATCHDOG_GRACE_SECS:-120}"
+# After a restart, refuse to restart again for this long (let it reconnect).
+COOLDOWN_SECS="${EXE_GATEWAY_WATCHDOG_COOLDOWN_SECS:-300}"
+
+STATE_DIR="${EXE_GATEWAY_WATCHDOG_STATE_DIR:-${STACK_DIR}/.gateway-watchdog}"
+STRIKE_FILE="${STATE_DIR}/strikes"
+LAST_RESTART_FILE="${STATE_DIR}/last-restart"
+
+mkdir -p "$STATE_DIR" 2>/dev/null || true
+
+ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+log() { echo "[$(ts)] gateway-watchdog: $*"; }
+
+# Read a single KEY=VALUE from the stack .env WITHOUT sourcing it (the .env holds
+# many secrets and arbitrary values; sourcing could execute / mangle them).
+read_env() {
+  local key="$1"
+  [[ -f "$ENV_FILE" ]] || return 0
+  # Last assignment wins; strip surrounding quotes; ignore comments.
+  sed -n "s/^[[:space:]]*${key}=//p" "$ENV_FILE" 2>/dev/null | tail -n1 \
+    | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
+}
+
+reset_strikes() { echo 0 > "$STRIKE_FILE" 2>/dev/null || true; }
+read_strikes()  { local s; s=$(cat "$STRIKE_FILE" 2>/dev/null || echo 0); [[ "$s" =~ ^[0-9]+$ ]] && echo "$s" || echo 0; }
+bump_strikes()  { local s; s=$(read_strikes); echo $((s + 1)) > "$STRIKE_FILE" 2>/dev/null || true; echo $((s + 1)); }
+
+# Container uptime in seconds (0 if not running / unknown).
+container_uptime_secs() {
+  local started running
+  running=$(docker inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo "false")
+  [[ "$running" == "true" ]] || { echo 0; return; }
+  started=$(docker inspect --format '{{.State.StartedAt}}' "$CONTAINER" 2>/dev/null || echo "")
+  [[ -n "$started" ]] || { echo 0; return; }
+  local start_epoch now_epoch
+  start_epoch=$(date -u -d "$started" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%S" "${started%%.*}" +%s 2>/dev/null || echo 0)
+  now_epoch=$(date -u +%s)
+  if [[ "$start_epoch" -gt 0 && "$now_epoch" -ge "$start_epoch" ]]; then
+    echo $((now_epoch - start_epoch))
+  else
+    echo 0
+  fi
+}
+
+seconds_since_last_restart() {
+  local last now
+  last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
+  [[ "$last" =~ ^[0-9]+$ ]] || last=0
+  now=$(date -u +%s)
+  echo $((now - last))
+}
+
+# ── Adapter-state probe ──────────────────────────────────────────────────────
+# Echoes exactly one of: connected | disconnected | inconclusive
+probe_adapter_state() {
+  local token unauth auth
+
+  # 1) Forward-compat: unauth /health MAY expose adapters_connected (gw bug 7e9fc713).
+  unauth=$(curl -sk --max-time "$TIMEOUT" "http://${HOST}:${PORT}/health" 2>/dev/null || echo "")
+  if [[ -n "$unauth" ]]; then
+    if echo "$unauth" | grep -Eq '"adapters_connected"[[:space:]]*:[[:space:]]*true'; then
+      echo "connected"; return
+    fi
+    if echo "$unauth" | grep -Eq '"adapters_connected"[[:space:]]*:[[:space:]]*false'; then
+      echo "disconnected"; return
+    fi
+    # No adapters_connected field → this gateway predates 7e9fc713. Fall through to
+    # the authenticated probe; the bare {status,uptime,adapterCount} body is NOT a
+    # connection verdict and must never be treated as one.
+  fi
+
+  # 2) Authenticated /admin/webhooks/health — reports real per-adapter state.
+  token=$(read_env EXE_GATEWAY_AUTH_TOKEN)
+  if [[ -z "$token" ]]; then
+    # FAIL SAFE: no token → we cannot make an authoritative judgement. Never restart.
+    echo "inconclusive"; return
+  fi
+
+  auth=$(curl -sk --max-time "$TIMEOUT" \
+    -H "Authorization: Bearer ${token}" \
+    "http://${HOST}:${PORT}/admin/webhooks/health" 2>/dev/null || echo "")
+
+  if [[ -z "$auth" ]]; then
+    # Couldn't reach the authed endpoint at all → inconclusive (could be a transient
+    # network blip; the HTTP layer may still be fine). Don't restart on this alone.
+    echo "inconclusive"; return
+  fi
+
+  # Real connection verdict. The authed health reports adapter connection via
+  # "connected":true and/or a per-account "whatsapp":{... "connected":true}.
+  if echo "$auth" | grep -Eq '"connected"[[:space:]]*:[[:space:]]*true'; then
+    echo "connected"; return
+  fi
+  if echo "$auth" | grep -Eq '"handlerRegistered"[[:space:]]*:[[:space:]]*true' \
+     && echo "$auth" | grep -Eq '"connected"[[:space:]]*:[[:space:]]*false'; then
+    echo "disconnected"; return
+  fi
+  # An authed body we can't classify (schema drift) is NOT grounds to restart.
+  echo "inconclusive"
+}
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+main() {
+  # Gateway not running at all? Let Docker's own `restart: unless-stopped` handle
+  # a crashed container — the watchdog only acts on a RUNNING-but-wedged gateway.
+  local uptime
+  uptime=$(container_uptime_secs)
+  if [[ "$uptime" -eq 0 ]]; then
+    log "container ${CONTAINER} not running (or uptime unknown) — deferring to Docker restart policy; not counting a strike."
+    reset_strikes
+    exit 0
+  fi
+
+  local state
+  state=$(probe_adapter_state)
+
+  case "$state" in
+    connected)
+      reset_strikes
+      log "OK adapters connected (uptime ${uptime}s)."
+      exit 0
+      ;;
+    inconclusive)
+      # NEVER restart on an inconclusive check. Log and reset so a transient
+      # unknown can't accumulate toward a restart.
+      reset_strikes
+      log "INCONCLUSIVE health check (token unset or unparseable body) — NOT restarting."
+      exit 0
+      ;;
+    disconnected)
+      : # fall through to strike logic below
+      ;;
+    *)
+      reset_strikes
+      log "unexpected probe result '${state}' — treating as inconclusive, NOT restarting."
+      exit 0
+      ;;
+  esac
+
+  # ── disconnected ──
+  # Startup grace: a gateway that is merely (re)starting / re-pairing WhatsApp must
+  # not be counted against. Require GRACE_SECS of uptime before any strike.
+  if [[ "$uptime" -lt "$GRACE_SECS" ]]; then
+    log "adapter disconnected but within startup grace (uptime ${uptime}s < ${GRACE_SECS}s) — not counting a strike."
+    exit 0
+  fi
+
+  # Backoff: don't restart again within the cooldown window — give the gateway time
+  # to reconnect after the previous restart.
+  local since_restart
+  since_restart=$(seconds_since_last_restart)
+  if [[ "$since_restart" -lt "$COOLDOWN_SECS" ]]; then
+    log "adapter disconnected but within post-restart cooldown (${since_restart}s < ${COOLDOWN_SECS}s) — backing off, not restarting."
+    exit 0
+  fi
+
+  local strikes
+  strikes=$(bump_strikes)
+  log "adapter DISCONNECTED — strike ${strikes}/${MAX_STRIKES} (uptime ${uptime}s)."
+
+  if [[ "$strikes" -ge "$MAX_STRIKES" ]]; then
+    log "ALERT strike threshold reached — restarting ${CONTAINER}."
+    if docker restart "$CONTAINER" >/dev/null 2>&1; then
+      date -u +%s > "$LAST_RESTART_FILE" 2>/dev/null || true
+      reset_strikes
+      log "restarted ${CONTAINER}; entering ${COOLDOWN_SECS}s cooldown."
+    else
+      log "ERROR docker restart ${CONTAINER} failed."
+    fi
+  fi
+  exit 0
+}
+
+main "$@"

package/deploy/compose/generate-env.ts

@@ -174,8 +174,14 @@ export function generateEnv(options: GenerateEnvOptions): string {
     "WHATSAPP_ACCESS_TOKEN=",
     `API_ROUTER_URL=${DEFAULT_API_ROUTER_URL}`,
     `API_ROUTER_KEY=${randomSecret(48)}`,
-    "# BYOK: to use your own API keys instead of the Exe API Router,",
-    "# set BYOK_ENABLED=true and provide ANTHROPIC_API_KEY below.",
+    "# Billing enforcement contract (bug df8823d0). These are the vars the gateway",
+    "# ACTUALLY reads to decide metered vs BYOK. Default: metered (fail-closed) —",
+    "# direct provider keys are rejected and all AI traffic is billed via the router.",
+    "EXE_BILLING_MODE=metered",
+    "EXE_ALLOW_DIRECT_PROVIDER_KEYS=false",
+    "# BYOK: to use your own API keys instead of the Exe API Router, set",
+    "# BYOK_ENABLED=true (the customer-facing alias for EXE_BILLING_MODE=byok +",
+    "# EXE_ALLOW_DIRECT_PROVIDER_KEYS=true) and provide ANTHROPIC_API_KEY below.",
     "# BYOK_ENABLED=false",
     "# ANTHROPIC_API_KEY=",
     `GATEWAY_HTTP_HOST_PORT=${GATEWAY_HTTP_HOST_PORT}`,
@@ -403,8 +409,14 @@ export function generateExampleEnv(): string {
     "WHATSAPP_ACCESS_TOKEN=CHANGEME_WHATSAPP_ACCESS_TOKEN",
     `API_ROUTER_URL=${DEFAULT_API_ROUTER_URL}`,
     "API_ROUTER_KEY=CHANGEME_API_ROUTER_KEY",
-    "# BYOK: to use your own API keys instead of the Exe API Router,",
-    "# set BYOK_ENABLED=true and provide ANTHROPIC_API_KEY below.",
+    "# Billing enforcement contract (bug df8823d0). These are the vars the gateway",
+    "# ACTUALLY reads to decide metered vs BYOK. Default: metered (fail-closed) —",
+    "# direct provider keys are rejected and all AI traffic is billed via the router.",
+    "EXE_BILLING_MODE=metered",
+    "EXE_ALLOW_DIRECT_PROVIDER_KEYS=false",
+    "# BYOK: to use your own API keys instead of the Exe API Router, set",
+    "# BYOK_ENABLED=true (the customer-facing alias for EXE_BILLING_MODE=byok +",
+    "# EXE_ALLOW_DIRECT_PROVIDER_KEYS=true) and provide ANTHROPIC_API_KEY below.",
     "# BYOK_ENABLED=false",
     "# ANTHROPIC_API_KEY=CHANGEME_ANTHROPIC_API_KEY",
     `GATEWAY_HTTP_HOST_PORT=${GATEWAY_HTTP_HOST_PORT}`,