@askexenow/exe-os 0.9.302 → 0.9.303

package/deploy/compose/README.md

@@ -6,9 +6,13 @@ Customer VPS deployments (Hygo/High/etc.) must start from `.env.customer.example
 
 Full production stack for a single client VPS: CRM + wiki + gateway + exed
 backed by Postgres, ClickHouse, and Redis. Pinned image tags, healthchecks on
-every service, named volumes for persistence, and host-nginx-friendly port
+all core services, named volumes for persistence, and host-nginx-friendly port
 publishing on `127.0.0.1`.
 
+> **Note:** `exe-otel-collector` does not have a container-level healthcheck
+> (scratch image with no shell) — it exposes a health extension on `:13133`
+> which is monitored externally.
+
 This is the **Lane A-2** deliverable from the v1.6 execution plan
 (`exe/output/v16-execution-plan.md`). Pairs with **Lane A-1** Ansible roles
 which provision the host and install nginx-tls; the `nginx-tls` role includes
@@ -56,8 +60,8 @@ nginx (Lane A-1 `nginx-tls`) can reverse-proxy them. Override the host port via
 
 `exe-monitor-agent` reports host/container health to the monitoring hub, but it
 needs **sensitive read-only host access** to do so: `/var/run/docker.sock`
-(root-equivalent — can inspect every container), `/proc`, `/sys`, and
-`/etc/os-release`. Because that access is effectively root on the box, the agent
+(root-equivalent — can inspect every container) and `/etc/os-release`.
+Because that access is effectively root on the box, the agent
 is **gated behind the `monitor-agent` compose profile** and does **not** start by
 default.
 

package/deploy/compose/add-routes.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# exe-os overlay-apply / post-deploy hook (add-routes.sh)
+#
+# Re-applies CUSTOMER OVERLAY files into the exe-gateway container after every
+# image upgrade / container recreate, then reloads the gateway so the overlays
+# take effect.
+#
+# ─────────────────────────────────────────────────────────────────────────────
+# BUG HISTORY (02bc1bb8, P1)
+# ─────────────────────────────────────────────────────────────────────────────
+# On a stack image upgrade the gateway container is RECREATED. Anything that lived
+# only inside the old container's writable layer (a customer overlay such as
+# po-intake.js, copied in by hand) is DROPPED with no warning. HYGO's working
+# po-intake.js overlay — 37 orders processed — was killed on v0.9.17 → v0.9.22.
+# The intended post-deploy hook (this file) shipped as a 0-BYTE empty file, and
+# /opt/exe-stack/overlay/ existed on the host but was never mounted into the
+# container, so nothing re-applied it.
+#
+# This script + the compose bind-mount (`./overlay:/data/overlay:ro`) close the
+# gap two ways:
+#   (a) overlays survive recreate via the persistent host bind-mount, AND
+#   (b) this hook re-applies them into the container's app dir + reloads after
+#       every recreate. Idempotent + safe: a no-op when no overlays exist.
+#
+# Mechanism:
+#   Host overlay dir  : ${OVERLAY_DIR:-/opt/exe-stack/overlay}
+#   In-container view : /data/overlay  (read-only bind-mount, always present)
+#   Applied into      : ${OVERLAY_TARGET:-/app/overlay} inside the container
+#
+# Overlays are loaded by the gateway from EXE_GATEWAY_OVERLAY_DIR (defaults to
+# /data/overlay — the bind-mount — so on a modern gateway no copy is even needed;
+# the copy below is belt-and-suspenders for gateway builds that load from /app).
+
+set -uo pipefail
+
+STACK_DIR="${EXE_STACK_DIR:-/opt/exe-stack}"
+OVERLAY_DIR="${OVERLAY_DIR:-${STACK_DIR}/overlay}"
+CONTAINER="${EXE_GATEWAY_CONTAINER:-exe-gateway}"
+# Where the gateway image expects overlays if it does NOT read the bind-mount directly.
+OVERLAY_TARGET="${OVERLAY_TARGET:-/app/overlay}"
+
+ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+log() { echo "[$(ts)] add-routes: $*"; }
+
+# Nothing to do if there is no overlay dir or it is empty. This is the common
+# (no customer overlay) case — exit cleanly so the post-deploy step is a no-op.
+if [[ ! -d "$OVERLAY_DIR" ]]; then
+  log "no overlay dir at ${OVERLAY_DIR} — nothing to apply."
+  exit 0
+fi
+
+shopt -s nullglob dotglob
+overlay_files=("$OVERLAY_DIR"/*)
+shopt -u nullglob dotglob
+if [[ ${#overlay_files[@]} -eq 0 ]]; then
+  log "overlay dir ${OVERLAY_DIR} is empty — nothing to apply."
+  exit 0
+fi
+
+# Gateway must be running to receive the overlay copy + reload.
+running=$(docker inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo "false")
+if [[ "$running" != "true" ]]; then
+  log "WARN gateway container ${CONTAINER} is not running — cannot apply overlays now. They remain at ${OVERLAY_DIR} and the read-only bind-mount (/data/overlay) still exposes them once the gateway is up."
+  exit 0
+fi
+
+log "applying ${#overlay_files[@]} overlay file(s) from ${OVERLAY_DIR} into ${CONTAINER}:${OVERLAY_TARGET}"
+
+# Ensure the in-container target dir exists, then copy each overlay file in.
+# `docker cp` works even on a read-only-bind-mounted source because we copy from
+# the HOST path, not from the (ro) /data/overlay mount.
+docker exec "$CONTAINER" mkdir -p "$OVERLAY_TARGET" >/dev/null 2>&1 || {
+  log "ERROR could not create ${OVERLAY_TARGET} inside ${CONTAINER}."
+  exit 1
+}
+
+applied=0
+for f in "${overlay_files[@]}"; do
+  [[ -f "$f" ]] || continue   # skip sub-directories for the flat copy
+  base=$(basename "$f")
+  if docker cp "$f" "${CONTAINER}:${OVERLAY_TARGET}/${base}" >/dev/null 2>&1; then
+    log "  applied ${base}"
+    applied=$((applied + 1))
+  else
+    log "  ERROR failed to apply ${base}"
+  fi
+done
+
+if [[ "$applied" -eq 0 ]]; then
+  log "no overlay files applied."
+  exit 0
+fi
+
+# Reload the gateway so the overlays take effect. A graceful in-container reload is
+# preferred; fall back to a container restart if the image exposes no reload hook.
+# (The restart here is INTENTIONAL + one-shot after a deploy — not the watchdog
+# loop — and only happens when overlays were actually applied.)
+if docker exec "$CONTAINER" sh -c 'test -x /app/reload.sh' >/dev/null 2>&1; then
+  log "reloading gateway via /app/reload.sh"
+  docker exec "$CONTAINER" /app/reload.sh >/dev/null 2>&1 || log "WARN /app/reload.sh exited non-zero"
+else
+  log "no in-container reload hook; restarting ${CONTAINER} once to load overlays"
+  docker restart "$CONTAINER" >/dev/null 2>&1 || log "WARN docker restart ${CONTAINER} failed"
+fi
+
+log "applied ${applied} overlay file(s)."
+exit 0

package/deploy/compose/docker-compose.yml

@@ -3,10 +3,10 @@
 # Services: exe-db (postgres) + clickhouse + redis + exe-crm + exe-wiki + exe-os + exe-gateway + exe-erp + otel-collector
 # ONE postgres (exe-db) — all services connect to it via DATABASE_URL.
 # Optional: exe-monitor-agent reports fleet health to the monitoring hub. It needs
-#   sensitive host access (docker.sock, /proc, /sys) so it is gated behind the
+#   sensitive host access (docker.sock, /etc/os-release) so it is gated behind the
 #   "monitor-agent" compose profile and stays OFF until the operator consents.
 #   See deploy/monitor/HOST-ACCESS-CONSENT.md.
-# All image tags pinned per-client via .env (no :latest). Healthchecks on every service.
+# All image tags pinned per-client via .env (no :latest). Healthchecks on all core services.
 # Named volumes for state; explicit subnets; depends_on with service_healthy gates.
 #
 # Validate without secrets:
@@ -26,7 +26,10 @@ services:
   # ------------------------------------------------------------------
 
   exe-db:
-    image: ${EXE_DB_IMAGE:-pgvector/pgvector:pg16}
+    # bug ee16e7ef: pin a versioned tag, not the mutable :pg16 rolling tag.
+    # pgvector/pgvector:pg16 can change upstream without notice; 0.8.0-pg16
+    # is the tested stable release. Override via EXE_DB_IMAGE in .env.
+    image: ${EXE_DB_IMAGE:-pgvector/pgvector:0.8.0-pg16}
     container_name: exe-db
     restart: unless-stopped
     # SECURITY (bug 67d62490): no blanket `env_file: .env`. Each service
@@ -236,17 +239,21 @@ services:
     image: ${MONITOR_HUB_IMAGE_TAG:-ghcr.io/askexe/exe-monitor-hub:v0.9.4}
     container_name: exe-monitor-hub
     restart: unless-stopped
-    # bug d3d49101: PocketBase's data dir for this hub is a RELATIVE path
-    # (DefaultDataDir="exe-monitor_data", see exe-monitor internal/cmd/hub).
-    # The scratch image runs the binary from / with no WORKDIR, so it would
-    # persist to /exe-monitor_data — NOT /app/pb_data and NOT /beszel_data.
-    # Pin it explicitly with --dir=/beszel_data so the binary writes to the same
-    # path the named volume is mounted at (and the same path the image VOLUME
-    # declares), so data survives container recreation.
+    # bug d3d49101 / 1fdb8be9: PocketBase's data dir for this hub is a RELATIVE
+    # path (DefaultDataDir="exe-monitor_data", see exe-monitor internal/cmd/hub).
+    # The image's WORKDIR is /app, so DefaultDataDir resolves to /app/exe-monitor_data.
+    # Pin it explicitly with --dir=/app/exe-monitor_data so the binary writes to
+    # the same path the named volume is mounted at, so data survives container
+    # recreation. Previous bug: mount was at /beszel_data but data went to
+    # /app/exe-monitor_data → writable layer → wiped on every recreate.
     # bug 182c6da3 / a125785d: EXE_MONITOR_KEY enables /api/exe-monitor/errors
     # ingestion (fail-closed in the hub when unset) and GATEWAY_ALERT_URL enables
     # error-spike alerts (silently disabled in the hub when unset).
-    command: ["serve", "--http=0.0.0.0:8090", "--dir=/beszel_data"]
+    # bug 1fdb8be9: the custom hub binary resolves DefaultDataDir="exe-monitor_data"
+    # relative to the WORKDIR (/app), so actual data lives at /app/exe-monitor_data.
+    # --dir must point there (NOT /beszel_data) and the volume must mount there too,
+    # otherwise container recreates wipe all monitor history.
+    command: ["serve", "--http=0.0.0.0:8090", "--dir=/app/exe-monitor_data"]
     environment:
       EXE_MONITOR_ADMIN_TOKEN: ${EXE_MONITOR_ADMIN_TOKEN:-}
       # bug a125785d: shared secret for POST /api/exe-monitor/errors. The hub
@@ -266,7 +273,11 @@ services:
     ports:
       - "127.0.0.1:${MONITOR_HUB_PORT:-8090}:8090"
     volumes:
-      - monitor_hub_data:/beszel_data
+      # bug 1fdb8be9: mount at the ACTUAL data dir the hub binary writes to.
+      # Was /beszel_data (stock Beszel path), but the exe-monitor fork uses
+      # /app/exe-monitor_data. Mismatch meant data lived in the writable layer
+      # and was lost on every container recreate / stack-update.
+      - monitor_hub_data:/app/exe-monitor_data
     networks:
       - backend
     healthcheck:
@@ -329,6 +340,11 @@ services:
       NODE_PORT: "3000"
       EXE_LICENSE_KEY: ${EXE_LICENSE_KEY:?EXE_LICENSE_KEY is required — purchase at https://askexe.com}
       SERVER_URL: ${CRM_SERVER_URL:-https://crm.askexe.com}
+      # bug b99d3762 / 83192765: the CRM error-forwarder POSTs to the monitor hub and must
+      # send X-Monitor-Key once the hub requires it on every path (mirrors gateway/erp).
+      ERROR_REPORTING_ENABLED: "true"
+      MONITOR_ERROR_URL: http://exe-monitor-hub:8090/api/exe-monitor/errors
+      MONITOR_API_KEY: ${MONITOR_API_KEY:-${EXE_MONITOR_KEY:-}}
       APP_SECRET: ${CRM_APP_SECRET:?CRM_APP_SECRET is required}
       EXE_CRM_ADMIN_TOKEN: ${EXE_CRM_ADMIN_TOKEN:-}
       EXE_CRM_ADMIN_EMAIL: ${EXE_CRM_ADMIN_EMAIL:-}
@@ -444,6 +460,11 @@ services:
       SERVER_PORT: "3001"
       EXE_LICENSE_KEY: ${EXE_LICENSE_KEY:?EXE_LICENSE_KEY is required — purchase at https://askexe.com}
       STORAGE_DIR: /app/server/storage
+      # bug b99d3762 / 83192765: the wiki error-reporter POSTs to the monitor hub and must
+      # send X-Monitor-Key once the hub requires it on every path (mirrors gateway/erp).
+      ERROR_REPORTING_ENABLED: "true"
+      MONITOR_ERROR_URL: http://exe-monitor-hub:8090/api/exe-monitor/errors
+      MONITOR_API_KEY: ${MONITOR_API_KEY:-${EXE_MONITOR_KEY:-}}
       DATABASE_URL: postgres://${POSTGRES_USER:-exe}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@exe-db:5432/${POSTGRES_DB:-exedb}?schema=${WIKI_DB_SCHEMA:-wiki}&sslmode=disable
       AUTH_TOKEN: ${WIKI_AUTH_TOKEN:?WIKI_AUTH_TOKEN is required}
       EXE_WIKI_ADMIN_TOKEN: ${EXE_WIKI_ADMIN_TOKEN:-}
@@ -597,6 +618,14 @@ services:
       - exe_os_data:/home/exed/.exe-os
     networks:
       - backend
+    healthcheck:
+      # The worker is a long-running node process (embed-worker.js). Verify the
+      # PID-1 node process is alive via /proc/1/cmdline.
+      test: ["CMD-SHELL", "test -f /proc/1/cmdline"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -637,6 +666,13 @@ services:
       NODE_ENV: production
       EXE_GATEWAY_HOME: /data
       EXE_GATEWAY_CONFIG: /data/gateway.json
+      # bug 02bc1bb8: customer overlay dir. The host ./overlay is bind-mounted
+      # read-only at /data/overlay (see volumes below) so customer overlays
+      # (e.g. po-intake.js) SURVIVE every image upgrade / container recreate
+      # instead of being silently dropped from the old container layer. The
+      # gateway loads overlays from this dir; the post-deploy add-routes.sh hook
+      # additionally re-applies them for gateway builds that load from /app.
+      EXE_GATEWAY_OVERLAY_DIR: /data/overlay
       DATABASE_URL: postgres://${POSTGRES_USER:-exe}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@exe-db:5432/${POSTGRES_DB:-exedb}?sslmode=disable
       EXE_GATEWAY_PORT: "3100"
       EXE_GATEWAY_HOST: "0.0.0.0"
@@ -675,6 +711,14 @@ services:
       - gateway_data:/data
       - gateway_whatsapp_auth:/app/auth_info
       - ./gateway.json:/data/gateway.json:ro
+      # bug 02bc1bb8: PERSISTENT customer-overlay bind-mount. Anything the
+      # customer drops in the host ./overlay dir (e.g. po-intake.js) is exposed
+      # read-only inside the container at /data/overlay and therefore SURVIVES
+      # image upgrades / `docker compose up` recreate — unlike files that only
+      # ever lived in the container's writable layer (which a recreate drops).
+      # bootstrapStackHost ensures ./overlay exists so docker never auto-creates
+      # the bind path as a stray file. Empty for customers with no overlay.
+      - ./overlay:/data/overlay:ro
     networks:
       - backend
       - frontend
@@ -718,8 +762,9 @@ services:
       exe-monitor-hub:
         condition: service_healthy
     environment:
-      # Reports to the local hub by default. stack-update auto-pairs this agent
-      # with the hub via /api/register-agent and writes TOKEN + KEY back to .env.
+      # Reports to the local hub by default. The agent must be paired manually
+      # from the hub UI (Settings → Add System) — the hub does not expose a
+      # REST registration endpoint. Write the generated TOKEN + KEY into .env.
       HUB_URL: ${MONITOR_HUB_URL:-http://exe-monitor-hub:8090}
       # TOKEN and KEY are written automatically by `exe-os stack-update` during
       # bootstrap. They start empty; the agent retries connecting until the hub
@@ -1154,6 +1199,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the socketio node process is listening on port 9000.
+      test: ["CMD-SHELL", "node -e \"const s=require('net').connect(9000,'127.0.0.1',()=>{s.destroy();process.exit(0)});s.setTimeout(4000,()=>{s.destroy();process.exit(1)});s.on('error',()=>process.exit(1))\""]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -1193,6 +1245,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the bench worker process is alive and Redis queue is reachable.
+      test: ["CMD-SHELL", "pgrep -f 'rq worker' > /dev/null"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:
@@ -1232,6 +1291,13 @@ services:
       - erp_assets:/home/frappe/frappe-bench/sites/assets
     networks:
       - backend
+    healthcheck:
+      # Verify the bench schedule process is alive.
+      test: ["CMD-SHELL", "pgrep -f 'bench schedule' > /dev/null"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     deploy:
       resources:
         limits:

package/deploy/compose/gateway-watchdog.sh

@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+# exe-os gateway watchdog — verifies the exe-gateway WhatsApp adapter is actually
+# CONNECTED (not just that the HTTP server answers) and restarts the gateway only
+# when it is genuinely wedged.
+#
+# Install (cron, every 2 min):
+#   chmod +x /opt/exe-stack/gateway-watchdog.sh
+#   crontab -l | { cat; echo "*/2 * * * * /opt/exe-stack/gateway-watchdog.sh 2>&1 | logger -t exe-gw-watchdog"; } | crontab -
+#
+# ─────────────────────────────────────────────────────────────────────────────
+# BUG HISTORY (facd5078, P0)
+# ─────────────────────────────────────────────────────────────────────────────
+# The previous watchdog curled the UNAUTHENTICATED http://localhost:3100/health
+# and grepped for `"connected":true`. But the unauth /health only returns
+# {status, uptime, adapterCount} — it carries NO adapter connection state — so the
+# grep ALWAYS failed → 3 strikes every ~6 min → `docker restart exe-gateway`
+# 24/7 (1,668 restarts in 72h on HYGO). `docker restart` does NOT increment the
+# container RestartCount, which masked the loop entirely.
+#
+# This rewrite makes the check ACCURATE and FAIL-SAFE:
+#   1. AUTHENTICATED probe of /admin/webhooks/health (Bearer EXE_GATEWAY_AUTH_TOKEN),
+#      which reports real per-adapter state. Preferred — correct regardless of
+#      gateway version.
+#   2. If the running gateway already exposes `adapters_connected` on the UNAUTH
+#      /health (exe-gateway bug 7e9fc713), that is honored too — but we do NOT
+#      hard-depend on it shipping first.
+#   3. FAIL SAFE: if EXE_GATEWAY_AUTH_TOKEN is unset, OR the health body cannot be
+#      parsed into a definite connected/disconnected verdict, the result is
+#      INCONCLUSIVE → we LOG and do NOT restart. A watchdog must never restart on
+#      an unknown.
+#   4. STARTUP GRACE: the gateway is given GRACE_SECS of container uptime before
+#      any strike is counted, so a gateway that is merely (re)starting / pairing
+#      WhatsApp is never restart-looped.
+#   5. BACKOFF: after a restart we refuse to restart again for COOLDOWN_SECS, so we
+#      never hammer a gateway that needs time to reconnect.
+
+set -uo pipefail
+
+# ── Config ───────────────────────────────────────────────────────────────────
+STACK_DIR="${EXE_STACK_DIR:-/opt/exe-stack}"
+ENV_FILE="${EXE_STACK_ENV_FILE:-${STACK_DIR}/.env}"
+CONTAINER="${EXE_GATEWAY_CONTAINER:-exe-gateway}"
+PORT="${EXE_GATEWAY_HTTP_PORT:-3100}"
+HOST="${EXE_GATEWAY_HOST:-127.0.0.1}"
+TIMEOUT="${EXE_GATEWAY_WATCHDOG_TIMEOUT:-5}"
+
+# Strikes before a restart. Each cron tick = one probe.
+MAX_STRIKES="${EXE_GATEWAY_WATCHDOG_STRIKES:-3}"
+# Don't count strikes until the container has been up at least this long.
+GRACE_SECS="${EXE_GATEWAY_WATCHDOG_GRACE_SECS:-120}"
+# After a restart, refuse to restart again for this long (let it reconnect).
+COOLDOWN_SECS="${EXE_GATEWAY_WATCHDOG_COOLDOWN_SECS:-300}"
+
+STATE_DIR="${EXE_GATEWAY_WATCHDOG_STATE_DIR:-${STACK_DIR}/.gateway-watchdog}"
+STRIKE_FILE="${STATE_DIR}/strikes"
+LAST_RESTART_FILE="${STATE_DIR}/last-restart"
+
+mkdir -p "$STATE_DIR" 2>/dev/null || true
+
+ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+log() { echo "[$(ts)] gateway-watchdog: $*"; }
+
+# Read a single KEY=VALUE from the stack .env WITHOUT sourcing it (the .env holds
+# many secrets and arbitrary values; sourcing could execute / mangle them).
+read_env() {
+  local key="$1"
+  [[ -f "$ENV_FILE" ]] || return 0
+  # Last assignment wins; strip surrounding quotes; ignore comments.
+  sed -n "s/^[[:space:]]*${key}=//p" "$ENV_FILE" 2>/dev/null | tail -n1 \
+    | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"
+}
+
+reset_strikes() { echo 0 > "$STRIKE_FILE" 2>/dev/null || true; }
+read_strikes()  { local s; s=$(cat "$STRIKE_FILE" 2>/dev/null || echo 0); [[ "$s" =~ ^[0-9]+$ ]] && echo "$s" || echo 0; }
+bump_strikes()  { local s; s=$(read_strikes); echo $((s + 1)) > "$STRIKE_FILE" 2>/dev/null || true; echo $((s + 1)); }
+
+# Container uptime in seconds (0 if not running / unknown).
+container_uptime_secs() {
+  local started running
+  running=$(docker inspect --format '{{.State.Running}}' "$CONTAINER" 2>/dev/null || echo "false")
+  [[ "$running" == "true" ]] || { echo 0; return; }
+  started=$(docker inspect --format '{{.State.StartedAt}}' "$CONTAINER" 2>/dev/null || echo "")
+  [[ -n "$started" ]] || { echo 0; return; }
+  local start_epoch now_epoch
+  start_epoch=$(date -u -d "$started" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%S" "${started%%.*}" +%s 2>/dev/null || echo 0)
+  now_epoch=$(date -u +%s)
+  if [[ "$start_epoch" -gt 0 && "$now_epoch" -ge "$start_epoch" ]]; then
+    echo $((now_epoch - start_epoch))
+  else
+    echo 0
+  fi
+}
+
+seconds_since_last_restart() {
+  local last now
+  last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
+  [[ "$last" =~ ^[0-9]+$ ]] || last=0
+  now=$(date -u +%s)
+  echo $((now - last))
+}
+
+# ── Adapter-state probe ──────────────────────────────────────────────────────
+# Echoes exactly one of: connected | disconnected | inconclusive
+probe_adapter_state() {
+  local token unauth auth
+
+  # 1) Forward-compat: unauth /health MAY expose adapters_connected (gw bug 7e9fc713).
+  unauth=$(curl -sk --max-time "$TIMEOUT" "http://${HOST}:${PORT}/health" 2>/dev/null || echo "")
+  if [[ -n "$unauth" ]]; then
+    if echo "$unauth" | grep -Eq '"adapters_connected"[[:space:]]*:[[:space:]]*true'; then
+      echo "connected"; return
+    fi
+    if echo "$unauth" | grep -Eq '"adapters_connected"[[:space:]]*:[[:space:]]*false'; then
+      echo "disconnected"; return
+    fi
+    # No adapters_connected field → this gateway predates 7e9fc713. Fall through to
+    # the authenticated probe; the bare {status,uptime,adapterCount} body is NOT a
+    # connection verdict and must never be treated as one.
+  fi
+
+  # 2) Authenticated /admin/webhooks/health — reports real per-adapter state.
+  token=$(read_env EXE_GATEWAY_AUTH_TOKEN)
+  if [[ -z "$token" ]]; then
+    # FAIL SAFE: no token → we cannot make an authoritative judgement. Never restart.
+    echo "inconclusive"; return
+  fi
+
+  auth=$(curl -sk --max-time "$TIMEOUT" \
+    -H "Authorization: Bearer ${token}" \
+    "http://${HOST}:${PORT}/admin/webhooks/health" 2>/dev/null || echo "")
+
+  if [[ -z "$auth" ]]; then
+    # Couldn't reach the authed endpoint at all → inconclusive (could be a transient
+    # network blip; the HTTP layer may still be fine). Don't restart on this alone.
+    echo "inconclusive"; return
+  fi
+
+  # Real connection verdict. The authed health reports adapter connection via
+  # "connected":true and/or a per-account "whatsapp":{... "connected":true}.
+  if echo "$auth" | grep -Eq '"connected"[[:space:]]*:[[:space:]]*true'; then
+    echo "connected"; return
+  fi
+  if echo "$auth" | grep -Eq '"handlerRegistered"[[:space:]]*:[[:space:]]*true' \
+     && echo "$auth" | grep -Eq '"connected"[[:space:]]*:[[:space:]]*false'; then
+    echo "disconnected"; return
+  fi
+  # An authed body we can't classify (schema drift) is NOT grounds to restart.
+  echo "inconclusive"
+}
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+main() {
+  # Gateway not running at all? Let Docker's own `restart: unless-stopped` handle
+  # a crashed container — the watchdog only acts on a RUNNING-but-wedged gateway.
+  local uptime
+  uptime=$(container_uptime_secs)
+  if [[ "$uptime" -eq 0 ]]; then
+    log "container ${CONTAINER} not running (or uptime unknown) — deferring to Docker restart policy; not counting a strike."
+    reset_strikes
+    exit 0
+  fi
+
+  local state
+  state=$(probe_adapter_state)
+
+  case "$state" in
+    connected)
+      reset_strikes
+      log "OK adapters connected (uptime ${uptime}s)."
+      exit 0
+      ;;
+    inconclusive)
+      # NEVER restart on an inconclusive check. Log and reset so a transient
+      # unknown can't accumulate toward a restart.
+      reset_strikes
+      log "INCONCLUSIVE health check (token unset or unparseable body) — NOT restarting."
+      exit 0
+      ;;
+    disconnected)
+      : # fall through to strike logic below
+      ;;
+    *)
+      reset_strikes
+      log "unexpected probe result '${state}' — treating as inconclusive, NOT restarting."
+      exit 0
+      ;;
+  esac
+
+  # ── disconnected ──
+  # Startup grace: a gateway that is merely (re)starting / re-pairing WhatsApp must
+  # not be counted against. Require GRACE_SECS of uptime before any strike.
+  if [[ "$uptime" -lt "$GRACE_SECS" ]]; then
+    log "adapter disconnected but within startup grace (uptime ${uptime}s < ${GRACE_SECS}s) — not counting a strike."
+    exit 0
+  fi
+
+  # Backoff: don't restart again within the cooldown window — give the gateway time
+  # to reconnect after the previous restart.
+  local since_restart
+  since_restart=$(seconds_since_last_restart)
+  if [[ "$since_restart" -lt "$COOLDOWN_SECS" ]]; then
+    log "adapter disconnected but within post-restart cooldown (${since_restart}s < ${COOLDOWN_SECS}s) — backing off, not restarting."
+    exit 0
+  fi
+
+  local strikes
+  strikes=$(bump_strikes)
+  log "adapter DISCONNECTED — strike ${strikes}/${MAX_STRIKES} (uptime ${uptime}s)."
+
+  if [[ "$strikes" -ge "$MAX_STRIKES" ]]; then
+    log "ALERT strike threshold reached — restarting ${CONTAINER}."
+    if docker restart "$CONTAINER" >/dev/null 2>&1; then
+      date -u +%s > "$LAST_RESTART_FILE" 2>/dev/null || true
+      reset_strikes
+      log "restarted ${CONTAINER}; entering ${COOLDOWN_SECS}s cooldown."
+    else
+      log "ERROR docker restart ${CONTAINER} failed."
+    fi
+  fi
+  exit 0
+}
+
+main "$@"

package/deploy/compose/setup.sh

@@ -52,11 +52,10 @@ info "Step 2: Generating .env file"
 if [[ -f .env ]]; then
   warn ".env already exists — skipping generation. Delete .env to regenerate."
 else
-  if command -v node >/dev/null 2>&1; then
-    node -e "
-      const { generateEnv } = require('../../dist/deploy/compose/generate-env.js');
-      console.log(generateEnv({ clientName: '$CLIENT', domain: '$DOMAIN', licenseKey: '${LICENSE:-}' || undefined }));
-    " > .env 2>/dev/null || {
+  if command -v node >/dev/null 2>&1 && command -v npx >/dev/null 2>&1; then
+    # generate-env.ts is a TypeScript ESM file shipped at deploy/compose/generate-env.ts.
+    # Run it via npx tsx (the project is pure ESM — require() is forbidden).
+    npx --yes tsx "$SCRIPT_DIR/generate-env.ts" "$CLIENT" "$DOMAIN" "${LICENSE:-}" > .env 2>/dev/null || {
       # Fallback: generate inline
       info "Generating secrets inline..."
       gen() { openssl rand -hex "$1"; }
@@ -177,8 +176,10 @@ fi
 # exe-monitor-agent reports fleet health to the monitoring hub, but to do so it
 # needs SENSITIVE read-only host access:
 #   - /var/run/docker.sock  (root-equivalent: enumerate/inspect ALL containers)
-#   - /proc, /sys           (host process list + system-wide CPU/mem/hardware)
 #   - /etc/os-release       (host OS identity)
+# (bug 107cde54: /proc and /sys mounts were REMOVED — the Beszel-based agent
+#  does not read bind-mounted host /proc or /sys, so they were pure attack
+#  surface with no metrics benefit.)
 # The service is gated behind the "monitor-agent" compose profile and stays OFF
 # until the operator explicitly acknowledges this scope here. Acknowledgement is
 # recorded by adding "monitor-agent" to COMPOSE_PROFILES in .env. Pre-answer in
@@ -196,8 +197,6 @@ else
   ├──────────────────────────────────────────────────────────────────────┤
   │  Enabling the monitor agent grants it READ-ONLY access to the host:    │
   │    • /var/run/docker.sock  — inspect ALL containers (root-equivalent)  │
-  │    • /proc                 — host process list + system CPU/mem stats   │
-  │    • /sys                  — host hardware/kernel metrics               │
   │    • /etc/os-release       — host OS name/version                       │
   │                                                                        │
   │  The Docker socket is effectively root on this machine. Only enable    │

package/deploy/stack-manifests/v0.9.json

@@ -1344,12 +1344,13 @@
       "breakingChanges": [],
       "services": {
         "exe-db": {
-          "image": "pgvector/pgvector:pg16",
+          "image": "pgvector/pgvector:0.8.0-pg16",
           "env": "EXE_DB_IMAGE",
           "composeService": "exe-db",
           "required": true,
           "category": "core",
           "description": "Postgres (pgvector) \u2014 shared system of record for all services",
+          "_comment": "bug ee16e7ef: pinned to versioned 0.8.0-pg16 tag \u2014 the mutable :pg16 rolling tag can change upstream without notice.",
           "migrations": {
             "command": "exec -T exe-db psql -v ON_ERROR_STOP=1 -U ${POSTGRES_USER:-exe} -d ${POSTGRES_DB:-exedb} -f /docker-entrypoint-initdb.d/01-init.sql"
           }
@@ -1438,6 +1439,48 @@
           "category": "core",
           "description": "GoTrue auth backend \u2014 issues SSO/login tokens for CRM/Wiki/ERP",
           "_comment": "bug 2976868a: the stack health gate previously omitted GoTrue, so SSO/login could be fully down while the gate reported green. Gate the backend directly."
+        },
+        "redis": {
+          "image": "redis:7.4-alpine",
+          "composeService": "redis",
+          "required": true,
+          "category": "infra",
+          "description": "Redis \u2014 session cache, job queues (CRM, gateway, ERP)",
+          "_comment": "bug 4948fc48: compose-pinned infra dep \u2014 no env override, image tag lives in docker-compose.yml."
+        },
+        "clickhouse": {
+          "image": "clickhouse/clickhouse-server:24.8.4.13-alpine",
+          "env": "CLICKHOUSE_IMAGE",
+          "composeService": "clickhouse",
+          "required": true,
+          "category": "infra",
+          "description": "ClickHouse \u2014 analytics and event storage (CRM)",
+          "_comment": "bug 4948fc48: compose-pinned infra dep required by CRM."
+        },
+        "cloudflared": {
+          "image": "cloudflare/cloudflared@sha256:ba461b8aa9c042156dbd39c38657fe7431bafa063220eab8d5330a523863da9f",
+          "composeService": "cloudflared",
+          "required": true,
+          "category": "infra",
+          "description": "Cloudflare Tunnel \u2014 secure ingress proxy (replaces nginx + SSL certs)",
+          "_comment": "bug 4948fc48: digest-pinned in compose, no env override."
+        },
+        "exe-sso-edge": {
+          "image": "nginx:alpine",
+          "composeService": "exe-sso-edge",
+          "required": true,
+          "category": "infra",
+          "description": "SSO edge \u2014 reverse proxy that gates CRM/wiki/ERP behind unified auth",
+          "_comment": "bug 4948fc48: stock nginx:alpine, config templated from DOMAIN."
+        },
+        "exe-otel-collector": {
+          "image": "otel/opentelemetry-collector-contrib:0.114.0",
+          "env": "OTEL_COLLECTOR_IMAGE",
+          "composeService": "exe-otel-collector",
+          "required": false,
+          "category": "observability",
+          "description": "OpenTelemetry Collector \u2014 receives OTLP traces/metrics/logs from all services",
+          "_comment": "bug 4948fc48: compose-pinned observability infra."
         }
       }
     }