@askexenow/exe-os 0.9.290 → 0.9.292

package/deploy/compose/.env.customer.example

@@ -2,6 +2,10 @@
 # Copy to .env before deployment and replace every CHANGEME_* value.
 # Values under # SET_MANUALLY must be provided by the operator.
 
+# --- Domain ---
+# Customer apex domain. Drives auth gateway templating and GoTrue From address.
+DOMAIN=CHANGEME_DOMAIN
+
 # --- Data Layer ---
 POSTGRES_USER=exe
 POSTGRES_PASSWORD=CHANGEME_POSTGRES_PASSWORD
@@ -18,8 +22,17 @@ GOTRUE_JWT_SECRET=CHANGEME_GOTRUE_JWT_SECRET
 GOTRUE_API_PORT=9999
 GOTRUE_SITE_URL=https://crm.CHANGEME_DOMAIN
 GOTRUE_EXTERNAL_URL=https://auth.CHANGEME_DOMAIN
+# SSO redirect allow-list (bug 66f8e10a) — app origins for unified SSO.
+GOTRUE_URI_ALLOW_LIST=https://crm.CHANGEME_DOMAIN,https://wiki.CHANGEME_DOMAIN,https://erp.CHANGEME_DOMAIN
 GOTRUE_DISABLE_SIGNUP=true
 GOTRUE_MAILER_AUTOCONFIRM=false
+# Auth-email From address — customer domain, not askexe.com (bug 133c9d5b).
+SMTP_FROM=noreply@CHANGEME_DOMAIN
+# To enable verified email confirmation, set SMTP_* (MAILER_AUTOCONFIRM stays false).
+SMTP_HOST=
+SMTP_PORT=587
+SMTP_USER=
+SMTP_PASS=
 
 # --- CRM ---
 CRM_IMAGE_TAG=update.askexe.com/askexe/exe-crm:v0.9.47
@@ -48,7 +61,7 @@ EXE_MCP_HOST_PORT=48739
 EXE_CLOUD_SYNC_TO_POSTGRES=true
 
 # --- ERP (Frappe/ERPNext fork) ---
-ERP_IMAGE_TAG=update.askexe.com/askexe/exe-erp:v0.2.0-final7
+ERP_IMAGE_TAG=update.askexe.com/askexe/exe-erp:v0.2.0-final8
 # Administrator password for the ERPNext site (created on first boot).
 ERP_ADMIN_PASSWORD=CHANGEME_ERP_ADMIN_PASSWORD
 ERP_SITE_NAME=erp.askexe.com

package/deploy/compose/.env.example

@@ -2,6 +2,10 @@
 # Copy to .env before deployment and replace every CHANGEME_* value.
 # Values under # SET_MANUALLY must be provided by the operator.
 
+# --- Domain ---
+# Customer apex domain. Drives auth gateway templating and GoTrue From address.
+DOMAIN=CHANGEME_DOMAIN
+
 # --- Data Layer ---
 POSTGRES_USER=exe
 POSTGRES_PASSWORD=CHANGEME_POSTGRES_PASSWORD
@@ -13,13 +17,48 @@ CLICKHOUSE_PASSWORD=CHANGEME_CLICKHOUSE_PASSWORD
 
 REDIS_PASSWORD=CHANGEME_REDIS_PASSWORD
 
+# --- Backups (bug 56a71515 off-box, fc71e7ca failure handling) ---
+# Off-box upload: the daily cron runs `backup.sh --upload-r2`. Without this the
+# encrypted archive only ever lands on the same disk as live data (disk death =
+# total loss). R2 creds are reused from the R2_MEDIA_* vars when set.
+# EXE_BACKUP_KEY=CHANGEME_EXE_BACKUP_KEY   # 256-bit key — encrypts archives at rest
+BACKUP_RETENTION_DAYS=7
+# Override the DB container name if your topology differs (default: exe-db).
+# BACKUP_DB_CONTAINER=exe-db
+# Failure alerting (no silent success). Set either/both:
+#   BACKUP_ALERT_WEBHOOK   — POST {host,stage,message} on any backup failure
+#   BACKUP_HEALTHCHECK_URL — ping on success, <url>/fail on failure (healthchecks.io)
+# BACKUP_ALERT_WEBHOOK=
+# BACKUP_HEALTHCHECK_URL=
+
+# WAL archiving / Point-In-Time Recovery (bug 5176aa4e) — OPT-IN.
+# Defaults below keep the stack booting exactly as before (archive_mode=off,
+# RPO = nightly pg_dump). Set PG_ARCHIVE_MODE=on to enable continuous WAL
+# archiving into the pg_wal_archive volume; back that volume off-box for PITR.
+# PG_ARCHIVE_MODE=off
+# PG_WAL_LEVEL=replica
+# PG_ARCHIVE_TIMEOUT=300
+# PG_MAX_WAL_SENDERS=3
+# Custom archive target (e.g. push WAL straight to off-box storage). Leave unset
+# to use the default copy into /var/lib/postgresql/wal_archive.
+# PG_ARCHIVE_COMMAND=
+
 # --- GoTrue (shared auth) ---
 GOTRUE_JWT_SECRET=CHANGEME_GOTRUE_JWT_SECRET
 GOTRUE_API_PORT=9999
 GOTRUE_SITE_URL=https://crm.CHANGEME_DOMAIN
 GOTRUE_EXTERNAL_URL=https://auth.CHANGEME_DOMAIN
+# SSO redirect allow-list (bug 66f8e10a) — app origins for unified SSO.
+GOTRUE_URI_ALLOW_LIST=https://crm.CHANGEME_DOMAIN,https://wiki.CHANGEME_DOMAIN,https://erp.CHANGEME_DOMAIN
 GOTRUE_DISABLE_SIGNUP=true
 GOTRUE_MAILER_AUTOCONFIRM=false
+# From address for auth emails — customer domain, not askexe.com (bug 133c9d5b).
+SMTP_FROM=noreply@CHANGEME_DOMAIN
+# To enable verified email confirmation, set SMTP_* below (MAILER_AUTOCONFIRM stays false).
+SMTP_HOST=
+SMTP_PORT=587
+SMTP_USER=
+SMTP_PASS=
 
 # --- CRM ---
 CRM_IMAGE_TAG=ghcr.io/askexe/exe-crm:v0.9.47
@@ -73,7 +112,7 @@ ERROR_REPORTING_ENABLED=true
 MONITOR_ERROR_URL=http://exe-monitor-hub:8090/api/exe-monitor/errors
 
 # --- ERP (optional — Exe ERP, Frappe/ERPNext fork) ---
-ERP_IMAGE_TAG=ghcr.io/askexe/exe-erp:v0.2.0-final7
+ERP_IMAGE_TAG=ghcr.io/askexe/exe-erp:v0.2.0-final8
 ERP_ADMIN_PASSWORD=CHANGEME_ERP_ADMIN_PASSWORD
 EXE_ERP_ADMIN_TOKEN=CHANGEME_EXE_ERP_ADMIN_TOKEN
 ERP_SITE_NAME=erp.askexe.com

package/deploy/compose/backup.sh

@@ -5,8 +5,19 @@
 #   ./backup.sh                          # Backup to /opt/exe-backups/
 #   ./backup.sh --output /tmp/backup     # Backup to custom dir
 #   ./backup.sh --download               # Create backup + print download command
-#   ./backup.sh --upload-r2              # Backup + upload to Cloudflare R2
-#   ./backup.sh --upload-s3 s3://bucket  # Backup + upload to S3
+#   ./backup.sh --upload-r2              # Backup + upload to Cloudflare R2 (off-box)
+#   ./backup.sh --upload-s3 s3://bucket  # Backup + upload to S3 (off-box)
+#
+# FAILURE HANDLING (bug fc71e7ca):
+#   The script aborts (non-zero exit) the moment a CRITICAL step fails — the
+#   postgres dump is gated on pg_dumpall's exit code, so it NEVER prints
+#   "backup ok" over a failed/empty dump. Failures are logged AND surface an
+#   alert via an optional webhook (BACKUP_ALERT_WEBHOOK) and/or a healthcheck
+#   ping (BACKUP_HEALTHCHECK_URL — e.g. healthchecks.io). No silent success.
+#
+# OFF-BOX (bug 56a71515): pass --upload-r2 (or --upload-s3) — or set the cron
+#   to do so — so the encrypted archive leaves the VPS disk. A backup that only
+#   ever lands on the same disk as live data does not survive disk/host loss.
 #
 set -euo pipefail
 
@@ -17,6 +28,51 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 DOWNLOAD_MODE=false
 UPLOAD_R2=false
 UPLOAD_S3=""
+# Real DB container name. Past bug fc71e7ca referenced a non-existent
+# 'crm-postgres'; the canonical stack DB container is 'exe-db'. Override via env.
+DB_CONTAINER="${BACKUP_DB_CONTAINER:-exe-db}"
+PG_USER="${POSTGRES_USER:-exe}"
+
+# ------------------------------------------------------------------
+# Alerting — surface failures loudly. Optional, configured via env:
+#   BACKUP_ALERT_WEBHOOK   — POST a JSON {host,stage,message} on failure
+#   BACKUP_HEALTHCHECK_URL — ping <url> on success, <url>/fail on failure
+#                            (compatible with healthchecks.io / Uptime Kuma)
+# ------------------------------------------------------------------
+HOST_NAME="$(hostname 2>/dev/null || echo unknown)"
+ALERT_WEBHOOK="${BACKUP_ALERT_WEBHOOK:-}"
+HEALTHCHECK_URL="${BACKUP_HEALTHCHECK_URL:-}"
+if [[ -z "$ALERT_WEBHOOK" && -f "$SCRIPT_DIR/.env" ]]; then
+  ALERT_WEBHOOK=$(grep -E '^BACKUP_ALERT_WEBHOOK=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2- || true)
+fi
+if [[ -z "$HEALTHCHECK_URL" && -f "$SCRIPT_DIR/.env" ]]; then
+  HEALTHCHECK_URL=$(grep -E '^BACKUP_HEALTHCHECK_URL=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2- || true)
+fi
+
+alert() {
+  # alert <stage> <message>
+  local stage="$1"; shift
+  local msg="$*"
+  echo "[backup][ERROR] stage=$stage: $msg" >&2
+  if [[ -n "$ALERT_WEBHOOK" ]] && command -v curl >/dev/null 2>&1; then
+    curl -fsS -m 10 -X POST "$ALERT_WEBHOOK" \
+      -H 'Content-Type: application/json' \
+      -d "{\"host\":\"$HOST_NAME\",\"stage\":\"$stage\",\"message\":\"$msg\"}" \
+      >/dev/null 2>&1 || echo "[backup] (alert webhook POST failed)" >&2
+  fi
+  if [[ -n "$HEALTHCHECK_URL" ]] && command -v curl >/dev/null 2>&1; then
+    curl -fsS -m 10 "${HEALTHCHECK_URL%/}/fail" >/dev/null 2>&1 || true
+  fi
+}
+
+# fail <stage> <message> — emit alert and exit non-zero. No silent success.
+fail() {
+  alert "$1" "$2"
+  exit 1
+}
+
+# Any unexpected error (set -e) routes through here so we still alert.
+trap 'fail "unexpected" "backup aborted at line $LINENO (see log above)"' ERR
 
 while [[ $# -gt 0 ]]; do
   case $1 in
@@ -34,12 +90,26 @@ mkdir -p "$ARCHIVE"
 
 echo "[backup] Starting exe-os full stack backup ($DATE)"
 
-# 1. Postgres dump (all databases)
-echo "[backup] Dumping postgres..."
-docker exec exe-db pg_dumpall -U exe > "$ARCHIVE/postgres.sql" 2>/dev/null || echo "[backup] Postgres dump failed"
-gzip "$ARCHIVE/postgres.sql" 2>/dev/null || true
+# 1. Postgres dump (all databases) — CRITICAL. Gate success on the exit code.
+#    Previously a failure here was swallowed by `|| echo` and the script still
+#    printed "backup ok" over an empty dump (bug fc71e7ca). Now we abort.
+echo "[backup] Dumping postgres from container '$DB_CONTAINER'..."
+if ! docker exec "$DB_CONTAINER" pg_dumpall -U "$PG_USER" > "$ARCHIVE/postgres.sql" 2> "$ARCHIVE/postgres.err"; then
+  cat "$ARCHIVE/postgres.err" >&2 || true
+  fail "pg_dumpall" "postgres dump failed (container=$DB_CONTAINER user=$PG_USER) — backup aborted, NOT marked ok"
+fi
+# Guard against a 0-byte / truncated dump that still exits 0.
+if [[ ! -s "$ARCHIVE/postgres.sql" ]]; then
+  fail "pg_dumpall" "postgres dump produced an empty file — refusing to report success"
+fi
+rm -f "$ARCHIVE/postgres.err"
+if ! gzip "$ARCHIVE/postgres.sql"; then
+  fail "gzip" "failed to compress postgres.sql"
+fi
+echo "[backup] Postgres dump OK ($(du -h "$ARCHIVE/postgres.sql.gz" | cut -f1))"
 
 # 2. Customer config files (.env, gateway.json, branding.json, clickhouse-config.xml, cloudflared)
+#    Non-fatal: copy what exists. These are also recoverable from the operator's saved .env.
 echo "[backup] Backing up config files..."
 cp "$SCRIPT_DIR/.env" "$ARCHIVE/env.bak" 2>/dev/null || true
 cp "$SCRIPT_DIR/gateway.json" "$ARCHIVE/gateway.json" 2>/dev/null || true
@@ -49,20 +119,22 @@ cp -r "$SCRIPT_DIR/cloudflared" "$ARCHIVE/cloudflared" 2>/dev/null || true
 
 # 3. Gateway auth state (Baileys creds — critical for WhatsApp connection)
 echo "[backup] Backing up gateway WhatsApp auth state..."
-docker cp exe-gateway:/data/. "$ARCHIVE/gateway-data/" 2>/dev/null || echo "[backup] Gateway data skipped"
+docker cp exe-gateway:/data/. "$ARCHIVE/gateway-data/" 2>/dev/null || echo "[backup] Gateway data skipped (container not running)"
 
 # 4. Wiki storage (uploaded documents)
 echo "[backup] Backing up wiki storage..."
-docker cp exe-wiki:/app/server/storage/. "$ARCHIVE/wiki-storage/" 2>/dev/null || echo "[backup] Wiki storage skipped"
+docker cp exe-wiki:/app/server/storage/. "$ARCHIVE/wiki-storage/" 2>/dev/null || echo "[backup] Wiki storage skipped (container not running)"
 
 # 5. exe-os daemon state (optional — SQLCipher DB is local-first, not critical for VPS)
 echo "[backup] Backing up exe-os state..."
-docker cp exe-os:/home/exed/.exe-os/. "$ARCHIVE/exe-os-data/" 2>/dev/null || echo "[backup] exe-os data skipped"
+docker cp exe-os:/home/exed/.exe-os/. "$ARCHIVE/exe-os-data/" 2>/dev/null || echo "[backup] exe-os data skipped (container not running)"
 
-# 6. Create single archive (compressed)
+# 6. Create single archive (compressed) — CRITICAL.
 echo "[backup] Creating archive..."
 TARFILE="$BACKUP_DIR/exe-backup-$DATE.tar.gz"
-tar -czf "$TARFILE" -C "$BACKUP_DIR" "exe-backup-$DATE" 2>/dev/null
+if ! tar -czf "$TARFILE" -C "$BACKUP_DIR" "exe-backup-$DATE"; then
+  fail "tar" "failed to create archive $TARFILE"
+fi
 rm -rf "$ARCHIVE"
 echo "[backup] Archive: $TARFILE ($(du -h "$TARFILE" | cut -f1))"
 
@@ -71,28 +143,30 @@ echo "[backup] Archive: $TARFILE ($(du -h "$TARFILE" | cut -f1))"
 # Without this key, R2 backups are unreadable — even AskExe cannot access them.
 BACKUP_KEY="${EXE_BACKUP_KEY:-}"
 if [[ -z "$BACKUP_KEY" && -f "$SCRIPT_DIR/.env" ]]; then
-  BACKUP_KEY=$(grep -E '^EXE_BACKUP_KEY=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2)
+  BACKUP_KEY=$(grep -E '^EXE_BACKUP_KEY=' "$SCRIPT_DIR/.env" 2>/dev/null | cut -d= -f2- || true)
 fi
 if [[ -n "$BACKUP_KEY" ]]; then
   ENCRYPTED_FILE="${TARFILE}.enc"
   echo "[backup] Encrypting with AES-256-CBC (customer-owned key)..."
-  openssl enc -aes-256-cbc -pbkdf2 -iter 100000 \
-    -in "$TARFILE" -out "$ENCRYPTED_FILE" \
-    -pass "pass:${BACKUP_KEY}" 2>/dev/null
-  if [[ -f "$ENCRYPTED_FILE" ]]; then
-    rm -f "$TARFILE"
-    TARFILE="$ENCRYPTED_FILE"
-    echo "[backup] Encrypted: $TARFILE ($(du -h "$TARFILE" | cut -f1))"
-  else
-    echo "[backup] Encryption failed — uploading compressed (unencrypted)"
+  if ! openssl enc -aes-256-cbc -pbkdf2 -iter 100000 \
+       -in "$TARFILE" -out "$ENCRYPTED_FILE" \
+       -pass "pass:${BACKUP_KEY}"; then
+    fail "encrypt" "openssl encryption failed — refusing to upload plaintext customer data"
   fi
+  rm -f "$TARFILE"
+  TARFILE="$ENCRYPTED_FILE"
+  echo "[backup] Encrypted: $TARFILE ($(du -h "$TARFILE" | cut -f1))"
 else
-  echo "[backup] WARNING: EXE_BACKUP_KEY not set — backup NOT encrypted. Set it in .env for E2E encryption."
+  echo "[backup] WARNING: EXE_BACKUP_KEY not set — backup NOT encrypted. Set it in .env for E2E encryption." >&2
 fi
 
-# 7. Retention — delete old backups
+# 7. Retention — delete old backups (match BOTH .tar.gz and encrypted .tar.gz.enc).
 echo "[backup] Cleaning backups older than $RETENTION_DAYS days..."
-find "$BACKUP_DIR" -name "exe-backup-*.tar.gz" -mtime "+$RETENTION_DAYS" -delete 2>/dev/null
+find "$BACKUP_DIR" -name "exe-backup-*.tar.gz" -mtime "+$RETENTION_DAYS" -delete 2>/dev/null || true
+find "$BACKUP_DIR" -name "exe-backup-*.tar.gz.enc" -mtime "+$RETENTION_DAYS" -delete 2>/dev/null || true
+
+# Track off-box result so we can refuse to claim success on an upload-only run.
+OFFBOX_OK=true
 
 # 8. Download instructions
 if $DOWNLOAD_MODE; then
@@ -100,58 +174,89 @@ if $DOWNLOAD_MODE; then
   echo "=== Download to your local machine ==="
   echo "Run on your local machine:"
   echo ""
-  HOSTNAME=$(hostname)
   IP=$(curl -s ifconfig.me 2>/dev/null || echo "<VPS_IP>")
-  echo "  scp root@$IP:$TARFILE ./exe-backup-$DATE.tar.gz"
+  echo "  scp root@$IP:$TARFILE ./$(basename "$TARFILE")"
   echo ""
   echo "Or via Tailscale:"
   TS_IP=$(tailscale ip -4 2>/dev/null || echo "<TAILSCALE_IP>")
-  echo "  scp root@$TS_IP:$TARFILE ./exe-backup-$DATE.tar.gz"
+  echo "  scp root@$TS_IP:$TARFILE ./$(basename "$TARFILE")"
   echo ""
   echo "To restore on a new VPS:"
-  echo "  tar -xzf exe-backup-$DATE.tar.gz"
+  echo "  tar -xzf exe-backup-$DATE.tar.gz   # (decrypt the .enc first if encrypted)"
   echo "  cp env.bak /opt/exe-stack/.env"
   echo "  cp gateway.json /opt/exe-stack/"
   echo "  cp branding.json /opt/exe-stack/"
   echo "  cp clickhouse-config.xml /opt/exe-stack/"
   echo "  cp -r cloudflared/ /opt/exe-stack/"
-  echo "  cat postgres.sql.gz | gunzip | docker exec -i exe-db psql -U exe"
+  echo "  cat postgres.sql.gz | gunzip | docker exec -i $DB_CONTAINER psql -U $PG_USER"
   echo "  docker compose up -d"
 fi
 
-# 9. Upload to R2 (Cloudflare) — uses same R2 credentials as media/cloud sync
+# 9. Upload to R2 (Cloudflare) — uses same R2 credentials as media/cloud sync.
+#    OFF-BOX target (bug 56a71515): this is what survives disk/host death.
 if $UPLOAD_R2; then
-  # Source .env for R2 credentials if not already in environment
   if [[ -z "${R2_ENDPOINT:-}" && -f "$SCRIPT_DIR/.env" ]]; then
     # Map R2_MEDIA_* vars to the aws-cli vars backup needs
-    eval "$(grep -E '^R2_MEDIA_' "$SCRIPT_DIR/.env" 2>/dev/null)"
+    eval "$(grep -E '^R2_MEDIA_' "$SCRIPT_DIR/.env" 2>/dev/null || true)"
     export R2_ENDPOINT="${R2_ENDPOINT:-${R2_MEDIA_ENDPOINT:-}}"
     export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-${R2_MEDIA_ACCESS_KEY:-}}"
     export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-${R2_MEDIA_SECRET_KEY:-}}"
   fi
 
   R2_BACKUP_BUCKET="${R2_BACKUP_BUCKET:-${R2_MEDIA_BUCKET:-exe-backups}}"
-  R2_BACKUP_PREFIX="${R2_BACKUP_PREFIX:-backups/$(hostname)}"
+  R2_BACKUP_PREFIX="${R2_BACKUP_PREFIX:-backups/$HOST_NAME}"
 
   if [[ -z "${R2_ENDPOINT:-}" ]]; then
-    echo "[backup] R2 upload skipped — R2_MEDIA_ENDPOINT not set in .env"
+    OFFBOX_OK=false
+    alert "r2-config" "R2 upload requested but R2_MEDIA_ENDPOINT not set in .env — backup stayed on-box only"
   elif command -v rclone >/dev/null 2>&1; then
-    rclone copy "$TARFILE" "r2:${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/" 2>&1 && echo "[backup] R2 upload complete → ${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/" || echo "[backup] R2 upload failed"
+    if rclone copy "$TARFILE" "r2:${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/"; then
+      echo "[backup] R2 upload complete → ${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/"
+    else
+      OFFBOX_OK=false
+      alert "r2-upload" "rclone upload to ${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/ failed — off-box copy missing"
+    fi
   elif command -v aws >/dev/null 2>&1; then
-    aws s3 cp "$TARFILE" "s3://${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/$(basename "$TARFILE")" \
-      --endpoint-url "${R2_ENDPOINT}" 2>&1 && echo "[backup] R2 upload complete" || echo "[backup] R2 upload failed"
+    if aws s3 cp "$TARFILE" "s3://${R2_BACKUP_BUCKET}/${R2_BACKUP_PREFIX}/$(basename "$TARFILE")" \
+         --endpoint-url "${R2_ENDPOINT}"; then
+      echo "[backup] R2 upload complete"
+    else
+      OFFBOX_OK=false
+      alert "r2-upload" "aws s3 cp to ${R2_BACKUP_BUCKET} failed — off-box copy missing"
+    fi
   else
-    echo "[backup] Install rclone or aws-cli for R2 upload. apt install awscli"
+    OFFBOX_OK=false
+    alert "r2-tooling" "neither rclone nor aws-cli installed — cannot push backup off-box. apt install rclone"
   fi
 fi
 
-# 10. Upload to S3
+# 10. Upload to S3 (off-box)
 if [[ -n "$UPLOAD_S3" ]]; then
   echo "[backup] Uploading to S3..."
-  aws s3 cp "$TARFILE" "$UPLOAD_S3/$(basename "$TARFILE")" 2>&1 && echo "[backup] S3 upload complete" || echo "[backup] S3 upload failed"
+  if ! command -v aws >/dev/null 2>&1; then
+    OFFBOX_OK=false
+    alert "s3-tooling" "aws-cli not installed — cannot upload to $UPLOAD_S3"
+  elif aws s3 cp "$TARFILE" "$UPLOAD_S3/$(basename "$TARFILE")"; then
+    echo "[backup] S3 upload complete"
+  else
+    OFFBOX_OK=false
+    alert "s3-upload" "aws s3 cp to $UPLOAD_S3 failed — off-box copy missing"
+  fi
+fi
+
+# 11. Final result. Only ping the healthcheck success URL if everything that was
+#     requested actually succeeded — including the off-box upload when asked for.
+if { $UPLOAD_R2 || [[ -n "$UPLOAD_S3" ]]; } && [[ "$OFFBOX_OK" != true ]]; then
+  fail "offbox" "local archive created but off-box upload did not complete — DR copy is missing"
+fi
+
+if [[ -n "$HEALTHCHECK_URL" ]] && command -v curl >/dev/null 2>&1; then
+  curl -fsS -m 10 "$HEALTHCHECK_URL" >/dev/null 2>&1 || echo "[backup] (success healthcheck ping failed)" >&2
 fi
 
-echo "[backup] Done."
+# Clear the ERR trap before the clean exit.
+trap - ERR
+echo "[backup] Done — backup OK."
 echo ""
 echo "Backups:"
-ls -lh "$BACKUP_DIR"/exe-backup-*.tar.gz 2>/dev/null | tail -5
+ls -lh "$BACKUP_DIR"/exe-backup-*.tar.gz* 2>/dev/null | tail -5