@askexenow/exe-os 0.9.268 → 0.9.269

package/deploy/compose/docker-compose.yml

@@ -543,8 +543,11 @@ services:
     restart: unless-stopped
     # Token mode: `tunnel run --token <TOKEN>` — no local credentials file needed.
     # The token embeds the tunnel ID + credentials; routes are managed in the CF dashboard.
+    # --metrics exposes a local HTTP endpoint used by the healthcheck.
     command: >
-      tunnel run
+      tunnel
+      --metrics localhost:2000
+      run
       --token ${CLOUDFLARE_TUNNEL_TOKEN}
     depends_on:
       exe-crm:
@@ -560,11 +563,15 @@ services:
         ipv4_address: 10.42.0.31
       frontend:
         ipv4_address: 10.43.0.31
-    # No Docker healthcheck here. `cloudflared tunnel run --token ...` does not
-    # expose a local health endpoint by default, and `cloudflared tunnel info`
-    # can report failure in token-mode even while public hostnames are serving.
-    # stack-update verifies app health before starting the tunnel; tunnel
-    # reachability should be verified externally by the routed hostnames.
+    healthcheck:
+      # `tunnel info` requires API credentials unavailable in token mode.
+      # The command above exposes cloudflared's local metrics endpoint; use it
+      # for container liveness without hitting the Cloudflare control plane.
+      test: ["CMD-SHELL", "pgrep -x cloudflared && wget -q --spider http://localhost:2000/ready || exit 1"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
     logging:
       driver: json-file
       options: { max-size: "10m", max-file: "3" }

package/dist/bin/stack-update.js

@@ -10,7 +10,7 @@ import {
   loadStackManifest,
   patchEnv,
   runStackUpdate
-} from "../chunk-2NIJORQE.js";
+} from "../chunk-UHKKI5MD.js";
 import {
   runVerifyStack
 } from "../chunk-IRHNV4GY.js";
@@ -20,7 +20,7 @@ import {
 import {
   logResult,
   runHealthGate
-} from "../chunk-HGTXRRBK.js";
+} from "../chunk-AHVGHSWX.js";
 import "../chunk-MOZ2YQ54.js";
 import "../chunk-VXIMSRTO.js";
 import "../chunk-LYH5HE24.js";

package/dist/bin/vps-health-gate.js

@@ -8,7 +8,7 @@ import {
   logResult,
   main,
   runHealthGate
-} from "../chunk-HGTXRRBK.js";
+} from "../chunk-AHVGHSWX.js";
 import "../chunk-MLKGABMK.js";
 export {
   checkCRM,

package/dist/chunk-AHVGHSWX.js

@@ -0,0 +1,230 @@
+// src/bin/vps-health-gate.ts
+import { spawnSync } from "child_process";
+import { appendFileSync, existsSync, mkdirSync, readdirSync } from "fs";
+import http from "http";
+import path from "path";
+var DEPLOY_LOG = "/opt/exe-stack/deploy-log.jsonl";
+var BACKUP_DIR = "/opt/exe-stack/backups";
+async function checkCRM() {
+  const start = Date.now();
+  try {
+    const status = await httpGet("http://localhost:3000/api");
+    return {
+      check: "crm",
+      status: status >= 200 && status < 400 ? "pass" : "fail",
+      message: status >= 200 && status < 400 ? `CRM healthy (HTTP ${status})` : `CRM unhealthy (HTTP ${status})`,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    return {
+      check: "crm",
+      status: "fail",
+      message: `CRM unreachable: ${err instanceof Error ? err.message : err}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+async function checkGateway() {
+  const start = Date.now();
+  try {
+    const status = await httpGet("http://localhost:3100/health");
+    return {
+      check: "gateway",
+      status: status >= 200 && status < 300 ? "pass" : "fail",
+      message: status >= 200 && status < 300 ? `Gateway healthy (HTTP ${status})` : `Gateway unhealthy (HTTP ${status})`,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    return {
+      check: "gateway",
+      status: "fail",
+      message: `Gateway unreachable: ${err instanceof Error ? err.message : err}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+function checkPostgres() {
+  const start = Date.now();
+  const databaseUrl = process.env.DATABASE_URL || "postgres://exe@localhost:5432/exedb";
+  const result = spawnSync("psql", [databaseUrl, "-c", "SELECT 1"], {
+    encoding: "utf8",
+    stdio: ["pipe", "pipe", "pipe"],
+    timeout: 1e4
+  });
+  return {
+    check: "postgres",
+    status: result.status === 0 ? "pass" : "fail",
+    message: result.status === 0 ? "Postgres responding" : `Postgres failed: ${result.stderr?.trim() || `exit ${result.status}`}`,
+    durationMs: Date.now() - start
+  };
+}
+function checkRawEvents() {
+  const start = Date.now();
+  const databaseUrl = process.env.DATABASE_URL || "postgres://exe@localhost:5432/exedb";
+  const result = spawnSync(
+    "psql",
+    [databaseUrl, "-t", "-c", "SELECT count(*) FROM raw.raw_events"],
+    {
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "pipe"],
+      timeout: 1e4
+    }
+  );
+  if (result.status !== 0) {
+    return {
+      check: "raw_events",
+      status: "fail",
+      message: `raw.raw_events query failed: ${result.stderr?.trim() || `exit ${result.status}`}`,
+      durationMs: Date.now() - start
+    };
+  }
+  const count = parseInt(result.stdout?.trim() || "0", 10);
+  return {
+    check: "raw_events",
+    status: count > 0 ? "pass" : "fail",
+    message: count > 0 ? `raw.raw_events has ${count} rows` : "raw.raw_events is empty",
+    durationMs: Date.now() - start
+  };
+}
+async function checkGoTrue() {
+  const start = Date.now();
+  try {
+    const status = await httpGet("http://localhost:9999/health");
+    return {
+      check: "gotrue",
+      status: status >= 200 && status < 300 ? "pass" : "fail",
+      message: status >= 200 && status < 300 ? `GoTrue healthy (HTTP ${status})` : `GoTrue unhealthy (HTTP ${status})`,
+      durationMs: Date.now() - start
+    };
+  } catch (err) {
+    return {
+      check: "gotrue",
+      status: "fail",
+      message: `GoTrue unreachable: ${err instanceof Error ? err.message : err}`,
+      durationMs: Date.now() - start
+    };
+  }
+}
+async function runHealthGate() {
+  console.log("[health-gate] Running post-deploy health checks...\n");
+  const results = [];
+  results.push(checkPostgres());
+  results.push(checkRawEvents());
+  results.push(await checkCRM());
+  results.push(await checkGateway());
+  results.push(await checkGoTrue());
+  const passed = results.every((r) => r.status === "pass");
+  for (const r of results) {
+    const icon = r.status === "pass" ? "\u2713" : "\u2717";
+    const color = r.status === "pass" ? "\x1B[32m" : "\x1B[31m";
+    console.log(`  ${color}${icon}\x1B[0m ${r.check.padEnd(12)} ${r.message} (${r.durationMs}ms)`);
+  }
+  console.log("");
+  const result = {
+    passed,
+    results,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+  };
+  return result;
+}
+function logResult(result) {
+  mkdirSync(path.dirname(DEPLOY_LOG), { recursive: true });
+  const line = JSON.stringify({
+    ...result,
+    type: "health_gate"
+  }) + "\n";
+  try {
+    appendFileSync(DEPLOY_LOG, line);
+    console.log(`[health-gate] Result logged to ${DEPLOY_LOG}`);
+  } catch (err) {
+    console.warn(`[health-gate] Failed to write deploy log: ${err instanceof Error ? err.message : err}`);
+  }
+}
+function restorePreDeployBackup() {
+  if (!existsSync(BACKUP_DIR)) return false;
+  const backups = readdirSync(BACKUP_DIR).filter((f) => f.startsWith("pg-pre-deploy-") && f.endsWith(".dump")).sort().reverse();
+  if (backups.length === 0) {
+    console.warn("[health-gate] No pre-deploy backup found to restore");
+    return false;
+  }
+  const latest = backups[0];
+  const filepath = path.join(BACKUP_DIR, latest);
+  const databaseUrl = process.env.DATABASE_URL || "postgres://exe@localhost:5432/exedb";
+  console.log(`[health-gate] Restoring pre-deploy backup: ${latest}`);
+  const result = spawnSync("pg_restore", ["-d", databaseUrl, "--clean", "--if-exists", filepath], {
+    encoding: "utf8",
+    stdio: ["pipe", "pipe", "pipe"],
+    timeout: 3e5
+  });
+  if (result.status !== 0) {
+    console.error(`[health-gate] pg_restore failed: ${result.stderr?.trim() || `exit ${result.status}`}`);
+    return false;
+  }
+  console.log("[health-gate] \u2713 Pre-deploy backup restored");
+  return true;
+}
+function httpGet(url) {
+  return new Promise((resolve, reject) => {
+    const req = http.request(url, { method: "GET", timeout: 1e4 }, (res) => {
+      res.resume();
+      resolve(res.statusCode ?? 0);
+    });
+    req.on("timeout", () => req.destroy(new Error("timeout")));
+    req.on("error", reject);
+    req.end();
+  });
+}
+async function main(args) {
+  if (args.includes("--help") || args.includes("-h")) {
+    console.log(`
+  exe-os vps-health-gate \u2014 Post-deploy health checks
+
+  Runs 5 checks: Postgres, raw_events, CRM, Gateway, GoTrue.
+  If any check fails, triggers rollback and exits with code 1.
+
+  Usage:
+    exe-os vps-health-gate              Run health gate
+    exe-os vps-health-gate --check-only Run checks without rollback on failure
+`);
+    return;
+  }
+  const checkOnly = args.includes("--check-only");
+  const result = await runHealthGate();
+  logResult(result);
+  if (result.passed) {
+    console.log("[health-gate] \u2713 All checks passed \u2014 deploy is healthy");
+    return;
+  }
+  const failed = result.results.filter((r) => r.status === "fail");
+  console.error(`[health-gate] \u2717 ${failed.length} check(s) failed`);
+  if (checkOnly) {
+    process.exitCode = 1;
+    return;
+  }
+  console.log("[health-gate] Starting rollback...");
+  restorePreDeployBackup();
+  try {
+    const { rollbackStackUpdate, defaultStackPaths } = await import("./stack-update-YJ433OLM.js");
+    const paths = defaultStackPaths();
+    await rollbackStackUpdate({
+      manifestRef: paths.manifestRef,
+      composeFile: paths.composeFile,
+      envFile: paths.envFile
+    });
+    console.log("[health-gate] \u2713 Stack rolled back to previous version");
+  } catch (err) {
+    console.error(`[health-gate] Stack rollback failed: ${err instanceof Error ? err.message : err}`);
+  }
+  process.exitCode = 1;
+}
+
+export {
+  checkCRM,
+  checkGateway,
+  checkPostgres,
+  checkRawEvents,
+  checkGoTrue,
+  runHealthGate,
+  logResult,
+  main
+};