@meshxdata/fops 0.1.48 → 0.1.50

package/src/plugins/bundled/fops-plugin-azure/lib/azure-provision.js

@@ -10,46 +10,14 @@ import {
   resolvePublicIp, subArgs, buildTags, fetchMyIp,
   sshCmd, waitForSsh, closeMux, fopsUpCmd, buildPublicUrl,
   runReconcilers, ensureOpenAiNetworkAccess,
-  reconcileOk, RECONCILE_LABEL_WIDTH,
+  reconcileOk, reconcileSection, RECONCILE_LABEL_WIDTH,
 } from "./azure-helpers.js";
+import { postStartChecks } from "./azure-provision-health.js";
+import { provisionVm } from "./azure-provision-init.js";
 
-let _tlsWarningSuppressed = false;
-async function tlsFetch(url, opts = {}) {
-  if (!_tlsWarningSuppressed) {
-    _tlsWarningSuppressed = true;
-    const origEmit = process.emitWarning;
-    process.emitWarning = (warning, ...args) => {
-      if (typeof warning === "string" && warning.includes("NODE_TLS_REJECT_UNAUTHORIZED")) return;
-      return origEmit.call(process, warning, ...args);
-    };
-  }
-  const prev = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
-  process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-  try {
-    return await fetch(url, opts);
-  } finally {
-    if (prev === undefined) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
-    else process.env.NODE_TLS_REJECT_UNAUTHORIZED = prev;
-  }
-}
-
-const URL_WAIT_MODE_HTTP_ANY = "http-any";
-const URL_WAIT_MODE_HTTP_2XX = "http-2xx";
-const URL_WAIT_PROGRESS_INTERVAL_MS = 15000;
-
-function normalizeUrlWaitMode(mode) {
-  const normalized = String(mode || URL_WAIT_MODE_HTTP_ANY).trim().toLowerCase();
-  if (normalized === URL_WAIT_MODE_HTTP_2XX) return URL_WAIT_MODE_HTTP_2XX;
-  return URL_WAIT_MODE_HTTP_ANY;
-}
-
-function isUrlReadyByMode(status, waitMode) {
-  if (waitMode === URL_WAIT_MODE_HTTP_2XX) {
-    return status >= 200 && status < 300;
-  }
-  // Any HTTP response means DNS + TLS + edge routing are reachable.
-  return Number.isInteger(status) && status >= 100 && status <= 599;
-}
+// Re-export for backwards compatibility
+export { postStartChecks } from "./azure-provision-health.js";
+export { provisionVm } from "./azure-provision-init.js";
 
 // ── Ensure GHCR on VM (wait for Docker, login with retries) ──────────────────
 
@@ -75,10 +43,14 @@ async function ensureGhcrOnVm(ssh, user, githubToken, { timeout = 60000 } = {})
 
 // ── Configure a fresh or restarted VM ───────────────────────────────────────
 
-export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s, traefik, dai, deferStartToReconcile } = {}) {
+export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s, traefik, dai, deferStartToReconcile, quiet } = {}) {
   const ssh = (cmd) => sshCmd(execa, ip, user, cmd);
 
-  console.log(chalk.dim("  Configuring VM..."));
+  if (!quiet) console.log(chalk.dim("  Configuring VM..."));
+
+  // Derive environment name from publicUrl (e.g., staging.meshx.app -> Staging)
+  const envName = (publicUrl || "").replace(/https?:\/\//, "").split(".")[0] || "Local";
+  const environmentName = envName.charAt(0).toUpperCase() + envName.slice(1);
 
   // Batch: sshd tuning + docker group + ownership + br_netfilter — single SSH round-trip
   const setupBatch = [
@@ -93,19 +65,23 @@ export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s
     "sudo chown -R azureuser:azureuser /opt/foundation-compose 2>/dev/null; true",
     // Ensure br_netfilter is loaded so k3s CoreDNS service-IP routing works
     "sudo modprobe br_netfilter 2>/dev/null; sudo sysctl -qw net.bridge.bridge-nf-call-iptables=1 2>/dev/null; true",
-    // Only inject FOUNDATION_PUBLIC_URL if not already set — never overwrite
-    `cd /opt/foundation-compose && grep -q '^FOUNDATION_PUBLIC_URL=' .env 2>/dev/null || echo 'FOUNDATION_PUBLIC_URL=${publicUrl}' >> .env`,
+    // Ensure FOUNDATION_PUBLIC_URL is correct (remove stale values, set correct one)
+    `cd /opt/foundation-compose && sed -i '/^FOUNDATION_PUBLIC_URL=/d' .env 2>/dev/null; echo 'FOUNDATION_PUBLIC_URL=${publicUrl}' >> .env`,
+    // Ensure ENVIRONMENT_NAME is correct
+    `cd /opt/foundation-compose && sed -i '/^ENVIRONMENT_NAME=/d' .env 2>/dev/null; echo 'ENVIRONMENT_NAME=${environmentName}' >> .env`,
   ].join("\n");
   await ssh(setupBatch);
 
   let ghcrOk = false;
   if (githubToken) {
-    console.log(chalk.dim("  Configuring GitHub/GHCR credentials..."));
+    if (!quiet) console.log(chalk.dim("  Configuring GitHub/GHCR credentials..."));
     ghcrOk = await ensureGhcrOnVm(ssh, user, githubToken);
-    if (ghcrOk) {
-      console.log(chalk.green("  ✓ GHCR credentials configured"));
-    } else {
-      console.log(chalk.yellow("  ⚠ GHCR login failed or deferred — will retry before compose pull"));
+    if (!quiet) {
+      if (ghcrOk) {
+        console.log(chalk.green("  ✓ GHCR credentials configured"));
+      } else {
+        console.log(chalk.yellow("  ⚠ GHCR login failed or deferred — will retry before compose pull"));
+      }
     }
   }
 
@@ -122,6 +98,9 @@ export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s
   // provisioning only takes effect on the NEXT login, so the current SSH
   // session won't have it yet.
 
+  // When quiet (reconciliation mode), skip service status checks — handled by vmReconcileFoundation
+  if (quiet) return;
+
   // Skip starting if an up process is already running (provisionVm or cloud-init may have launched one)
   const { stdout: upProcs } = await ssh(
     "pgrep -af 'fops up|docker compose up' 2>/dev/null | grep -v pgrep | head -1 || true"
@@ -306,6 +285,7 @@ const VM_RECONCILERS = [
   { name: "vm-security",     fn: vmReconcileSecurity },
   { name: "ssh-reachability", fn: vmReconcileSsh },
   { name: "dns-credentials", fn: vmReconcileDnsCreds },
+  { name: "repo",            fn: vmReconcileRepo },
   { name: "fops-cli",        fn: vmReconcileFopsCli },
   { name: "foundation",      fn: vmReconcileFoundation },
   { name: "post-start",      fn: vmReconcilePostStart },
@@ -334,6 +314,7 @@ export async function reconcileVm(execa, opts) {
 // ── Step: Power state ────────────────────────────────────────────────────────
 
 async function vmReconcilePower(ctx) {
+  reconcileSection("Infrastructure");
   const { execa, vmName, rg, sub } = ctx;
   const { stdout: ivJson } = await execa("az", [
     "vm", "get-instance-view", "-g", rg, "-n", vmName, "--output", "json",
@@ -502,6 +483,7 @@ async function vmReconcileNetworking(ctx) {
 // ── Step: NSG / firewall rules ───────────────────────────────────────────────
 
 async function vmReconcileNsg(ctx) {
+  reconcileSection("Firewall");
   const { execa, vmName, rg, sub, nicCode, nicJson, nicName, location, port, traefik } = ctx;
 
   let nsgName = "";
@@ -745,6 +727,7 @@ async function vmReconcileOpenAiNetwork(ctx) {
 // ── Step: VM security settings ───────────────────────────────────────────────
 
 async function vmReconcileSecurity(ctx) {
+  reconcileSection("Security");
   const { execa, vmName, rg, sub, location } = ctx;
   const { stdout: vmJson, exitCode } = await execa("az", [
     "vm", "show", "-g", rg, "-n", vmName, "--output", "json",
@@ -1047,8 +1030,8 @@ async function removeSshBypassViaRunCommand(execa, rg, vmName, sourceCidr, sub)
 // ── Step: SSH reachability ───────────────────────────────────────────────────
 
 async function vmReconcileSsh(ctx) {
+  reconcileSection("Connectivity");
   const { execa, ip, adminUser, port, desiredUrl, vmName, rg, sub } = ctx;
-  console.log(chalk.dim("  Checking SSH..."));
 
   // Always fetch the knock sequence fresh from the Azure VM tag — local state can drift
   // after state recovery or manual changes, causing knocks with the wrong sequence.
@@ -1209,13 +1192,45 @@ async function vmReconcileDnsCreds(ctx) {
   const { execa, ip, adminUser, port, desiredUrl, cfToken, githubToken, k3s, traefik } = ctx;
   ctx.publicUrl = desiredUrl || buildPublicUrl(ip, port);
   await syncDns(cfToken, ctx.publicUrl, ip);
-  await configureVm(execa, ip, adminUser, ctx.publicUrl, { githubToken, k3s, traefik, deferStartToReconcile: true });
+  await configureVm(execa, ip, adminUser, ctx.publicUrl, { githubToken, k3s, traefik, deferStartToReconcile: true, quiet: true });
+}
+
+// ── Step: Ensure foundation-compose repo exists ──────────────────────────────
+
+async function vmReconcileRepo(ctx) {
+  if (ctx.done) return;
+  const { execa, ip, adminUser } = ctx;
+  const ssh = (cmd, timeout = 30000) => sshCmd(execa, ip, adminUser, cmd, timeout);
+
+  const { stdout: exists } = await ssh("[ -d /opt/foundation-compose/.git ] && echo yes || echo no");
+  if (exists?.trim() === "yes") {
+    reconcileOk("Repository", "/opt/foundation-compose");
+    return;
+  }
+
+  console.log(chalk.yellow("  ↻ Repository missing — cloning foundation-compose..."));
+  const cloneResult = await ssh([
+    "sudo rm -rf /opt/foundation-compose",
+    "sudo git clone --branch main --depth 1 https://github.com/meshxdata/foundation-compose.git /opt/foundation-compose",
+  ].join(" && "), 120000);
+
+  if (cloneResult.exitCode !== 0) {
+    console.log(chalk.red("  ✗ Failed to clone repository"));
+    return;
+  }
+
+  console.log(chalk.dim("  Initializing submodules..."));
+  await ssh("cd /opt/foundation-compose && sudo git submodule update --init --recursive --depth 1", 300000);
+  await ssh("sudo chown -R azureuser:azureuser /opt/foundation-compose");
+  await ssh("mkdir -p /opt/foundation-compose/credentials && touch /opt/foundation-compose/credentials/kubeconfig.yaml");
+  reconcileOk("Repository", "cloned and initialized");
 }
 
 // ── Step: Ensure fops CLI is available on the VM ─────────────────────────────
 
 async function vmReconcileFopsCli(ctx) {
   if (ctx.done) return;
+  reconcileSection("Services");
   const { execa, ip, adminUser } = ctx;
   const ssh = (cmd, timeout = 30000) => sshCmd(execa, ip, adminUser, cmd, timeout);
 
@@ -1375,400 +1390,7 @@ async function vmReconcilePostStart(ctx) {
   await postStartChecks(execa, ip, adminUser, { publicUrl, waitMode });
 }
 
-// ── Post-start: postgres ready + grant admin ────────────────────────────────
-
-const POSTGRES_INITIAL_DELAY_MS = 45000;  // give compose time to start containers after nohup fops up
-const POSTGRES_POLL_INTERVAL_MS = 10000;
-const POSTGRES_MAX_WAIT_MS = 600000;      // 10 min for first boot / heavy stack or recovery
-const POSTGRES_RECOVERY_HINT_INTERVAL_MS = 60000;  // remind every 60s that we're waiting on recovery
-
-export async function postStartChecks(execa, ip, adminUser, { maxWait = POSTGRES_MAX_WAIT_MS, publicUrl, waitMode } = {}) {
-  banner("Post-start checks");
-
-  const ssh = (cmd, timeout = 30000) => sshCmd(execa, ip, adminUser, cmd, timeout);
-
-  hint("Waiting for Postgres…");
-  await new Promise((r) => setTimeout(r, POSTGRES_INITIAL_DELAY_MS));
-  const pgStart = Date.now();
-  let pgReady = false;
-  let lastRecoveryHintAt = -POSTGRES_RECOVERY_HINT_INTERVAL_MS;  // so first recovery is reported immediately
-  let lastEventCheckAt = 0;
-  const EVENT_CHECK_INTERVAL_MS = 15000;  // show docker events every 15s while waiting
-
-  while (Date.now() - pgStart < maxWait) {
-    const { exitCode, stdout, stderr } = await ssh(
-      "cd /opt/foundation-compose && sudo docker compose exec -T postgres psql -U foundation -d foundation -c 'SELECT 1' 2>&1",
-      15000
-    );
-    if (exitCode === 0) { pgReady = true; break; }
-    const out = (stdout || "") + (stderr || "");
-    const inRecovery = /recovery|not yet accepting connections|Consistent recovery state has not been yet reached/i.test(out);
-    if (inRecovery && Date.now() - lastRecoveryHintAt >= POSTGRES_RECOVERY_HINT_INTERVAL_MS) {
-      console.log(WARN("  Postgres in recovery (database was not properly shut down). Waiting for it to accept connections…"));
-      lastRecoveryHintAt = Date.now();
-    }
-
-    // Show recent docker events while waiting
-    if (Date.now() - lastEventCheckAt >= EVENT_CHECK_INTERVAL_MS) {
-      const { stdout: events } = await ssh(
-        "docker events --since 30s --until 0s --format '{{.Type}}|{{.Action}}|{{.Actor.Attributes.name}}|{{.Actor.Attributes.image}}' 2>/dev/null | tail -8",
-        10000
-      ).catch(() => ({ stdout: "" }));
-      if (events?.trim()) {
-        const eventIcons = { pull: "📥", start: "▶", create: "✦", die: "✗", kill: "⚡", stop: "■" };
-        const lines = events.trim().split("\n").map((line) => {
-          const [type, action, name, image] = line.split("|");
-          const icon = eventIcons[action] || "·";
-          const svc = name || image?.split("/").pop()?.split(":")[0] || "";
-          return `${icon} ${action.padEnd(7)} ${svc}`;
-        });
-        const elapsed = Math.round((Date.now() - pgStart) / 1000);
-        console.log(DIM(`  [${elapsed}s] Docker activity:`));
-        for (const line of lines) console.log(DIM(`    ${line}`));
-      }
-      lastEventCheckAt = Date.now();
-    }
-
-    await new Promise((r) => setTimeout(r, POSTGRES_POLL_INTERVAL_MS));
-  }
-  if (!pgReady) {
-    console.log(WARN("  ⚠ Postgres not ready after timeout (may still be in recovery after unclean shutdown)"));
-
-    const { stdout: composePs } = await ssh(
-      "cd /opt/foundation-compose && sudo docker compose ps -a --format '{{.Service}}\t{{.State}}\t{{.Status}}' 2>/dev/null | head -20"
-    );
-    const states = composePs?.trim()?.split("\n").map((line) => line.split("\t")[1]).filter(Boolean) ?? [];
-    const allCreated = states.length > 0 && states.every((s) => s === "created");
-    if (composePs?.trim()) {
-      hint("Container status:");
-      for (const line of composePs.trim().split("\n")) hint(`  ${line}`);
-      if (allCreated) {
-        hint("Containers still in 'created' — stack may still be starting. Wait a few minutes then: fops azure deploy");
-      }
-    } else {
-      hint("No containers found — docker compose may not have started.");
-    }
-
-    const { stdout: pullErrors } = await ssh(
-      "grep -i -E 'error|denied|unauthorized|manifest unknown|pull access denied' /tmp/fops-up.log 2>/dev/null | tail -5"
-    );
-    if (pullErrors?.trim()) {
-      console.log(WARN("  Possible pull errors:"));
-      for (const line of pullErrors.trim().split("\n")) hint(`  ${line}`);
-    }
-
-    const { stdout: ghcrCheck } = await ssh(
-      "sudo cat /root/.docker/config.json 2>/dev/null | grep -q ghcr.io && echo 'ok' || echo 'missing'"
-    );
-    if (ghcrCheck?.trim() !== "ok") {
-      console.log(WARN("  ⚠ GHCR credentials missing from /root/.docker/config.json"));
-      hint("Re-run: fops azure up (will re-configure credentials)");
-    }
-
-    const { stdout: upLogTail } = await ssh(
-      "tail -80 /tmp/fops-up.log 2>/dev/null || echo '(log not found or empty)'"
-    );
-    if (upLogTail?.trim()) {
-      console.log(WARN("  Last lines of /tmp/fops-up.log:"));
-      for (const line of upLogTail.trim().split("\n").slice(-40)) {
-        console.log(chalk.dim(`    ${line}`));
-      }
-    }
-
-    hint(`\nDebug: ssh ${adminUser}@${ip} "tail -100 /tmp/fops-up.log"`);
-    hint("Retry: fops azure deploy\n");
-    return;
-  }
-  console.log(OK("  ✓ Postgres ready"));
-
-  hint("Waiting for backend migrations…");
-  const migStart = Date.now();
-  let migReady = false;
-  while (Date.now() - migStart < 120000) {
-    const { exitCode: migCheck } = await ssh(
-      `cd /opt/foundation-compose && sudo docker compose exec -T postgres psql -U foundation -d foundation -c "SELECT 1 FROM \\"user\\" LIMIT 1" >/dev/null 2>&1`
-    );
-    if (migCheck === 0) { migReady = true; break; }
-    await new Promise((r) => setTimeout(r, 5000));
-  }
-  if (!migReady) {
-    hint("Retrying migration check in 30s…");
-    await new Promise((r) => setTimeout(r, 30000));
-    const { exitCode: retryCheck } = await ssh(
-      `cd /opt/foundation-compose && sudo docker compose exec -T postgres psql -U foundation -d foundation -c "SELECT 1 FROM \\"user\\" LIMIT 1" >/dev/null 2>&1`
-    );
-    if (retryCheck === 0) migReady = true;
-  }
-  if (!migReady) {
-    console.log(WARN("  ⚠ Migrations not complete — skipping grant-admin (schema not ready)"));
-    hint("Run manually after stack is up:  fops azure grant admin");
-  } else {
-  const operatorEmail = process.env.FOUNDATION_USERNAME || process.env.FOUNDATION_OPERATOR_EMAIL || process.env.QA_USERNAME || "compose@meshx.io";
-  hint(`Ensuring ${operatorEmail} user + Foundation Admin…`);
-  const ensureUserSql = [
-    // Ensure the default operator user exists
-    `INSERT INTO "user" (identifier, urn, username, first_name, last_name, email, is_system)`,
-    `  VALUES (gen_random_uuid(), 'urn:meshx:user:operator', $op, 'Foundation', 'Operator', $op, false)`.replace(/\$op/g, `'${operatorEmail.replace(/'/g, "''")}'`),
-    `  ON CONFLICT (username) DO NOTHING;`,
-    // Grant Foundation Admin to operator user
-    `INSERT INTO role_member (identifier, role_id)`,
-    `  SELECT u.identifier, r.id FROM "user" u CROSS JOIN "role" r`,
-    `  WHERE u.username = '${operatorEmail.replace(/'/g, "''")}' AND r.name = 'Foundation Admin'`,
-    `  ON CONFLICT (role_id, identifier) DO NOTHING;`,
-    // Grant Foundation Admin to all other non-system users too
-    `INSERT INTO role_member (identifier, role_id)`,
-    `  SELECT u.identifier, r.id FROM "user" u CROSS JOIN "role" r`,
-    `  WHERE NOT u.is_system`,
-    `    AND u.username NOT IN ('admin@meshx.io', 'scheduler@meshx.io', 'opa@meshx.io')`,
-    `    AND r.name = 'Foundation Admin'`,
-    `  ON CONFLICT (role_id, identifier) DO NOTHING;`,
-  ].join("\n");
-  const b64Grant = Buffer.from(ensureUserSql).toString("base64");
-  let grantCode = -1;
-  let grantOut = "";
-  for (let attempt = 0; attempt < 2; attempt++) {
-    if (attempt === 1) {
-      hint("Grant failed; retrying in 20s…");
-      await new Promise((r) => setTimeout(r, 20000));
-    }
-    const result = await ssh(
-      `cd /opt/foundation-compose && echo '${b64Grant}' | base64 -d | sudo docker compose exec -T postgres psql -U foundation -d foundation -v ON_ERROR_STOP=1 2>&1`
-    );
-    grantOut = result.stdout || "";
-    grantCode = result.exitCode;
-    if (grantCode === 0) break;
-  }
-  if (grantCode === 0) {
-    console.log(OK(`  ✓ ${operatorEmail} — Foundation Admin granted`));
-  } else {
-    console.log(WARN("  ⚠ Admin grant failed — run manually:  fops azure grant admin"));
-    if (grantOut?.trim()) hint(grantOut.trim());
-  }
-
-  const orgSql = `INSERT INTO user_organization (user_id, organization_id) SELECT u.id, o.id FROM "user" u CROSS JOIN organization o WHERE o.name = 'root' ON CONFLICT DO NOTHING;`;
-  const b64Org = Buffer.from(orgSql).toString("base64");
-  let orgCode = -1;
-  for (let attempt = 0; attempt < 2; attempt++) {
-    if (attempt === 1) await new Promise((r) => setTimeout(r, 15000));
-    const result = await ssh(
-      `cd /opt/foundation-compose && echo '${b64Org}' | base64 -d | sudo docker compose exec -T postgres psql -U foundation -d foundation -v ON_ERROR_STOP=1 2>&1`
-    );
-    orgCode = result.exitCode;
-    if (orgCode === 0) break;
-  }
-  if (orgCode === 0) {
-    console.log(OK("  ✓ Root organization membership ensured"));
-  }
-  }
-
-  // Wait for the public URL to become reachable (Traefik + DNS propagation)
-  if (publicUrl) {
-    const urlWaitMode = normalizeUrlWaitMode(waitMode);
-    hint(`Waiting for ${publicUrl} to respond (mode: ${urlWaitMode})…`);
-    const urlMaxWait = 300000; // 5 min — Traefik + DNS can be slow after start
-    const urlStart = Date.now();
-    let lastProgressAt = 0;
-    let lastProbe = "not probed yet";
-    let urlOk = false;
-    while (Date.now() - urlStart < urlMaxWait) {
-      try {
-        const r = await tlsFetch(publicUrl, { signal: AbortSignal.timeout(8000), redirect: "follow" });
-        lastProbe = `HTTP ${r.status}`;
-        if (isUrlReadyByMode(r.status, urlWaitMode)) {
-          urlOk = true;
-          break;
-        }
-      } catch (err) {
-        // DNS not ready or connection refused — retry
-        lastProbe = err?.name || err?.message || "network error";
-      }
-      const now = Date.now();
-      if (now - lastProgressAt >= URL_WAIT_PROGRESS_INTERVAL_MS) {
-        const elapsed = Math.round((now - urlStart) / 1000);
-        hint(`Still waiting for ${publicUrl} [${elapsed}s] — last probe: ${lastProbe}`);
-        lastProgressAt = now;
-      }
-      await new Promise(r => setTimeout(r, 5000));
-    }
-    if (urlOk) {
-      console.log(OK(`  ✓ ${publicUrl} is reachable (${lastProbe})`));
-    } else {
-      console.log(WARN(`  ⚠ ${publicUrl} not responding yet (${lastProbe}) — Traefik, DNS, or edge config may still be propagating`));
-      hint(`Try stricter mode if needed: fops azure up --wait-mode ${URL_WAIT_MODE_HTTP_2XX}`);
-      hint("Check:  curl -sk " + publicUrl);
-    }
-  }
-
-  console.log("");
-}
-
-// ═══════════════════════════════════════════════════════════════════════════
-// Provision — install Docker, Node, fops, clone repo on a fresh Ubuntu VM
-// ═══════════════════════════════════════════════════════════════════════════
-
-export async function provisionVm(execa, ip, adminUser, { githubToken, branch = "main", fopsVersion = "latest" } = {}) {
-  banner("Provisioning VM");
-  const ssh = (cmd, timeout = 120000) => sshCmd(execa, ip, adminUser, cmd, timeout);
-
-  async function runScript(label, script, timeout = 180000) {
-    hint(`${label}…`);
-    const b64 = Buffer.from(script).toString("base64");
-    const { exitCode, stdout } = await ssh(
-      `echo '${b64}' | base64 -d | sudo bash -e 2>&1`, timeout,
-    );
-    if (exitCode !== 0) {
-      console.log(WARN(`  ⚠ ${label} had issues`));
-      if (stdout?.trim()) hint(stdout.trim().split("\n").pop());
-    } else {
-      console.log(OK(`  ✓ ${label}`));
-    }
-    return exitCode;
-  }
-
-  const waitAptLock = "while fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; do echo 'Waiting for apt/dpkg lock…'; sleep 5; done";
-  await runScript("Waiting for cloud-init", [
-    "cloud-init status --wait 2>/dev/null || true",
-    "while fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; do sleep 3; done",
-  ].join("\n"), 300000);
-
-  await runScript("Installing system packages", [
-    waitAptLock,
-    "export DEBIAN_FRONTEND=noninteractive",
-    "apt-get update -y -qq",
-    "apt-get install -y -qq apt-transport-https ca-certificates curl gnupg lsb-release jq git make unzip zsh software-properties-common python3-venv python3-pip",
-  ].join("\n"), 300000);
-
-  await runScript("Installing Docker", [
-    waitAptLock,
-    "export DEBIAN_FRONTEND=noninteractive",
-    "install -m 0755 -d /etc/apt/keyrings",
-    "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg",
-    "chmod a+r /etc/apt/keyrings/docker.gpg",
-    `echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list`,
-    "apt-get update -qq",
-    "apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin",
-    "systemctl enable docker && systemctl start docker",
-    `usermod -aG docker ${adminUser}`,
-  ].join("\n"), 300000);
-
-  await runScript("Configuring br_netfilter for k3s DNS", [
-    "modprobe br_netfilter",
-    "echo br_netfilter > /etc/modules-load.d/br_netfilter.conf",
-    "sysctl -w net.bridge.bridge-nf-call-iptables=1",
-    "echo 'net.bridge.bridge-nf-call-iptables = 1' > /etc/sysctl.d/99-br-netfilter.conf",
-  ].join("\n"));
-
-  await runScript("Installing GitHub CLI", [
-    waitAptLock,
-    "set +e",
-    "curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg 2>/dev/null",
-    "chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg 2>/dev/null",
-    "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" > /etc/apt/sources.list.d/github-cli.list",
-    "for _ in 1 2 3 4 5; do if apt-get update -qq && apt-get install -y -qq gh; then break; fi; echo 'Retrying in 10s…'; sleep 10; done",
-    "set -e",
-    "command -v gh >/dev/null 2>&1 || (echo 'gh not found after install attempts' && exit 1)",
-  ].join("\n"), 120000);
-
-  const fopsPkg = fopsVersion === "latest"
-    ? "@meshxdata/fops"
-    : `@meshxdata/fops@${fopsVersion}`;
-  await runScript("Installing Node.js + fops", [
-    waitAptLock,
-    `curl -fsSL https://deb.nodesource.com/setup_20.x | bash -`,
-    "apt-get install -y -qq nodejs",
-    `npm install -g ${fopsPkg}`,
-    `FOPS_DIR=$(npm root -g)/@meshxdata/fops`,
-    `if [ ! -d "$FOPS_DIR/node_modules/commander" ]; then`,
-    `  echo "Dependencies missing — installing inside package dir…"`,
-    `  cd "$FOPS_DIR" && npm install --omit=dev`,
-    `fi`,
-    `fops --version || { echo "fops binary not on PATH — linking manually"; ln -sf "$(npm root -g)/@meshxdata/fops/fops.mjs" /usr/local/bin/fops; fops --version; }`,
-  ].join("\n"), 300000);
-
-  if (githubToken) {
-    await runScript("Configuring GitHub credentials", [
-      `printf 'machine github.com login x-access-token password %s\\n' '${githubToken}' > /root/.netrc && chmod 600 /root/.netrc`,
-      `printf 'machine github.com login x-access-token password %s\\n' '${githubToken}' > /home/${adminUser}/.netrc && chmod 600 /home/${adminUser}/.netrc && chown ${adminUser}:${adminUser} /home/${adminUser}/.netrc`,
-      `echo '${githubToken}' | docker login ghcr.io -u x-access-token --password-stdin`,
-      `mkdir -p /home/${adminUser}/.docker && cp /root/.docker/config.json /home/${adminUser}/.docker/config.json 2>/dev/null && chown -R ${adminUser}:${adminUser} /home/${adminUser}/.docker || true`,
-    ].join("\n"));
-  }
-
-  await runScript("Cloning foundation-compose", [
-    "rm -rf /opt/foundation-compose",
-    `git clone --branch ${branch} --depth 1 --recurse-submodules https://github.com/meshxdata/foundation-compose.git /opt/foundation-compose`,
-    "mkdir -p /opt/foundation-compose/credentials",
-    "touch /opt/foundation-compose/credentials/kubeconfig.yaml",
-    `chown -R ${adminUser}:${adminUser} /opt/foundation-compose`,
-  ].join("\n"), 300000);
-
-  await runScript("Installing fops-api systemd service", [
-    `cat > /etc/systemd/system/fops-api.service <<'UNIT'`,
-    "[Unit]",
-    "Description=fops API server",
-    "After=network.target docker.service",
-    "Wants=docker.service",
-    "",
-    "[Service]",
-    "Type=simple",
-    `User=${adminUser}`,
-    "WorkingDirectory=/opt/foundation-compose",
-    "Environment=COMPOSE_ROOT=/opt/foundation-compose",
-    "EnvironmentFile=-/opt/foundation-compose/.env",
-    "ExecStart=/usr/bin/env fops serve --host 127.0.0.1 --port 4100",
-    "Restart=always",
-    "RestartSec=10",
-    "Environment=NODE_ENV=production",
-    "",
-    "[Install]",
-    "WantedBy=multi-user.target",
-    "UNIT",
-    "systemctl daemon-reload",
-    "systemctl enable --now fops-api",
-  ].join("\n"));
-
-  const cloudInitSrc = path.resolve(
-    import.meta.dirname, "..", "..", "..", "..", "..", "packer", "scripts", "cloud-init-foundation.sh",
-  );
-  let cloudInitScript = "";
-  try {
-    cloudInitScript = fs.readFileSync(cloudInitSrc, "utf8");
-  } catch {
-    hint("cloud-init-foundation.sh not found locally — skipping per-boot script");
-  }
-  if (cloudInitScript) {
-    const b64 = Buffer.from(cloudInitScript).toString("base64");
-    await ssh(
-      `echo '${b64}' | base64 -d | sudo tee /var/lib/cloud/scripts/per-boot/foundation-startup.sh > /dev/null && sudo chmod +x /var/lib/cloud/scripts/per-boot/foundation-startup.sh`,
-    );
-    console.log(OK("  ✓ Cloud-init per-boot script installed"));
-  }
+// postStartChecks is now in azure-provision-health.js and re-exported above for backwards compatibility
 
-  await runScript("Setting MOTD", [
-    "chmod -x /etc/update-motd.d/* 2>/dev/null || true",
-    `cat > /etc/motd <<'MOTD'
 
-    ___                      _       _   _
-   / __\\__  _   _ _ __   __| | __ _| |_(_) ___  _ __
-  / _\\/ _ \\| | | | '_ \\ / _\` |/ _\` | __| |/ _ \\| '_ \\
- / / | (_) | |_| | | | | (_| | (_| | |_| | (_) | | | |
- \\/   \\___/ \\__,_|_| |_|\\__,_|\\__,_|\\__|_|\\___/|_| |_|
-
-  Data Mesh Platform
-
-  Quick start:
-    fops status          Show running services
-    fops up              Start the platform
-    fops down            Stop the platform
-    fops logs <service>  Tail service logs
-    fops doctor          Diagnose issues
-
-  Project dir:   /opt/foundation-compose
-
-MOTD`,
-  ].join("\n"));
-
-  await ssh("sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/*", 30000);
-
-  console.log(OK("\n  ✓ Provisioning complete"));
-}
+// provisionVm is now in azure-provision-init.js and re-exported above for backwards compatibility

package/src/plugins/bundled/fops-plugin-azure/lib/azure-results.js

@@ -450,6 +450,17 @@ export async function resultsShow(opts = {}) {
       }
     }
   }
+  if (result.failedTests?.length) {
+    console.log(ERR(`\n  Failed tests (${result.failedTests.length}):`));
+    for (const { test, reason } of result.failedTests.slice(0, 20)) {
+      const shortTest = test.replace(/^tests\//, "");
+      const reasonStr = reason ? DIM(` - ${reason.slice(0, 60)}`) : "";
+      console.log(ERR(`    • ${shortTest}${reasonStr}`));
+    }
+    if (result.failedTests.length > 20) {
+      console.log(DIM(`    ... and ${result.failedTests.length - 20} more`));
+    }
+  }
   if (result.pushedBy) kvLine("Pushed by", result.pushedBy);
   if (result.pushedAt) kvLine("Pushed at", result.pushedAt);
   console.log("");