@meshxdata/fops 0.1.45 → 0.1.47

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-core.js

@@ -0,0 +1,1185 @@
+/**
+ * azure-aks-core.js – Core AKS CLI commands and helpers
+ *
+ * Dependencies: azure.js, azure-aks-naming.js, azure-aks-state.js,
+ *               azure-aks-reconcilers.js, azure-aks-flux.js, azure-aks-network.js,
+ *               azure-aks-postgres.js, azure-aks-ingress.js
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import chalk from "chalk";
+import {
+  DEFAULTS, DIM, OK, WARN, ERR, LABEL, ACCENT,
+  banner, hint, kvLine,
+  lazyExeca, ensureAzCli, ensureAzAuth, subArgs, buildTags, fetchMyIp,
+  readState,
+  resolveGithubToken,
+} from "./azure.js";
+import { AKS_DEFAULTS, PG_REPLICA_REGIONS, timeSince } from "./azure-aks-naming.js";
+import {
+  readAksClusters,
+  readClusterState,
+  writeClusterState,
+  clearClusterState,
+  requireCluster,
+} from "./azure-aks-state.js";
+import { reconcileCluster } from "./azure-aks-reconcilers.js";
+import {
+  TEMPLATE_DEFAULTS,
+  provisionFluxFromTemplate,
+  bootstrapFlux,
+  reconcileFluxPrereqs,
+} from "./azure-aks-flux.js";
+import { reconcileNetworkAccess } from "./azure-aks-network.js";
+import { reconcilePostgres, aksPostgresReplicaCreate, reconcileEventHubs } from "./azure-aks-postgres.js";
+import { clusterDomain } from "./azure-aks-ingress.js";
+import { printClusterInfo } from "./azure-aks-stacks.js";
+
+/** Resolve a module path under the CLI's src/ directory (works from source and ~/.fops/plugins). */
+function resolveCliSrc(relPath) {
+  const thisDir = path.dirname(fileURLToPath(import.meta.url));
+  const fromSource = path.resolve(thisDir, "../../../..", relPath);
+  if (fs.existsSync(fromSource)) return pathToFileURL(fromSource).href;
+  const fopsBin = process.argv[1];
+  if (fopsBin) {
+    try {
+      const cliRoot = path.dirname(fs.realpathSync(fopsBin));
+      const fromCli = path.resolve(cliRoot, "src", relPath);
+      if (fs.existsSync(fromCli)) return pathToFileURL(fromCli).href;
+    } catch { /* fall through */ }
+  }
+  return "../../../../" + relPath;
+}
+
+// ── K8s version resolver ─────────────────────────────────────────────────────
+
+/**
+ * Query the latest GA (non-preview) Kubernetes version available in a region.
+ * Falls back to AKS_DEFAULTS.kubernetesVersion if the query fails.
+ */
+export async function resolveK8sVersion(execa, { location, subscription } = {}) {
+  try {
+    const args = [
+      "aks", "get-versions",
+      "--location", location || DEFAULTS.location,
+      "--output", "json",
+    ];
+    if (subscription) args.push("--subscription", subscription);
+
+    const { stdout } = await execa("az", args, { timeout: 30000 });
+    const data = JSON.parse(stdout);
+
+    // data.values is an array of { version, isPreview, patchVersions }
+    const gaVersions = (data.values || [])
+      .filter((v) => !v.isPreview)
+      .map((v) => v.version)
+      .sort((a, b) => {
+        const pa = a.split(".").map(Number);
+        const pb = b.split(".").map(Number);
+        return pb[0] - pa[0] || pb[1] - pa[1] || (pb[2] || 0) - (pa[2] || 0);
+      });
+
+    if (gaVersions.length > 0) return gaVersions[0];
+  } catch { /* fall through to default */ }
+  return AKS_DEFAULTS.kubernetesVersion;
+}
+
+// ── CLI prerequisite checkers ────────────────────────────────────────────────
+
+export async function ensureFluxCli(execa) {
+  try {
+    await execa("flux", ["--version"], { timeout: 10000 });
+  } catch {
+    console.error(ERR("\n  Flux CLI is not installed."));
+    hint("Install:  brew install fluxcd/tap/flux");
+    hint("Or:       curl -s https://fluxcd.io/install.sh | sudo bash\n");
+    process.exit(1);
+  }
+}
+
+export async function ensureKubectl(execa) {
+  try {
+    await execa("kubectl", ["version", "--client", "--output=json"], { timeout: 10000 });
+  } catch {
+    console.error(ERR("\n  kubectl is not installed."));
+    hint("Install:  brew install kubectl\n");
+    process.exit(1);
+  }
+}
+
+// ── Shared internals ──────────────────────────────────────────────────────────
+
+export async function ensureGhcrPullSecret(execa, { clusterName, githubToken, namespace = "default" }) {
+  if (!githubToken) return;
+
+  banner("GHCR Pull Secret");
+  hint("Creating image pull secret for ghcr.io…");
+
+  const secretName = "ghcr-pull-secret";
+
+  // Create the secret in the target namespace
+  const { exitCode } = await execa("kubectl", [
+    "create", "secret", "docker-registry", secretName,
+    "--docker-server=ghcr.io",
+    "--docker-username=x-access-token",
+    `--docker-password=${githubToken}`,
+    "--namespace", namespace,
+    "--context", clusterName,
+    "--dry-run=client", "-o", "yaml",
+  ], { timeout: 15000, reject: false }).then(async (dryRun) => {
+    if (dryRun.exitCode !== 0) return dryRun;
+    // Pipe through kubectl apply so it's idempotent
+    return execa("kubectl", [
+      "apply", "-f", "-", "--context", clusterName,
+    ], { input: dryRun.stdout, timeout: 15000, reject: false });
+  });
+
+  if (exitCode === 0) {
+    console.log(OK(`  ✓ Pull secret "${secretName}" in namespace "${namespace}"`));
+  } else {
+    console.log(WARN(`  ⚠ Could not create pull secret — create manually:`));
+    hint(`  kubectl create secret docker-registry ${secretName} --docker-server=ghcr.io --docker-username=x-access-token --docker-password=<token>`);
+  }
+
+  // Also create in foundation namespace (app pods need it)
+  const foundationNsCode = await execa("kubectl", [
+    "create", "namespace", "foundation",
+    "--context", clusterName, "--dry-run=client", "-o", "yaml",
+  ], { timeout: 10000, reject: false }).then(async (ns) => {
+    if (ns.exitCode !== 0) return ns;
+    return execa("kubectl", ["apply", "-f", "-", "--context", clusterName],
+      { input: ns.stdout, timeout: 10000, reject: false });
+  });
+
+  if (foundationNsCode.exitCode === 0) {
+    const { exitCode: fnCode } = await execa("kubectl", [
+      "create", "secret", "docker-registry", secretName,
+      "--docker-server=ghcr.io",
+      "--docker-username=x-access-token",
+      `--docker-password=${githubToken}`,
+      "--namespace", "foundation",
+      "--context", clusterName,
+      "--dry-run=client", "-o", "yaml",
+    ], { timeout: 15000, reject: false }).then(async (dryRun) => {
+      if (dryRun.exitCode !== 0) return dryRun;
+      return execa("kubectl", ["apply", "-f", "-", "--context", clusterName],
+        { input: dryRun.stdout, timeout: 15000, reject: false });
+    });
+    if (fnCode === 0) {
+      console.log(OK(`  ✓ Pull secret "${secretName}" in namespace "foundation"`));
+    }
+  }
+
+  // Also create in flux-system namespace (Flux needs it for image automation)
+  const { exitCode: fluxNsCode } = await execa("kubectl", [
+    "create", "namespace", "flux-system",
+    "--context", clusterName, "--dry-run=client", "-o", "yaml",
+  ], { timeout: 10000, reject: false }).then(async (ns) => {
+    if (ns.exitCode !== 0) return ns;
+    return execa("kubectl", ["apply", "-f", "-", "--context", clusterName],
+      { input: ns.stdout, timeout: 10000, reject: false });
+  });
+
+  if (fluxNsCode === 0) {
+    const { exitCode: fsCode } = await execa("kubectl", [
+      "create", "secret", "docker-registry", secretName,
+      "--docker-server=ghcr.io",
+      "--docker-username=x-access-token",
+      `--docker-password=${githubToken}`,
+      "--namespace", "flux-system",
+      "--context", clusterName,
+      "--dry-run=client", "-o", "yaml",
+    ], { timeout: 15000, reject: false }).then(async (dryRun) => {
+      if (dryRun.exitCode !== 0) return dryRun;
+      return execa("kubectl", ["apply", "-f", "-", "--context", clusterName],
+        { input: dryRun.stdout, timeout: 15000, reject: false });
+    });
+    if (fsCode === 0) {
+      console.log(OK(`  ✓ Pull secret "${secretName}" in namespace "flux-system"`));
+    }
+  }
+}
+
+export async function getCredentials(execa, { clusterName, rg, sub, admin } = {}) {
+  hint("Fetching kubeconfig…");
+  const args = [
+    "aks", "get-credentials",
+    "--resource-group", rg,
+    "--name", clusterName,
+    "--overwrite-existing",
+    "--output", "none",
+    ...subArgs(sub),
+  ];
+  if (admin) args.push("--admin");
+  await execa("az", args, { timeout: 30000 });
+  console.log(OK(`  ✓ Kubeconfig merged`));
+}
+
+// ── aks up ────────────────────────────────────────────────────────────────────
+
+export async function aksUp(opts = {}) {
+  const execa = await lazyExeca();
+  const clusterName = opts.clusterName || AKS_DEFAULTS.clusterName;
+  const rg = opts.resourceGroup || AKS_DEFAULTS.resourceGroup;
+  const location = opts.location || AKS_DEFAULTS.location;
+  const nodeCount = opts.nodeCount || AKS_DEFAULTS.nodeCount;
+  const minCount = opts.minCount || AKS_DEFAULTS.minCount;
+  const maxCount = opts.maxCount || AKS_DEFAULTS.maxCount;
+  const nodeVmSize = opts.nodeVmSize || AKS_DEFAULTS.nodeVmSize;
+  const tier = opts.tier || AKS_DEFAULTS.tier;
+  const networkPlugin = opts.networkPlugin || AKS_DEFAULTS.networkPlugin;
+  const sub = opts.profile;
+
+  await ensureAzCli(execa);
+  await ensureKubectl(execa);
+  const account = await ensureAzAuth(execa, { subscription: sub });
+
+  // Resolve K8s version: use explicit opt, otherwise auto-detect latest GA for region
+  const k8sVersion = opts.kubernetesVersion ||
+    await resolveK8sVersion(execa, { location, subscription: sub });
+
+  banner(`AKS Up: ${clusterName}`);
+  kvLine("Account",    DIM(`${account.name} (${account.id})`));
+  kvLine("Location",   DIM(location));
+  kvLine("Nodes",      DIM(`${nodeCount} x ${nodeVmSize} (autoscale ${minCount}–${maxCount})`));
+  kvLine("K8s",        DIM(k8sVersion));
+  kvLine("Tier",       DIM(tier));
+
+  // Check if cluster already exists
+  const { exitCode: exists } = await execa("az", [
+    "aks", "show", "-g", rg, "-n", clusterName, "--output", "none",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (exists === 0) {
+    console.log(WARN(`\n  Cluster "${clusterName}" already exists — reconciling…`));
+
+    const maxPods = opts.maxPods || 110;
+    const ctx = { execa, clusterName, rg, sub, opts, minCount, maxCount, maxPods };
+    await reconcileCluster(ctx);
+
+    // Re-provision templates if --reprovision flag is set
+    if (opts.reprovision) {
+      const githubToken = resolveGithubToken(opts);
+      if (!githubToken) {
+        console.log(WARN("  ⚠ Cannot reprovision — no GitHub token found."));
+        hint("Authenticate with:  gh auth login   (writes to ~/.netrc)");
+      } else {
+        const fluxRepo = opts.fluxRepo ?? AKS_DEFAULTS.fluxRepo;
+        const fluxOwner = opts.fluxOwner ?? AKS_DEFAULTS.fluxOwner;
+        const fluxBranch = opts.fluxBranch ?? AKS_DEFAULTS.fluxBranch;
+        const templateRepo = opts.templateRepo ?? TEMPLATE_DEFAULTS.templateRepo;
+        const templateOwner = opts.templateOwner ?? TEMPLATE_DEFAULTS.templateOwner;
+        const templateBranch = opts.templateBranch ?? TEMPLATE_DEFAULTS.templateBranch;
+        const environment = opts.environment || "demo";
+
+        banner("Re-provisioning cluster templates");
+        try {
+          const result = await provisionFluxFromTemplate(execa, {
+            clusterName,
+            region: location,
+            domain: clusterDomain(clusterName),
+            keyvaultUrl: opts.keyvaultUrl || `https://fops-${clusterName}-kv.vault.azure.net`,
+            environment,
+            githubToken,
+            fluxRepo,
+            fluxOwner,
+            fluxBranch,
+            templateRepo,
+            templateOwner,
+            templateBranch,
+            dryRun: opts.dryRun,
+            noCommit: opts.noCommit,
+          });
+          if (result && result.committed !== false) {
+            console.log(OK("  ✓ Templates re-provisioned and committed"));
+          }
+        } catch (err) {
+          console.log(WARN(`  ⚠ Template re-provisioning failed: ${(err.message || "").split("\n")[0]}`));
+        }
+      }
+    }
+
+    const tracked = readClusterState(clusterName);
+    if (tracked) printClusterInfo(tracked);
+    return tracked;
+  }
+
+  // Create resource group
+  hint(`Ensuring resource group ${rg}…`);
+  await execa("az", [
+    "group", "create", "--name", rg, "--location", location,
+    "--output", "none", ...subArgs(sub),
+  ], { timeout: 30000 });
+
+  // Create AKS cluster
+  banner(`Creating AKS cluster "${clusterName}"`);
+
+  // Detect operator IP to lock down the API server
+  hint("Detecting your public IP…");
+  const myIp = await fetchMyIp();
+  if (myIp) {
+    console.log(OK(`  ✓ API server will be scoped to ${myIp}`));
+  } else {
+    console.log(WARN("  ⚠ Could not detect public IP — API server will be open to all"));
+    hint("Lock it down later: az aks update -g <rg> -n <name> --api-server-authorized-ip-ranges <ip>/32");
+  }
+
+  hint("This takes 5–10 minutes…\n");
+
+  const tags = buildTags(clusterName, {
+    createdBy: account.user?.name || "fops",
+    type: "aks",
+  });
+  const tagStr = Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(" ");
+
+  const maxPods = opts.maxPods || 110;
+
+  // Zone redundancy: query available zones for VM size in region
+  const useZones = opts.zones !== false && (opts.zones === true || nodeCount >= 3);
+  let zones = [];
+  if (useZones) {
+    try {
+      const { stdout: skuJson } = await execa("az", [
+        "vm", "list-skus",
+        "--location", location,
+        "--size", nodeVmSize,
+        "--resource-type", "virtualMachines",
+        "--query", "[0].locationInfo[0].zones",
+        "--output", "json",
+        ...subArgs(sub),
+      ], { timeout: 30000 });
+      const availableZones = JSON.parse(skuJson || "[]");
+      zones = Array.isArray(availableZones) ? availableZones.sort() : [];
+    } catch {
+      // Fallback to no zones if query fails
+      zones = [];
+    }
+  }
+
+  const createArgs = [
+    "aks", "create",
+    "--resource-group", rg,
+    "--name", clusterName,
+    "--location", location,
+    "--node-count", String(nodeCount),
+    "--node-vm-size", nodeVmSize,
+    "--max-pods", String(maxPods),
+    "--enable-cluster-autoscaler",
+    "--min-count", String(minCount),
+    "--max-count", String(maxCount),
+    "--kubernetes-version", k8sVersion,
+    "--tier", tier,
+    "--network-plugin", networkPlugin,
+    "--generate-ssh-keys",
+    "--enable-managed-identity",
+    "--enable-oidc-issuer",
+    "--enable-workload-identity",
+    "--ssh-access", "disabled",
+    "--tags", ...tagStr.split(" "),
+    "--output", "json",
+    ...subArgs(sub),
+  ];
+
+  // Add availability zones for HA
+  if (zones.length > 0) {
+    createArgs.push("--zones", ...zones);
+    console.log(OK(`  ✓ Zone redundancy enabled (zones ${zones.join(", ")})`));
+  }
+
+  if (myIp) {
+    createArgs.push("--api-server-authorized-ip-ranges", `${myIp}/32`);
+  }
+
+  let cluster;
+  try {
+    const { stdout: clusterJson } = await execa("az", createArgs, { timeout: 900000 });
+    cluster = JSON.parse(clusterJson);
+  } catch (err) {
+    const msg = (err.stderr || err.message || "").replace(/^.*ERROR:\s*/m, "");
+    console.error(ERR(`\n  ✗ Cluster creation failed:\n    ${msg.split("\n")[0]}`));
+    throw err;
+  }
+  console.log(OK(`  ✓ AKS cluster created`));
+
+  // Get kubeconfig
+  await getCredentials(execa, { clusterName, rg, sub });
+
+  // Save state
+  writeClusterState(clusterName, {
+    resourceGroup: rg,
+    location,
+    nodeCount,
+    nodeVmSize,
+    kubernetesVersion: cluster.kubernetesVersion || k8sVersion,
+    subscriptionId: account.id,
+    fqdn: cluster.fqdn,
+    provisioningState: cluster.provisioningState,
+    zones: zones.length > 0 ? zones : undefined,
+    createdAt: new Date().toISOString(),
+  });
+
+  // Create GHCR pull secret so the cluster can pull private images
+  const githubToken = resolveGithubToken(opts);
+  if (githubToken) {
+    await ensureGhcrPullSecret(execa, { clusterName, githubToken });
+  }
+
+  // Bootstrap Flux — defaults to meshxdata/platform-flux-template
+  const fluxRepo = opts.fluxRepo ?? AKS_DEFAULTS.fluxRepo;
+  const fluxOwner = opts.fluxOwner ?? AKS_DEFAULTS.fluxOwner;
+  const fluxBranch = opts.fluxBranch ?? AKS_DEFAULTS.fluxBranch;
+  const fluxPath = opts.fluxPath || `clusters/${clusterName}`;
+
+  // Template-based provisioning options
+  const templateRepo = opts.templateRepo ?? TEMPLATE_DEFAULTS.templateRepo;
+  const templateOwner = opts.templateOwner ?? TEMPLATE_DEFAULTS.templateOwner;
+  const templateBranch = opts.templateBranch ?? TEMPLATE_DEFAULTS.templateBranch;
+  const environment = opts.environment || "demo";
+
+  // Get Azure identity info for ExternalSecrets
+  let azureIdentityId = "";
+  let azureTenantId = "";
+  try {
+    const { stdout: identityJson } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", clusterName,
+      "--query", "identityProfile.kubeletidentity.clientId", "-o", "tsv",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+    azureIdentityId = (identityJson || "").trim();
+    const { stdout: tenantJson } = await execa("az", [
+      "account", "show", "--query", "tenantId", "-o", "tsv",
+    ], { reject: false, timeout: 10000 });
+    azureTenantId = (tenantJson || "").trim();
+  } catch { /* ignore */ }
+
+  if (opts.noFlux) {
+    console.log("");
+    hint("Flux not bootstrapped (--no-flux).");
+    hint("Bootstrap later:  fops azure aks flux bootstrap <name>");
+  } else if (!githubToken) {
+    console.log("");
+    console.log(WARN("  ⚠ Skipping Flux bootstrap — no GitHub token found."));
+    hint("Authenticate with:  gh auth login   (writes to ~/.netrc)");
+    hint("Or set GITHUB_TOKEN, or pass --github-token, then run:");
+    hint(`  fops azure aks flux bootstrap ${clusterName}`);
+  } else {
+    // Provision cluster manifests from template and commit to flux repo
+    let templateProvisioned = false;
+    if (!opts.noTemplate) {
+      try {
+        const result = await provisionFluxFromTemplate(execa, {
+          clusterName,
+          region: location,
+          domain: clusterDomain(clusterName),
+          keyvaultUrl: opts.keyvaultUrl || `https://fops-${clusterName}-kv.vault.azure.net`,
+          environment,
+          azureIdentityId,
+          azureTenantId,
+          githubToken,
+          fluxRepo,
+          fluxOwner,
+          fluxBranch,
+          templateRepo,
+          templateOwner,
+          templateBranch,
+          dryRun: opts.dryRun,
+          noCommit: opts.noCommit,
+        });
+        templateProvisioned = result && result.committed !== false;
+      } catch (err) {
+        console.log(WARN(`  ⚠ Template provisioning failed: ${(err.message || "").split("\n")[0]}`));
+        hint("Falling back to legacy Flux bootstrap");
+      }
+    }
+
+    // Bootstrap Flux to point to the cluster path
+    await bootstrapFlux(execa, {
+      clusterName, rg, sub,
+      githubToken,
+      repo: fluxRepo,
+      owner: fluxOwner,
+      path: fluxPath,
+      branch: fluxBranch,
+    });
+    writeClusterState(clusterName, {
+      flux: {
+        repo: fluxRepo, owner: fluxOwner, path: fluxPath, branch: fluxBranch,
+        templateProvisioned,
+      },
+    });
+  }
+
+  // Pre-install CRDs and fix webhook scheduling so Flux kustomizations can reconcile
+  if (!opts.noFlux) {
+    try {
+      await reconcileFluxPrereqs({ execa, clusterName, rg, sub, opts });
+    } catch (err) {
+      console.log(WARN(`  ⚠ Flux prereqs: ${(err.message || "").split("\n")[0]}`));
+      hint("Run again with: fops azure aks up " + clusterName);
+    }
+  }
+
+  // Configure network access (Key Vault, Storage Account service endpoints)
+  try {
+    const { stdout: freshJson } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", clusterName, "--output", "json", ...subArgs(sub),
+    ], { timeout: 30000 });
+    const freshCluster = JSON.parse(freshJson);
+    await reconcileNetworkAccess({ execa, clusterName, rg, sub, cluster: freshCluster, opts });
+  } catch (err) {
+    console.log(WARN(`  ⚠ Network access config: ${(err.message || "").split("\n")[0]}`));
+  }
+
+  // Provision Postgres Flexible Server in the AKS VNet
+  if (opts.noPostgres !== true) {
+    try {
+      // Re-fetch cluster to get nodeResourceGroup
+      const { stdout: freshJson } = await execa("az", [
+        "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+        ...subArgs(sub),
+      ], { timeout: 30000 });
+      const freshCluster = JSON.parse(freshJson);
+      const pgCtx = { execa, clusterName, rg, sub, cluster: freshCluster, opts };
+      await reconcilePostgres(pgCtx);
+
+      // Create geo-replica for HA (default: enabled when in a supported region)
+      const geoReplicaRegion = PG_REPLICA_REGIONS[location];
+      const createGeoReplica = opts.geoReplica !== false && geoReplicaRegion;
+      if (createGeoReplica) {
+        try {
+          console.log(OK(`  ✓ Creating geo-replica in ${geoReplicaRegion} for HA…`));
+          await aksPostgresReplicaCreate({
+            clusterName,
+            profile: sub,
+            region: geoReplicaRegion,
+          });
+        } catch (replicaErr) {
+          console.log(WARN(`  ⚠ Geo-replica creation failed: ${replicaErr.message?.split("\n")[0] || replicaErr}`));
+          hint(`  Create manually: fops azure aks postgres replica create --region ${geoReplicaRegion}`);
+        }
+      }
+    } catch (err) {
+      const msg = err.message || "";
+      const stderr = err.stderr ? err.stderr.trim().split("\n").slice(-2).join(" ") : "";
+      console.log(WARN(`  ⚠ Postgres provisioning failed: ${msg.split("\n")[0]}`));
+      if (stderr && !msg.includes(stderr)) hint(stderr);
+      hint("Retry with: fops azure aks up " + clusterName);
+    }
+  }
+
+  // Provision Event Hubs (managed Kafka) if requested
+  if (opts.managedKafka === true) {
+    try {
+      const { stdout: freshJson } = await execa("az", [
+        "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+        ...subArgs(sub),
+      ], { timeout: 30000 });
+      const freshCluster = JSON.parse(freshJson);
+      const ehCtx = { execa, clusterName, rg, sub, cluster: freshCluster, opts };
+      await reconcileEventHubs(ehCtx);
+    } catch (err) {
+      const msg = err.message || "";
+      console.log(WARN(`  ⚠ Event Hubs provisioning failed: ${msg.split("\n")[0]}`));
+      hint("Retry with: fops azure aks up " + clusterName + " --managed-kafka");
+    }
+  }
+
+  const info = readClusterState(clusterName);
+  printClusterInfo(info);
+  return info;
+}
+
+// ── aks down ──────────────────────────────────────────────────────────────────
+
+export async function aksDown(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const name = opts.clusterName;
+  let clusterName, rg;
+
+  // Try local state first
+  const tracked = readClusterState(name);
+  if (tracked?.clusterName) {
+    clusterName = tracked.clusterName;
+    rg = tracked.resourceGroup;
+  } else if (name) {
+    // Not in local state — probe Azure directly for residual clusters
+    rg = AKS_DEFAULTS.resourceGroup;
+    const { exitCode, stdout } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", name, "--output", "json", ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+
+    if (exitCode === 0 && stdout) {
+      clusterName = name;
+      const info = JSON.parse(stdout);
+      rg = info.resourceGroup || rg;
+      console.log(WARN(`\n  Cluster "${name}" not in local state but found in Azure (residual).`));
+    } else {
+      // Also try listing all clusters in the default RG
+      const { stdout: listJson } = await execa("az", [
+        "aks", "list", "-g", rg, "--output", "json", ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      const clusters = listJson ? JSON.parse(listJson) : [];
+      const match = clusters.find((c) => c.name === name);
+      if (match) {
+        clusterName = match.name;
+        rg = match.resourceGroup || rg;
+        console.log(WARN(`\n  Cluster "${name}" not in local state but found in Azure (residual).`));
+      } else {
+        console.error(ERR(`\n  Cluster "${name}" not found in local state or Azure (RG: ${rg}).`));
+        hint("List Azure clusters:  az aks list -g foundation-aks-rg -o table");
+        hint("List tracked:         fops azure aks list\n");
+        process.exit(1);
+      }
+    }
+  } else {
+    // No name given, require tracked state
+    requireCluster(name);
+    return; // unreachable, requireCluster exits
+  }
+
+  banner(`Destroying AKS cluster "${clusterName}"`);
+  kvLine("RG", DIM(rg));
+
+  if (!opts.yes) {
+    const { confirm } = await import(resolveCliSrc("ui/confirm.js"));
+    const ok = await confirm(`  Destroy cluster "${clusterName}" and all workloads?`);
+    if (!ok) { console.log(DIM("\n  Cancelled.\n")); return; }
+  }
+
+  hint("This takes 3–5 minutes…\n");
+
+  // Delete Postgres Flexible Server first (releases subnet delegation blocking VNet deletion)
+  const pgServerNameVal = `fops-${clusterName}-psql`;
+  const { exitCode: pgExists } = await execa("az", [
+    "postgres", "flexible-server", "show",
+    "--name", pgServerNameVal, "--resource-group", rg,
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (pgExists === 0) {
+    console.log(DIM(`  Deleting Postgres server "${pgServerNameVal}"…`));
+    await execa("az", [
+      "postgres", "flexible-server", "delete",
+      "--name", pgServerNameVal, "--resource-group", rg,
+      "--yes", "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 300000 });
+    console.log(OK("  ✓ Postgres server deleted"));
+  }
+
+  // Delete Private DNS zone (also blocks VNet deletion)
+  const dnsZone = `${pgServerNameVal}.private.postgres.database.azure.com`;
+  const { exitCode: dnsExists } = await execa("az", [
+    "network", "private-dns", "zone", "show",
+    "-g", rg, "-n", dnsZone, "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+
+  if (dnsExists === 0) {
+    console.log(DIM(`  Deleting Private DNS zone "${dnsZone}"…`));
+    await execa("az", [
+      "network", "private-dns", "zone", "delete",
+      "-g", rg, "-n", dnsZone, "--yes", "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 60000 });
+    console.log(OK("  ✓ Private DNS zone deleted"));
+  }
+
+  // Delete KeyVault (best-effort cleanup)
+  const kvNameVal = `fops-${clusterName}-kv`.replace(/[^a-zA-Z0-9-]/g, "").slice(0, 24);
+  await execa("az", [
+    "keyvault", "delete", "--name", kvNameVal, "--resource-group", rg,
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 60000 });
+
+  await execa("az", [
+    "aks", "delete", "--resource-group", rg, "--name", clusterName,
+    "--yes", "--no-wait", "--output", "none",
+    ...subArgs(sub),
+  ], { timeout: 300000 });
+  console.log(OK("  ✓ Cluster deletion initiated"));
+
+  // Remove kubeconfig context
+  try {
+    await execa("kubectl", ["config", "delete-context", clusterName], { reject: false, timeout: 10000 });
+    await execa("kubectl", ["config", "delete-cluster", clusterName], { reject: false, timeout: 10000 });
+    console.log(OK("  ✓ Kubeconfig context removed"));
+  } catch { /* best-effort */ }
+
+  clearClusterState(clusterName);
+  console.log(OK("\n  ✓ Done.") + DIM("  State cleared.\n"));
+}
+
+// ── aks list ──────────────────────────────────────────────────────────────────
+
+export async function aksList(opts = {}) {
+  let { activeCluster, clusters } = readAksClusters();
+  let names = Object.keys(clusters);
+
+  banner("AKS Clusters");
+
+  // If no clusters tracked locally, try to discover fops-managed clusters from Azure
+  if (names.length === 0) {
+    const execa = await lazyExeca();
+    try {
+      await ensureAzCli(execa);
+      await ensureAzAuth(execa, { subscription: opts.profile });
+    } catch {
+      hint("No clusters tracked.");
+      hint("Create one:  fops azure aks up <name>\n");
+      return;
+    }
+
+    hint("No clusters tracked locally — checking Azure for fops-managed clusters…\n");
+
+    try {
+      // Query all AKS clusters and filter by managed=fops tag
+      const { stdout, exitCode } = await execa("az", [
+        "aks", "list",
+        "--query", "[?tags.managed=='fops']",
+        "--output", "json",
+        ...subArgs(opts.profile),
+      ], { timeout: 60000, reject: false });
+
+      if (exitCode === 0 && stdout?.trim()) {
+        const discovered = JSON.parse(stdout);
+        if (discovered.length > 0) {
+          for (const cl of discovered) {
+            const name = cl.name;
+            const info = {
+              resourceGroup: cl.resourceGroup,
+              location: cl.location,
+              kubernetesVersion: cl.kubernetesVersion,
+              fqdn: cl.fqdn,
+              nodeCount: cl.agentPoolProfiles?.reduce((s, p) => s + (p.count || 0), 0) || 0,
+              nodeVmSize: cl.agentPoolProfiles?.[0]?.vmSize || "unknown",
+              subscriptionId: cl.id?.split("/")[2],
+              createdAt: cl.provisioningState === "Succeeded" ? new Date().toISOString() : null,
+            };
+            writeClusterState(name, info);
+            console.log(OK(`  + Discovered ${name} (${cl.location})`));
+          }
+          console.log("");
+          // Re-read after discovery
+          const updated = readAksClusters();
+          activeCluster = updated.activeCluster;
+          clusters = updated.clusters;
+          names = Object.keys(clusters);
+        }
+      }
+    } catch {
+      // Discovery failed, continue with empty list
+    }
+
+    if (names.length === 0) {
+      hint("No fops-managed clusters found in Azure.");
+      hint("Create one:  fops azure aks up <name>\n");
+      return;
+    }
+  }
+
+  // Refresh each tracked cluster from Azure so RG, Location, Nodes, FQDN, etc. are current
+  try {
+    const execa = await lazyExeca();
+    await ensureAzCli(execa);
+    await ensureAzAuth(execa, { subscription: opts.profile });
+    for (const name of names) {
+      const cl = clusters[name];
+      let azCluster = null;
+      if (cl.resourceGroup) {
+        const { stdout, exitCode } = await execa("az", [
+          "aks", "show", "-g", cl.resourceGroup, "-n", name, "--output", "json",
+          ...subArgs(opts.profile),
+        ], { timeout: 15000, reject: false });
+        if (exitCode === 0 && stdout?.trim()) azCluster = JSON.parse(stdout);
+      }
+      if (!azCluster) {
+        const { stdout, exitCode } = await execa("az", [
+          "aks", "list", "--output", "json", ...subArgs(opts.profile),
+        ], { timeout: 60000, reject: false });
+        if (exitCode === 0 && stdout?.trim()) {
+          const list = JSON.parse(stdout);
+          azCluster = list.find((c) => c.name === name) || null;
+        }
+      }
+      if (azCluster) {
+        const info = {
+          resourceGroup: azCluster.resourceGroup,
+          location: azCluster.location,
+          kubernetesVersion: azCluster.kubernetesVersion,
+          fqdn: azCluster.fqdn,
+          nodeCount: azCluster.agentPoolProfiles?.reduce((s, p) => s + (p.count || 0), 0) ?? null,
+          nodeVmSize: azCluster.agentPoolProfiles?.[0]?.vmSize || null,
+          subscriptionId: azCluster.id?.split("/")[2],
+        };
+        if (!cl.createdAt && azCluster.provisioningState === "Succeeded") {
+          info.createdAt = new Date().toISOString();
+        }
+        writeClusterState(name, info);
+      }
+    }
+    const updated = readAksClusters();
+    clusters = updated.clusters;
+  } catch {
+    // Keep existing state if refresh fails (e.g. az not logged in)
+  }
+
+  for (const name of names) {
+    const cl = clusters[name];
+    const active = name === activeCluster ? OK(" ◀ active") : "";
+    console.log(`\n  ${LABEL(name)}${active}`);
+    kvLine("  RG",       DIM(cl.resourceGroup || "—"), { pad: 12 });
+    kvLine("  Location", DIM(cl.location || "—"), { pad: 12 });
+    kvLine("  Nodes",    DIM(`${cl.nodeCount || "?"} x ${cl.nodeVmSize || "?"}`), { pad: 12 });
+    kvLine("  K8s",      DIM(cl.kubernetesVersion || "—"), { pad: 12 });
+    if (cl.zones?.length > 0) {
+      kvLine("  Zones",  DIM(cl.zones.join(", ")), { pad: 12 });
+    }
+    kvLine("  FQDN",     DIM(cl.fqdn || "—"), { pad: 12 });
+    if (cl.flux) {
+      kvLine("  Flux",   DIM(`${cl.flux.owner}/${cl.flux.repo} (${cl.flux.path})`), { pad: 12 });
+    }
+    kvLine("  Created",  DIM(cl.createdAt ? new Date(cl.createdAt).toLocaleString() : "—"), { pad: 12 });
+    if (cl.domain) {
+      kvLine("  Domain",   DIM(cl.domain), { pad: 12 });
+    }
+    if (cl.qa) {
+      const qaAge = timeSince(cl.qa.at);
+      const qaLabel = cl.qa.passed ? OK(`✓ passed (${qaAge} ago)`) : ERR(`✗ failed (${qaAge} ago)`);
+      kvLine("  QA",       qaLabel, { pad: 12 });
+    }
+  }
+  console.log("");
+}
+
+// ── aks status ────────────────────────────────────────────────────────────────
+
+export async function aksStatus(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+
+  banner(`AKS Status: ${clusterName}`);
+
+  // Cluster info from Azure
+  const { stdout: showJson } = await execa("az", [
+    "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+    ...subArgs(sub),
+  ], { timeout: 30000 });
+  const info = JSON.parse(showJson);
+
+  kvLine("State",      info.provisioningState === "Succeeded" ? OK(info.provisioningState) : WARN(info.provisioningState));
+  kvLine("Power",      info.powerState?.code === "Running" ? OK(info.powerState.code) : WARN(info.powerState?.code || "unknown"));
+  kvLine("K8s",        DIM(info.kubernetesVersion));
+  kvLine("FQDN",       DIM(info.fqdn));
+  kvLine("Location",   DIM(info.location));
+  kvLine("Tier",       DIM(info.sku?.tier || "—"));
+
+  // Node pools
+  const pools = info.agentPoolProfiles || [];
+  console.log(`\n  ${LABEL("Node Pools")}`);
+  for (const pool of pools) {
+    const status = pool.provisioningState === "Succeeded" ? OK("ready") : WARN(pool.provisioningState);
+    const zones = pool.availabilityZones?.length > 0 ? ` zones:${pool.availabilityZones.join(",")}` : " no-zones";
+    const scaling = pool.enableAutoScaling ? ` (${pool.minCount}-${pool.maxCount})` : "";
+    console.log(`    ${pool.name}: ${pool.count}${scaling} x ${pool.vmSize} [${status}]${DIM(zones)}`);
+  }
+
+  // Flux status (if available)
+  try {
+    await execa("flux", ["--version"], { timeout: 5000 });
+    console.log(`\n  ${LABEL("Flux Status")}`);
+    const { stdout: fluxStatus } = await execa("flux", [
+      "get", "all", "--context", clusterName, "--no-header",
+    ], { timeout: 30000, reject: false });
+    if (fluxStatus?.trim()) {
+      for (const line of fluxStatus.trim().split("\n").slice(0, 20)) {
+        console.log(`    ${DIM(line)}`);
+      }
+    } else {
+      hint("  Flux not installed or no resources found.");
+    }
+  } catch {
+    hint("  Flux CLI not available — skipping Flux status.");
+  }
+
+  console.log("");
+}
+
+// ── aks config versions ──────────────────────────────────────────────────────
+
+const AKS_FOUNDATION_COMPONENTS = {
+  "foundation-backend":        { label: "Backend",   short: "be" },
+  "foundation-frontend":       { label: "Frontend",  short: "fe" },
+  "foundation-processor":      { label: "Processor", short: "pr" },
+  "foundation-watcher":        { label: "Watcher",   short: "wa" },
+  "foundation-scheduler":      { label: "Scheduler", short: "sc" },
+  "foundation-storage-engine": { label: "Storage",   short: "se" },
+};
+
+export async function aksConfigVersions(opts = {}) {
+  const execa = await lazyExeca();
+  const { clusterName } = requireCluster(opts.clusterName);
+
+  banner(`Service Versions: ${clusterName}`);
+
+  const kubectl = (args) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 15000, reject: false });
+
+  const { stdout, exitCode } = await kubectl([
+    "get", "deployments", "-n", "foundation",
+    "-o", "jsonpath={range .items[*]}{.metadata.name}={.spec.template.spec.containers[0].image}{\"\\n\"}{end}",
+  ]);
+
+  if (exitCode !== 0 || !stdout?.trim()) {
+    hint("Could not read deployments. Is kubeconfig merged?");
+    hint(`  Run: fops azure aks kubeconfig ${clusterName}\n`);
+    return;
+  }
+
+  const versions = {};
+  for (const line of stdout.trim().split("\n")) {
+    const eq = line.indexOf("=");
+    if (eq < 0) continue;
+    const name = line.slice(0, eq).trim();
+    const image = line.slice(eq + 1).trim();
+    const comp = AKS_FOUNDATION_COMPONENTS[name];
+    if (!comp) continue;
+    const colon = image.lastIndexOf(":");
+    const tag = colon >= 0 ? image.slice(colon + 1) : "latest";
+    versions[name] = { ...comp, image, tag };
+  }
+
+  if (Object.keys(versions).length === 0) {
+    hint("No Foundation services found in the 'foundation' namespace.\n");
+    return;
+  }
+
+  const maxLabel = Math.max(...Object.values(versions).map(v => v.label.length));
+  for (const [, v] of Object.entries(versions)) {
+    console.log(`  ${ACCENT(v.label.padEnd(maxLabel + 2))} ${chalk.white(v.tag)}  ${DIM(v.image)}`);
+  }
+  console.log("");
+
+  // Check for version drift
+  const tags = [...new Set(Object.values(versions).map(v => v.tag))];
+  if (tags.length === 1) {
+    console.log(OK(`  ✓ All services on ${tags[0]}`));
+  } else {
+    console.log(WARN(`  ⚠ Version drift detected — ${tags.length} different tags in use`));
+  }
+  console.log("");
+}
+
+// ── aks kubeconfig ────────────────────────────────────────────────────────────
+
+export async function aksKubeconfig(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+
+  await getCredentials(execa, { clusterName, rg, sub, admin: opts.admin });
+  console.log(OK(`\n  ✓ Kubeconfig merged for "${clusterName}"`));
+  hint(`kubectl config use-context ${clusterName}\n`);
+}
+
+// ── aks node-pool add ─────────────────────────────────────────────────────────
+
+export async function aksNodePoolAdd(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+
+  const poolName = opts.poolName;
+  const nodeCount = opts.nodeCount || 3;
+  const vmSize = opts.nodeVmSize || AKS_DEFAULTS.nodeVmSize;
+
+  if (!poolName) {
+    console.error(ERR("\n  --pool-name is required.\n"));
+    process.exit(1);
+  }
+
+  banner(`Adding node pool "${poolName}" to ${clusterName}`);
+  kvLine("Size",  DIM(`${nodeCount} x ${vmSize}`));
+  hint("This takes a few minutes…\n");
+
+  const args = [
+    "aks", "nodepool", "add",
+    "--resource-group", rg,
+    "--cluster-name", clusterName,
+    "--name", poolName,
+    "--node-count", String(nodeCount),
+    "--node-vm-size", vmSize,
+    "--ssh-access", "disabled",
+    "--output", "none",
+    ...subArgs(sub),
+  ];
+  if (opts.mode) args.push("--mode", opts.mode);
+  if (opts.labels) args.push("--labels", opts.labels);
+  if (opts.taints) args.push("--node-taints", opts.taints);
+  if (opts.maxPods) args.push("--max-pods", String(opts.maxPods));
+
+  // Zone support - can specify "1,2,3" or use --zones flag
+  const zones = opts.zones === true ? ["1", "2", "3"] : (opts.zones ? opts.zones.split(",") : []);
+  if (zones.length > 0) {
+    args.push("--zones", ...zones);
+    kvLine("Zones", zones.join(", "));
+  }
+
+  await execa("az", args, { timeout: 600000 });
+  const zoneInfo = zones.length > 0 ? ` (zones ${zones.join(",")})` : "";
+  console.log(OK(`  ✓ Node pool "${poolName}" added${zoneInfo}\n`));
+}
+
+// ── aks node-pool remove ──────────────────────────────────────────────────────
+
+export async function aksNodePoolRemove(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+  const poolName = opts.poolName;
+
+  if (!poolName) {
+    console.error(ERR("\n  Pool name is required.\n"));
+    process.exit(1);
+  }
+
+  banner(`Removing node pool "${poolName}" from ${clusterName}`);
+  hint("This takes a few minutes…\n");
+
+  await execa("az", [
+    "aks", "nodepool", "delete",
+    "--resource-group", rg,
+    "--cluster-name", clusterName,
+    "--name", poolName,
+    "--output", "none",
+    ...subArgs(sub),
+  ], { timeout: 600000 });
+  console.log(OK(`  ✓ Node pool "${poolName}" removed\n`));
+}
+
+// ── aks grant admin ──────────────────────────────────────────────────────────
+
+export async function aksGrantAdmin(opts = {}) {
+  const execa = await lazyExeca();
+  const { clusterName } = requireCluster(opts.clusterName);
+  const email = opts.email;
+
+  if (!email) {
+    console.error(ERR("\n  Email is required.\n"));
+    process.exit(1);
+  }
+
+  banner(`Granting admin to ${email}`);
+
+  const kubectl = async (args, execaOpts = {}) => {
+    const { stdout, stderr, exitCode } = await execa("kubectl", ["--context", clusterName, ...args], {
+      timeout: 60000, reject: false, ...execaOpts,
+    });
+    return { stdout, stderr, exitCode };
+  };
+
+  // Read postgres host from HelmRelease values (try config.postgres.host first, then env.postgres.host)
+  let { stdout: pgHost } = await kubectl([
+    "get", "helmrelease", "foundation-backend", "-n", "foundation",
+    "-o", "jsonpath={.spec.values.config.postgres.host}",
+  ]);
+  if (!pgHost) {
+    ({ stdout: pgHost } = await kubectl([
+      "get", "helmrelease", "foundation-backend", "-n", "foundation",
+      "-o", "jsonpath={.spec.values.env.postgres.host}",
+    ]));
+  }
+
+  if (!pgHost) {
+    console.log(ERR("  ✗ Could not find postgres host in foundation-backend HelmRelease"));
+    return;
+  }
+
+  // Read password from the postgres secret
+  const { stdout: pwB64 } = await kubectl([
+    "get", "secret", "postgres", "-n", "foundation",
+    "-o", "jsonpath={.data.password}",
+  ]);
+  if (!pwB64) {
+    console.log(ERR("  ✗ No postgres secret found"));
+    return;
+  }
+  const pgPass = Buffer.from(pwB64, "base64").toString();
+
+  // SQL to grant Foundation Admin role (role_id=1) to user by email
+  // Uses the role_member table which links users to roles via identifier
+  const escapedEmail = email.replace(/'/g, "''");
+  const sql = `INSERT INTO role_member (role_id, identifier) SELECT 1, u.identifier FROM "user" u WHERE u.email = '${escapedEmail}' AND NOT EXISTS (SELECT 1 FROM role_member rm WHERE rm.role_id = 1 AND rm.identifier = u.identifier);`;
+  // Escape double quotes for shell - "user" becomes \\"user\\"
+  const escapedSql = sql.replace(/"/g, '\\"');
+  const script = `psql -c "${escapedSql}" && echo DONE`;
+
+  const jobManifest = JSON.stringify({
+    apiVersion: "batch/v1", kind: "Job",
+    metadata: { name: "fops-grant-admin", namespace: "foundation" },
+    spec: {
+      backoffLimit: 2, ttlSecondsAfterFinished: 60,
+      template: {
+        spec: {
+          restartPolicy: "Never",
+          containers: [{
+            name: "psql",
+            image: "postgres:16-alpine",
+            env: [
+              { name: "PGHOST", value: pgHost },
+              { name: "PGUSER", value: "foundation" },
+              { name: "PGDATABASE", value: "foundation" },
+              { name: "PGPASSWORD", value: pgPass },
+              { name: "PGSSLMODE", value: "require" },
+            ],
+            command: ["sh", "-c", script],
+          }],
+        },
+      },
+    },
+  });
+
+  // Delete old job if it exists
+  await kubectl(["delete", "job", "fops-grant-admin", "-n", "foundation", "--ignore-not-found"]);
+  await new Promise(r => setTimeout(r, 2000));
+
+  const { exitCode, stderr } = await kubectl(["apply", "-f", "-"], { input: jobManifest });
+  if (exitCode !== 0) {
+    console.log(ERR(`  ✗ grant-admin job failed: ${(stderr || "").split("\n")[0]}`));
+    return;
+  }
+
+  // Wait for job to complete
+  const { exitCode: waitCode } = await execa("kubectl", [
+    "--context", clusterName,
+    "wait", "--for=condition=complete", "job/fops-grant-admin",
+    "-n", "foundation", "--timeout=60s",
+  ], { timeout: 70000, reject: false });
+
+  if (waitCode === 0) {
+    console.log(OK(`  ✓ Admin role granted to ${email}`));
+  } else {
+    const { stdout: logs } = await kubectl(["logs", "job/fops-grant-admin", "-n", "foundation", "--tail=10"]);
+    if (logs?.includes("DONE")) {
+      console.log(OK(`  ✓ Admin role granted to ${email}`));
+    } else {
+      console.log(WARN("  ⚠ grant-admin job didn't complete — check: kubectl logs job/fops-grant-admin -n foundation"));
+      console.log(DIM(logs || ""));
+    }
+  }
+
+  await kubectl(["delete", "job", "fops-grant-admin", "-n", "foundation", "--ignore-not-found"]);
+}