@meshxdata/fops 0.1.45 → 0.1.46

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-postgres.js

@@ -0,0 +1,768 @@
+/**
+ * azure-aks-postgres.js - PostgreSQL Flexible Server and replicas
+ *
+ * Depends on: azure-aks-naming.js, azure-aks-state.js, azure-aks-network.js
+ */
+
+import chalk from "chalk";
+import { DEFAULTS, OK, WARN, ERR, DIM, hint, banner, kvLine, subArgs } from "./azure.js";
+import { pgServerName, generatePassword, PG_DEFAULTS, PG_REPLICA_REGIONS, EH_DEFAULTS, ehNamespaceName } from "./azure-aks-naming.js";
+import { readClusterState, writeClusterState, requireCluster } from "./azure-aks-state.js";
+import { findAvailableSubnetCidr, findAksVnet } from "./azure-aks-network.js";
+
+// ── Service databases ─────────────────────────────────────────────────────────
+
+export const PG_SERVICE_DBS = ["foundation", "processor", "scheduler", "watcher", "mlflow"];
+
+// ── Postgres reconciliation ───────────────────────────────────────────────────
+
+export async function reconcilePostgres(ctx) {
+  const { execa, clusterName, rg, sub, cluster } = ctx;
+  const location = cluster.location || DEFAULTS.location;
+  const serverName = pgServerName(clusterName);
+
+  // 1. Check if server already exists
+  const { exitCode: pgExists, stdout: pgJson } = await execa("az", [
+    "postgres", "flexible-server", "show",
+    "--name", serverName, "--resource-group", rg,
+    "--output", "json", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  let fqdn;
+  let adminPassword;
+
+  if (pgExists === 0 && pgJson?.trim()) {
+    const pg = JSON.parse(pgJson);
+    fqdn = pg.fullyQualifiedDomainName;
+    console.log(OK(`  ✓ Postgres Flexible Server "${serverName}" exists (${fqdn})`));
+
+    // Reconcile storage auto-grow
+    const autoGrow = pg.storage?.autoGrow;
+    if (autoGrow !== "Enabled") {
+      hint("Enabling storage auto-grow…");
+      await execa("az", [
+        "postgres", "flexible-server", "update",
+        "--name", serverName, "--resource-group", rg,
+        "--storage-auto-grow", "Enabled",
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 120000 });
+      console.log(OK("  ✓ Storage auto-grow enabled"));
+    }
+
+    adminPassword = readClusterState(clusterName)?.postgres?.adminPassword;
+    if (!adminPassword) {
+      console.log(WARN("  ⚠ Postgres admin password not in local state — secret sync skipped"));
+      hint(`  Password was set at creation time. If lost, reset with:`);
+      hint(`  az postgres flexible-server update -g ${rg} -n ${serverName} --admin-password <new>`);
+      return;
+    }
+  } else {
+    // 2. Resolve the AKS VNet + create a delegated subnet for Postgres
+    const nodeRg = cluster.nodeResourceGroup;
+    if (!nodeRg) {
+      console.log(WARN("  ⚠ Could not determine node resource group — skipping Postgres"));
+      return;
+    }
+
+    hint(`Resolving AKS VNet in ${nodeRg}…`);
+    const { stdout: vnetListJson, exitCode: vnetCode } = await execa("az", [
+      "network", "vnet", "list", "-g", nodeRg, "--output", "json",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (vnetCode !== 0 || !vnetListJson?.trim()) {
+      console.log(WARN("  ⚠ No VNet found in node resource group — skipping Postgres"));
+      return;
+    }
+
+    const vnets = JSON.parse(vnetListJson);
+    const vnet = vnets[0];
+    if (!vnet) {
+      console.log(WARN("  ⚠ No VNet found — skipping Postgres"));
+      return;
+    }
+
+    const vnetName = vnet.name;
+    const pgSubnetName = "postgres-subnet";
+
+    // Check if the postgres subnet already exists
+    const { exitCode: subnetExists } = await execa("az", [
+      "network", "vnet", "subnet", "show",
+      "-g", nodeRg, "--vnet-name", vnetName, "-n", pgSubnetName,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (subnetExists !== 0) {
+      const vnetPrefix = vnet.addressSpace?.addressPrefixes?.[0] || "10.224.0.0/12";
+      const pgCidr = await findAvailableSubnetCidr(execa, nodeRg, vnetName, vnetPrefix, sub);
+
+      hint(`Creating subnet "${pgSubnetName}" (${pgCidr}) in ${vnetName}…`);
+      const { exitCode: createCode, stderr: createErr } = await execa("az", [
+        "network", "vnet", "subnet", "create",
+        "-g", nodeRg, "--vnet-name", vnetName, "-n", pgSubnetName,
+        "--address-prefixes", pgCidr,
+        "--delegations", "Microsoft.DBforPostgreSQL/flexibleServers",
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 60000 });
+      if (createCode !== 0) {
+        const detail = (createErr || "").split("\n").filter(l => l.trim()).slice(-2).join(" ");
+        throw new Error(`Subnet creation failed (${pgCidr}): ${detail || "exit code " + createCode}`);
+      }
+      console.log(OK(`  ✓ Subnet "${pgSubnetName}" created`));
+    }
+
+    // Get subnet ID
+    const { stdout: subnetJson } = await execa("az", [
+      "network", "vnet", "subnet", "show",
+      "-g", nodeRg, "--vnet-name", vnetName, "-n", pgSubnetName,
+      "--output", "json", ...subArgs(sub),
+    ], { timeout: 15000 });
+    const subnetId = JSON.parse(subnetJson).id;
+
+    // Create a private DNS zone for Postgres in the VNet
+    const dnsZone = `${serverName}.private.postgres.database.azure.com`;
+    const { exitCode: dnsExists } = await execa("az", [
+      "network", "private-dns", "zone", "show",
+      "-g", rg, "-n", dnsZone, "--output", "none",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (dnsExists !== 0) {
+      hint(`Creating private DNS zone ${dnsZone}…`);
+      await execa("az", [
+        "network", "private-dns", "zone", "create",
+        "-g", rg, "-n", dnsZone, "--output", "none",
+        ...subArgs(sub),
+      ], { timeout: 60000 });
+    }
+
+    // 3. Create the Flexible Server
+    adminPassword = generatePassword();
+    console.log(chalk.yellow(`  ↻ Creating Postgres Flexible Server "${serverName}"…`));
+    hint("This takes 3–5 minutes…");
+
+    await execa("az", [
+      "postgres", "flexible-server", "create",
+      "--name", serverName,
+      "--resource-group", rg,
+      "--location", location,
+      "--admin-user", PG_DEFAULTS.adminUser,
+      "--admin-password", adminPassword,
+      "--sku-name", PG_DEFAULTS.sku,
+      "--tier", PG_DEFAULTS.tier,
+      "--version", PG_DEFAULTS.version,
+      "--storage-size", String(PG_DEFAULTS.storageSizeGb),
+      "--storage-auto-grow", "Enabled",
+      "--subnet", subnetId,
+      "--private-dns-zone", dnsZone,
+      "--yes",
+      "--output", "json", ...subArgs(sub),
+    ], { timeout: 600000 });
+
+    // Read the created server to get the FQDN
+    const { stdout: createdJson } = await execa("az", [
+      "postgres", "flexible-server", "show",
+      "--name", serverName, "--resource-group", rg,
+      "--output", "json", ...subArgs(sub),
+    ], { timeout: 15000 });
+    fqdn = JSON.parse(createdJson).fullyQualifiedDomainName;
+
+    console.log(OK(`  ✓ Postgres Flexible Server created (${fqdn})`));
+
+    // Save password + FQDN to local state
+    writeClusterState(clusterName, {
+      postgres: { serverName, fqdn, adminUser: PG_DEFAULTS.adminUser, adminPassword },
+    });
+  }
+
+  // 4. Allowlist pg_trgm extension (required by backend search migration)
+  const { stdout: extVal } = await execa("az", [
+    "postgres", "flexible-server", "parameter", "show",
+    "--resource-group", rg, "--server-name", serverName,
+    "--name", "azure.extensions", "--query", "value", "-o", "tsv",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  const currentExts = (extVal || "").trim().split(",").map(e => e.trim()).filter(Boolean);
+  if (!currentExts.includes("PG_TRGM") && !currentExts.includes("pg_trgm")) {
+    const newExts = [...currentExts, "pg_trgm"].join(",");
+    hint("Allowlisting pg_trgm extension…");
+    await execa("az", [
+      "postgres", "flexible-server", "parameter", "set",
+      "--resource-group", rg, "--server-name", serverName,
+      "--name", "azure.extensions", "--value", newExts,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 60000 });
+    console.log(OK("  ✓ pg_trgm extension allowlisted"));
+  }
+
+  // 5. Sync the K8s "postgres" secret into the foundation namespace
+  if (fqdn && adminPassword) {
+    await syncPostgresSecret(execa, { clusterName, fqdn, adminUser: PG_DEFAULTS.adminUser, adminPassword });
+  }
+}
+
+// ── Sync Postgres secret to K8s ───────────────────────────────────────────────
+
+export async function syncPostgresSecret(execa, { clusterName, fqdn, adminUser, adminPassword }) {
+  const namespaces = ["foundation"];
+  for (const ns of namespaces) {
+    // Ensure namespace exists
+    await execa("kubectl", [
+      "--context", clusterName,
+      "create", "namespace", ns, "--dry-run=client", "-o", "yaml",
+    ], { reject: false, timeout: 10000 }).then(({ stdout }) =>
+      execa("kubectl", ["--context", clusterName, "apply", "-f", "-"], { input: stdout, timeout: 10000, reject: false })
+    );
+
+    // Create or update the postgres secret
+    const { exitCode } = await execa("kubectl", [
+      "--context", clusterName, "-n", ns,
+      "create", "secret", "generic", "postgres",
+      "--from-literal=host=" + fqdn,
+      "--from-literal=superUserPassword=" + adminPassword,
+      "--from-literal=user=" + adminUser,
+      "--from-literal=password=" + adminPassword,
+      "--dry-run=client", "-o", "yaml",
+    ], { timeout: 10000 }).then(({ stdout }) =>
+      execa("kubectl", [
+        "--context", clusterName, "-n", ns, "apply", "-f", "-",
+      ], { input: stdout, timeout: 10000, reject: false })
+    );
+
+    if (exitCode === 0) {
+      console.log(OK(`  ✓ Secret "postgres" synced to ${ns} namespace`));
+    } else {
+      console.log(WARN(`  ⚠ Could not sync postgres secret to ${ns}`));
+    }
+  }
+}
+
+// ── Postgres databases reconciliation ─────────────────────────────────────────
+
+export async function reconcilePgDatabases(ctx) {
+  const { execa, clusterName } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 60000, reject: false, ...opts });
+
+  const pgServer = pgServerName(clusterName);
+  const pgHost = `${pgServer}.postgres.database.azure.com`;
+
+  // Read password from the postgres secret
+  const { stdout: pwB64 } = await kubectl([
+    "get", "secret", "postgres", "-n", "foundation",
+    "-o", "jsonpath={.data.password}",
+  ]);
+  if (!pwB64) {
+    console.log(WARN("  ⚠ No postgres secret found — skipping DB setup"));
+    return;
+  }
+  const pgPass = Buffer.from(pwB64, "base64").toString();
+
+  // Run a psql job to create all databases and roles
+  const sqlLines = PG_SERVICE_DBS.map(db => [
+    `SELECT 'CREATE DATABASE ${db}' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${db}');`,
+    `DO \\$\\$ BEGIN IF NOT EXISTS (SELECT FROM pg_database WHERE datname = '${db}') THEN EXECUTE 'CREATE DATABASE ${db}'; END IF; IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${db}') THEN CREATE ROLE ${db} LOGIN PASSWORD '${pgPass}'; END IF; END \\$\\$;`,
+    `GRANT ALL ON DATABASE ${db} TO ${db};`,
+  ]).flat();
+
+  const script = sqlLines.map(s => `psql -c "${s}"`).join(" && ") + " && echo DONE";
+
+  const jobManifest = JSON.stringify({
+    apiVersion: "batch/v1", kind: "Job",
+    metadata: { name: "fops-pg-setup", namespace: "foundation" },
+    spec: {
+      backoffLimit: 2, ttlSecondsAfterFinished: 60,
+      template: {
+        spec: {
+          restartPolicy: "Never",
+          containers: [{
+            name: "psql",
+            image: "postgres:16-alpine",
+            env: [
+              { name: "PGHOST", value: pgHost },
+              { name: "PGUSER", value: "foundation" },
+              { name: "PGDATABASE", value: "postgres" },
+              { name: "PGPASSWORD", value: pgPass },
+              { name: "PGSSLMODE", value: "require" },
+            ],
+            command: ["sh", "-c", script],
+          }],
+        },
+      },
+    },
+  });
+
+  // Delete old job if it exists
+  await kubectl(["delete", "job", "fops-pg-setup", "-n", "foundation", "--ignore-not-found"]);
+  await new Promise(r => setTimeout(r, 2000));
+
+  const { exitCode, stderr } = await kubectl(["apply", "-f", "-"], { input: jobManifest });
+  if (exitCode !== 0) {
+    console.log(WARN(`  ⚠ pg-setup job failed: ${(stderr || "").split("\n")[0]}`));
+    return;
+  }
+
+  // Wait for job to complete (max 60s)
+  const { exitCode: waitCode } = await execa("kubectl", [
+    "--context", clusterName,
+    "wait", "--for=condition=complete", "job/fops-pg-setup",
+    "-n", "foundation", "--timeout=60s",
+  ], { timeout: 70000, reject: false });
+
+  if (waitCode === 0) {
+    console.log(OK(`  ✓ Postgres databases ready (${PG_SERVICE_DBS.join(", ")})`));
+  } else {
+    const { stdout: logs } = await kubectl([
+      "logs", "job/fops-pg-setup", "-n", "foundation", "--tail=5",
+    ]);
+    if (logs?.includes("DONE")) {
+      console.log(OK(`  ✓ Postgres databases ready (${PG_SERVICE_DBS.join(", ")})`));
+    } else {
+      console.log(WARN("  ⚠ pg-setup job didn't complete — check: kubectl logs job/fops-pg-setup -n foundation"));
+    }
+  }
+
+  await kubectl(["delete", "job", "fops-pg-setup", "-n", "foundation", "--ignore-not-found"]);
+}
+
+// ── Postgres read replicas ────────────────────────────────────────────────────
+
+export async function aksPostgresReplicaCreate(opts = {}) {
+  const { lazyExeca, ensureAzCli, ensureAzAuth } = await import("./azure.js");
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+  const sourceServer = pgServerName(clusterName);
+  const targetRegion = opts.region || PG_REPLICA_REGIONS[opts.sourceRegion] || "westeurope";
+  const replicaName = opts.replicaName || `${sourceServer}-replica-${targetRegion.replace(/[^a-z]/g, "")}`;
+
+  banner("Postgres Read Replica");
+  kvLine("Source",  sourceServer);
+  kvLine("Replica", replicaName);
+  kvLine("Region",  targetRegion);
+
+  // Get source server resource ID
+  const { stdout: sourceJson, exitCode: sourceExists } = await execa("az", [
+    "postgres", "flexible-server", "show",
+    "--name", sourceServer, "--resource-group", rg,
+    "--output", "json", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (sourceExists !== 0) {
+    console.error(ERR(`\n  ✗ Source server "${sourceServer}" not found`));
+    process.exit(1);
+  }
+
+  const source = JSON.parse(sourceJson);
+  const sourceId = source.id;
+
+  // Check if replica already exists
+  const { exitCode: replicaExists } = await execa("az", [
+    "postgres", "flexible-server", "show",
+    "--name", replicaName, "--resource-group", rg,
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (replicaExists === 0) {
+    console.log(OK(`\n  ✓ Replica "${replicaName}" already exists`));
+    return;
+  }
+
+  hint("Creating read replica (this takes 10-15 minutes)…\n");
+
+  const { exitCode, stderr } = await execa("az", [
+    "postgres", "flexible-server", "replica", "create",
+    "--replica-name", replicaName,
+    "--resource-group", rg,
+    "--source-server", sourceId,
+    "--location", targetRegion,
+    "--output", "json", ...subArgs(sub),
+  ], { timeout: 900000, reject: false });
+
+  if (exitCode !== 0) {
+    const errMsg = (stderr || "").split("\n").find(l => l.includes("ERROR")) || stderr;
+    console.error(ERR(`\n  ✗ Replica creation failed: ${errMsg}`));
+    process.exit(1);
+  }
+
+  // Get replica FQDN
+  const { stdout: replicaJson } = await execa("az", [
+    "postgres", "flexible-server", "show",
+    "--name", replicaName, "--resource-group", rg,
+    "--output", "json", ...subArgs(sub),
+  ], { timeout: 30000 });
+  const replica = JSON.parse(replicaJson);
+
+  console.log(OK(`\n  ✓ Read replica created`));
+  kvLine("FQDN",     replica.fullyQualifiedDomainName);
+  kvLine("Region",   replica.location);
+  kvLine("State",    replica.state);
+
+  // Save replica info to state
+  const cl = readClusterState(clusterName);
+  const replicas = cl.postgres?.replicas || [];
+  replicas.push({
+    name: replicaName,
+    fqdn: replica.fullyQualifiedDomainName,
+    region: targetRegion,
+    createdAt: new Date().toISOString(),
+  });
+  writeClusterState(clusterName, {
+    postgres: { ...cl.postgres, replicas },
+  });
+
+  hint(`\n  Connect: psql "host=${replica.fullyQualifiedDomainName} user=foundation sslmode=require"`);
+  hint("  Note: Replica is read-only. Use for read scaling or DR.\n");
+}
+
+export async function aksPostgresReplicaList(opts = {}) {
+  const { lazyExeca, ensureAzCli, ensureAzAuth, LABEL } = await import("./azure.js");
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+  const sourceServer = pgServerName(clusterName);
+
+  banner(`Postgres Replicas: ${sourceServer}`);
+
+  const { stdout: replicasJson } = await execa("az", [
+    "postgres", "flexible-server", "replica", "list",
+    "--name", sourceServer, "--resource-group", rg,
+    "--output", "json", ...subArgs(sub),
+  ], { timeout: 60000 });
+
+  const replicas = JSON.parse(replicasJson || "[]");
+
+  if (replicas.length === 0) {
+    console.log(DIM("\n  No replicas found.\n"));
+    hint(`Create one: fops azure aks postgres replica create --region westeurope`);
+    return;
+  }
+
+  console.log("");
+  for (const r of replicas) {
+    const state = r.state === "Ready" ? OK(r.state) : WARN(r.state);
+    console.log(`  ${LABEL(r.name)}`);
+    kvLine("    FQDN",     DIM(r.fullyQualifiedDomainName), { pad: 10 });
+    kvLine("    Region",   DIM(r.location), { pad: 10 });
+    kvLine("    State",    state, { pad: 10 });
+    kvLine("    Role",     DIM(r.replicationRole || "Replica"), { pad: 10 });
+    console.log("");
+  }
+}
+
+export async function aksPostgresReplicaPromote(opts = {}) {
+  const { lazyExeca, ensureAzCli, ensureAzAuth } = await import("./azure.js");
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const { resourceGroup: rg } = requireCluster(opts.clusterName);
+  const replicaName = opts.replicaName;
+
+  if (!replicaName) {
+    console.error(ERR("\n  --replica-name is required"));
+    process.exit(1);
+  }
+
+  banner("Promote Replica");
+  kvLine("Replica", replicaName);
+
+  console.log(WARN("\n  ⚠ WARNING: Promoting a replica breaks replication permanently."));
+  console.log(WARN("    The replica becomes a standalone server."));
+  console.log(WARN("    This is typically used for disaster recovery failover.\n"));
+
+  if (!opts.yes) {
+    const readline = await import("node:readline");
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const answer = await new Promise(resolve => {
+      rl.question(`  Promote "${replicaName}" to standalone? [y/N] `, resolve);
+    });
+    rl.close();
+    if (answer.toLowerCase() !== "y") {
+      console.log(DIM("  Cancelled.\n"));
+      return;
+    }
+  }
+
+  hint("Promoting replica (this takes a few minutes)…\n");
+
+  const { exitCode, stderr } = await execa("az", [
+    "postgres", "flexible-server", "replica", "stop-replication",
+    "--name", replicaName, "--resource-group", rg,
+    "--yes", "--output", "none", ...subArgs(sub),
+  ], { timeout: 600000, reject: false });
+
+  if (exitCode !== 0) {
+    console.error(ERR(`\n  ✗ Promotion failed: ${stderr}`));
+    process.exit(1);
+  }
+
+  console.log(OK(`\n  ✓ Replica "${replicaName}" promoted to standalone server`));
+  hint("  Update your application connection strings to point to the new primary.\n");
+}
+
+export async function aksPostgresReplicaDelete(opts = {}) {
+  const { lazyExeca, ensureAzCli, ensureAzAuth } = await import("./azure.js");
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+  const replicaName = opts.replicaName;
+
+  if (!replicaName) {
+    console.error(ERR("\n  --replica-name is required"));
+    process.exit(1);
+  }
+
+  banner("Delete Replica");
+  kvLine("Replica", replicaName);
+
+  if (!opts.yes) {
+    const readline = await import("node:readline");
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const answer = await new Promise(resolve => {
+      rl.question(`  Delete replica "${replicaName}"? [y/N] `, resolve);
+    });
+    rl.close();
+    if (answer.toLowerCase() !== "y") {
+      console.log(DIM("  Cancelled.\n"));
+      return;
+    }
+  }
+
+  hint("Deleting replica…\n");
+
+  const { exitCode, stderr } = await execa("az", [
+    "postgres", "flexible-server", "delete",
+    "--name", replicaName, "--resource-group", rg,
+    "--yes", "--output", "none", ...subArgs(sub),
+  ], { timeout: 300000, reject: false });
+
+  if (exitCode !== 0) {
+    console.error(ERR(`\n  ✗ Delete failed: ${stderr}`));
+    process.exit(1);
+  }
+
+  // Remove from state
+  const cl = readClusterState(clusterName);
+  if (cl.postgres?.replicas) {
+    cl.postgres.replicas = cl.postgres.replicas.filter(r => r.name !== replicaName);
+    writeClusterState(clusterName, { postgres: cl.postgres });
+  }
+
+  console.log(OK(`\n  ✓ Replica "${replicaName}" deleted\n`));
+}
+
+// ── Event Hubs (managed Kafka) ─────────────────────────────────────────────────
+
+export async function reconcileEventHubs(ctx) {
+  if (!ctx.opts?.managedKafka) return;
+
+  const { execa, clusterName, rg, sub, cluster } = ctx;
+  const location = cluster.location || DEFAULTS.location;
+  const nsName = ehNamespaceName(clusterName);
+
+  // 1. Check if namespace already exists
+  const { exitCode: ehExists, stdout: ehJson } = await execa("az", [
+    "eventhubs", "namespace", "show",
+    "--name", nsName, "--resource-group", rg,
+    "--output", "json", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  let endpoint = "";
+  if (ehExists === 0 && ehJson) {
+    try {
+      const ns = JSON.parse(ehJson);
+      endpoint = ns.serviceBusEndpoint || "";
+      console.log(OK(`  ✓ Event Hubs namespace "${nsName}" exists`));
+    } catch {}
+  } else {
+    // Create namespace
+    console.log(chalk.yellow(`  ↻ Creating Event Hubs namespace "${nsName}"…`));
+    try {
+      const { stdout: createdJson } = await execa("az", [
+        "eventhubs", "namespace", "create",
+        "--name", nsName,
+        "--resource-group", rg,
+        "--location", location,
+        "--sku", EH_DEFAULTS.sku,
+        "--capacity", String(EH_DEFAULTS.capacity),
+        "--enable-kafka", "true",
+        "--output", "json", ...subArgs(sub),
+      ], { timeout: 300000 });
+      const ns = JSON.parse(createdJson);
+      endpoint = ns.serviceBusEndpoint || "";
+      console.log(OK(`  ✓ Event Hubs namespace created`));
+    } catch (err) {
+      const msg = (err.stderr || err.message || "").split("\n")[0];
+      console.log(WARN(`  ⚠ Event Hubs creation failed: ${msg}`));
+      return;
+    }
+  }
+
+  // 2. Disable public network access
+  hint("Disabling public access…");
+  await execa("az", [
+    "eventhubs", "namespace", "update",
+    "--name", nsName, "--resource-group", rg,
+    "--public-network-access", "Disabled",
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 120000 });
+
+  // 3. Create private endpoint if not exists
+  const nodeRg = cluster.nodeResourceGroup;
+  const vnetName = await findAksVnet(execa, { nodeRg, sub });
+  if (!vnetName) {
+    console.log(WARN("  ⚠ Could not find AKS VNet — skipping private endpoint"));
+    return;
+  }
+
+  const peName = `${nsName}-pe`;
+  const { exitCode: peExists } = await execa("az", [
+    "network", "private-endpoint", "show",
+    "--name", peName, "--resource-group", rg,
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (peExists !== 0) {
+    hint("Creating private endpoint…");
+
+    // Get Event Hubs resource ID
+    const { stdout: ehIdJson } = await execa("az", [
+      "eventhubs", "namespace", "show",
+      "--name", nsName, "--resource-group", rg,
+      "--query", "id", "-o", "tsv", ...subArgs(sub),
+    ], { timeout: 15000 });
+    const ehId = ehIdJson.trim();
+
+    // Create subnet for private endpoints if not exists
+    const peSubnetName = "private-endpoints";
+    const { exitCode: subnetExists } = await execa("az", [
+      "network", "vnet", "subnet", "show",
+      "--resource-group", nodeRg, "--vnet-name", vnetName, "--name", peSubnetName,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (subnetExists !== 0) {
+      await execa("az", [
+        "network", "vnet", "subnet", "create",
+        "--resource-group", nodeRg, "--vnet-name", vnetName, "--name", peSubnetName,
+        "--address-prefixes", "10.225.0.0/24",
+        "--disable-private-endpoint-network-policies", "true",
+        "--output", "none", ...subArgs(sub),
+      ], { timeout: 60000 });
+    }
+
+    // Create private endpoint
+    await execa("az", [
+      "network", "private-endpoint", "create",
+      "--name", peName,
+      "--resource-group", rg,
+      "--vnet-name", vnetName,
+      "--subnet", peSubnetName,
+      "--private-connection-resource-id", ehId,
+      "--group-id", "namespace",
+      "--connection-name", `${nsName}-connection`,
+      "--output", "none", ...subArgs(sub),
+    ], { timeout: 120000 });
+    console.log(OK("  ✓ Private endpoint created"));
+
+    // 4. Create private DNS zone and link
+    const dnsZoneName = "privatelink.servicebus.windows.net";
+    const { exitCode: zoneExists } = await execa("az", [
+      "network", "private-dns", "zone", "show",
+      "--resource-group", rg, "--name", dnsZoneName,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (zoneExists !== 0) {
+      await execa("az", [
+        "network", "private-dns", "zone", "create",
+        "--resource-group", rg, "--name", dnsZoneName,
+        "--output", "none", ...subArgs(sub),
+      ], { timeout: 60000 });
+    }
+
+    // Link DNS zone to VNet
+    const linkName = `${clusterName}-eh-link`;
+    const { exitCode: linkExists } = await execa("az", [
+      "network", "private-dns", "link", "vnet", "show",
+      "--resource-group", rg, "--zone-name", dnsZoneName, "--name", linkName,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (linkExists !== 0) {
+      const { stdout: vnetIdJson } = await execa("az", [
+        "network", "vnet", "show",
+        "--resource-group", nodeRg, "--name", vnetName,
+        "--query", "id", "-o", "tsv", ...subArgs(sub),
+      ], { timeout: 15000 });
+      await execa("az", [
+        "network", "private-dns", "link", "vnet", "create",
+        "--resource-group", rg, "--zone-name", dnsZoneName, "--name", linkName,
+        "--virtual-network", vnetIdJson.trim(),
+        "--registration-enabled", "false",
+        "--output", "none", ...subArgs(sub),
+      ], { timeout: 60000 });
+    }
+
+    // Create DNS record for the private endpoint
+    const { stdout: peIpJson } = await execa("az", [
+      "network", "private-endpoint", "show",
+      "--name", peName, "--resource-group", rg,
+      "--query", "customDnsConfigs[0].ipAddresses[0]", "-o", "tsv", ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+    const peIp = (peIpJson || "").trim();
+
+    if (peIp) {
+      await execa("az", [
+        "network", "private-dns", "record-set", "a", "add-record",
+        "--resource-group", rg, "--zone-name", dnsZoneName,
+        "--record-set-name", nsName, "--ipv4-address", peIp,
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      console.log(OK("  ✓ Private DNS configured"));
+    }
+  } else {
+    console.log(OK("  ✓ Private endpoint already exists"));
+  }
+
+  // 5. Get connection string and create K8s secret
+  const { stdout: connStrJson } = await execa("az", [
+    "eventhubs", "namespace", "authorization-rule", "keys", "list",
+    "--resource-group", rg, "--namespace-name", nsName, "--name", "RootManageSharedAccessKey",
+    "--query", "primaryConnectionString", "-o", "tsv", ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  const connStr = (connStrJson || "").trim();
+
+  if (connStr) {
+    const kafkaBootstrap = `${nsName}.servicebus.windows.net:9093`;
+    const kubectl = (args, opts = {}) =>
+      execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+    await kubectl([
+      "-n", "foundation", "create", "secret", "generic", "kafka-eventhubs",
+      "--from-literal=bootstrap-servers=" + kafkaBootstrap,
+      "--from-literal=connection-string=" + connStr,
+      "--from-literal=sasl-username=$ConnectionString",
+      "--from-literal=sasl-password=" + connStr,
+      "--dry-run=client", "-o", "yaml",
+    ]).then(({ stdout }) =>
+      kubectl(["-n", "foundation", "apply", "-f", "-"], { input: stdout })
+    );
+    console.log(OK(`  ✓ Kafka connection secret created (${kafkaBootstrap})`));
+
+    writeClusterState(clusterName, {
+      eventHubs: { namespace: nsName, bootstrap: kafkaBootstrap },
+    });
+  }
+}