@meshxdata/fops 0.1.48 → 0.1.50

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-core.js

@@ -17,7 +17,7 @@ import {
   readState,
   resolveGithubToken,
 } from "./azure.js";
-import { AKS_DEFAULTS, PG_REPLICA_REGIONS, timeSince } from "./azure-aks-naming.js";
+import { AKS_DEFAULTS, PG_REPLICA_REGIONS, HA_REPLICA_REGIONS, timeSince } from "./azure-aks-naming.js";
 import {
   readAksClusters,
   readClusterState,
@@ -34,6 +34,7 @@ import {
 } from "./azure-aks-flux.js";
 import { reconcileNetworkAccess } from "./azure-aks-network.js";
 import { reconcilePostgres, aksPostgresReplicaCreate, reconcileEventHubs } from "./azure-aks-postgres.js";
+import { reconcileStorageReplication, reconcileStorageAccount } from "./azure-aks-storage.js";
 import { clusterDomain } from "./azure-aks-ingress.js";
 import { printClusterInfo } from "./azure-aks-stacks.js";
 
@@ -217,6 +218,177 @@ export async function getCredentials(execa, { clusterName, rg, sub, admin } = {}
   console.log(OK(`  ✓ Kubeconfig merged`));
 }
 
+// ── aks standby create (internal helper) ──────────────────────────────────────
+
+async function aksStandbyCreate(execa, opts) {
+  const {
+    primaryClusterName,
+    standbyClusterName,
+    rg,
+    location,
+    nodeCount,
+    minCount,
+    maxCount,
+    nodeVmSize,
+    tier,
+    networkPlugin,
+    k8sVersion,
+    sub,
+    githubToken,
+    fluxRepo,
+    fluxOwner,
+    fluxBranch,
+    templateRepo,
+    templateOwner,
+    templateBranch,
+    environment,
+    account,
+  } = opts;
+
+  banner(`Creating Standby AKS: ${standbyClusterName}`);
+  kvLine("Region",     DIM(location));
+  kvLine("Primary",    DIM(primaryClusterName));
+
+  // Detect operator IP
+  const myIp = await fetchMyIp();
+
+  // Zone redundancy for standby
+  let zones = [];
+  try {
+    const { stdout: skuJson } = await execa("az", [
+      "vm", "list-skus", "--location", location, "--size", nodeVmSize,
+      "--resource-type", "virtualMachines", "--query", "[0].locationInfo[0].zones",
+      "--output", "json", ...subArgs(sub),
+    ], { timeout: 30000 });
+    zones = JSON.parse(skuJson || "[]").sort();
+  } catch { zones = []; }
+
+  const tags = buildTags(standbyClusterName, {
+    createdBy: account?.user?.name || "fops",
+    type: "aks-standby",
+    primaryCluster: primaryClusterName,
+  });
+  const tagStr = Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(" ");
+
+  const createArgs = [
+    "aks", "create",
+    "--resource-group", rg,
+    "--name", standbyClusterName,
+    "--location", location,
+    "--node-count", String(nodeCount),
+    "--node-vm-size", nodeVmSize,
+    "--max-pods", "110",
+    "--enable-cluster-autoscaler",
+    "--min-count", String(minCount),
+    "--max-count", String(maxCount),
+    "--kubernetes-version", k8sVersion,
+    "--tier", tier,
+    "--network-plugin", networkPlugin,
+    "--generate-ssh-keys",
+    "--enable-managed-identity",
+    "--enable-oidc-issuer",
+    "--enable-workload-identity",
+    "--ssh-access", "disabled",
+    "--tags", ...tagStr.split(" "),
+    "--output", "json",
+    ...subArgs(sub),
+  ];
+
+  if (zones.length > 0) createArgs.push("--zones", ...zones);
+  if (myIp) createArgs.push("--api-server-authorized-ip-ranges", `${myIp}/32`);
+
+  hint("Creating standby cluster (5-10 minutes)…\n");
+  const { stdout: clusterJson } = await execa("az", createArgs, { timeout: 900000 });
+  const cluster = JSON.parse(clusterJson);
+  console.log(OK(`  ✓ Standby AKS cluster created in ${location}`));
+
+  // Get kubeconfig
+  await getCredentials(execa, { clusterName: standbyClusterName, rg, sub });
+
+  // Save standby state
+  writeClusterState(standbyClusterName, {
+    resourceGroup: rg,
+    location,
+    nodeCount,
+    nodeVmSize,
+    kubernetesVersion: cluster.kubernetesVersion || k8sVersion,
+    subscriptionId: account?.id,
+    fqdn: cluster.fqdn,
+    provisioningState: cluster.provisioningState,
+    zones: zones.length > 0 ? zones : undefined,
+    isStandby: true,
+    primaryCluster: primaryClusterName,
+    createdAt: new Date().toISOString(),
+  });
+
+  // Bootstrap Flux with same config as primary
+  if (githubToken) {
+    const fluxPath = `clusters/${standbyClusterName}`;
+
+    // Provision templates for standby cluster
+    try {
+      await provisionFluxFromTemplate(execa, {
+        clusterName: standbyClusterName,
+        region: location,
+        domain: clusterDomain(standbyClusterName),
+        keyvaultUrl: `https://fops-${standbyClusterName}-kv.vault.azure.net`,
+        environment,
+        githubToken,
+        fluxRepo,
+        fluxOwner,
+        fluxBranch,
+        templateRepo,
+        templateOwner,
+        templateBranch,
+      });
+    } catch (err) {
+      console.log(WARN(`  ⚠ Template provisioning: ${(err.message || "").split("\n")[0]}`));
+    }
+
+    await bootstrapFlux(execa, {
+      clusterName: standbyClusterName, rg, sub,
+      githubToken,
+      repo: fluxRepo,
+      owner: fluxOwner,
+      path: fluxPath,
+      branch: fluxBranch,
+    });
+
+    writeClusterState(standbyClusterName, {
+      flux: { repo: fluxRepo, owner: fluxOwner, path: fluxPath, branch: fluxBranch },
+    });
+
+    // Pre-install CRDs and fix webhook scheduling
+    try {
+      await reconcileFluxPrereqs({ execa, clusterName: standbyClusterName, rg, sub, opts: {} });
+    } catch { /* best effort */ }
+  }
+
+  // Set up Key Vault and network access for standby
+  try {
+    const { stdout: freshJson } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", standbyClusterName, "--output", "json", ...subArgs(sub),
+    ], { timeout: 30000 });
+    const freshCluster = JSON.parse(freshJson);
+    await reconcileNetworkAccess({ execa, clusterName: standbyClusterName, rg, sub, cluster: freshCluster, opts: {} });
+  } catch { /* best effort */ }
+
+  // Create storage account in standby region (uses HA storage)
+  try {
+    const { stdout: freshJson } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", standbyClusterName, "--output", "json", ...subArgs(sub),
+    ], { timeout: 30000 });
+    const freshCluster = JSON.parse(freshJson);
+    await reconcileStorageAccount({ execa, clusterName: standbyClusterName, rg, sub, cluster: freshCluster, opts: {} });
+  } catch { /* best effort */ }
+
+  console.log(OK(`\n  ✓ Standby cluster "${standbyClusterName}" ready`));
+  hint(`  Flux will sync workloads from ${fluxOwner}/${fluxRepo}`);
+  hint(`  Configure postgres secret to point to read replica for read-only mode\n`);
+
+  return { clusterName: standbyClusterName, cluster };
+}
+
 // ── aks up ────────────────────────────────────────────────────────────────────
 
 export async function aksUp(opts = {}) {
@@ -246,6 +418,10 @@ export async function aksUp(opts = {}) {
   kvLine("Nodes",      DIM(`${nodeCount} x ${nodeVmSize} (autoscale ${minCount}–${maxCount})`));
   kvLine("K8s",        DIM(k8sVersion));
   kvLine("Tier",       DIM(tier));
+  if (opts.ha) {
+    const haRegion = HA_REPLICA_REGIONS[location] || "northeurope";
+    kvLine("HA",       OK(`enabled (replica: ${haRegion})`));
+  }
 
   // Check if cluster already exists
   const { exitCode: exists } = await execa("az", [
@@ -302,6 +478,89 @@ export async function aksUp(opts = {}) {
       }
     }
 
+    // Set up HA for existing cluster if --ha flag is set
+    if (opts.ha === true) {
+      const haRegion = HA_REPLICA_REGIONS[location];
+      if (haRegion) {
+        // Storage replication
+        try {
+          const { stdout: freshJson } = await execa("az", [
+            "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+            ...subArgs(sub),
+          ], { timeout: 30000 });
+          const freshCluster = JSON.parse(freshJson);
+          const storageCtx = { execa, clusterName, rg, sub, cluster: freshCluster, opts };
+          await reconcileStorageReplication(storageCtx);
+        } catch (err) {
+          console.log(WARN(`  ⚠ Storage HA: ${(err.message || "").split("\n")[0]}`));
+        }
+
+        // Postgres read replica
+        try {
+          hint(`Checking Postgres replica in ${haRegion}…`);
+          await aksPostgresReplicaCreate({
+            clusterName,
+            profile: sub,
+            region: haRegion,
+          });
+        } catch (err) {
+          if (!err.message?.includes("already exists")) {
+            console.log(WARN(`  ⚠ Postgres replica: ${(err.message || "").split("\n")[0]}`));
+          }
+        }
+
+        // Standby AKS cluster
+        const standbyClusterName = `${clusterName}-standby`;
+        try {
+          const { exitCode: standbyExists } = await execa("az", [
+            "aks", "show", "-g", rg, "-n", standbyClusterName, "--output", "none",
+            ...subArgs(sub),
+          ], { reject: false, timeout: 30000 });
+
+          if (standbyExists === 0) {
+            console.log(OK(`  ✓ Standby cluster "${standbyClusterName}" already exists`));
+          } else {
+            const githubToken = resolveGithubToken(opts);
+            await aksStandbyCreate(execa, {
+              primaryClusterName: clusterName,
+              standbyClusterName,
+              rg,
+              location: haRegion,
+              nodeCount,
+              minCount,
+              maxCount,
+              nodeVmSize,
+              tier,
+              networkPlugin,
+              k8sVersion,
+              sub,
+              githubToken,
+              fluxRepo: opts.fluxRepo ?? AKS_DEFAULTS.fluxRepo,
+              fluxOwner: opts.fluxOwner ?? AKS_DEFAULTS.fluxOwner,
+              fluxBranch: opts.fluxBranch ?? AKS_DEFAULTS.fluxBranch,
+              templateRepo: opts.templateRepo ?? TEMPLATE_DEFAULTS.templateRepo,
+              templateOwner: opts.templateOwner ?? TEMPLATE_DEFAULTS.templateOwner,
+              templateBranch: opts.templateBranch ?? TEMPLATE_DEFAULTS.templateBranch,
+              environment: opts.environment || "demo",
+              account,
+            });
+          }
+
+          writeClusterState(clusterName, {
+            ha: {
+              enabled: true,
+              standbyCluster: standbyClusterName,
+              standbyRegion: haRegion,
+              configuredAt: new Date().toISOString(),
+            },
+          });
+        } catch (err) {
+          console.log(WARN(`  ⚠ Standby cluster: ${(err.message || "").split("\n")[0]}`));
+          hint(`  Create manually: fops azure aks up ${standbyClusterName} --location ${haRegion}`);
+        }
+      }
+    }
+
     const tracked = readClusterState(clusterName);
     if (tracked) printClusterInfo(tracked);
     return tracked;
@@ -545,12 +804,13 @@ export async function aksUp(opts = {}) {
       const pgCtx = { execa, clusterName, rg, sub, cluster: freshCluster, opts };
       await reconcilePostgres(pgCtx);
 
-      // Create geo-replica for HA (default: enabled when in a supported region)
-      const geoReplicaRegion = PG_REPLICA_REGIONS[location];
-      const createGeoReplica = opts.geoReplica !== false && geoReplicaRegion;
-      if (createGeoReplica) {
+      // Create geo-replica for HA when --ha flag is set (uses HA_REPLICA_REGIONS)
+      // or when --geo-replica is explicitly requested (uses PG_REPLICA_REGIONS)
+      const haRegion = opts.ha ? HA_REPLICA_REGIONS[location] : null;
+      const geoReplicaRegion = haRegion || (opts.geoReplica !== false ? PG_REPLICA_REGIONS[location] : null);
+      if (geoReplicaRegion && (opts.ha || opts.geoReplica !== false)) {
         try {
-          console.log(OK(`  ✓ Creating geo-replica in ${geoReplicaRegion} for HA…`));
+          hint(`Creating Postgres read replica in ${geoReplicaRegion} for HA…`);
           await aksPostgresReplicaCreate({
             clusterName,
             profile: sub,
@@ -570,6 +830,81 @@ export async function aksUp(opts = {}) {
     }
   }
 
+  // Set up cross-region storage replication when --ha flag is set
+  if (opts.ha === true) {
+    try {
+      const { stdout: freshJson } = await execa("az", [
+        "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+        ...subArgs(sub),
+      ], { timeout: 30000 });
+      const freshCluster = JSON.parse(freshJson);
+      const storageCtx = { execa, clusterName, rg, sub, cluster: freshCluster, opts };
+      await reconcileStorageReplication(storageCtx);
+    } catch (err) {
+      const msg = err.message || "";
+      console.log(WARN(`  ⚠ Storage HA setup failed: ${msg.split("\n")[0]}`));
+      hint("Retry with: fops azure aks up " + clusterName + " --ha");
+    }
+
+    // Create standby AKS cluster in HA region
+    const haRegion = HA_REPLICA_REGIONS[location];
+    if (haRegion) {
+      const standbyClusterName = `${clusterName}-standby`;
+      hint(`\nSetting up standby AKS cluster in ${haRegion}…`);
+
+      try {
+        // Check if standby cluster already exists
+        const { exitCode: standbyExists } = await execa("az", [
+          "aks", "show", "-g", rg, "-n", standbyClusterName, "--output", "none",
+          ...subArgs(sub),
+        ], { reject: false, timeout: 30000 });
+
+        if (standbyExists === 0) {
+          console.log(OK(`  ✓ Standby cluster "${standbyClusterName}" already exists`));
+        } else {
+          // Create standby cluster with same config but in HA region
+          await aksStandbyCreate(execa, {
+            primaryClusterName: clusterName,
+            standbyClusterName,
+            rg,
+            location: haRegion,
+            nodeCount,
+            minCount,
+            maxCount,
+            nodeVmSize,
+            tier,
+            networkPlugin,
+            k8sVersion,
+            sub,
+            githubToken,
+            fluxRepo: opts.fluxRepo ?? AKS_DEFAULTS.fluxRepo,
+            fluxOwner: opts.fluxOwner ?? AKS_DEFAULTS.fluxOwner,
+            fluxBranch: opts.fluxBranch ?? AKS_DEFAULTS.fluxBranch,
+            templateRepo: opts.templateRepo ?? TEMPLATE_DEFAULTS.templateRepo,
+            templateOwner: opts.templateOwner ?? TEMPLATE_DEFAULTS.templateOwner,
+            templateBranch: opts.templateBranch ?? TEMPLATE_DEFAULTS.templateBranch,
+            environment: opts.environment || "demo",
+            account,
+          });
+        }
+
+        // Save HA cluster info to state
+        writeClusterState(clusterName, {
+          ha: {
+            enabled: true,
+            standbyCluster: standbyClusterName,
+            standbyRegion: haRegion,
+            configuredAt: new Date().toISOString(),
+          },
+        });
+      } catch (err) {
+        const msg = err.message || "";
+        console.log(WARN(`  ⚠ Standby cluster creation failed: ${msg.split("\n")[0]}`));
+        hint(`  Create manually: fops azure aks up ${standbyClusterName} --location ${haRegion}`);
+      }
+    }
+  }
+
   // Provision Event Hubs (managed Kafka) if requested
   if (opts.managedKafka === true) {
     try {
@@ -852,6 +1187,12 @@ export async function aksList(opts = {}) {
       const qaLabel = cl.qa.passed ? OK(`✓ passed (${qaAge} ago)`) : ERR(`✗ failed (${qaAge} ago)`);
       kvLine("  QA",       qaLabel, { pad: 12 });
     }
+    if (cl.ha?.enabled) {
+      kvLine("  HA",       OK(`✓ ${cl.ha.standbyCluster} (${cl.ha.standbyRegion})`), { pad: 12 });
+    }
+    if (cl.isStandby) {
+      kvLine("  Role",     WARN(`standby for ${cl.primaryCluster}`), { pad: 12 });
+    }
   }
   console.log("");
 }