@meshxdata/fops 0.1.44 → 0.1.46

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-reconcilers.js

@@ -0,0 +1,538 @@
+/**
+ * azure-aks-reconcilers.js - Reconciler arrays and orchestration
+ *
+ * Depends on: azure-aks-naming.js, azure-aks-state.js, azure-aks-network.js,
+ *             azure-aks-secrets.js, azure-aks-storage.js, azure-aks-postgres.js,
+ *             azure-aks-flux.js, azure-aks-ingress.js
+ */
+
+import {
+  OK, WARN, ERR, DIM,
+  banner, hint, kvLine, subArgs,
+  lazyExeca, runReconcilers,
+} from "./azure.js";
+import { AKS_DEFAULTS } from "./azure-aks-naming.js";
+import { readClusterState, requireCluster } from "./azure-aks-state.js";
+import { reconcileApiServerIp } from "./azure-aks-network.js";
+import { reconcileSecretStore, reconcileK8sSecrets, reconcileVaultUnseal } from "./azure-aks-secrets.js";
+import { reconcileStorageAccount, reconcileStorageEngine, reconcileHelmRepos, reconcileHelmValues, reconcileAcrWebhooks } from "./azure-aks-storage.js";
+import { reconcilePostgres, reconcilePgDatabases } from "./azure-aks-postgres.js";
+import { reconcileFluxStep, reconcileFluxPrereqs, suspendManagedKustomizations } from "./azure-aks-flux.js";
+import { reconcileIngressIp, reconcileFrontendAuth } from "./azure-aks-ingress.js";
+
+// ── fops-api deployment ──────────────────────────────────────────────────────
+
+const FOPS_API_NS = "fops-system";
+
+function fopsApiManifests(clusterName) {
+  const labels = { app: "fops-api", "app.kubernetes.io/managed-by": "fops" };
+  return [
+    {
+      apiVersion: "v1", kind: "Namespace",
+      metadata: { name: FOPS_API_NS, labels: { "app.kubernetes.io/managed-by": "fops" } },
+    },
+    {
+      apiVersion: "v1", kind: "ServiceAccount",
+      metadata: { name: "fops-api", namespace: FOPS_API_NS, labels },
+    },
+    {
+      apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRoleBinding",
+      metadata: { name: "fops-api-viewer", labels },
+      subjects: [{ kind: "ServiceAccount", name: "fops-api", namespace: FOPS_API_NS }],
+      roleRef: { kind: "ClusterRole", name: "view", apiGroup: "rbac.authorization.k8s.io" },
+    },
+    {
+      apiVersion: "apps/v1", kind: "Deployment",
+      metadata: { name: "fops-api", namespace: FOPS_API_NS, labels },
+      spec: {
+        replicas: 1,
+        selector: { matchLabels: { app: "fops-api" } },
+        template: {
+          metadata: { labels },
+          spec: {
+            serviceAccountName: "fops-api",
+            containers: [{
+              name: "fops-api",
+              image: "node:20-alpine",
+              command: ["sh", "-c", "npm install -g @meshxdata/fops && fops serve --host 0.0.0.0 --port 4100"],
+              ports: [{ name: "http", containerPort: 4100 }],
+              env: [
+                { name: "NODE_ENV", value: "production" },
+                { name: "FOPS_CLUSTER", value: clusterName },
+              ],
+              resources: {
+                requests: { cpu: "100m", memory: "256Mi" },
+                limits:   { cpu: "500m", memory: "512Mi" },
+              },
+              readinessProbe: {
+                httpGet: { path: "/api/health", port: 4100 },
+                initialDelaySeconds: 30, periodSeconds: 10,
+              },
+              livenessProbe: {
+                httpGet: { path: "/api/health", port: 4100 },
+                initialDelaySeconds: 60, periodSeconds: 30,
+              },
+            }],
+          },
+        },
+      },
+    },
+    {
+      apiVersion: "v1", kind: "Service",
+      metadata: { name: "fops-api", namespace: FOPS_API_NS, labels },
+      spec: {
+        selector: { app: "fops-api" },
+        ports: [{ name: "http", port: 4100, targetPort: 4100 }],
+      },
+    },
+  ];
+}
+
+// ── Internal reconcilers ──────────────────────────────────────────────────────
+
+async function reconcileAddons(ctx) {
+  const { execa, clusterName, rg, sub } = ctx;
+  const cluster = ctx.cluster || {};
+  const baseArgs = ["-g", rg, "-n", clusterName, ...subArgs(sub)];
+
+  const defenderOn = cluster.securityProfile?.defender?.securityMonitoring?.enabled === true;
+  const monitoringOn = cluster.addonProfiles?.omsagent?.enabled === true;
+  const azurePolicyOn = cluster.addonProfiles?.azurepolicy?.enabled === true;
+  const fileDriverOn = cluster.storageProfile?.fileCSIDriver?.enabled !== false;
+
+  if (!defenderOn && !monitoringOn && !azurePolicyOn && !fileDriverOn) {
+    console.log(OK("  ✓ Unwanted addons already disabled"));
+    return;
+  }
+
+  let changed = false;
+
+  if (defenderOn) {
+    hint("Disabling Defender for Containers…");
+    const r = await execa("az", [
+      "aks", "update", ...baseArgs, "--disable-defender", "--yes", "--output", "none",
+    ], { reject: false, timeout: 300000 });
+    if (r.exitCode === 0) { console.log(OK("  ✓ Defender disabled")); changed = true; }
+    else { console.log(WARN(`  ⚠ Defender: ${(r.stderr || "").split("\n")[0]}`)); }
+  }
+
+  const addons = [];
+  if (monitoringOn) addons.push("monitoring");
+  if (azurePolicyOn) addons.push("azure-policy");
+
+  if (addons.length > 0) {
+    hint(`Disabling ${addons.join(", ")}…`);
+    const r = await execa("az", [
+      "aks", "disable-addons", ...baseArgs, "--addons", addons.join(","),
+      "--output", "none",
+    ], { reject: false, timeout: 300000 });
+    if (r.exitCode === 0) { console.log(OK(`  ✓ Disabled: ${addons.join(", ")}`)); changed = true; }
+    else { console.log(WARN(`  ⚠ Addons: ${(r.stderr || "").split("\n")[0]}`)); }
+  }
+
+  const fileDriverExplicitlyOn = cluster.storageProfile?.fileCSIDriver?.enabled === true;
+  if (fileDriverExplicitlyOn) {
+    hint("Disabling Azure File CSI driver (not needed)…");
+    const r = await execa("az", [
+      "aks", "update", ...baseArgs, "--disable-file-driver", "--yes", "--output", "none",
+    ], { reject: false, timeout: 300000 });
+    if (r.exitCode === 0) { console.log(OK("  ✓ Azure File driver disabled (−8 pods)")); changed = true; }
+    else { console.log(WARN(`  ⚠ File driver: ${(r.stderr || "").split("\n")[0]}`)); }
+  } else if (cluster.storageProfile?.fileCSIDriver?.enabled === false) {
+    console.log(OK("  ✓ Azure File driver already disabled"));
+  }
+
+  if (changed) {
+    try {
+      const { stdout } = await execa("az", [
+        "aks", "show", ...baseArgs, "--output", "json",
+      ], { timeout: 30000 });
+      ctx.cluster = JSON.parse(stdout);
+    } catch {}
+  }
+}
+
+async function reconcileAutoscaler(ctx) {
+  const { execa, clusterName, rg, sub, minCount, maxCount, cluster } = ctx;
+  const pool = (cluster.agentPoolProfiles || []).find(p => p.mode === "System")
+    || (cluster.agentPoolProfiles || [])[0];
+  if (!pool) return;
+
+  if (!pool.enableAutoScaling) {
+    hint(`Enabling autoscaler on pool "${pool.name}" (${minCount}–${maxCount})…`);
+    await execa("az", [
+      "aks", "nodepool", "update",
+      "--resource-group", rg,
+      "--cluster-name", clusterName,
+      "--name", pool.name,
+      "--enable-cluster-autoscaler",
+      "--min-count", String(minCount),
+      "--max-count", String(maxCount),
+      "--output", "none", ...subArgs(sub),
+    ], { timeout: 120000 });
+    console.log(OK(`  ✓ Autoscaler enabled on "${pool.name}" (${minCount}–${maxCount})`));
+  } else {
+    console.log(OK(`  ✓ Autoscaler already enabled on "${pool.name}" (${pool.minCount}–${pool.maxCount})`));
+  }
+}
+
+async function reconcileSpotPool(ctx) {
+  const { execa, clusterName, rg, sub, cluster, maxPods } = ctx;
+  const pools = cluster.agentPoolProfiles || [];
+  const spotPool = pools.find(p => p.scaleSetPriority === "Spot");
+
+  if (spotPool) {
+    const count = spotPool.count || 0;
+    const scaling = spotPool.enableAutoScaling
+      ? `autoscale ${spotPool.minCount}–${spotPool.maxCount}`
+      : `${count} nodes`;
+    console.log(OK(`  ✓ Spot pool "${spotPool.name}" present (${scaling}, max-pods ${spotPool.maxPods})`));
+    return;
+  }
+
+  hint("Creating spot node pool for Spark workloads…");
+  const sysPool = pools.find(p => p.mode === "System") || pools[0];
+  const vmSize = sysPool?.vmSize || "Standard_D8s_v3";
+  const zones = sysPool?.availabilityZones || [];
+
+  const createArgs = [
+    "aks", "nodepool", "add",
+    "--resource-group", rg,
+    "--cluster-name", clusterName,
+    "--name", "spot",
+    "--node-count", "1",
+    "--node-vm-size", vmSize,
+    "--max-pods", String(maxPods || 110),
+    "--priority", "Spot",
+    "--eviction-policy", "Delete",
+    "--spot-max-price", "-1",
+    "--enable-cluster-autoscaler",
+    "--min-count", "1",
+    "--max-count", "20",
+    "--labels", "kubernetes.azure.com/scalesetpriority=spot",
+    "--node-taints", "kubernetes.azure.com/scalesetpriority=spot:NoSchedule",
+    "--ssh-access", "disabled",
+    "--output", "none",
+    ...subArgs(sub),
+  ];
+
+  if (zones.length > 0) {
+    createArgs.push("--zones", ...zones);
+  }
+
+  const { exitCode, stderr } = await execa("az", createArgs, { timeout: 300000, reject: false });
+
+  if (exitCode === 0) {
+    const zoneInfo = zones.length > 0 ? `, zones ${zones.join(",")}` : "";
+    console.log(OK(`  ✓ Spot pool created (1 node, autoscale 0–3${zoneInfo})`));
+  } else {
+    const lines = (stderr || "").split("\n").filter(l => !l.startsWith("WARNING:") && l.trim());
+    const errMsg = lines[0] || "unknown error";
+    console.log(WARN(`  ⚠ Spot pool creation failed: ${errMsg}`));
+  }
+}
+
+async function reconcileDescheduler(ctx) {
+  const { execa, clusterName } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  const { exitCode } = await kubectl(["get", "cronjob", "descheduler", "-n", "kube-system"]);
+  if (exitCode === 0) {
+    console.log(OK("  ✓ Descheduler already installed"));
+    return;
+  }
+
+  hint("Installing descheduler…");
+
+  const { exitCode: helmCheck } = await execa("helm", ["version", "--short"], { reject: false, timeout: 10000 });
+  if (helmCheck !== 0) {
+    console.log(WARN("  ⚠ Helm not found — skipping descheduler install"));
+    return;
+  }
+
+  const { exitCode: repoAdd } = await execa("helm", [
+    "repo", "add", "descheduler", "https://kubernetes-sigs.github.io/descheduler/",
+  ], { reject: false, timeout: 30000 });
+  if (repoAdd !== 0) {
+    await execa("helm", ["repo", "update", "descheduler"], { reject: false, timeout: 30000 });
+  }
+
+  const { exitCode: installCode, stderr } = await execa("helm", [
+    "upgrade", "--install", "descheduler", "descheduler/descheduler",
+    "--namespace", "kube-system",
+    "--kube-context", clusterName,
+    "--set", "schedule=*/5 * * * *",
+    "--set", "deschedulerPolicy.strategies.RemoveDuplicates.enabled=true",
+    "--set", "deschedulerPolicy.strategies.LowNodeUtilization.enabled=true",
+    "--set", "deschedulerPolicy.strategies.RemovePodsHavingTooManyRestarts.enabled=true",
+    "--wait", "--timeout", "120s",
+  ], { timeout: 180000, reject: false });
+
+  if (installCode === 0) {
+    console.log(OK("  ✓ Descheduler installed (runs every 5 min)"));
+  } else {
+    const errMsg = (stderr || "").split("\n")[0];
+    console.log(WARN(`  ⚠ Descheduler install failed: ${errMsg}`));
+  }
+}
+
+async function reconcileKubeconfig(ctx) {
+  const { getCredentials } = await import("./azure-aks-core.js");
+  await getCredentials(ctx.execa, { clusterName: ctx.clusterName, rg: ctx.rg, sub: ctx.sub });
+}
+
+async function reconcileFopsApi(ctx) {
+  const { execa, clusterName } = ctx;
+
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { reject: false, timeout: 30000, ...opts });
+
+  const { stdout: existing, exitCode } = await kubectl([
+    "get", "deployment", "fops-api", "-n", FOPS_API_NS,
+    "-o", "jsonpath={.status.readyReplicas}",
+  ]);
+
+  if (exitCode === 0 && parseInt(existing) > 0) {
+    console.log(OK("  ✓ fops-api deployment running"));
+    return;
+  }
+
+  hint("Deploying fops-api to cluster…");
+
+  const manifests = fopsApiManifests(clusterName);
+  const yaml = manifests.map(m => JSON.stringify(m)).join("\n---\n");
+
+  const result = await execa("kubectl", [
+    "--context", clusterName, "apply", "-f", "-",
+  ], { input: yaml, reject: false, timeout: 30000 });
+
+  if (result.exitCode === 0) {
+    console.log(OK(`  ✓ fops-api deployed to ${FOPS_API_NS} namespace`));
+    hint(`  Access: kubectl --context ${clusterName} port-forward svc/fops-api -n ${FOPS_API_NS} 4100:4100`);
+  } else {
+    const errMsg = (result.stderr || result.stdout || "").trim().split("\n")[0];
+    console.log(WARN(`  ⚠ fops-api deploy failed: ${errMsg}`));
+  }
+}
+
+// ── AKS Reconcilers array ─────────────────────────────────────────────────────
+
+export const AKS_RECONCILERS = [
+  { name: "api-server-ip", fn: reconcileApiServerIp },
+  { name: "addons",        fn: reconcileAddons },
+  { name: "autoscaler",    fn: reconcileAutoscaler },
+  { name: "spot-pool",     fn: reconcileSpotPool },
+  { name: "kubeconfig",    fn: reconcileKubeconfig },
+  { name: "suspend-flux",  fn: suspendManagedKustomizations },
+  { name: "descheduler",   fn: reconcileDescheduler },
+  { name: "postgres",      fn: reconcilePostgres },
+  { name: "pg-databases",  fn: reconcilePgDatabases },
+  { name: "secret-store",  fn: reconcileSecretStore },
+  { name: "storage-account", fn: reconcileStorageAccount },
+  { name: "k8s-secrets",   fn: reconcileK8sSecrets },
+  { name: "storage-engine", fn: reconcileStorageEngine },
+  { name: "flux",          fn: reconcileFluxStep },
+  { name: "helm-repos",    fn: reconcileHelmRepos },
+  { name: "flux-prereqs",  fn: reconcileFluxPrereqs },
+  { name: "acr-webhooks",  fn: reconcileAcrWebhooks },
+  { name: "helm-values",   fn: reconcileHelmValues },
+  { name: "vault-unseal",  fn: reconcileVaultUnseal },
+  { name: "ingress-ip",   fn: reconcileIngressIp },
+  { name: "frontend-auth", fn: reconcileFrontendAuth },
+  { name: "fops-api",      fn: reconcileFopsApi },
+];
+
+// ── Cluster reconciler ────────────────────────────────────────────────────────
+
+export async function reconcileCluster(ctx) {
+  const { execa, clusterName, rg, sub } = ctx;
+
+  try {
+    const { stdout } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", clusterName, "--output", "json",
+      ...subArgs(sub),
+    ], { timeout: 30000 });
+    ctx.cluster = JSON.parse(stdout);
+  } catch (err) {
+    console.log(WARN(`  ⚠ Could not fetch cluster details: ${(err.message || "").split("\n")[0]}`));
+    ctx.cluster = {};
+  }
+
+  await runReconcilers(AKS_RECONCILERS, ctx);
+}
+
+// ── AKS Doctor ────────────────────────────────────────────────────────────────
+
+export async function aksDoctor(opts = {}) {
+  const execa = await lazyExeca();
+  const cl = requireCluster(opts.clusterName);
+  const clusterName = cl.clusterName;
+  const rg = cl.resourceGroup || AKS_DEFAULTS.resourceGroup;
+  const sub = cl.subscription || opts.profile;
+
+  banner(`AKS Doctor: ${clusterName}`);
+
+  const { getCredentials, ensureGhcrPullSecret } = await import("./azure-aks-core.js");
+  await getCredentials(execa, { clusterName, rg, sub });
+
+  let fixed = 0;
+  let issues = 0;
+  const pgServices = ["foundation-backend", "foundation-processor", "foundation-watcher", "foundation-scheduler"];
+
+  // 1. Check and fix GHCR pull secret
+  hint("Checking GHCR pull secret…");
+  const githubToken = opts.githubToken || process.env.GITHUB_TOKEN;
+  if (!githubToken) {
+    console.log(WARN("  ⚠ No GitHub token found — set GITHUB_TOKEN or pass --github-token"));
+    issues++;
+  } else {
+    await ensureGhcrPullSecret(execa, { clusterName, githubToken, namespace: "default" });
+    fixed++;
+  }
+
+  // 2. Check and fix SecretStore identity
+  hint("Checking SecretStore identity configuration…");
+  const { stdout: ssJson } = await execa("kubectl", [
+    "get", "secretstore", "azure-secretsmanager", "-n", "foundation",
+    "-o", "json", "--context", clusterName,
+  ], { reject: false, timeout: 15000 });
+
+  if (ssJson) {
+    try {
+      const ss = JSON.parse(ssJson);
+      const identityId = ss?.spec?.provider?.azurekv?.identityId || "";
+      const tenantId = ss?.spec?.provider?.azurekv?.tenantId || "";
+
+      if (!identityId || !tenantId) {
+        hint("  Fetching AKS managed identity…");
+        const { stdout: aksId } = await execa("az", [
+          "aks", "show", "-g", rg, "-n", clusterName,
+          "--query", "identityProfile.kubeletidentity.clientId", "-o", "tsv",
+          ...subArgs(sub),
+        ], { reject: false, timeout: 30000 });
+        const { stdout: tenant } = await execa("az", [
+          "account", "show", "--query", "tenantId", "-o", "tsv",
+        ], { reject: false, timeout: 10000 });
+
+        const newIdentityId = (aksId || "").trim();
+        const newTenantId = (tenant || "").trim();
+
+        if (newIdentityId && newTenantId) {
+          hint("  Patching SecretStore with identity values…");
+          await execa("kubectl", [
+            "patch", "secretstore", "azure-secretsmanager", "-n", "foundation",
+            "--type=merge", "-p", JSON.stringify({
+              spec: { provider: { azurekv: { identityId: newIdentityId, tenantId: newTenantId }}}
+            }),
+            "--context", clusterName,
+          ], { reject: false, timeout: 15000 });
+          console.log(OK(`  ✓ SecretStore patched with identity: ${newIdentityId.slice(0, 8)}…`));
+          fixed++;
+        } else {
+          console.log(WARN("  ⚠ Could not retrieve AKS identity"));
+          issues++;
+        }
+      } else {
+        console.log(OK(`  ✓ SecretStore has identity configured`));
+      }
+    } catch { /* ignore parse errors */ }
+  }
+
+  // 3. Force refresh ExternalSecrets
+  hint("Refreshing ExternalSecrets…");
+  const { stdout: esList } = await execa("kubectl", [
+    "get", "externalsecret", "-n", "foundation", "-o", "name", "--context", clusterName,
+  ], { reject: false, timeout: 15000 });
+
+  if (esList) {
+    const esNames = esList.trim().split("\n").filter(Boolean);
+    for (const es of esNames) {
+      await execa("kubectl", [
+        "annotate", es, "-n", "foundation",
+        `force-sync=${Date.now()}`, "--overwrite", "--context", clusterName,
+      ], { reject: false, timeout: 10000 });
+    }
+    console.log(OK(`  ✓ Refreshed ${esNames.length} ExternalSecrets`));
+  }
+
+  // 4. Check ExternalSecrets sync status
+  hint("Checking ExternalSecrets sync status…");
+  const { stdout: esStatusJson } = await execa("kubectl", [
+    "get", "externalsecret", "-n", "foundation", "-o", "json", "--context", clusterName,
+  ], { reject: false, timeout: 15000 });
+
+  if (esStatusJson) {
+    try {
+      const esData = JSON.parse(esStatusJson);
+      const failedEs = (esData.items || []).filter(es => {
+        const ready = (es.status?.conditions || []).find(c => c.type === "Ready");
+        return ready && ready.status !== "True";
+      });
+      if (failedEs.length > 0) {
+        console.log(WARN(`  ⚠ ${failedEs.length} ExternalSecret(s) have sync errors:`));
+        for (const es of failedEs) {
+          const ready = (es.status?.conditions || []).find(c => c.type === "Ready");
+          const msg = ready?.message || "Unknown error";
+          console.log(WARN(`    - ${es.metadata.name}: ${msg.slice(0, 80)}…`));
+        }
+        issues += failedEs.length;
+      } else {
+        console.log(OK(`  ✓ All ExternalSecrets synced successfully`));
+      }
+    } catch { /* ignore */ }
+  }
+
+  // 5. Check PGSSLMODE in ConfigMaps for Azure PostgreSQL
+  hint("Checking PostgreSQL SSL configuration…");
+  for (const svc of pgServices) {
+    const { stdout: cmJson } = await execa("kubectl", [
+      "get", "configmap", svc, "-n", "foundation", "-o", "json", "--context", clusterName,
+    ], { reject: false, timeout: 10000 });
+
+    if (cmJson) {
+      try {
+        const cm = JSON.parse(cmJson);
+        const hasPgsslmode = cm.data?.PGSSLMODE || cm.data?.MX_POSTGRES_SSLMODE;
+        if (!hasPgsslmode) {
+          hint(`  Patching ${svc} ConfigMap with PGSSLMODE=require…`);
+          await execa("kubectl", [
+            "patch", "configmap", svc, "-n", "foundation", "--type=merge",
+            "-p", JSON.stringify({ data: { PGSSLMODE: "require", MX_POSTGRES_SSLMODE: "require" } }),
+            "--context", clusterName,
+          ], { reject: false, timeout: 10000 });
+          console.log(OK(`  ✓ Added PGSSLMODE to ${svc}`));
+          fixed++;
+        }
+      } catch { /* ignore */ }
+    }
+  }
+
+  // 6. Reconcile Flux
+  hint("Reconciling Flux…");
+  await execa("flux", [
+    "reconcile", "source", "git", "flux-system", "--context", clusterName,
+  ], { reject: false, timeout: 60000 });
+  console.log(OK("  ✓ Flux reconcile triggered"));
+
+  // 7. Restart deployments to pick up ConfigMap changes
+  if (fixed > 0) {
+    hint("Restarting affected deployments…");
+    for (const svc of pgServices) {
+      await execa("kubectl", [
+        "rollout", "restart", "deployment", svc, "-n", "foundation", "--context", clusterName,
+      ], { reject: false, timeout: 15000 });
+    }
+    console.log(OK("  ✓ Deployments restarted"));
+  }
+
+  // Summary
+  console.log("");
+  if (issues > 0) {
+    console.log(WARN(`  Doctor found ${issues} issue(s) that need manual attention.`));
+  }
+  if (fixed > 0) {
+    console.log(OK(`  ✓ Fixed ${fixed} issue(s) automatically.`));
+  }
+  hint("Run 'kubectl get pods -n foundation' to check pod status.");
+}