@meshxdata/fops 0.1.45 → 0.1.47

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-state.js

@@ -0,0 +1,145 @@
+/**
+ * azure-aks-state.js - Cluster and stack state management
+ *
+ * Depends on: azure-aks-naming.js
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { ERR, hint, readState, saveState } from "./azure.js";
+import { AKS_DEFAULTS } from "./azure-aks-naming.js";
+
+// ── Cluster state ─────────────────────────────────────────────────────────────
+// State layout: { azure: { ..., activeCluster: "<name>", clusters: { ... } } }
+
+export function readAksClusters() {
+  const state = readState();
+  const az = state.azure || {};
+  return {
+    activeCluster: az.activeCluster,
+    clusters: az.clusters || {},
+  };
+}
+
+export function readClusterState(name) {
+  const { activeCluster, clusters } = readAksClusters();
+  if (name) return clusters[name] || null;
+  if (activeCluster && clusters[activeCluster]) return clusters[activeCluster];
+  return null;
+}
+
+export function writeClusterState(name, patch) {
+  const state = readState();
+  const az = state.azure || {};
+  const clusters = az.clusters || {};
+  clusters[name] = { ...clusters[name], ...patch, clusterName: name };
+  az.clusters = clusters;
+  az.activeCluster = name;
+  state.azure = az;
+  saveState(state);
+}
+
+export function clearClusterState(name) {
+  const state = readState();
+  const az = state.azure || {};
+  const clusters = az.clusters || {};
+  delete clusters[name];
+  if (az.activeCluster === name) {
+    const remaining = Object.keys(clusters);
+    az.activeCluster = remaining.length > 0 ? remaining[0] : undefined;
+  }
+  az.clusters = clusters;
+  state.azure = az;
+  saveState(state);
+}
+
+// ── Stack state management ────────────────────────────────────────────────────
+
+export function readStackState(clusterName, namespace) {
+  const cl = readClusterState(clusterName);
+  if (!cl) return null;
+  const stacks = cl.stacks || {};
+  return stacks[namespace] || null;
+}
+
+export function writeStackState(clusterName, namespace, patch) {
+  const cl = readClusterState(clusterName) || {};
+  const stacks = cl.stacks || {};
+  stacks[namespace] = {
+    ...stacks[namespace],
+    ...patch,
+    namespace,
+    updatedAt: new Date().toISOString(),
+  };
+  if (!stacks[namespace].createdAt) {
+    stacks[namespace].createdAt = stacks[namespace].updatedAt;
+  }
+  writeClusterState(clusterName, { stacks });
+}
+
+export function deleteStackState(clusterName, namespace) {
+  const cl = readClusterState(clusterName);
+  if (!cl) return;
+  const stacks = cl.stacks || {};
+  delete stacks[namespace];
+  writeClusterState(clusterName, { stacks });
+}
+
+export function listStacks(clusterName) {
+  const cl = readClusterState(clusterName);
+  if (!cl) return [];
+  const stacks = cl.stacks || {};
+  return Object.values(stacks);
+}
+
+// ── Flux config resolution ────────────────────────────────────────────────────
+
+/** Read flux defaults from project root: .fops.json or config/azure-flux.json (azure.fluxOwner, etc.). */
+export function readProjectFluxConfig(projectRoot) {
+  if (!projectRoot || !fs.existsSync(projectRoot)) return null;
+  const tryFile = (p) => {
+    if (!fs.existsSync(p)) return null;
+    try {
+      const raw = JSON.parse(fs.readFileSync(p, "utf8"));
+      const az = raw?.azure || raw;
+      const out = {};
+      if (az.fluxOwner != null) out.fluxOwner = az.fluxOwner;
+      if (az.fluxRepo != null) out.fluxRepo = az.fluxRepo;
+      if (az.fluxPath != null) out.fluxPath = az.fluxPath;
+      if (az.fluxBranch != null) out.fluxBranch = az.fluxBranch;
+      return Object.keys(out).length ? out : null;
+    } catch { return null; }
+  };
+  return tryFile(path.join(projectRoot, ".fops.json")) || tryFile(path.join(projectRoot, "config", "azure-flux.json")) || null;
+}
+
+/** Resolve effective Flux repo: CLI opts > cluster state > global azure > project config > AKS_DEFAULTS. */
+export function resolveFluxConfig(clusterName, opts) {
+  const state = readState();
+  const az = state.azure || {};
+  const tracked = readClusterState(clusterName);
+  const project = readProjectFluxConfig(az.projectRoot || state.projectRoot);
+  return {
+    fluxRepo: opts?.fluxRepo ?? tracked?.flux?.repo ?? az.fluxRepo ?? project?.fluxRepo ?? AKS_DEFAULTS.fluxRepo,
+    fluxOwner: opts?.fluxOwner ?? tracked?.flux?.owner ?? az.fluxOwner ?? project?.fluxOwner ?? AKS_DEFAULTS.fluxOwner,
+    fluxPath: opts?.fluxPath || tracked?.flux?.path || az.fluxPath || project?.fluxPath || AKS_DEFAULTS.fluxPath,
+    fluxBranch: opts?.fluxBranch ?? tracked?.flux?.branch ?? az.fluxBranch ?? project?.fluxBranch ?? AKS_DEFAULTS.fluxBranch,
+  };
+}
+
+// ── Cluster requirement check ─────────────────────────────────────────────────
+
+export function requireCluster(name) {
+  const cl = readClusterState(name);
+  if (!cl || !cl.clusterName) {
+    const label = name ? `"${name}"` : "(none active)";
+    console.error(ERR(`\n  No AKS cluster tracked: ${label}`));
+    hint("Create one:  fops azure aks up <name>");
+    hint("List:        fops azure aks list\n");
+    process.exit(1);
+  }
+  return {
+    ...cl,
+    resourceGroup: cl.resourceGroup ?? AKS_DEFAULTS.resourceGroup,
+  };
+}

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-storage.js

@@ -0,0 +1,496 @@
+/**
+ * azure-aks-storage.js - Storage account, Helm repos, and storage engine
+ *
+ * Depends on: azure-aks-naming.js, azure-aks-state.js
+ */
+
+import crypto from "node:crypto";
+import { OK, WARN, hint, subArgs } from "./azure.js";
+import { pgServerName } from "./azure-aks-naming.js";
+import { readClusterState } from "./azure-aks-state.js";
+
+// ── Helm Repositories ─────────────────────────────────────────────────────────
+
+export const HELM_REPOS = [
+  {
+    name: "foundation", namespace: "flux-system",
+    spec: { type: "oci", interval: "1m0s", url: "oci://meshxregistry.azurecr.io/foundation-charts", secretRef: { name: "meshxregistry-helm-secret" } },
+  },
+  {
+    name: "hive-metastore", namespace: "flux-system",
+    spec: { type: "oci", interval: "1m0s", url: "oci://meshxregistry.azurecr.io/foundation-charts", secretRef: { name: "meshxregistry-helm-secret" } },
+  },
+  {
+    name: "trinodb", namespace: "flux-system",
+    spec: { interval: "30m", url: "https://trinodb.github.io/charts" },
+  },
+];
+
+// ── Old Postgres hosts for migration ──────────────────────────────────────────
+
+export const OLD_PG_HOSTS = [
+  "az-vel-app-data-demo-uaen-psql.postgres.database.azure.com",
+];
+
+// ── Storage Account reconciliation ────────────────────────────────────────────
+
+export async function reconcileStorageAccount(ctx) {
+  const { execa, clusterName, rg, sub } = ctx;
+  const storageAccountName = `fops${clusterName.replace(/-/g, "")}`.toLowerCase().slice(0, 24);
+  const vaultName = `fops-${clusterName}-kv`;
+  const containers = ["foundation", "vault"];
+
+  hint(`Reconciling Azure Storage Account "${storageAccountName}"…`);
+
+  // 1. Check if storage account exists
+  const { exitCode: saExists } = await execa("az", [
+    "storage", "account", "show",
+    "--name", storageAccountName,
+    "--resource-group", rg,
+    "--output", "none",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (saExists !== 0) {
+    // Create storage account
+    hint(`Creating Storage Account "${storageAccountName}"…`);
+    const { exitCode, stderr } = await execa("az", [
+      "storage", "account", "create",
+      "--name", storageAccountName,
+      "--resource-group", rg,
+      "--sku", "Standard_LRS",
+      "--kind", "StorageV2",
+      "--https-only", "true",
+      "--min-tls-version", "TLS1_2",
+      "--allow-blob-public-access", "false",
+      "--output", "none",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 120000 });
+
+    if (exitCode !== 0) {
+      console.log(WARN(`  ⚠ Storage Account creation failed: ${(stderr || "").split("\n")[0]}`));
+      return;
+    }
+    console.log(OK(`  ✓ Storage Account "${storageAccountName}" created`));
+  } else {
+    console.log(OK(`  ✓ Storage Account "${storageAccountName}" exists`));
+  }
+
+  // 2. Get storage account key
+  const { stdout: keyJson } = await execa("az", [
+    "storage", "account", "keys", "list",
+    "--account-name", storageAccountName,
+    "--resource-group", rg,
+    "--query", "[0].value",
+    "--output", "tsv",
+    ...subArgs(sub),
+  ], { timeout: 30000 });
+  const storageKey = keyJson?.trim();
+
+  if (!storageKey) {
+    console.log(WARN("  ⚠ Could not retrieve storage account key"));
+    return;
+  }
+
+  // 3. Create containers
+  for (const container of containers) {
+    const { exitCode: containerExists } = await execa("az", [
+      "storage", "container", "show",
+      "--name", container,
+      "--account-name", storageAccountName,
+      "--account-key", storageKey,
+      "--output", "none",
+    ], { reject: false, timeout: 30000 });
+
+    if (containerExists !== 0) {
+      await execa("az", [
+        "storage", "container", "create",
+        "--name", container,
+        "--account-name", storageAccountName,
+        "--account-key", storageKey,
+        "--output", "none",
+      ], { reject: false, timeout: 30000 });
+      console.log(OK(`  ✓ Container "${container}" created`));
+    } else {
+      console.log(OK(`  ✓ Container "${container}" exists`));
+    }
+  }
+
+  // 4. Generate storage-engine auth credentials
+  const authIdentity = "storage-engine";
+  const authCredential = crypto.randomBytes(24).toString("base64").replace(/[^a-zA-Z0-9]/g, "");
+
+  // 5. Store credentials in Key Vault
+  const secrets = [
+    { name: "AZURE-STORAGE-ACCOUNT", value: storageAccountName },
+    { name: "AZURE-STORAGE-KEY", value: storageKey },
+    // storage-engine secrets as JSON
+    { name: "foundation-storage-engine-secrets", value: JSON.stringify({
+      AZURE_STORAGE_ACCOUNT_NAME: storageAccountName,
+      AZURE_STORAGE_ACCOUNT_KEY: storageKey,
+    })},
+    { name: "foundation-storage-engine-auth", value: JSON.stringify({
+      AUTH_IDENTITY: authIdentity,
+      AUTH_CREDENTIAL: authCredential,
+    })},
+  ];
+
+  for (const { name, value } of secrets) {
+    const { exitCode } = await execa("az", [
+      "keyvault", "secret", "set",
+      "--vault-name", vaultName,
+      "--name", name,
+      "--value", value,
+      "--output", "none",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+
+    if (exitCode === 0) {
+      console.log(OK(`  ✓ Secret "${name}" stored in Key Vault`));
+    }
+  }
+
+  // 6. Grant AKS kubelet identity Storage Blob Data Contributor role
+  try {
+    const { stdout: aksJson } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", clusterName,
+      "--query", "identityProfile.kubeletidentity.objectId",
+      "-o", "tsv",
+      ...subArgs(sub),
+    ], { timeout: 30000 });
+    const kubeletOid = aksJson?.trim();
+
+    if (kubeletOid) {
+      const { stdout: saId } = await execa("az", [
+        "storage", "account", "show",
+        "--name", storageAccountName,
+        "--resource-group", rg,
+        "--query", "id",
+        "-o", "tsv",
+        ...subArgs(sub),
+      ], { timeout: 30000 });
+
+      const { exitCode: roleExists } = await execa("az", [
+        "role", "assignment", "list",
+        "--assignee", kubeletOid,
+        "--scope", saId?.trim(),
+        "--role", "Storage Blob Data Contributor",
+        "--query", "[0]",
+        "-o", "tsv",
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+
+      if (roleExists !== 0 || !saId) {
+        await execa("az", [
+          "role", "assignment", "create",
+          "--assignee-object-id", kubeletOid,
+          "--assignee-principal-type", "ServicePrincipal",
+          "--role", "Storage Blob Data Contributor",
+          "--scope", saId?.trim(),
+          ...subArgs(sub),
+        ], { reject: false, timeout: 60000 });
+        console.log(OK("  ✓ AKS kubelet identity granted Storage Blob Data Contributor"));
+      } else {
+        console.log(OK("  ✓ AKS kubelet identity already has Storage Blob access"));
+      }
+    }
+  } catch (err) {
+    console.log(WARN(`  ⚠ Could not grant AKS identity blob access: ${err.message?.split("\n")[0]}`));
+  }
+}
+
+// ── Storage engine deployment ─────────────────────────────────────────────────
+
+export async function reconcileStorageEngine(ctx) {
+  const { execa, clusterName } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  // Check if deployment already exists
+  const { exitCode } = await kubectl([
+    "get", "deployment", "foundation-storage-engine", "-n", "foundation",
+  ]);
+  if (exitCode === 0) {
+    console.log(OK("  ✓ Storage engine deployment exists"));
+    return;
+  }
+
+  hint("Creating foundation-storage-engine deployment…");
+  const manifest = JSON.stringify({
+    apiVersion: "apps/v1", kind: "Deployment",
+    metadata: { name: "foundation-storage-engine", namespace: "foundation", labels: { app: "foundation-storage-engine" } },
+    spec: {
+      replicas: 1,
+      selector: { matchLabels: { app: "foundation-storage-engine" } },
+      template: {
+        metadata: { labels: { app: "foundation-storage-engine" } },
+        spec: {
+          containers: [{
+            name: "storage-engine",
+            image: "minio/minio:RELEASE.2024-11-07T00-52-20Z",
+            args: ["server", "/data", "--console-address", ":9001"],
+            env: [
+              { name: "MINIO_ROOT_USER", value: "minio" },
+              { name: "MINIO_ROOT_PASSWORD", value: "minio123" },
+            ],
+            ports: [
+              { containerPort: 9000, name: "api" },
+              { containerPort: 9001, name: "console" },
+            ],
+            volumeMounts: [{ name: "data", mountPath: "/data" }],
+            resources: { requests: { cpu: "100m", memory: "256Mi" }, limits: { cpu: "500m", memory: "512Mi" } },
+            readinessProbe: { httpGet: { path: "/minio/health/ready", port: 9000 }, initialDelaySeconds: 5, periodSeconds: 10 },
+          }],
+          volumes: [{ name: "data", emptyDir: {} }],
+        },
+      },
+    },
+  });
+  await kubectl(["apply", "-f", "-"], { input: manifest });
+
+  // Service: foundation-storage-engine (port 8080 → 9000)
+  const svcManifest = JSON.stringify({
+    apiVersion: "v1", kind: "Service",
+    metadata: { name: "foundation-storage-engine", namespace: "foundation" },
+    spec: {
+      selector: { app: "foundation-storage-engine" },
+      ports: [
+        { port: 8080, targetPort: 9000, name: "api" },
+        { port: 9000, targetPort: 9000, name: "s3" },
+      ],
+    },
+  });
+  await kubectl(["apply", "-f", "-"], { input: svcManifest });
+
+  // Also patch the existing "minio" service to point here (Vault uses minio.foundation.svc)
+  const minioSvcManifest = JSON.stringify({
+    apiVersion: "v1", kind: "Service",
+    metadata: { name: "minio", namespace: "foundation" },
+    spec: {
+      selector: { app: "foundation-storage-engine" },
+      ports: [
+        { port: 80, targetPort: 9000, name: "http" },
+        { port: 9000, targetPort: 9000, name: "s3" },
+        { port: 8080, targetPort: 9000, name: "api" },
+      ],
+    },
+  });
+  await kubectl(["apply", "-f", "-"], { input: minioSvcManifest });
+
+  // Wait for the deployment to be ready, then create the vault bucket
+  await kubectl(["rollout", "status", "deployment/foundation-storage-engine", "-n", "foundation", "--timeout=60s"], { timeout: 70000 });
+
+  // Create the vault bucket via a one-shot mc pod
+  const mcJobYaml = JSON.stringify({
+    apiVersion: "batch/v1", kind: "Job",
+    metadata: { name: "fops-mc-init", namespace: "foundation" },
+    spec: {
+      backoffLimit: 3, ttlSecondsAfterFinished: 60,
+      template: {
+        spec: {
+          restartPolicy: "Never",
+          containers: [{
+            name: "mc",
+            image: "minio/mc:latest",
+            command: ["sh", "-c", "mc alias set local http://foundation-storage-engine:8080 minio minio123 && mc mb local/vault --ignore-existing"],
+          }],
+        },
+      },
+    },
+  });
+  await kubectl(["delete", "job", "fops-mc-init", "-n", "foundation", "--ignore-not-found"]);
+  await kubectl(["apply", "-f", "-"], { input: mcJobYaml });
+  await kubectl(["wait", "--for=condition=complete", "job/fops-mc-init", "-n", "foundation", "--timeout=60s"], { timeout: 70000 });
+
+  console.log(OK("  ✓ Storage engine deployed with vault bucket"));
+}
+
+// ── Helm repos reconciliation ─────────────────────────────────────────────────
+
+export async function reconcileHelmRepos(ctx) {
+  const { execa, clusterName } = ctx;
+  const tracked = readClusterState(clusterName);
+  if (!tracked?.flux) return;
+
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { reject: false, timeout: 30000, ...opts });
+
+  // Detect which API version the cluster supports
+  const { stdout: crdJson } = await kubectl([
+    "get", "crd", "helmrepositories.source.toolkit.fluxcd.io",
+    "-o", "jsonpath={.spec.versions[*].name}",
+  ]);
+  const versions = (crdJson || "").split(/\s+/).filter(Boolean);
+  const apiVersion = versions.includes("v1")
+    ? "source.toolkit.fluxcd.io/v1"
+    : versions.includes("v1beta2")
+      ? "source.toolkit.fluxcd.io/v1beta2"
+      : "source.toolkit.fluxcd.io/v1";
+
+  // Ensure the ACR helm secret exists in flux-system for HelmRepository auth
+  const { exitCode: secretExists } = await kubectl([
+    "get", "secret", "meshxregistry-helm-secret", "-n", "flux-system",
+  ]);
+  if (secretExists !== 0) {
+    hint("Creating ACR helm secret in flux-system…");
+    const secretNs = ["acr-cache-system", "foundation"].find(async ns => {
+      const { exitCode } = await kubectl(["get", "secret", "meshxregistry-helm-secret", "-n", ns]);
+      return exitCode === 0;
+    });
+    if (secretNs) {
+      const { stdout: secretYaml } = await kubectl([
+        "get", "secret", "meshxregistry-helm-secret", "-n", secretNs, "-o", "json",
+      ]);
+      if (secretYaml) {
+        try {
+          const secret = JSON.parse(secretYaml);
+          delete secret.metadata.namespace;
+          delete secret.metadata.resourceVersion;
+          delete secret.metadata.uid;
+          delete secret.metadata.creationTimestamp;
+          if (secret.metadata.annotations) {
+            delete secret.metadata.annotations["kubectl.kubernetes.io/last-applied-configuration"];
+          }
+          const clean = JSON.stringify(secret);
+          await execa("kubectl", [
+            "--context", clusterName, "apply", "-n", "flux-system", "-f", "-",
+          ], { input: clean, reject: false, timeout: 10000 });
+          console.log(OK("  ✓ ACR helm secret replicated to flux-system"));
+        } catch { /* reflector should handle this eventually */ }
+      }
+    }
+  }
+
+  let updated = 0;
+  let unchanged = 0;
+  let failed = 0;
+
+  const repos = ctx.opts?.dai ? HELM_REPOS : HELM_REPOS.filter(r => !r.daiOnly);
+  for (const repo of repos) {
+    const specLines = [];
+    for (const [k, v] of Object.entries(repo.spec)) {
+      if (k === "secretRef") {
+        specLines.push(`  secretRef:`);
+        specLines.push(`    name: ${v.name}`);
+      } else {
+        specLines.push(`  ${k}: "${v}"`);
+      }
+    }
+
+    const yaml = [
+      `apiVersion: ${apiVersion}`,
+      `kind: HelmRepository`,
+      `metadata:`,
+      `  name: ${repo.name}`,
+      `  namespace: ${repo.namespace}`,
+      `spec:`,
+      ...specLines,
+    ].join("\n");
+
+    const applyResult = await execa("kubectl", [
+      "--context", clusterName, "apply", "-f", "-",
+    ], { input: yaml, reject: false, timeout: 15000 });
+
+    if (applyResult.exitCode === 0) {
+      const out = (applyResult.stdout || "").trim();
+      if (out.includes("configured")) updated++;
+      else unchanged++;
+    } else {
+      failed++;
+      const errMsg = (applyResult.stderr || applyResult.stdout || "unknown error").trim().split("\n")[0];
+      console.log(WARN(`  ⚠ HelmRepository ${repo.namespace}/${repo.name}: ${errMsg}`));
+    }
+  }
+
+  // Verify they actually exist
+  const { stdout: verify } = await kubectl([
+    "get", "helmrepository.source.toolkit.fluxcd.io", "-n", "flux-system",
+    "--no-headers", "-o", "custom-columns=NAME:.metadata.name",
+  ]);
+  const actual = (verify || "").trim().split("\n").filter(Boolean);
+  if (updated > 0) {
+    console.log(OK(`  ✓ ${actual.length} HelmRepository source(s) in flux-system (${updated} updated, ${unchanged} unchanged)`));
+  } else if (failed === 0) {
+    console.log(OK(`  ✓ All ${actual.length} HelmRepository sources up to date`));
+  }
+  if (actual.length < HELM_REPOS.length) {
+    const missing = HELM_REPOS.filter(r => !actual.includes(r.name));
+    for (const m of missing) {
+      console.log(WARN(`  ⚠ Missing: ${m.name}`));
+    }
+  }
+}
+
+// ── Helm values reconciliation ────────────────────────────────────────────────
+
+export async function reconcileHelmValues(ctx) {
+  const { execa, clusterName } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  const pgServer = pgServerName(clusterName);
+  const correctHost = `${pgServer}.postgres.database.azure.com`;
+
+  // List HelmReleases in foundation namespace
+  const { stdout: hrJson } = await kubectl(["get", "helmrelease", "-n", "foundation", "-o", "json"]);
+  if (!hrJson) return;
+
+  const hrs = JSON.parse(hrJson).items || [];
+  let patched = 0;
+
+  for (const hr of hrs) {
+    const name = hr.metadata?.name;
+    const valsStr = JSON.stringify(hr.spec?.values || {});
+    let needsPatch = false;
+
+    for (const oldHost of OLD_PG_HOSTS) {
+      if (valsStr.includes(oldHost)) {
+        needsPatch = true;
+        break;
+      }
+    }
+    if (!needsPatch) continue;
+
+    let newVals = valsStr;
+    for (const oldHost of OLD_PG_HOSTS) {
+      newVals = newVals.replaceAll(oldHost, correctHost);
+    }
+
+    const patch = JSON.stringify({ spec: { values: JSON.parse(newVals) } });
+    const tmpFile = `/tmp/fops-hr-patch-${name}.json`;
+    const { writeFileSync, unlinkSync } = await import("node:fs");
+    writeFileSync(tmpFile, patch);
+
+    const { exitCode } = await kubectl([
+      "patch", "helmrelease", name, "-n", "foundation",
+      "--type", "merge", "--patch-file", tmpFile,
+    ]);
+    try { unlinkSync(tmpFile); } catch {}
+
+    if (exitCode === 0) patched++;
+  }
+
+  if (patched > 0) {
+    console.log(OK(`  ✓ Patched postgres host in ${patched} HelmRelease(s) → ${correctHost}`));
+  } else {
+    console.log(OK("  ✓ HelmRelease postgres hosts are correct"));
+  }
+}
+
+// ── ACR webhooks cleanup ──────────────────────────────────────────────────────
+
+export async function reconcileAcrWebhooks(ctx) {
+  const { execa, clusterName } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 15000, reject: false, ...opts });
+
+  for (const wh of ["acr-pod-webhook", "acr-helm-webhook"]) {
+    const { exitCode } = await kubectl([
+      "get", "mutatingwebhookconfiguration", wh,
+    ]);
+    if (exitCode === 0) {
+      await kubectl(["delete", "mutatingwebhookconfiguration", wh]);
+      console.log(OK(`  ✓ Removed orphaned ${wh}`));
+    }
+  }
+}