@meshxdata/fops 0.1.45 → 0.1.47

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-secrets.js

@@ -0,0 +1,849 @@
+/**
+ * azure-aks-secrets.js - Key Vault, SecretStore, and secrets management
+ *
+ * Depends on: azure-aks-naming.js, azure-aks-state.js
+ */
+
+import { DEFAULTS, OK, WARN, DIM, hint, subArgs } from "./azure.js";
+import { kvName, generatePassword, PG_DEFAULTS } from "./azure-aks-naming.js";
+import { writeClusterState } from "./azure-aks-state.js";
+
+// ── Secret Store Configuration ────────────────────────────────────────────────
+
+export const SECRET_STORE_NAMESPACES_BASE = ["foundation"];
+export const SECRET_STORE_NAMESPACES_DAI = ["dai"];
+export const SECRET_STORE_NAME = "azure-secretsmanager";
+
+// ── Key Vault seed secrets mapping ────────────────────────────────────────────
+
+export const KV_SEED_SECRETS = [
+  { name: "foundation-secrets", envKeys: {
+    password: "POSTGRES_PASSWORD",
+    secret_key: "MX_SECRET_KEY",
+    "auth0.secret": "AUTH0_SECRET",
+    "auth0.client_id": "AUTH0_CLIENT_ID",
+    "auth0.client_secret": "AUTH0_CLIENT_SECRET",
+    "auth0.domain": "AUTH0_DOMAIN",
+    "auth0.audience": "AUTH0_AUDIENCE",
+    "auth0.issuer_base_url": "AUTH0_ISSUER_BASE_URL",
+    "auth0.base_url": "AUTH0_BASE_URL",
+  }},
+  { name: "auth0", envKeys: {
+    client_id: "AUTH0_CLIENT_ID",
+    client_secret: "AUTH0_CLIENT_SECRET",
+  }},
+  { name: "foundation-trino-jwt", envKeys: {
+    "jwt-secret.pem": "MX_SECRET_KEY",
+    secret_key: "MX_SECRET_KEY",
+  }},
+  { name: "foundation-nats", envKeys: {
+    "nkeys-secret": "MX_SECRET_KEY",
+    secret_key: "MX_SECRET_KEY",
+  }},
+  { name: "foundation-storage-engine-auth", envKeys: {
+    AUTH_IDENTITY: "AUTH_IDENTITY",
+    AUTH_CREDENTIAL: "AUTH_CREDENTIAL",
+  }},
+  { name: "foundation-storage-engine-secrets", envKeys: {
+    AZURE_ACCOUNT_NAME: "AZURE_STORAGE_ACCOUNT",
+    AZURE_ACCOUNT_KEY: "AZURE_STORAGE_KEY",
+    AZURE_SAS_TOKEN: "AZURE_STORAGE_SAS_TOKEN",
+  }},
+  { name: "dai-secrets", daiOnly: true, envKeys: {
+    AUTH0_CLIENT_ID: "AUTH0_CLIENT_ID",
+    AUTH0_CLIENT_SECRET: "AUTH0_CLIENT_SECRET",
+    AUTH0_SECRET: "AUTH0_SECRET",
+    NATS_NKEYS_SECRET: "MX_SECRET_KEY",
+    AZURE_OPENAI_API_KEY: "MX_OPENAI_API_KEY",
+  }},
+  { name: "foundation-scheduler-secrets", envKeys: {
+    secretKey: "MX_SECRET_KEY",
+  }},
+];
+
+// ── SecretStore reconciliation ────────────────────────────────────────────────
+
+export async function reconcileSecretStore(ctx) {
+  const { execa, clusterName, rg, sub, opts } = ctx;
+  const vaultName = kvName(clusterName);
+  const location = ctx.cluster?.location || DEFAULTS.location;
+  const ssNamespaces = opts?.dai
+    ? [...SECRET_STORE_NAMESPACES_BASE, ...SECRET_STORE_NAMESPACES_DAI]
+    : SECRET_STORE_NAMESPACES_BASE;
+
+  const kubectl = (args, kopts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...kopts });
+
+  // 1. Ensure Key Vault exists
+  const { exitCode: kvExists } = await execa("az", [
+    "keyvault", "show", "--name", vaultName, "--output", "none",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (kvExists !== 0) {
+    hint(`Creating Key Vault "${vaultName}"…`);
+    const { exitCode, stderr } = await execa("az", [
+      "keyvault", "create",
+      "--name", vaultName,
+      "--resource-group", rg,
+      "--location", location,
+      "--enable-rbac-authorization", "true",
+      "--output", "none",
+      ...subArgs(sub),
+    ], { timeout: 120000, reject: false });
+    if (exitCode !== 0) {
+      console.log(WARN(`  ⚠ Key Vault creation failed: ${(stderr || "").split("\n")[0]}`));
+      return;
+    }
+    console.log(OK(`  ✓ Key Vault "${vaultName}" created`));
+  } else {
+    console.log(OK(`  ✓ Key Vault "${vaultName}" exists`));
+  }
+
+  // 2. Get Key Vault resource ID and ensure SP has Secrets Officer role
+  const { stdout: kvJson } = await execa("az", [
+    "keyvault", "show", "--name", vaultName, "--query", "id", "-o", "tsv",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+  const kvId = (kvJson || "").trim();
+
+  const spClientId = await getSpClientId(kubectl);
+  if (spClientId && kvId) {
+    const { stdout: spObjId } = await execa("az", [
+      "ad", "sp", "show", "--id", spClientId, "--query", "id", "-o", "tsv",
+    ], { reject: false, timeout: 30000 });
+    const objectId = (spObjId || "").trim();
+
+    if (objectId) {
+      const { exitCode: roleExists } = await execa("az", [
+        "role", "assignment", "list",
+        "--assignee", objectId,
+        "--role", "Key Vault Secrets Officer",
+        "--scope", kvId,
+        "--query", "[0].id", "-o", "tsv",
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+
+      const hasRole = roleExists === 0;
+      if (!hasRole) {
+        await execa("az", [
+          "role", "assignment", "create",
+          "--assignee-object-id", objectId,
+          "--assignee-principal-type", "ServicePrincipal",
+          "--role", "Key Vault Secrets Officer",
+          "--scope", kvId,
+          ...subArgs(sub),
+        ], { reject: false, timeout: 30000 });
+        console.log(OK("  ✓ SP granted Key Vault Secrets Officer role"));
+      }
+    }
+  }
+
+  // 2b. Grant AKS kubelet identity access to Key Vault (for ExternalSecrets with ManagedIdentity auth)
+  if (kvId) {
+    const { stdout: kubeletId } = await execa("az", [
+      "aks", "show", "-g", rg, "-n", clusterName,
+      "--query", "identityProfile.kubeletidentity.objectId", "-o", "tsv",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+    const kubeletObjectId = (kubeletId || "").trim();
+
+    if (kubeletObjectId) {
+      const { stdout: hasKubeletRole } = await execa("az", [
+        "role", "assignment", "list",
+        "--assignee", kubeletObjectId,
+        "--role", "Key Vault Secrets User",
+        "--scope", kvId,
+        "--query", "[0].id", "-o", "tsv",
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+
+      if (!hasKubeletRole?.trim()) {
+        await execa("az", [
+          "role", "assignment", "create",
+          "--assignee-object-id", kubeletObjectId,
+          "--assignee-principal-type", "ServicePrincipal",
+          "--role", "Key Vault Secrets User",
+          "--scope", kvId,
+          ...subArgs(sub),
+        ], { reject: false, timeout: 30000 });
+        console.log(OK("  ✓ AKS kubelet identity granted Key Vault Secrets User role"));
+      }
+    }
+  }
+
+  // 3. Ensure azure-secret-sp exists in each target namespace
+  const { stdout: spSecretJson } = await kubectl([
+    "get", "secret", "azure-secret-sp", "-n", "foundation", "-o", "json",
+  ]);
+  const spSecret = spSecretJson ? JSON.parse(spSecretJson) : null;
+
+  if (!spSecret || !spSecret.data?.ClientID) {
+    console.log(WARN("  ⚠ Secret 'azure-secret-sp' not found in foundation — SecretStore needs SP credentials"));
+    hint("Create it with: kubectl create secret generic azure-secret-sp -n foundation --from-literal=ClientID=<sp-client-id> --from-literal=ClientSecret=<sp-secret>");
+    return;
+  }
+
+  // Replicate azure-secret-sp to other namespaces
+  for (const ns of ssNamespaces) {
+    await kubectl(["create", "namespace", ns, "--dry-run=client", "-o", "yaml"]);
+    const { exitCode: nsExists } = await kubectl(["get", "namespace", ns]);
+    if (nsExists !== 0) {
+      await kubectl(["create", "namespace", ns]);
+    }
+    if (ns === "foundation") continue;
+    const { exitCode: spExists } = await kubectl(["get", "secret", "azure-secret-sp", "-n", ns]);
+    if (spExists !== 0) {
+      await kubectl([
+        "create", "secret", "generic", "azure-secret-sp",
+        "-n", ns,
+        "--from-literal", `ClientID=${Buffer.from(spSecret.data.ClientID, "base64").toString()}`,
+        "--from-literal", `ClientSecret=${Buffer.from(spSecret.data.ClientSecret, "base64").toString()}`,
+      ]);
+      console.log(OK(`  ✓ Replicated azure-secret-sp to ${ns}`));
+    }
+  }
+
+  // 4. Get tenant ID
+  const { stdout: tenantId } = await execa("az", [
+    "account", "show", "--query", "tenantId", "-o", "tsv",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+
+  // 5. Create SecretStore in each namespace
+  const apiVersion = await detectEsApiVersion(kubectl);
+
+  for (const ns of ssNamespaces) {
+    const { exitCode: ssExists } = await kubectl([
+      "get", "secretstore", SECRET_STORE_NAME, "-n", ns,
+    ]);
+    if (ssExists === 0) continue;
+
+    const manifest = `apiVersion: ${apiVersion}
+kind: SecretStore
+metadata:
+  name: ${SECRET_STORE_NAME}
+  namespace: ${ns}
+spec:
+  provider:
+    azurekv:
+      authType: ServicePrincipal
+      vaultUrl: https://${vaultName}.vault.azure.net
+      tenantId: ${(tenantId || "").trim()}
+      authSecretRef:
+        clientId:
+          name: azure-secret-sp
+          key: ClientID
+        clientSecret:
+          name: azure-secret-sp
+          key: ClientSecret
+`;
+    const { exitCode: applyCode, stderr } = await kubectl(
+      ["apply", "-f", "-"],
+      { input: manifest },
+    );
+    if (applyCode === 0) {
+      console.log(OK(`  ✓ SecretStore "${SECRET_STORE_NAME}" created in ${ns}`));
+    } else {
+      console.log(WARN(`  ⚠ SecretStore creation failed in ${ns}: ${(stderr || "").split("\n")[0]}`));
+    }
+  }
+
+  // 6. Seed Key Vault secrets from a running VM if the vault is empty
+  await seedKeyVaultFromVm(execa, { vaultName, sub, dai: opts?.dai });
+}
+
+// ── K8s secrets reconciliation ────────────────────────────────────────────────
+
+export async function reconcileK8sSecrets(ctx) {
+  const { execa, clusterName, rg, sub } = ctx;
+  const { pgServerName } = await import("./azure-aks-naming.js");
+  const { PG_SERVICE_DBS } = await import("./azure-aks-postgres.js");
+
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  // Read current postgres password
+  const { stdout: pwB64 } = await kubectl([
+    "get", "secret", "postgres", "-n", "foundation",
+    "-o", "jsonpath={.data.password}",
+  ]);
+  if (!pwB64) return;
+  let pgPass = Buffer.from(pwB64, "base64").toString();
+  const pgHost = `${pgServerName(clusterName)}.postgres.database.azure.com`;
+  const serverName = pgServerName(clusterName);
+
+  // Check for URL-unsafe characters that break pogo-migrate connection strings
+  const URL_UNSAFE = /[^a-zA-Z0-9]/;
+  if (URL_UNSAFE.test(pgPass)) {
+    console.log(WARN("  ⚠ Postgres password contains URL-unsafe characters — regenerating…"));
+    const newPass = generatePassword();
+
+    // Update the Azure Flexible Server admin password
+    await execa("az", [
+      "postgres", "flexible-server", "update",
+      "--name", serverName, "--resource-group", rg,
+      "--admin-password", newPass,
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 120000 });
+
+    // Update all database role passwords via psql job
+    const sqlStatements = PG_SERVICE_DBS
+      .map(role => `ALTER ROLE ${role} WITH PASSWORD '${newPass}';`)
+      .join(" ");
+    const jobYaml = JSON.stringify({
+      apiVersion: "batch/v1", kind: "Job",
+      metadata: { name: "fops-pg-repass", namespace: "foundation" },
+      spec: {
+        backoffLimit: 2, ttlSecondsAfterFinished: 120,
+        template: {
+          spec: {
+            restartPolicy: "Never",
+            containers: [{
+              name: "psql", image: "postgres:16-alpine",
+              command: ["sh", "-c", `export PGPASSWORD='${newPass}' PGHOST='${pgHost}' PGUSER=foundation PGSSLMODE=require; psql -d foundation -c "${sqlStatements}"`],
+            }],
+          },
+        },
+      },
+    });
+    await kubectl(["delete", "job", "fops-pg-repass", "-n", "foundation", "--ignore-not-found"]);
+    await kubectl(["apply", "-f", "-"], { input: jobYaml, timeout: 60000 });
+    // Wait for the job to complete
+    await kubectl(["wait", "--for=condition=complete", "job/fops-pg-repass", "-n", "foundation", "--timeout=60s"], { timeout: 70000 });
+
+    pgPass = newPass;
+
+    // Update local state
+    writeClusterState(clusterName, {
+      postgres: { adminPassword: newPass },
+    });
+    console.log(OK("  ✓ Postgres password regenerated (URL-safe)"));
+  }
+
+  // Ensure postgres secret has all required keys (mlflow needs mlflow-username etc.)
+  const { stdout: existingKeys } = await kubectl([
+    "get", "secret", "postgres", "-n", "foundation",
+    "-o", "jsonpath={.data}",
+  ]);
+  const keys = existingKeys ? Object.keys(JSON.parse(existingKeys)) : [];
+  const requiredKeys = [
+    "host", "user", "password", "superUserPassword",
+    "postgres-password", "mlflow-username", "mlflow-password",
+  ];
+  const missing = requiredKeys.filter(k => !keys.includes(k));
+  const needsUpdate = missing.length > 0 || URL_UNSAFE.test(Buffer.from(pwB64, "base64").toString());
+
+  if (needsUpdate) {
+    if (missing.length) hint(`Adding missing keys to postgres secret: ${missing.join(", ")}`);
+    const args = [
+      "create", "secret", "generic", "postgres", "-n", "foundation",
+      "--from-literal", `host=${pgHost}`,
+      "--from-literal", `user=foundation`,
+      "--from-literal", `password=${pgPass}`,
+      "--from-literal", `superUserPassword=${pgPass}`,
+      "--from-literal", `postgres-password=${pgPass}`,
+      "--from-literal", `mlflow-username=mlflow`,
+      "--from-literal", `mlflow-password=${pgPass}`,
+      "--dry-run=client", "-o", "yaml",
+    ];
+    const { stdout: secretYaml } = await kubectl(args);
+    if (secretYaml) {
+      await kubectl(["apply", "-f", "-"], { input: secretYaml });
+      console.log(OK("  ✓ Postgres secret updated with all required keys"));
+    }
+  }
+
+  // Ensure OPA keypair secret exists
+  const { exitCode: opaExists } = await kubectl([
+    "get", "secret", "foundation-opa-keypair", "-n", "foundation",
+  ]);
+  if (opaExists !== 0) {
+    hint("Creating OPA keypair secret…");
+    // Use the OPA AWS-style keys from the VM env or generate placeholders
+    const { listVms } = await import("./azure-state.js");
+    const { vms: vmMap } = listVms();
+    const vms = Object.entries(vmMap || {}).filter(([, v]) => v.ip);
+
+    let opaAccessKey = "placeholder";
+    let opaSecretKey = "placeholder";
+
+    if (vms.length > 0) {
+      const [, vm] = vms[0];
+      const { sshCmd } = await import("./azure-helpers.js");
+      const user = vm.adminUser || "azureuser";
+      const { stdout: envContent } = await sshCmd(
+        execa, vm.ip, user,
+        "grep -E '^OPA_(ACCESS_KEY_ID|SECRET_ACCESS_KEY)=' /opt/foundation-compose/.env 2>/dev/null",
+        15000,
+      );
+      if (envContent) {
+        const m1 = envContent.match(/OPA_ACCESS_KEY_ID=(.+)/);
+        const m2 = envContent.match(/OPA_SECRET_ACCESS_KEY=(.+)/);
+        if (m1) opaAccessKey = m1[1].trim();
+        if (m2) opaSecretKey = m2[1].trim();
+      }
+    }
+
+    const { stdout: opaYaml } = await kubectl([
+      "create", "secret", "generic", "foundation-opa-keypair", "-n", "foundation",
+      "--from-literal", `OPA_ACCESS_KEY_ID=${opaAccessKey}`,
+      "--from-literal", `OPA_SECRET_ACCESS_KEY=${opaSecretKey}`,
+      "--dry-run=client", "-o", "yaml",
+    ]);
+    if (opaYaml) {
+      await kubectl(["apply", "-f", "-"], { input: opaYaml });
+      console.log(OK("  ✓ OPA keypair secret created"));
+    }
+  } else {
+    console.log(OK("  ✓ OPA keypair secret exists"));
+  }
+
+  // Ensure foundation-postgres ExternalName service exists (hive-metastore needs this)
+  const { exitCode: pgSvcExists } = await kubectl([
+    "get", "service", "foundation-postgres", "-n", "foundation",
+  ]);
+  if (pgSvcExists !== 0) {
+    const svcYaml = JSON.stringify({
+      apiVersion: "v1",
+      kind: "Service",
+      metadata: { name: "foundation-postgres", namespace: "foundation" },
+      spec: {
+        type: "ExternalName",
+        externalName: pgHost,
+      },
+    });
+    await kubectl(["apply", "-f", "-"], { input: svcYaml });
+    console.log(OK("  ✓ foundation-postgres ExternalName service created"));
+  }
+}
+
+// ── Vault unseal config reconciliation ────────────────────────────────────────
+
+export async function reconcileVaultUnseal(ctx) {
+  const { execa, clusterName, rg, sub } = ctx;
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  const correctKv = kvName(clusterName);
+
+  // Check if Vault CR exists
+  const { exitCode, stdout } = await kubectl([
+    "get", "vault", "vault", "-n", "foundation",
+    "-o", "jsonpath={.spec.unsealConfig.azure.keyVaultName}",
+  ]);
+  if (exitCode !== 0) return;
+
+  const currentKv = (stdout || "").trim();
+  if (currentKv === correctKv) {
+    console.log(OK(`  ✓ Vault unseal config → ${correctKv}`));
+    return;
+  }
+
+  hint(`Patching Vault unsealConfig: ${currentKv || "(unset)"} → ${correctKv}`);
+  await kubectl([
+    "patch", "vault", "vault", "-n", "foundation",
+    "--type", "merge", "-p",
+    JSON.stringify({ spec: { unsealConfig: { azure: { keyVaultName: correctKv } } } }),
+  ]);
+
+  // Ensure the Vault SP has Secrets Officer on the KV
+  const spClientId = await getSpClientId(kubectl);
+  if (spClientId) {
+    // Also check envsConfig for AZURE_CLIENT_ID which vault uses directly
+    const { stdout: vaultSpId } = await kubectl([
+      "get", "vault", "vault", "-n", "foundation",
+      "-o", "jsonpath={.spec.envsConfig[?(@.name=='AZURE_CLIENT_ID')].value}",
+    ]);
+    const targetSp = (vaultSpId || "").trim() || spClientId;
+
+    const { stdout: kvId } = await execa("az", [
+      "keyvault", "show", "--name", correctKv, "--query", "id", "-o", "tsv",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 15000 });
+
+    if (kvId?.trim()) {
+      await execa("az", [
+        "role", "assignment", "create",
+        "--assignee", targetSp,
+        "--role", "Key Vault Secrets Officer",
+        "--scope", kvId.trim(),
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+    }
+  }
+
+  // Restart vault pod to pick up the new unseal config
+  await kubectl(["delete", "pod", "vault-0", "-n", "foundation", "--ignore-not-found"]);
+  console.log(OK(`  ✓ Vault unseal config patched → ${correctKv}`));
+}
+
+// ── Seed Key Vault from VM ────────────────────────────────────────────────────
+
+export async function seedKeyVaultFromVm(execa, { vaultName, sub, dai }) {
+  // Check if vault already has secrets
+  const { stdout: existingSecrets } = await execa("az", [
+    "keyvault", "secret", "list", "--vault-name", vaultName,
+    "--query", "length(@)", "-o", "tsv",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (parseInt(existingSecrets || "0", 10) > 0) {
+    console.log(OK(`  ✓ Key Vault has ${existingSecrets.trim()} secret(s) — skipping seed`));
+    return;
+  }
+
+  // Try to pull .env from a running VM
+  const { listVms } = await import("./azure-state.js");
+  const { vms: vmMap } = listVms();
+  const vms = Object.entries(vmMap || {}).filter(([, v]) => v.ip);
+  if (vms.length === 0) {
+    hint("No VMs tracked — seed Key Vault manually or run fops azure aks seed-secrets");
+    return;
+  }
+
+  const [vmName, vm] = vms[0];
+  hint(`Seeding Key Vault from VM "${vmName}" (${vm.ip})…`);
+
+  const { sshCmd } = await import("./azure-helpers.js");
+  const user = vm.adminUser || "azureuser";
+  const { stdout: envContent, exitCode } = await sshCmd(
+    execa, vm.ip, user,
+    "cat /opt/foundation-compose/.env 2>/dev/null",
+    30000,
+  );
+  if (exitCode !== 0 || !envContent) {
+    console.log(WARN(`  ⚠ Could not read .env from ${vmName} — seed manually`));
+    return;
+  }
+
+  const envMap = {};
+  for (const line of envContent.split("\n")) {
+    const m = line.match(/^([A-Z0-9_]+)=(.+)$/);
+    if (m) envMap[m[1]] = m[2].replace(/^["']|["']$/g, "");
+  }
+
+  let seeded = 0;
+  const secrets = dai ? KV_SEED_SECRETS : KV_SEED_SECRETS.filter(s => !s.daiOnly);
+  for (const { name: secretName, envKeys } of secrets) {
+    const secretObj = {};
+    for (const [prop, envKey] of Object.entries(envKeys)) {
+      if (envMap[envKey]) secretObj[prop] = envMap[envKey];
+    }
+    if (Object.keys(secretObj).length === 0) continue;
+
+    const value = JSON.stringify(secretObj);
+    const { exitCode: setCode } = await execa("az", [
+      "keyvault", "secret", "set",
+      "--vault-name", vaultName,
+      "--name", secretName,
+      "--value", value,
+      "--content-type", "application/json",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+
+    if (setCode === 0) seeded++;
+  }
+
+  if (seeded > 0) {
+    console.log(OK(`  ✓ Seeded ${seeded} secret(s) into Key Vault from VM "${vmName}"`));
+  } else {
+    console.log(WARN("  ⚠ No matching env vars found on VM — seed secrets manually"));
+  }
+}
+
+// ── Helper functions ──────────────────────────────────────────────────────────
+
+export async function getSpClientId(kubectl) {
+  const { stdout } = await kubectl([
+    "get", "secret", "azure-secret-sp", "-n", "foundation",
+    "-o", "jsonpath={.data.ClientID}",
+  ]);
+  if (!stdout) return null;
+  return Buffer.from(stdout, "base64").toString();
+}
+
+export async function detectEsApiVersion(kubectl) {
+  const { exitCode } = await kubectl([
+    "api-resources", "--api-group=external-secrets.io",
+    "-o", "name",
+  ]);
+  if (exitCode !== 0) return "external-secrets.io/v1beta1";
+  const { stdout } = await kubectl([
+    "get", "crd", "secretstores.external-secrets.io",
+    "-o", "jsonpath={.spec.versions[*].name}",
+  ]);
+  const versions = (stdout || "").split(/\s+/);
+  if (versions.includes("v1")) return "external-secrets.io/v1";
+  if (versions.includes("v1beta1")) return "external-secrets.io/v1beta1";
+  return "external-secrets.io/v1";
+}
+
+// ── Vault auto-unseal bootstrap ──────────────────────────────────────────────
+
+export const VAULT_UNSEAL_KEY_NAME = "vault-unseal";
+
+export async function aksVaultInit(opts = {}) {
+  const { lazyExeca, ensureAzCli, ensureAzAuth, banner, subArgs } = await import("./azure.js");
+  const { requireCluster, writeClusterState } = await import("./azure-aks-state.js");
+  const { kvName } = await import("./azure-aks-naming.js");
+
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+
+  const { clusterName, resourceGroup: rg } = requireCluster(opts.clusterName);
+  const vaultName = kvName(clusterName);
+
+  banner(`Vault Auto-Unseal Bootstrap: ${clusterName}`);
+
+  const kubectl = (args, kopts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 60000, reject: false, ...kopts });
+
+  // 1. Ensure Key Vault exists
+  const { exitCode: kvExists } = await execa("az", [
+    "keyvault", "show", "--name", vaultName, "--output", "none",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (kvExists !== 0) {
+    console.log(WARN(`  ⚠ Key Vault "${vaultName}" not found. Run "fops azure aks doctor" first.`));
+    return;
+  }
+  console.log(OK(`  ✓ Key Vault "${vaultName}" exists`));
+
+  // 2. Get tenant ID
+  const { stdout: tenantIdRaw } = await execa("az", [
+    "account", "show", "--query", "tenantId", "-o", "tsv", ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  const tenantId = (tenantIdRaw || "").trim();
+
+  // 3. Get Key Vault resource ID
+  const { stdout: kvIdRaw } = await execa("az", [
+    "keyvault", "show", "--name", vaultName, "--query", "id", "-o", "tsv", ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  const kvId = (kvIdRaw || "").trim();
+
+  // 4. Create or verify the unseal key in Key Vault
+  const { exitCode: keyExists } = await execa("az", [
+    "keyvault", "key", "show", "--vault-name", vaultName, "--name", VAULT_UNSEAL_KEY_NAME,
+    "--output", "none", ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+
+  if (keyExists !== 0) {
+    hint(`Creating Key Vault key "${VAULT_UNSEAL_KEY_NAME}" for Vault auto-unseal…`);
+    const { exitCode: createCode, stderr } = await execa("az", [
+      "keyvault", "key", "create",
+      "--vault-name", vaultName,
+      "--name", VAULT_UNSEAL_KEY_NAME,
+      "--kty", "RSA",
+      "--size", "2048",
+      "--ops", "wrapKey", "unwrapKey",
+      "--output", "none", ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+    if (createCode !== 0) {
+      console.log(WARN(`  ⚠ Failed to create unseal key: ${(stderr || "").split("\n")[0]}`));
+      return;
+    }
+    console.log(OK(`  ✓ Created Key Vault key "${VAULT_UNSEAL_KEY_NAME}"`));
+  } else {
+    console.log(OK(`  ✓ Key Vault key "${VAULT_UNSEAL_KEY_NAME}" exists`));
+  }
+
+  // 5. Grant AKS kubelet identity Crypto User role on Key Vault
+  const { stdout: kubeletIdRaw } = await execa("az", [
+    "aks", "show", "-g", rg, "-n", clusterName,
+    "--query", "identityProfile.kubeletidentity.objectId", "-o", "tsv", ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+  const kubeletObjectId = (kubeletIdRaw || "").trim();
+
+  if (kubeletObjectId && kvId) {
+    const { stdout: hasCryptoRole } = await execa("az", [
+      "role", "assignment", "list",
+      "--assignee", kubeletObjectId,
+      "--role", "Key Vault Crypto User",
+      "--scope", kvId,
+      "--query", "[0].id", "-o", "tsv", ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+
+    if (!hasCryptoRole?.trim()) {
+      await execa("az", [
+        "role", "assignment", "create",
+        "--assignee-object-id", kubeletObjectId,
+        "--assignee-principal-type", "ServicePrincipal",
+        "--role", "Key Vault Crypto User",
+        "--scope", kvId,
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      console.log(OK("  ✓ Granted AKS kubelet identity Key Vault Crypto User role"));
+    } else {
+      console.log(OK("  ✓ AKS kubelet identity has Key Vault Crypto User role"));
+    }
+  }
+
+  // 6. Check current Vault pod status
+  const { stdout: vaultPodJson } = await kubectl([
+    "get", "pod", "-n", "foundation", "-l", "app.kubernetes.io/name=vault",
+    "-o", "json",
+  ]);
+  const vaultPods = vaultPodJson ? JSON.parse(vaultPodJson).items : [];
+  const vaultPod = vaultPods[0];
+
+  if (!vaultPod) {
+    console.log(WARN("  ⚠ No Vault pod found. Deploy Vault first via Flux."));
+    return;
+  }
+
+  // 7. Check if Vault is already initialized
+  const { stdout: vaultStatusRaw, exitCode: statusCode } = await kubectl([
+    "exec", "-n", "foundation", vaultPod.metadata.name, "--",
+    "vault", "status", "-format=json",
+  ]);
+
+  let vaultStatus = null;
+  try { vaultStatus = JSON.parse(vaultStatusRaw || "{}"); } catch {}
+
+  const isInitialized = vaultStatus?.initialized === true;
+  const isSealed = vaultStatus?.sealed !== false;
+
+  console.log(DIM(`  Vault status: initialized=${isInitialized}, sealed=${isSealed}`));
+
+  // 8. Update Vault CR or HelmRelease with auto-unseal config
+  const { exitCode: vaultCrExists } = await kubectl([
+    "get", "vault", "vault", "-n", "foundation",
+  ]);
+
+  if (vaultCrExists === 0) {
+    // Vault Operator CR mode - patch unsealConfig
+    hint("Patching Vault CR with Azure Key Vault auto-unseal config…");
+    const patch = {
+      spec: {
+        unsealConfig: {
+          azure: {
+            keyVaultName: vaultName,
+            keyName: VAULT_UNSEAL_KEY_NAME,
+            useManagedIdentity: true,
+          },
+        },
+      },
+    };
+    await kubectl([
+      "patch", "vault", "vault", "-n", "foundation",
+      "--type", "merge", "-p", JSON.stringify(patch),
+    ]);
+    console.log(OK("  ✓ Vault CR patched with auto-unseal config"));
+  } else {
+    // HelmRelease mode - check for ConfigMap/values
+    hint("Creating Vault auto-unseal ConfigMap for Helm values…");
+    const vaultConfig = {
+      seal: {
+        azurekeyvault: {
+          tenant_id: tenantId,
+          vault_name: vaultName,
+          key_name: VAULT_UNSEAL_KEY_NAME,
+        },
+      },
+    };
+    const configMapYaml = `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vault-auto-unseal-config
+  namespace: foundation
+  labels:
+    app.kubernetes.io/name: vault
+data:
+  vault-auto-unseal.hcl: |
+    seal "azurekeyvault" {
+      tenant_id = "${tenantId}"
+      vault_name = "${vaultName}"
+      key_name = "${VAULT_UNSEAL_KEY_NAME}"
+    }
+`;
+    await kubectl(["apply", "-f", "-"], { input: configMapYaml });
+    console.log(OK("  ✓ Vault auto-unseal ConfigMap created"));
+  }
+
+  // 9. Initialize Vault if not yet initialized
+  if (!isInitialized) {
+    hint("Initializing Vault with recovery keys…");
+    const { stdout: initOutput, exitCode: initCode } = await kubectl([
+      "exec", "-n", "foundation", vaultPod.metadata.name, "--",
+      "vault", "operator", "init",
+      "-recovery-shares=5",
+      "-recovery-threshold=3",
+      "-format=json",
+    ]);
+
+    if (initCode !== 0) {
+      console.log(WARN(`  ⚠ Vault initialization failed. May need to restart pod after config change.`));
+      hint("Restart Vault: kubectl delete pod -n foundation -l app.kubernetes.io/name=vault");
+      return;
+    }
+
+    let initResult;
+    try { initResult = JSON.parse(initOutput); } catch {
+      console.log(WARN("  ⚠ Could not parse Vault init output"));
+      return;
+    }
+
+    console.log(OK("  ✓ Vault initialized with auto-unseal"));
+
+    // 10. Store recovery keys in Key Vault
+    const recoveryKeys = initResult.recovery_keys_b64 || initResult.recovery_keys || [];
+    const rootToken = initResult.root_token;
+
+    if (recoveryKeys.length > 0) {
+      hint("Storing recovery keys in Azure Key Vault…");
+      const recoverySecret = JSON.stringify({
+        recovery_keys: recoveryKeys,
+        root_token: rootToken,
+        initialized_at: new Date().toISOString(),
+        cluster: clusterName,
+      });
+
+      await execa("az", [
+        "keyvault", "secret", "set",
+        "--vault-name", vaultName,
+        "--name", "vault-recovery-keys",
+        "--value", recoverySecret,
+        "--content-type", "application/json",
+        "--output", "none", ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      console.log(OK("  ✓ Recovery keys stored in Key Vault as 'vault-recovery-keys'"));
+    }
+
+    // Store in local cluster state too
+    writeClusterState(clusterName, {
+      vault: {
+        initialized: true,
+        autoUnseal: true,
+        keyVaultName: vaultName,
+        unsealKeyName: VAULT_UNSEAL_KEY_NAME,
+        initializedAt: new Date().toISOString(),
+      },
+    });
+  } else {
+    console.log(OK("  ✓ Vault already initialized"));
+
+    // Update state
+    writeClusterState(clusterName, {
+      vault: {
+        initialized: true,
+        autoUnseal: true,
+        keyVaultName: vaultName,
+        unsealKeyName: VAULT_UNSEAL_KEY_NAME,
+      },
+    });
+  }
+
+  // 11. Restart Vault pod to pick up new config
+  if (isSealed || !isInitialized) {
+    hint("Restarting Vault pod to apply auto-unseal config…");
+    await kubectl(["delete", "pod", "-n", "foundation", "-l", "app.kubernetes.io/name=vault"]);
+    console.log(OK("  ✓ Vault pod restarted"));
+    hint("Vault should auto-unseal within ~30 seconds. Check: kubectl get pod -n foundation -l app.kubernetes.io/name=vault");
+  }
+
+  console.log(OK("\n  ✓ Vault auto-unseal bootstrap complete\n"));
+}