@meshxdata/fops 0.1.51 → 0.1.53

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-core.js

@@ -1057,64 +1057,55 @@ export async function aksList(opts = {}) {
 
   banner("AKS Clusters");
 
-  // If no clusters tracked locally, try to discover fops-managed clusters from Azure
-  if (names.length === 0) {
+  // Always discover fops-managed clusters from Azure so we pick up clusters
+  // created by teammates or missing from local state.
+  try {
     const execa = await lazyExeca();
-    try {
-      await ensureAzCli(execa);
-      await ensureAzAuth(execa, { subscription: opts.profile });
-    } catch {
-      hint("No clusters tracked.");
-      hint("Create one:  fops azure aks up <name>\n");
-      return;
-    }
-
-    hint("No clusters tracked locally — checking Azure for fops-managed clusters…\n");
+    await ensureAzCli(execa);
+    await ensureAzAuth(execa, { subscription: opts.profile });
 
-    try {
-      // Query all AKS clusters and filter by managed=fops tag
-      const { stdout, exitCode } = await execa("az", [
-        "aks", "list",
-        "--query", "[?tags.managed=='fops']",
-        "--output", "json",
-        ...subArgs(opts.profile),
-      ], { timeout: 60000, reject: false });
-
-      if (exitCode === 0 && stdout?.trim()) {
-        const discovered = JSON.parse(stdout);
-        if (discovered.length > 0) {
-          for (const cl of discovered) {
-            const name = cl.name;
-            const info = {
-              resourceGroup: cl.resourceGroup,
-              location: cl.location,
-              kubernetesVersion: cl.kubernetesVersion,
-              fqdn: cl.fqdn,
-              nodeCount: cl.agentPoolProfiles?.reduce((s, p) => s + (p.count || 0), 0) || 0,
-              nodeVmSize: cl.agentPoolProfiles?.[0]?.vmSize || "unknown",
-              subscriptionId: cl.id?.split("/")[2],
-              createdAt: cl.provisioningState === "Succeeded" ? new Date().toISOString() : null,
-            };
-            writeClusterState(name, info);
-            console.log(OK(`  + Discovered ${name} (${cl.location})`));
-          }
-          console.log("");
-          // Re-read after discovery
-          const updated = readAksClusters();
-          activeCluster = updated.activeCluster;
-          clusters = updated.clusters;
-          names = Object.keys(clusters);
-        }
+    const { stdout, exitCode } = await execa("az", [
+      "aks", "list",
+      "--query", "[?tags.managed=='fops']",
+      "--output", "json",
+      ...subArgs(opts.profile),
+    ], { timeout: 60000, reject: false });
+
+    if (exitCode === 0 && stdout?.trim()) {
+      const discovered = JSON.parse(stdout);
+      let added = 0;
+      for (const cl of discovered) {
+        if (clusters[cl.name]) continue; // already tracked
+        const info = {
+          resourceGroup: cl.resourceGroup,
+          location: cl.location,
+          kubernetesVersion: cl.kubernetesVersion,
+          fqdn: cl.fqdn,
+          nodeCount: cl.agentPoolProfiles?.reduce((s, p) => s + (p.count || 0), 0) || 0,
+          nodeVmSize: cl.agentPoolProfiles?.[0]?.vmSize || "unknown",
+          subscriptionId: cl.id?.split("/")[2],
+          createdAt: cl.provisioningState === "Succeeded" ? new Date().toISOString() : null,
+        };
+        writeClusterState(cl.name, info);
+        console.log(OK(`  + Discovered ${cl.name} (${cl.location})`));
+        added++;
+      }
+      if (added > 0) {
+        console.log("");
+        const updated = readAksClusters();
+        activeCluster = updated.activeCluster;
+        clusters = updated.clusters;
+        names = Object.keys(clusters);
       }
-    } catch {
-      // Discovery failed, continue with empty list
     }
+  } catch {
+    // az not available or not authenticated — continue with local state
+  }
 
-    if (names.length === 0) {
-      hint("No fops-managed clusters found in Azure.");
-      hint("Create one:  fops azure aks up <name>\n");
-      return;
-    }
+  if (names.length === 0) {
+    hint("No clusters tracked.");
+    hint("Create one:  fops azure aks up <name>\n");
+    return;
   }
 
   // Refresh each tracked cluster from Azure so RG, Location, Nodes, FQDN, etc. are current
@@ -1250,6 +1241,15 @@ export async function aksStatus(opts = {}) {
     hint("  Flux CLI not available — skipping Flux status.");
   }
 
+  // External Secrets health check
+  console.log(`\n  ${LABEL("External Secrets")}`);
+  try {
+    const { validateExternalSecretsHealth } = await import("./azure-aks-secrets.js");
+    await validateExternalSecretsHealth({ execa, clusterName, rg, sub });
+  } catch (e) {
+    hint(`  Could not check External Secrets: ${e.message}`);
+  }
+
   console.log("");
 }
 

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-secrets.js

@@ -172,6 +172,32 @@ export async function reconcileSecretStore(ctx) {
     }
   }
 
+  // 2c. Check for External Secrets managed identity (ext-* prefix) and grant Key Vault access
+  const extSecretsIdentity = await detectExternalSecretsIdentity(execa, clusterName, sub);
+  if (extSecretsIdentity && kvId) {
+    const { stdout: hasExtRole } = await execa("az", [
+      "role", "assignment", "list",
+      "--assignee", extSecretsIdentity.clientId,
+      "--role", "Key Vault Secrets User",
+      "--scope", kvId,
+      "--query", "[0].id", "-o", "tsv",
+      ...subArgs(sub),
+    ], { reject: false, timeout: 30000 });
+
+    if (!hasExtRole?.trim()) {
+      await execa("az", [
+        "role", "assignment", "create",
+        "--assignee", extSecretsIdentity.clientId,
+        "--role", "Key Vault Secrets User",
+        "--scope", kvId,
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      console.log(OK(`  ✓ External Secrets identity granted Key Vault Secrets User role`));
+    }
+    // Store the identity ID for SecretStore configuration
+    ctx.extSecretsIdentityId = extSecretsIdentity.clientId;
+  }
+
   // 3. Ensure azure-secret-sp exists in each target namespace
   const { stdout: spSecretJson } = await kubectl([
     "get", "secret", "azure-secret-sp", "-n", "foundation", "-o", "json",
@@ -579,6 +605,131 @@ export async function detectEsApiVersion(kubectl) {
   return "external-secrets.io/v1";
 }
 
+/**
+ * Detect External Secrets managed identity (ext-* prefix) for clusters with multiple identities.
+ * When AKS has multiple user-assigned identities, SecretStore needs to specify which one to use.
+ */
+export async function detectExternalSecretsIdentity(execa, clusterName, sub) {
+  const { subArgs } = await import("./azure.js");
+
+  // List all managed identities that match the external-secrets pattern
+  const { stdout: identitiesJson } = await execa("az", [
+    "identity", "list",
+    "--query", `[?contains(name, '${clusterName}')].{name:name,clientId:clientId}`,
+    "-o", "json",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  let identities = [];
+  try { identities = JSON.parse(identitiesJson || "[]"); } catch {}
+
+  // Look for ext-* identity (External Secrets workload identity)
+  const extIdentity = identities.find(i => i.name?.startsWith("ext-") && i.name?.includes(clusterName));
+  if (extIdentity) {
+    return { name: extIdentity.name, clientId: extIdentity.clientId };
+  }
+
+  // If multiple identities exist but no ext-* found, warn about potential issues
+  if (identities.length > 1) {
+    const { WARN, hint } = await import("./azure.js");
+    console.log(WARN(`  ⚠ Multiple managed identities found for ${clusterName} but no ext-* identity detected`));
+    hint("External Secrets may fail with 'Multiple user assigned identities exist' error");
+    hint("Create a dedicated identity: az identity create -n ext-<cluster> -g <rg>");
+  }
+
+  return null;
+}
+
+/**
+ * Validate External Secrets health - checks SecretStore config and ExternalSecret status.
+ * Reports issues like missing identityId when multiple identities exist.
+ */
+export async function validateExternalSecretsHealth(ctx) {
+  const { execa, clusterName, sub } = ctx;
+  const { OK, WARN, DIM, hint, subArgs } = await import("./azure.js");
+
+  const kubectl = (args, opts = {}) =>
+    execa("kubectl", ["--context", clusterName, ...args], { timeout: 30000, reject: false, ...opts });
+
+  const issues = [];
+
+  // Check SecretStore status
+  const { stdout: ssJson } = await kubectl([
+    "get", "secretstore", SECRET_STORE_NAME, "-n", "foundation", "-o", "json",
+  ]);
+  if (!ssJson) {
+    issues.push({ level: "error", msg: "SecretStore not found in foundation namespace" });
+    return issues;
+  }
+
+  const ss = JSON.parse(ssJson);
+  const ssReady = ss.status?.conditions?.find(c => c.type === "Ready")?.status === "True";
+  const authType = ss.spec?.provider?.azurekv?.authType;
+  const identityId = ss.spec?.provider?.azurekv?.identityId;
+
+  // Check if using ManagedIdentity auth without identityId when multiple identities exist
+  if (authType === "ManagedIdentity" && !identityId) {
+    const extIdentity = await detectExternalSecretsIdentity(execa, clusterName, sub);
+    if (extIdentity) {
+      issues.push({
+        level: "warn",
+        msg: `SecretStore uses ManagedIdentity but identityId is empty`,
+        fix: `Set identityId to "${extIdentity.clientId}" in clusters/${clusterName}/config/secret-store.yaml`,
+      });
+    }
+  }
+
+  // Check ExternalSecret status
+  const { stdout: esJson } = await kubectl([
+    "get", "externalsecret", "-n", "foundation", "-o", "json",
+  ]);
+  const externalSecrets = esJson ? JSON.parse(esJson).items : [];
+
+  for (const es of externalSecrets) {
+    const ready = es.status?.conditions?.find(c => c.type === "Ready");
+    if (ready?.status !== "True") {
+      const msg = ready?.message || "Unknown error";
+      if (msg.includes("Multiple user assigned identities exist")) {
+        const extIdentity = await detectExternalSecretsIdentity(execa, clusterName, sub);
+        issues.push({
+          level: "error",
+          msg: `ExternalSecret "${es.metadata.name}" failing: Multiple identities detected`,
+          fix: extIdentity
+            ? `Add identityId: "${extIdentity.clientId}" to SecretStore spec`
+            : "Create ext-* managed identity and grant Key Vault access",
+        });
+      } else if (msg.includes("Forbidden") || msg.includes("not authorized")) {
+        issues.push({
+          level: "error",
+          msg: `ExternalSecret "${es.metadata.name}" failing: Key Vault access denied`,
+          fix: "Run: fops azure aks doctor --fix to grant Key Vault permissions",
+        });
+      } else {
+        issues.push({
+          level: "error",
+          msg: `ExternalSecret "${es.metadata.name}" failing: ${msg.substring(0, 100)}`,
+        });
+      }
+    }
+  }
+
+  // Report findings
+  if (issues.length === 0) {
+    console.log(OK("  ✓ External Secrets healthy"));
+  } else {
+    for (const issue of issues) {
+      if (issue.level === "error") {
+        console.log(WARN(`  ✗ ${issue.msg}`));
+      } else {
+        console.log(WARN(`  ⚠ ${issue.msg}`));
+      }
+      if (issue.fix) hint(`    Fix: ${issue.fix}`);
+    }
+  }
+
+  return issues;
+}
+
 // ── Vault auto-unseal bootstrap ──────────────────────────────────────────────
 
 export const VAULT_UNSEAL_KEY_NAME = "vault-unseal";

package/src/plugins/bundled/fops-plugin-azure/lib/azure-aks-storage.js

@@ -38,7 +38,7 @@ export async function reconcileStorageAccount(ctx) {
   const { execa, clusterName, rg, sub } = ctx;
   const storageAccountName = `fops${clusterName.replace(/-/g, "")}`.toLowerCase().slice(0, 24);
   const vaultName = `fops-${clusterName}-kv`;
-  const containers = ["foundation", "vault"];
+  const containers = ["foundation", "vault", "loki"];
 
   hint(`Reconciling Azure Storage Account "${storageAccountName}"…`);
 
@@ -571,7 +571,7 @@ export async function reconcileStorageReplication(ctx) {
 
   const sourceAccountName = `fops${clusterName.replace(/-/g, "")}`.toLowerCase().slice(0, 24);
   const destAccountName = `fops${clusterName.replace(/-/g, "")}ha`.toLowerCase().slice(0, 24);
-  const containers = ["foundation", "vault"];
+  const containers = ["foundation", "vault", "loki"];
 
   hint(`Setting up cross-region storage replication (${location} → ${replicaRegion})…`);
 

package/src/plugins/bundled/fops-plugin-azure/lib/azure-cost.js

@@ -24,22 +24,50 @@ async function az(args, opts = {}) {
   }
 }
 
+// In-memory cache for cost queries (TTL: 1 hour)
+const _costCache = new Map();
+const COST_CACHE_TTL = 60 * 60 * 1000; // 1 hour
+
 async function costQuery(scope, dataset) {
-  try {
-    const body = JSON.stringify({ ...dataset });
-    const { stdout, stderr } = await execa("az", [
-      "rest", "--method", "POST",
-      "--url", `https://management.azure.com${scope}/providers/Microsoft.CostManagement/query?api-version=2023-11-01`,
-      "--body", body,
-      "--output", "json",
-    ], { timeout: 120_000, reject: false });
-    if (stderr?.includes("Please run 'az login'") || stderr?.includes("AADSTS")) {
-      return { error: stderr.split("\n")[0] };
+  const cacheKey = JSON.stringify({ scope, dataset });
+  const cached = _costCache.get(cacheKey);
+  if (cached && Date.now() - cached.ts < COST_CACHE_TTL) {
+    return cached.data;
+  }
+
+  const maxRetries = 3;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const body = JSON.stringify({ ...dataset });
+      const { stdout, stderr } = await execa("az", [
+        "rest", "--method", "POST",
+        "--url", `https://management.azure.com${scope}/providers/Microsoft.CostManagement/query?api-version=2023-11-01`,
+        "--body", body,
+        "--output", "json",
+      ], { timeout: 120_000, reject: false });
+
+      if (stderr?.includes("Please run 'az login'") || stderr?.includes("AADSTS")) {
+        return { error: stderr.split("\n")[0] + "\nMake sure you are logged into Azure (az login) and have Cost Management access." };
+      }
+
+      // Handle 429 rate limiting
+      if (stderr?.includes("429") || stderr?.includes("Too many requests") || stderr?.includes("Too Many Requests")) {
+        const wait = Math.pow(2, attempt + 1) * 5000; // 10s, 20s, 40s
+        if (attempt < maxRetries - 1) {
+          await new Promise((r) => setTimeout(r, wait));
+          continue;
+        }
+        return { error: `Rate limited by Azure Cost Management API after ${maxRetries} retries. Try again in a few minutes.` };
+      }
+
+      const result = JSON.parse(stdout || "{}");
+      _costCache.set(cacheKey, { data: result, ts: Date.now() });
+      return result;
+    } catch (err) {
+      if (attempt === maxRetries - 1) return { error: err.message };
     }
-    return JSON.parse(stdout || "{}");
-  } catch (err) {
-    return { error: err.message };
   }
+  return { error: "Cost query failed after retries" };
 }
 
 function formatCost(amount, currency = "USD") {
@@ -402,16 +430,18 @@ export async function registerCostTools(api) {
         ? allVms.filter(v => v.powerState?.toLowerCase().includes(input.state))
         : allVms;
 
-      // Rough monthly cost estimates (USD, Pay-As-You-Go, select regions)
+      // Rough monthly cost estimates (USD, Pay-As-You-Go)
       const costs = {
-        Standard_B2s: 30, Standard_B4ms: 60, Standard_B2ms: 60,
-        Standard_D2s_v3: 70, Standard_D4s_v3: 140, Standard_D8s_v3: 281,
-        Standard_D16s_v3: 562, Standard_D32s_v3: 1124,
-        Standard_D2s_v5: 70, Standard_D4s_v5: 140, Standard_D8s_v5: 281,
-        Standard_D16s_v5: 562, Standard_D32s_v5: 1124,
-        Standard_E2s_v3: 92, Standard_E4s_v3: 184, Standard_E8s_v3: 368,
-        Standard_E16s_v3: 736, Standard_E32s_v3: 1472,
-        Standard_F2s_v2: 62, Standard_F4s_v2: 124, Standard_F8s_v2: 248,
+        // B-series (burstable)
+        Standard_B1s: 8, Standard_B2s: 30, Standard_B2ms: 60, Standard_B4ms: 120,
+        // D-series (general purpose) — v3/v4/v5 similar pricing
+        Standard_D2s_v3: 70, Standard_D4s_v3: 140, Standard_D8s_v3: 281, Standard_D16s_v3: 562, Standard_D32s_v3: 1124,
+        Standard_D2s_v5: 70, Standard_D4s_v5: 140, Standard_D8s_v5: 281, Standard_D16s_v5: 562, Standard_D32s_v5: 1124, Standard_D64s_v5: 2249,
+        // E-series (memory optimized)
+        Standard_E2s_v3: 92, Standard_E4s_v3: 184, Standard_E8s_v3: 368, Standard_E16s_v3: 736, Standard_E32s_v3: 1472,
+        Standard_E2s_v5: 92, Standard_E4s_v5: 184, Standard_E8s_v5: 368, Standard_E16s_v5: 736, Standard_E32s_v5: 1472, Standard_E64s_v5: 2621,
+        // F-series (compute optimized)
+        Standard_F2s_v2: 62, Standard_F4s_v2: 124, Standard_F8s_v2: 248, Standard_F16s_v2: 496, Standard_F32s_v2: 992,
       };
 
       let output = "Azure VMs\n" + "=".repeat(75) + "\n";

package/src/plugins/bundled/fops-plugin-azure/lib/azure-fleet.js

@@ -71,20 +71,28 @@ async function forEachVm({
     return { results: [], vms };
   }
 
-  const targets = opts.vmName ? [opts.vmName] : names;
-  for (const t of targets) {
+  const allTargets = opts.vmName ? [opts.vmName] : names;
+  for (const t of allTargets) {
     if (!vms[t]) {
       console.error(ERR(`\n  VM "${t}" not tracked. Run: fops azure list\n`));
       process.exit(1);
     }
   }
 
+  // Filter out VMs without public IPs (e.g., local stack) unless explicitly targeted
+  const skippedVms = opts.vmName ? [] : allTargets.filter(t => !vms[t].publicIp);
+  const targets = opts.vmName ? allTargets : allTargets.filter(t => vms[t].publicIp);
+
   banner(title);
-  hint(`${targets.length} VM(s)${concurrency ? ` (concurrency: ${concurrency})` : ""}…\n`);
+  if (targets.length === 0) {
+    hint("No VMs with public IPs to target.\n");
+    return { results: [], vms, activeVm: listVms().activeVm };
+  }
+  hint(`${targets.length} VM(s)${skippedVms.length ? ` (${skippedVms.length} skipped: no public IP)` : ""}${concurrency ? ` (concurrency: ${concurrency})` : ""}…\n`);
 
   async function runOne(name) {
     const vm = vms[name];
-    if (!vm.publicIp) return { name, ok: false, reason: "no public IP" };
+    if (!vm.publicIp) return { name, ok: false, reason: "no public IP (local stack?)" };
 
     try {
       await knockForVm(vm);

package/src/plugins/bundled/fops-plugin-azure/lib/azure-helpers.js

@@ -854,16 +854,20 @@ export function fopsUpCmd(publicUrl, { k3s, traefik, dai } = {}) {
   ].join("; ");
 
   const debugPostamble = [
-    `echo \\\"=== fops up finished at \\$(date -Iseconds) with exit code \\$? ===\\\" >> ${logFile}`,
+    `echo \\\"=== fops up finished at \\$(date -Iseconds) with exit code \\$_fops_rc ===\\\" >> ${logFile}`,
     `echo \\\"--- Container status ---\\\" >> ${logFile}`,
     `docker compose ps --format 'table {{.Name}}\\t{{.Status}}' >> ${logFile} 2>&1`,
     `echo \\\"--- Recent docker events ---\\\" >> ${logFile}`,
     `tail -50 ${eventsLog} >> ${logFile} 2>&1 || true`,
+    `exit \\$_fops_rc`,
   ].join("; ");
 
+  // Fail fast if Docker is not installed
+  const dockerGuard = `command -v docker >/dev/null 2>&1 || { echo \\\"ERROR: Docker is not installed — cannot start Foundation\\\" >> ${logFile}; echo \\\"ERROR: Docker is not installed\\\" >&2; exit 1; }`;
+
   // Run from project dir with FOUNDATION_ROOT set explicitly (sudo can reset cwd)
   const envSetup = `export PATH=/usr/local/bin:/usr/bin:\\$PATH FOUNDATION_ROOT=/opt/foundation-compose`;
-  return `bash -c "cd /opt/foundation-compose && ${envSetup}; ${debugPreamble}; ${quietPull}; if command -v fops >/dev/null 2>&1; then ${profileEnv}${fopsCmd}; else echo 'fops not found — falling back to docker compose'; ${composeCmd}; fi; ${debugPostamble}"`;
+  return `bash -c "cd /opt/foundation-compose && ${envSetup}; ${dockerGuard}; ${debugPreamble}; ${quietPull}; if command -v fops >/dev/null 2>&1; then ${profileEnv}${fopsCmd}; else echo 'fops not found — falling back to docker compose'; ${composeCmd}; fi; _fops_rc=\\$?; ${debugPostamble}"`;
 }
 
 /** Build remote "fops up [component] [branch]" args (same as local fops up). For foreground run on VM. */

package/src/plugins/bundled/fops-plugin-azure/lib/azure-ops.js

@@ -321,6 +321,71 @@ export async function azureTrinoStatus(opts = {}) {
   console.log("");
 }
 
+// ── ping ─────────────────────────────────────────────────────────────────────
+
+/**
+ * Check Foundation backend /api/ping/json health endpoint on a VM.
+ */
+export async function azurePing(opts = {}) {
+  const execa = await lazyExeca();
+  const state = requireVmState(opts.vmName);
+  const { vmName } = state;
+  const ip = state.publicIp;
+  const adminUser = DEFAULTS.adminUser;
+
+  if (!ip) {
+    console.log(WARN(`  VM ${vmName} has no public IP (probably stopped)`));
+    return;
+  }
+
+  await knockForVm(state);
+  const sshOk = await waitForSsh(execa, ip, adminUser, 10000);
+  if (!sshOk) {
+    console.log(WARN("\n  ⚠ SSH not reachable"));
+    return;
+  }
+
+  const pingToken = opts.token || process.env.FOPS_PING_TOKEN || "";
+  const tokenHeader = pingToken ? `-H "X-Ping-Token: ${pingToken}"` : "";
+  const { stdout, exitCode } = await sshCmd(execa, ip, adminUser,
+    `curl -sf ${tokenHeader} http://localhost:9001/api/ping/json 2>/dev/null || echo '{}'`,
+    15000,
+  );
+
+  let ping;
+  try {
+    ping = JSON.parse(stdout.trim() || "{}");
+  } catch {
+    console.log(ERR(`  Failed to parse ping response: ${stdout}`));
+    return;
+  }
+
+  banner(`Ping: ${vmName}`);
+
+  if (ping.ok === undefined) {
+    console.log(WARN("  No response from backend /api/ping/json"));
+    hint("Backend may be down or starting up");
+    console.log("");
+    return;
+  }
+
+  const overall = ping.ok ? OK("✓ healthy") : ERR("✗ unhealthy");
+  kvLine("Status", overall);
+  if (ping.tag) kvLine("Tag", DIM(ping.tag));
+
+  if (ping.checks) {
+    console.log("");
+    console.log(ACCENT("  Checks:"));
+    for (const [name, check] of Object.entries(ping.checks)) {
+      const status = check.ok ? OK("✓") : ERR("✗");
+      const latency = check.latency_ms !== undefined ? DIM(` (${check.latency_ms}ms)`) : "";
+      const err = check.error ? ERR(` — ${check.error}`) : "";
+      console.log(`    ${status} ${name}${latency}${err}`);
+    }
+  }
+  console.log("");
+}
+
 /**
  * Run VM diagnostics: show config versions, then run make download and print
  * full output so image-pull failures (e.g. after config versions change) can be diagnosed.
@@ -1295,6 +1360,41 @@ export async function azureList(opts = {}) {
     }
   } catch { /* az not available or not authenticated */ }
 
+  // Always discover AKS clusters from Azure (tag managed=fops)
+  try {
+    const execa = await lazyExeca();
+    const { writeClusterState } = await import("./azure-aks-state.js");
+    const { stdout, exitCode } = await execa("az", [
+      "aks", "list",
+      "--query", "[?tags.managed=='fops']",
+      "--output", "json",
+      ...subArgs(opts.subscription),
+    ], { timeout: 60000, reject: false });
+    if (exitCode === 0 && stdout?.trim()) {
+      const discovered = JSON.parse(stdout);
+      let added = 0;
+      for (const cl of discovered) {
+        if (aksClusters[cl.name]) continue;
+        writeClusterState(cl.name, {
+          resourceGroup: cl.resourceGroup,
+          location: cl.location,
+          kubernetesVersion: cl.kubernetesVersion,
+          fqdn: cl.fqdn,
+          nodeCount: cl.agentPoolProfiles?.reduce((s, p) => s + (p.count || 0), 0) || 0,
+          nodeVmSize: cl.agentPoolProfiles?.[0]?.vmSize || "unknown",
+          subscriptionId: cl.id?.split("/")[2],
+        });
+        added++;
+      }
+      if (added > 0) {
+        console.log(OK(`  ✓ Re-discovered ${added} AKS cluster(s) from Azure`) + DIM(" (tag managed=fops)\n"));
+        fullState = readState();
+        aksClusters = (fullState.azure || {}).clusters || {};
+        hasAks = Object.keys(aksClusters).length > 0;
+      }
+    }
+  } catch { /* az not available or AKS discovery failed */ }
+
   // JSON output mode - early return with structured data
   if (opts.json) {
     const output = {
@@ -1568,10 +1668,9 @@ export async function azureList(opts = {}) {
       const hasPrimary = primaryName && clusterNames.includes(primaryName);
       const prefix = isStandby && hasPrimary ? "  └─" : "";
       const dot = active ? OK("●") : DIM("○");
-      const displayName = isStandby && hasPrimary
-        ? `${cr.name} ${DIM("(HA standby)")}`
-        : cr.name;
-      const cNameTxt = active ? OK(displayName.padEnd(maxCName + 13)) : LABEL(displayName.padEnd(maxCName + 13));
+      const paddedName = cr.name.padEnd(maxCName);
+      const standbySuffix = isStandby && hasPrimary ? ` ${DIM("(HA standby)")}` : "";
+      const cNameTxt = active ? OK(paddedName) + standbySuffix : LABEL(paddedName) + standbySuffix;
       const loc = (cl?.location || cr.location || "–").padEnd(10);
       const nodes = cr.nodes != null ? `${cr.nodes} x ${cr.sizes || "?"}` : "–";
       const k8s = (cr.kubernetesVersion || "–").padEnd(6);
@@ -1640,12 +1739,19 @@ export function printServiceMatrix(results, nameWidth) {
   const withSvc = results.filter(r => r.services && Object.keys(r.services).length > 0);
   if (withSvc.length === 0) return;
 
+  // Resolve display value for a service entry (supports both string and {tag,sha} formats)
+  const svcVal = (entry) => {
+    if (!entry) return null;
+    if (typeof entry === "string") return entry;
+    return entry.sha || entry.tag || null;
+  };
+
   // Find the majority value per column to highlight drift
   const majority = {};
   for (const svc of SVC_ORDER) {
     const counts = {};
     for (const r of withSvc) {
-      const v = r.services?.[svc];
+      const v = svcVal(r.services?.[svc]);
       if (v) counts[v] = (counts[v] || 0) + 1;
     }
     const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
@@ -1660,7 +1766,7 @@ export function printServiceMatrix(results, nameWidth) {
   for (const r of withSvc) {
     const nameTxt = LABEL(r.name.padEnd(nameWidth));
     const cells = SVC_ORDER.map(svc => {
-      const v = r.services?.[svc] || "–";
+      const v = svcVal(r.services?.[svc]) || "–";
       const display = v.padEnd(colW);
       if (v === "–") return DIM(display);
       if (v !== majority[svc]) return WARN(display);
@@ -1671,7 +1777,7 @@ export function printServiceMatrix(results, nameWidth) {
 
   // Check for drift
   const hasDrift = SVC_ORDER.some(svc => {
-    const vals = withSvc.map(r => r.services?.[svc]).filter(Boolean);
+    const vals = withSvc.map(r => svcVal(r.services?.[svc])).filter(Boolean);
     return new Set(vals).size > 1;
   });
   if (hasDrift) {