@meshxdata/fops 0.1.52 → 0.1.54

package/src/plugins/bundled/fops-plugin-azure/lib/azure-provision.js

@@ -44,7 +44,7 @@ async function ensureGhcrOnVm(ssh, user, githubToken, { timeout = 60000 } = {})
 // ── Configure a fresh or restarted VM ───────────────────────────────────────
 
 export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s, traefik, dai, deferStartToReconcile, quiet } = {}) {
-  const ssh = (cmd) => sshCmd(execa, ip, user, cmd);
+  const ssh = (cmd, timeout) => sshCmd(execa, ip, user, cmd, timeout);
 
   if (!quiet) console.log(chalk.dim("  Configuring VM..."));
 
@@ -72,6 +72,68 @@ export async function configureVm(execa, ip, user, publicUrl, { githubToken, k3s
   ].join("\n");
   await ssh(setupBatch);
 
+  // Verify Docker is installed — if missing, install it before anything else
+  const { exitCode: dockerCheck } = await ssh("sudo docker info >/dev/null 2>&1");
+  if (dockerCheck !== 0) {
+    if (!quiet) console.log(chalk.yellow("  ⚠ Docker not found — installing..."));
+    // Repo setup is idempotent (tolerates partial prior attempts); install+start uses &&
+    const repoSetup = [
+      "export DEBIAN_FRONTEND=noninteractive",
+      "while fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; do sleep 3; done",
+      "sudo install -m 0755 -d /etc/apt/keyrings",
+      "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg",
+      "sudo chmod a+r /etc/apt/keyrings/docker.gpg",
+      `echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null`,
+    ].join("; ");
+    const installAndStart = [
+      "sudo apt-get update -qq",
+      "sudo apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin",
+      "sudo systemctl enable docker && sudo systemctl start docker",
+      `sudo usermod -aG docker ${user}`,
+    ].join(" && ");
+    const { exitCode: installExit } = await ssh(`${repoSetup}; ${installAndStart}`, 300000);
+    if (installExit === 0) {
+      if (!quiet) console.log(chalk.green("  ✓ Docker installed"));
+    } else {
+      console.log(chalk.red("  ✗ Docker installation failed — container operations will not work"));
+      console.log(chalk.dim(`    SSH in and check: ssh ${user}@${ip} "sudo apt-get install -y docker-ce"`));
+    }
+  }
+
+  // Verify Node.js + fops CLI are installed — if missing, install them
+  const { stdout: fopsWhich } = await ssh("command -v fops 2>/dev/null || echo MISSING");
+  if (!fopsWhich?.trim() || fopsWhich.includes("MISSING")) {
+    if (!quiet) console.log(chalk.yellow("  ⚠ fops CLI not found — installing Node.js + fops..."));
+    const installNode = [
+      "export DEBIAN_FRONTEND=noninteractive",
+      "if ! command -v node >/dev/null 2>&1; then curl -fsSL https://deb.nodesource.com/setup_20.x | sudo bash - && sudo apt-get install -y -qq nodejs; fi",
+    ].join("; ");
+    await ssh(installNode, 120000);
+    // Install fops globally, retry with sudo if needed
+    let fopsInstalled = false;
+    const { exitCode: userInstall } = await ssh("npm install -g @meshxdata/fops@latest 2>&1", 300000);
+    if (userInstall === 0) {
+      fopsInstalled = true;
+    } else {
+      const { exitCode: sudoInstall } = await ssh(
+        "sudo bash -c 'D=\"$(npm root -g)/@meshxdata\"; rm -rf \"$D\" 2>/dev/null; npm install -g @meshxdata/fops@latest' 2>&1",
+        300000,
+      );
+      fopsInstalled = sudoInstall === 0;
+    }
+    // Ensure fops is on PATH
+    await ssh(
+      'MJS="$(npm root -g 2>/dev/null)/@meshxdata/fops/fops.mjs"; [ -f "$MJS" ] || MJS="$(sudo npm root -g 2>/dev/null)/@meshxdata/fops/fops.mjs"; [ -f "$MJS" ] && sudo ln -sf "$MJS" /usr/local/bin/fops; true',
+      15000,
+    );
+    if (fopsInstalled) {
+      if (!quiet) console.log(chalk.green("  ✓ fops CLI installed"));
+    } else {
+      console.log(chalk.red("  ✗ fops CLI installation failed"));
+      console.log(chalk.dim(`    SSH in and check: ssh ${user}@${ip} "sudo npm install -g @meshxdata/fops@latest"`));
+    }
+  }
+
   let ghcrOk = false;
   if (githubToken) {
     if (!quiet) console.log(chalk.dim("  Configuring GitHub/GHCR credentials..."));
@@ -458,9 +520,12 @@ async function vmReconcileNetworking(ctx) {
       console.log(chalk.yellow("  ⚠ No NSG attached to NIC"));
     }
 
+    // Accelerated networking — only supported on D/E/F/M series (2+ vCPU), not B-series
+    const vmSize = (iv.hardwareProfile?.vmSize || "").toLowerCase();
+    const supportsAccelNet = !vmSize.startsWith("standard_b") && !vmSize.startsWith("standard_a");
     if (nic.enableAcceleratedNetworking) {
       reconcileOk("Accelerated networking", "enabled");
-    } else {
+    } else if (supportsAccelNet) {
       console.log(chalk.yellow("  ↻ Accelerated networking not enabled — enabling…"));
       const { exitCode: anCode } = await execa("az", [
         "network", "nic", "update", "-g", rg, "-n", ctx.nicName,
@@ -469,8 +534,9 @@ async function vmReconcileNetworking(ctx) {
       ], { reject: false, timeout: 30000 });
       console.log(anCode === 0
         ? chalk.green(`  ✓ ${"Accelerated networking".padEnd(RECONCILE_LABEL_WIDTH)} — enabled`)
-        : chalk.yellow("  ⚠ Could not enable accelerated networking (VM size may not support it)"));
+        : chalk.yellow("  ⚠ Could not enable accelerated networking"));
     }
+    // B-series and A-series VMs don't support accelerated networking — skip silently
   }
 
   if (!ctx.ip) ctx.ip = await resolvePublicIp(execa, rg, vmName);
@@ -832,10 +898,16 @@ async function vmReconcileSecurity(ctx) {
     reconcileOk("Boot diagnostics", "enabled");
   } else {
     console.log(chalk.yellow("  ↻ Boot diagnostics not enabled — enabling..."));
-    const { exitCode: bdCode } = await execa("az", [
-      "vm", "boot-diagnostics", "enable", "-g", rg, "-n", vmName, "--output", "none",
-      ...subArgs(sub),
-    ], { reject: false, timeout: 30000 });
+    let bdCode = 1;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const res = await execa("az", [
+        "vm", "boot-diagnostics", "enable", "-g", rg, "-n", vmName, "--output", "none",
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      bdCode = res.exitCode;
+      if (bdCode === 0) break;
+      if (attempt < 2) await new Promise((r) => setTimeout(r, 5000));
+    }
     if (bdCode === 0) {
       await new Promise((r) => setTimeout(r, 2000));
       const { stdout: bdJson } = await execa("az", [
@@ -880,20 +952,25 @@ async function vmReconcileSecurity(ctx) {
     console.log(amCode === 0
       ? chalk.green(`  ✓ ${"Antimalware extension".padEnd(RECONCILE_LABEL_WIDTH)} — installed`)
       : chalk.yellow("  ⚠ Could not install antimalware extension"));
-  } else {
-    console.log(chalk.dim(`  Antimalware extension — Windows only (skipped on Linux)`));
   }
+  // Linux VMs don't need antimalware — skip silently
 
   if (isTrustedLaunch) {
     if (hasGuestAttestation) {
       reconcileOk("Guest Attestation extension", "installed");
     } else {
       console.log(chalk.yellow("  ↻ Guest Attestation missing — installing…"));
-      const { exitCode: gaCode } = await execa("az", [
-        "vm", "extension", "set", "-g", rg, "--vm-name", vmName,
-        "-n", "GuestAttestation", "--publisher", "Microsoft.Azure.Security.LinuxAttestation",
-        "--output", "none", ...subArgs(sub),
-      ], { reject: false, timeout: 120000 });
+      let gaCode = 1;
+      for (let attempt = 0; attempt < 3; attempt++) {
+        const res = await execa("az", [
+          "vm", "extension", "set", "-g", rg, "--vm-name", vmName,
+          "-n", "GuestAttestation", "--publisher", "Microsoft.Azure.Security.LinuxAttestation",
+          "--output", "none", ...subArgs(sub),
+        ], { reject: false, timeout: 120000 });
+        gaCode = res.exitCode;
+        if (gaCode === 0) break;
+        if (attempt < 2) await new Promise((r) => setTimeout(r, 10000));
+      }
       console.log(gaCode === 0
         ? chalk.green(`  ✓ ${"Guest Attestation extension".padEnd(RECONCILE_LABEL_WIDTH)} — installed`)
         : chalk.yellow("  ⚠ Could not install Guest Attestation extension"));

package/src/plugins/bundled/fops-plugin-azure/lib/azure-service.js

@@ -0,0 +1,507 @@
+/**
+ * AzureService — structured JSON API surface for cross-plugin consumption.
+ * Wraps existing azure lib functions to return clean data (no console.log side effects).
+ */
+
+import fs from "node:fs";
+import nodePath from "node:path";
+import os from "node:os";
+import { listVms, readVmState } from "./azure-state.js";
+import { readAksClusters, readClusterState } from "./azure-aks-state.js";
+import { lazyExeca, resolveCliSrc } from "./azure-helpers.js";
+import { readCache } from "./azure-sync.js";
+
+// ── Persistent cost cache ────────────────────────────────────────────
+const COST_CACHE_DIR = nodePath.join(os.homedir(), ".fops", "costs");
+const COST_CACHE_PATH = nodePath.join(COST_CACHE_DIR, "cache.json");
+const COST_CACHE_TTL = 4 * 60 * 60 * 1000; // 4 hours
+
+function readCostCache(days) {
+  try {
+    const raw = JSON.parse(fs.readFileSync(COST_CACHE_PATH, "utf8"));
+    if (raw.days === days && raw.cachedAt && Date.now() - raw.cachedAt < COST_CACHE_TTL) {
+      return raw;
+    }
+  } catch {}
+  return null;
+}
+
+function writeCostCache(data) {
+  try {
+    fs.mkdirSync(COST_CACHE_DIR, { recursive: true });
+    fs.writeFileSync(COST_CACHE_PATH, JSON.stringify({ ...data, cachedAt: Date.now() }));
+  } catch {}
+}
+
+export class AzureService {
+  /**
+   * List all tracked VMs with their state.
+   * @returns {{ activeVm?: string, vms: object[] }}
+   */
+  listVms() {
+    const { activeVm, vms } = listVms();
+    const list = Object.entries(vms || {}).map(([name, vm]) => ({
+      name,
+      resourceGroup: vm.resourceGroup,
+      location: vm.location,
+      publicIp: vm.publicIp,
+      publicUrl: vm.publicUrl,
+      subscriptionId: vm.subscriptionId,
+      active: name === activeVm,
+      createdAt: vm.createdAt || vm.discoveredAt || null,
+    }));
+    return { activeVm, vms: list };
+  }
+
+  /**
+   * List all tracked AKS clusters.
+   * @returns {{ activeCluster?: string, clusters: object[] }}
+   */
+  listClusters() {
+    const { activeCluster, clusters } = readAksClusters();
+    const cache = readCache();
+    const cachedClusters = cache?.clusters || {};
+
+    const list = Object.entries(clusters || {}).map(([name, c]) => {
+      const cached = cachedClusters[name] || {};
+      return {
+        name,
+        resourceGroup: c.resourceGroup,
+        location: c.location,
+        kubernetesVersion: cached.kubernetesVersion || c.kubernetesVersion || null,
+        fqdn: cached.fqdn || c.fqdn || null,
+        active: name === activeCluster,
+        // HA pairing (from state or inferred by sync)
+        isStandby: cached.isStandby || c.isStandby || false,
+        primaryCluster: cached.primaryCluster || c.primaryCluster || null,
+        ha: cached.ha || c.ha || null,
+        // Storage (from state or discovered by sync)
+        storageHA: cached.storageHA || c.storageHA || null,
+        storageAccount: cached.storageAccount || null,
+        // Key Vault (from state or discovered by sync)
+        vault: cached.vault || (c.vault ? {
+          keyVaultName: c.vault.keyVaultName || null,
+          autoUnseal: c.vault.autoUnseal || false,
+          initialized: c.vault.initialized || false,
+        } : null),
+        // Postgres (from state or discovered by sync)
+        postgres: cached.postgres || (c.postgres ? { serverName: c.postgres.serverName, fqdn: c.postgres.fqdn } : null),
+        // Extras
+        domain: c.domain || null,
+        ingressIp: c.ingressIp || null,
+        nodeCount: cached.nodes || c.nodeCount || null,
+        nodeVmSize: cached.sizes || c.nodeVmSize || null,
+      };
+    });
+    return { activeCluster, clusters: list };
+  }
+
+  /**
+   * Get detailed info for a single VM.
+   * @param {string} name
+   * @returns {object|null}
+   */
+  getVmDetail(name) {
+    const vm = readVmState(name);
+    if (!vm) return null;
+    return { ...vm, type: "vm" };
+  }
+
+  /**
+   * Get detailed info for a single AKS cluster.
+   * @param {string} name
+   * @returns {object|null}
+   */
+  getClusterDetail(name) {
+    const c = readClusterState(name);
+    if (!c) return null;
+    return { ...c, type: "cluster" };
+  }
+
+  /**
+   * Start a deallocated VM.
+   * @param {string} name
+   */
+  async startVm(name) {
+    const { azureStart } = await import("./azure-vm-lifecycle.js");
+    await azureStart({ vmName: name });
+    return { ok: true, action: "start", vm: name };
+  }
+
+  /**
+   * Stop (deallocate) a VM.
+   * @param {string} name
+   */
+  async stopVm(name) {
+    const { azureStop } = await import("./azure-vm-lifecycle.js");
+    await azureStop({ vmName: name });
+    return { ok: true, action: "stop", vm: name };
+  }
+
+  /**
+   * Provision a new VM.
+   * @param {object} opts - Same options as azureUp
+   */
+  async provisionVm(opts) {
+    const { azureUp } = await import("./azure-vm-lifecycle.js");
+    const result = await azureUp(opts);
+    return { ok: true, action: "provision", ...result };
+  }
+
+  /**
+   * Destroy a VM.
+   * @param {string} name
+   */
+  async deleteVm(name) {
+    const { azureDown } = await import("./azure-vm-lifecycle.js");
+    await azureDown({ vmName: name });
+    return { ok: true, action: "delete", vm: name };
+  }
+
+  /**
+   * Resize a VM.
+   * @param {string} name
+   * @param {object} opts
+   */
+  async resizeVm(name, opts = {}) {
+    const { azureResize } = await import("./azure-vm-lifecycle.js");
+    await azureResize({ vmName: name, ...opts });
+    return { ok: true, action: "resize", vm: name };
+  }
+
+  /**
+   * Provision a new AKS cluster.
+   * @param {object} opts - Same options as aksUp
+   */
+  async provisionCluster(opts) {
+    const { aksUp } = await import("./azure-aks-core.js");
+    const result = await aksUp(opts);
+    return { ok: true, action: "provision", ...result };
+  }
+
+  /**
+   * Destroy an AKS cluster.
+   * @param {string} name
+   */
+  async deleteCluster(name) {
+    const { aksDown } = await import("./azure-aks-core.js");
+    await aksDown({ clusterName: name });
+    return { ok: true, action: "delete", cluster: name };
+  }
+
+  /**
+   * Sync/discover resources from Azure.
+   */
+  async syncResources(opts = {}) {
+    const { azureSync } = await import("./azure-sync.js");
+    const result = await azureSync({ quiet: true, ...opts });
+    return result;
+  }
+
+  /**
+   * List feature flags for a VM.
+   * Uses sync cache if available, falls back to SSH.
+   * @param {string} name - VM name
+   * @returns {{ flags: { name, label, value, services }[] }}
+   */
+  async listFeatureFlags(name) {
+    const { KNOWN_FLAGS } = await import(resolveCliSrc("feature-flags.js"));
+
+    // Try cache first (populated by sync)
+    const cache = readCache();
+    const cached = cache?.vms?.[name];
+    if (cached?.flags) {
+      const flags = [];
+      const seen = new Set();
+      for (const [flagName, value] of Object.entries(cached.flags)) {
+        seen.add(flagName);
+        flags.push({ name: flagName, label: KNOWN_FLAGS[flagName] || flagName, value, services: [] });
+      }
+      for (const [flagName, label] of Object.entries(KNOWN_FLAGS)) {
+        if (!seen.has(flagName)) flags.push({ name: flagName, label, value: false, services: [] });
+      }
+      return { flags: flags.sort((a, b) => a.name.localeCompare(b.name)), fromCache: true };
+    }
+
+    // Fallback: SSH into VM
+    const vm = readVmState(name);
+    if (!vm?.publicIp) throw new Error(`VM "${name}" not found or has no IP`);
+    const execa = await lazyExeca();
+    const { knockForVm, sshCmd, DEFAULTS } = await import("./azure.js");
+    await knockForVm(vm);
+    const { stdout, exitCode } = await sshCmd(execa, vm.publicIp, DEFAULTS.adminUser,
+      "cat /opt/foundation-compose/docker-compose.yaml", 30000);
+    if (exitCode !== 0 || !stdout?.trim()) throw new Error("Could not read docker-compose.yaml from VM");
+
+    const { parseComposeFlagsFromContent } = await import(resolveCliSrc("feature-flags.js"));
+    const composeFlags = parseComposeFlagsFromContent(stdout);
+
+    const flags = [];
+    const seen = new Set();
+    for (const [flagName, info] of Object.entries(composeFlags)) {
+      seen.add(flagName);
+      flags.push({ name: flagName, label: KNOWN_FLAGS[flagName] || flagName, value: info.value, services: [...info.services] });
+    }
+    for (const [flagName, label] of Object.entries(KNOWN_FLAGS)) {
+      if (!seen.has(flagName)) flags.push({ name: flagName, label, value: false, services: [] });
+    }
+    return { flags: flags.sort((a, b) => a.name.localeCompare(b.name)), fromCache: false };
+  }
+
+  /**
+   * Set feature flags on a VM and restart affected services.
+   * @param {string} name - VM name
+   * @param {object} flagValues - { MX_FF_NAME: true/false, ... }
+   */
+  async setFeatureFlags(name, flagValues) {
+    const vm = readVmState(name);
+    if (!vm?.publicIp) throw new Error(`VM "${name}" not found or has no IP`);
+    const execa = await lazyExeca();
+    const { knockForVm, sshCmd, DEFAULTS } = await import("./azure.js");
+    await knockForVm(vm);
+    const ssh = (cmd, timeout = 30000) => sshCmd(execa, vm.publicIp, DEFAULTS.adminUser, cmd, timeout);
+
+    const { stdout: composeContent, exitCode } = await ssh("cat /opt/foundation-compose/docker-compose.yaml");
+    if (exitCode !== 0) throw new Error("Could not read docker-compose.yaml");
+
+    const { KNOWN_FLAGS, parseComposeFlagsFromContent, applyComposeFlagChanges } = await import(resolveCliSrc("feature-flags.js"));
+    const composeFlags = parseComposeFlagsFromContent(composeContent);
+
+    const changes = [];
+    const affectedServices = new Set();
+    for (const [flagName, newValue] of Object.entries(flagValues)) {
+      const flag = composeFlags[flagName];
+      if (!flag) continue;
+      if (flag.value !== newValue) {
+        for (const line of flag.lines) changes.push({ lineNum: line.lineNum, newValue: String(newValue) });
+        for (const svc of flag.services) affectedServices.add(svc);
+      }
+    }
+
+    if (changes.length === 0) return { ok: true, changed: 0, restarted: [] };
+
+    const updatedContent = applyComposeFlagChanges(composeContent, changes);
+    const b64 = Buffer.from(updatedContent).toString("base64");
+    const { exitCode: writeCode } = await ssh(`echo '${b64}' | base64 -d > /opt/foundation-compose/docker-compose.yaml`);
+    if (writeCode !== 0) throw new Error("Failed to write docker-compose.yaml on VM");
+
+    console.log(`  ✓ Updated ${changes.length} flag value(s) on ${name}`);
+
+    const serviceList = [...affectedServices];
+    if (serviceList.length > 0) {
+      console.log(`  Restarting: ${serviceList.join(", ")}`);
+      await ssh(`cd /opt/foundation-compose && docker compose up -d --remove-orphans ${serviceList.join(" ")}`, 120000);
+      console.log(`  ✓ Services restarted`);
+    }
+
+    return { ok: true, changed: changes.length, restarted: serviceList };
+  }
+
+  /**
+   * Deploy stack to a VM (pull + restart).
+   * @param {string} name - VM name
+   * @param {object} opts
+   */
+  async deployStack(name, opts = {}) {
+    const { azureDeploy } = await import("./azure-ops.js");
+    await azureDeploy({ vmName: name, ...opts });
+    return { ok: true, action: "deploy", vm: name };
+  }
+
+  /**
+   * Get cached sync data.
+   * @returns {object|null}
+   */
+  getCachedStatus() {
+    return readCache();
+  }
+
+  /**
+   * Get fleet overview from sync cache.
+   * @returns {object}
+   */
+  getFleet() {
+    const cache = readCache();
+    if (!cache?.vms) return { vms: {}, clusters: {}, updatedAt: null };
+    return {
+      vms: cache.vms || {},
+      clusters: cache.clusters || {},
+      updatedAt: cache.updatedAt || null,
+    };
+  }
+
+  /**
+   * Get cost data for tracked VMs and clusters.
+   * @param {object} opts
+   * @param {number} opts.days - Number of days to look back (default: 30)
+   * @returns {{ vmCosts, clusterCosts, currency, days, error? }}
+   */
+  async getCosts(opts = {}) {
+    const days = opts.days || 30;
+    const execa = await lazyExeca();
+    const { ensureAzAuth, ensureAzCli } = await import("./azure.js");
+    await ensureAzCli(execa);
+
+    const { vms } = this.listVms();
+    const vmNames = vms.map((v) => v.name);
+
+    const { clusters } = this.listClusters();
+    const cache = readCache();
+    const cachedClusters = cache?.clusters || {};
+    const clusterRgs = clusters.map((c) => {
+      const cached = cachedClusters[c.name] || {};
+      const rg = c.resourceGroup || cached.resourceGroup;
+      const loc = c.location || cached.location;
+      return {
+        name: c.name,
+        rgs: [rg, `mc_${rg}_${c.name}_${loc}`].filter(Boolean),
+      };
+    });
+
+    const end = new Date();
+    const start = new Date();
+    start.setDate(start.getDate() - days);
+    const startDate = start.toISOString().split("T")[0];
+    const endDate = end.toISOString().split("T")[0];
+    const timePeriod = { from: startDate, to: endDate };
+    const baseDataset = {
+      granularity: "None",
+      aggregation: { totalCost: { name: "Cost", function: "Sum" } },
+    };
+
+    try {
+      const account = await ensureAzAuth(execa);
+      const subId = account.id;
+
+      const costQuery = async (body) => {
+        const { stdout } = await execa("az", [
+          "rest", "--method", "POST",
+          "--url", `https://management.azure.com/subscriptions/${subId}/providers/Microsoft.CostManagement/query?api-version=2023-11-01`,
+          "--body", JSON.stringify(body),
+          "--output", "json",
+        ], { timeout: 90000 });
+        const result = JSON.parse(stdout);
+        return result.rows || result.properties?.rows || [];
+      };
+
+      const vmCosts = {};
+      const clusterCosts = {};
+      let currency = "USD";
+
+      if (vmNames.length > 0) {
+        const rows = await costQuery({
+          type: "ActualCost", timeframe: "Custom", timePeriod,
+          dataset: {
+            ...baseDataset,
+            grouping: [{ type: "Dimension", name: "ResourceId" }],
+            filter: { dimensions: { name: "ResourceType", operator: "In", values: [
+              "microsoft.compute/virtualmachines", "microsoft.compute/disks",
+              "microsoft.network/publicipaddresses", "microsoft.network/networkinterfaces",
+              "microsoft.network/networksecuritygroups",
+            ]}},
+          },
+        });
+
+        const lowerNames = vmNames.map((n) => n.toLowerCase());
+        for (const row of rows) {
+          const amount = parseFloat(row[0]) || 0;
+          const resourceId = row[1] || "";
+          if (row[2]) currency = row[2];
+          if (amount < 0.001) continue;
+          const resourceName = resourceId.split("/").pop().toLowerCase();
+          const owner = lowerNames.find((n) => resourceName === n || resourceName.startsWith(n + "_") || resourceName.startsWith(n + "publicip") || resourceName === n + "-nsg" || resourceName === n + "-nic");
+          if (owner) vmCosts[owner] = (vmCosts[owner] || 0) + amount;
+        }
+      }
+
+      if (clusterRgs.length > 0) {
+        const rows = await costQuery({
+          type: "ActualCost", timeframe: "Custom", timePeriod,
+          dataset: {
+            ...baseDataset,
+            grouping: [{ type: "Dimension", name: "ResourceGroupName" }],
+          },
+        });
+
+        const lowerRgs = new Map(clusterRgs.map(({ name, rgs }) => [name, rgs.map((r) => r.toLowerCase())]));
+        for (const row of rows) {
+          const amount = parseFloat(row[0]) || 0;
+          const rg = (row[1] || "").toLowerCase();
+          if (row[2]) currency = row[2];
+          if (amount < 0.001 || !rg) continue;
+          for (const [clusterName, rgList] of lowerRgs) {
+            if (rgList.includes(rg)) clusterCosts[clusterName] = (clusterCosts[clusterName] || 0) + amount;
+          }
+        }
+      }
+
+      // Also get cost by service type for summary
+      const serviceRows = await costQuery({
+        type: "ActualCost", timeframe: "Custom", timePeriod,
+        dataset: {
+          ...baseDataset,
+          grouping: [{ type: "Dimension", name: "ServiceName" }],
+        },
+      });
+
+      const byService = {};
+      for (const row of serviceRows) {
+        const amount = parseFloat(row[0]) || 0;
+        const svc = row[1] || "Other";
+        if (row[2]) currency = row[2];
+        if (amount > 0.01) byService[svc] = amount;
+      }
+
+      const result = { vmCosts, clusterCosts, byService, currency, days };
+      writeCostCache(result);
+      return result;
+    } catch (e) {
+      // Serve from cache on failure (429, network errors, etc.)
+      const cached = readCostCache(days);
+      if (cached) {
+        return { ...cached, fromCache: true };
+      }
+      return { vmCosts: {}, clusterCosts: {}, byService: {}, currency: "USD", days, error: e.message };
+    }
+  }
+
+  /**
+   * Grant Foundation Admin role to users on a VM.
+   * @param {string} name - VM name
+   * @param {object} opts - { username?, auth0Sub? }
+   */
+  async grantAdmin(name, opts = {}) {
+    const { azureGrantAdmin } = await import("./azure-ops.js");
+    await azureGrantAdmin({ vmName: name, ...opts });
+    return { ok: true, action: "grant-admin", vm: name };
+  }
+
+  /**
+   * Get live VM status from Azure.
+   * @param {string} name
+   */
+  async getVmLiveStatus(name) {
+    const vm = readVmState(name);
+    if (!vm) return null;
+    try {
+      const execa = await lazyExeca();
+      const rg = vm.resourceGroup;
+      const sub = vm.subscriptionId;
+      const args = ["vm", "show", "--name", name, "--resource-group", rg, "--show-details", "-o", "json"];
+      if (sub) args.push("--subscription", sub);
+      const { stdout, exitCode } = await execa("az", args, { timeout: 30000, reject: false });
+      if (exitCode !== 0) return { name, status: "unknown", error: "az vm show failed" };
+      const data = JSON.parse(stdout);
+      return {
+        name,
+        powerState: data.powerState || null,
+        provisioningState: data.provisioningState || null,
+        vmSize: data.hardwareProfile?.vmSize || null,
+        osType: data.storageProfile?.osDisk?.osType || null,
+        location: data.location || null,
+      };
+    } catch (e) {
+      return { name, status: "error", error: e.message };
+    }
+  }
+}