@meshxdata/fops 0.1.36 → 0.1.37

package/src/plugins/bundled/fops-plugin-azure/lib/azure-auth.js

@@ -0,0 +1,454 @@
+/**
+ * Auth chain and landscape-fetch utilities for the Azure plugin.
+ * Extracted from index.js to keep the top-level plugin file small.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import pathMod from "node:path";
+import chalk from "chalk";
+
+export function hashContent(text) {
+  return crypto.createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+
+/**
+ * Resolve Foundation credentials from env → .env → ~/.fops.json.
+ * Returns { bearerToken } or { user, password } or null.
+ */
+export function resolveFoundationCreds() {
+  if (process.env.BEARER_TOKEN?.trim()) return { bearerToken: process.env.BEARER_TOKEN.trim() };
+  if (process.env.QA_USERNAME?.trim() && process.env.QA_PASSWORD)
+    return { user: process.env.QA_USERNAME.trim(), password: process.env.QA_PASSWORD };
+  try {
+    const raw = JSON.parse(fs.readFileSync(pathMod.join(os.homedir(), ".fops.json"), "utf8"));
+    const cfg = raw?.plugins?.entries?.["fops-plugin-foundation"]?.config || {};
+    if (cfg.bearerToken?.trim()) return { bearerToken: cfg.bearerToken.trim() };
+    if (cfg.user?.trim() && cfg.password) return { user: cfg.user.trim(), password: cfg.password };
+  } catch { /* no fops.json */ }
+  return null;
+}
+
+// VMs use Traefik with self-signed certs — skip TLS verification for VM API calls.
+let _tlsWarningSuppressed = false;
+export function suppressTlsWarning() {
+  if (_tlsWarningSuppressed) return;
+  _tlsWarningSuppressed = true;
+  const origEmit = process.emitWarning;
+  process.emitWarning = (warning, ...args) => {
+    if (typeof warning === "string" && warning.includes("NODE_TLS_REJECT_UNAUTHORIZED")) return;
+    return origEmit.call(process, warning, ...args);
+  };
+}
+
+export async function vmFetch(url, opts = {}) {
+  suppressTlsWarning();
+  const prev = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+  process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+  try {
+    return await fetch(url, { signal: AbortSignal.timeout(10_000), ...opts });
+  } finally {
+    if (prev === undefined) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+    else process.env.NODE_TLS_REJECT_UNAUTHORIZED = prev;
+  }
+}
+
+/**
+ * Resolve Auth0 config from ~/.fops.json → .env files.
+ * Returns { domain, clientId, clientSecret, audience } or null.
+ */
+export function resolveAuth0Config() {
+  try {
+    const raw = JSON.parse(fs.readFileSync(pathMod.join(os.homedir(), ".fops.json"), "utf8"));
+    const a0 = raw?.plugins?.entries?.["fops-plugin-foundation"]?.config?.auth0;
+    if (a0?.domain && a0?.clientId) return a0;
+
+    const projectRoot = raw?.projectRoot || "";
+    const envCandidates = [
+      ...(projectRoot ? [pathMod.join(projectRoot, ".env")] : []),
+      pathMod.resolve(".env"),
+      pathMod.resolve("..", ".env"),
+    ];
+    for (const ep of envCandidates) {
+      try {
+        const lines = fs.readFileSync(ep, "utf8").split("\n");
+        const get = (k) => {
+          const ln = lines.find((l) => l.startsWith(`${k}=`));
+          return ln ? ln.slice(k.length + 1).trim().replace(/^["']|["']$/g, "") : "";
+        };
+        const domain = get("MX_AUTH0_DOMAIN") || get("AUTH0_DOMAIN");
+        const clientId = get("MX_AUTH0_CLIENT_ID") || get("AUTH0_CLIENT_ID");
+        const clientSecret = get("MX_AUTH0_CLIENT_SECRET") || get("AUTH0_CLIENT_SECRET");
+        const audience = get("MX_AUTH0_AUDIENCE") || get("AUTH0_AUDIENCE");
+        if (domain && clientId) return { domain, clientId, clientSecret, audience };
+      } catch { /* try next */ }
+    }
+  } catch { /* no fops.json */ }
+  return null;
+}
+
+/**
+ * Authenticate against a remote VM's Foundation API and return a bearer token.
+ * Tries the backend /iam/login first, then falls back to Auth0 ROPC.
+ */
+export async function authenticateVm(vmUrl, ip, creds) {
+  if (creds.bearerToken) return creds.bearerToken;
+
+  const hasDomain = vmUrl && !vmUrl.match(/^https?:\/\/\d+\.\d+\.\d+\.\d+/);
+  const apiUrls = hasDomain
+    ? [`${vmUrl}/api`, ...(ip ? [`http://${ip}:9001/api`] : [])]
+    : [`${vmUrl}/api`, `https://${ip}:3002/api`, `http://${ip}:9001/api`];
+
+  for (const apiBase of apiUrls) {
+    try {
+      const resp = await vmFetch(`${apiBase}/iam/login`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ user: creds.user, password: creds.password }),
+      });
+      if (resp.ok) {
+        const data = await resp.json();
+        const token = data.access_token || data.token;
+        if (token) return token;
+      }
+    } catch { /* try next URL */ }
+  }
+
+  // Auth0 ROPC fallback
+  const auth0 = resolveAuth0Config();
+  if (auth0 && creds.user && creds.password) {
+    try {
+      const body = {
+        grant_type: "password",
+        client_id: auth0.clientId,
+        username: creds.user,
+        password: creds.password,
+        scope: "openid",
+      };
+      if (auth0.clientSecret) body.client_secret = auth0.clientSecret;
+      if (auth0.audience) body.audience = auth0.audience;
+
+      const resp = await fetch(`https://${auth0.domain}/oauth/token`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(10_000),
+      });
+      if (resp.ok) {
+        const data = await resp.json();
+        if (data.access_token) return data.access_token;
+      }
+    } catch { /* auth0 fallback failed */ }
+  }
+
+  return null;
+}
+
+export function isJwt(token) {
+  return token && token.split(".").length === 3;
+}
+
+/**
+ * Resolve a valid JWT bearer token for a remote VM/cluster.
+ * Auth chain: local bearer → pre-auth /iam/login → Auth0 ROPC → SSH fetch from VM.
+ * Returns { bearerToken, user, password, useTokenMode } or throws.
+ */
+export async function resolveRemoteAuth(opts = {}) {
+  const {
+    apiUrl, ip, vmState, execaFn: execa, sshCmd: ssh,
+    knockForVm: knock, suppressTlsWarning: suppressTls,
+  } = opts;
+  const log = opts.log || console.log;
+
+  const creds = resolveFoundationCreds();
+  let qaUser = creds?.user || process.env.QA_USERNAME || process.env.FOUNDATION_USERNAME || "operator@local";
+  let qaPass = creds?.password || process.env.QA_PASSWORD || "";
+  let bearerToken = creds?.bearerToken || "";
+
+  // 1) Use local bearer if it's a valid JWT
+  if (bearerToken && isJwt(bearerToken)) {
+    return { bearerToken, qaUser, qaPass, useTokenMode: true };
+  }
+  bearerToken = "";
+
+  // 2) Pre-auth against the backend /iam/login
+  if (qaUser && qaPass && apiUrl) {
+    try {
+      if (suppressTls) suppressTls();
+      const prev = process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+      try {
+        const resp = await fetch(`${apiUrl}/iam/login`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ username: qaUser, password: qaPass }),
+          signal: AbortSignal.timeout(10_000),
+        });
+        if (resp.ok) {
+          const data = await resp.json();
+          bearerToken = data.access_token || data.token || "";
+          if (bearerToken) {
+            log(chalk.green(`  ✓ Authenticated as ${qaUser}`));
+            return { bearerToken, qaUser, qaPass, useTokenMode: true };
+          }
+        } else {
+          log(chalk.dim(`  Local creds rejected: HTTP ${resp.status}`));
+        }
+      } finally {
+        if (prev === undefined) delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+        else process.env.NODE_TLS_REJECT_UNAUTHORIZED = prev;
+      }
+    } catch (e) {
+      log(chalk.dim(`  Pre-auth failed: ${e.cause?.code || e.message}`));
+    }
+  }
+
+  // 3) Auth0 ROPC with local config
+  if (qaUser && qaPass) {
+    const auth0Cfg = resolveAuth0Config();
+    if (auth0Cfg) {
+      try {
+        log(chalk.dim(`  Trying Auth0 ROPC (${auth0Cfg.domain})…`));
+        const body = {
+          grant_type: "password", client_id: auth0Cfg.clientId,
+          username: qaUser, password: qaPass, scope: "openid",
+        };
+        if (auth0Cfg.clientSecret) body.client_secret = auth0Cfg.clientSecret;
+        if (auth0Cfg.audience) body.audience = auth0Cfg.audience;
+        const resp = await fetch(`https://${auth0Cfg.domain}/oauth/token`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(body),
+          signal: AbortSignal.timeout(10_000),
+        });
+        if (resp.ok) {
+          const data = await resp.json();
+          if (data.access_token) {
+            log(chalk.green(`  ✓ Authenticated as ${qaUser} via Auth0`));
+            return { bearerToken: data.access_token, qaUser, qaPass, useTokenMode: true };
+          }
+        } else {
+          log(chalk.dim(`  Auth0 rejected: HTTP ${resp.status}`));
+        }
+      } catch (e) {
+        log(chalk.dim(`  Auth0 failed: ${e.message}`));
+      }
+    }
+  }
+
+  // 4) SSH fallback — fetch credentials from VM and authenticate
+  if (ip && ssh && execa) {
+    if (knock && vmState) await knock(vmState);
+    log(chalk.dim("  Fetching credentials from remote VM…"));
+    try {
+      const sshUser = vmState?.adminUser || "azureuser";
+      const { stdout } = await ssh(
+        execa, ip, sshUser,
+        "grep -E '^(BEARER_TOKEN|QA_USERNAME|QA_PASSWORD|MX_AUTH0_DOMAIN|MX_AUTH0_CLIENT_ID|MX_AUTH0_CLIENT_SECRET|MX_AUTH0_AUDIENCE)=' /opt/foundation-compose/.env | head -20",
+        15_000,
+      );
+      const remoteEnv = {};
+      for (const line of (stdout || "").split("\n")) {
+        const m = line.match(/^([A-Z_]+)=(.*)$/);
+        if (m) remoteEnv[m[1]] = m[2].trim().replace(/^["']|["']$/g, "");
+      }
+
+      const remoteToken = remoteEnv.BEARER_TOKEN;
+      if (remoteToken && isJwt(remoteToken)) {
+        log(chalk.green("  ✓ Got JWT bearer token from VM"));
+        return { bearerToken: remoteToken, qaUser, qaPass, useTokenMode: true };
+      }
+
+      if (remoteEnv.MX_AUTH0_DOMAIN && remoteEnv.MX_AUTH0_CLIENT_ID) {
+        const user = remoteEnv.QA_USERNAME || qaUser;
+        const pass = remoteEnv.QA_PASSWORD || qaPass;
+        if (user && pass) {
+          log(chalk.dim(`  Authenticating as ${user} via VM's Auth0…`));
+          const body = {
+            grant_type: "password", client_id: remoteEnv.MX_AUTH0_CLIENT_ID,
+            username: user, password: pass, scope: "openid",
+          };
+          if (remoteEnv.MX_AUTH0_CLIENT_SECRET) body.client_secret = remoteEnv.MX_AUTH0_CLIENT_SECRET;
+          if (remoteEnv.MX_AUTH0_AUDIENCE) body.audience = remoteEnv.MX_AUTH0_AUDIENCE;
+          try {
+            const resp = await fetch(`https://${remoteEnv.MX_AUTH0_DOMAIN}/oauth/token`, {
+              method: "POST",
+              headers: { "Content-Type": "application/json" },
+              body: JSON.stringify(body),
+              signal: AbortSignal.timeout(10_000),
+            });
+            if (resp.ok) {
+              const data = await resp.json();
+              if (data.access_token) {
+                log(chalk.green(`  ✓ Authenticated via VM's Auth0`));
+                return { bearerToken: data.access_token, qaUser: user, qaPass: pass, useTokenMode: true };
+              }
+            }
+          } catch (e) {
+            log(chalk.dim(`  VM Auth0 login failed: ${e.message}`));
+          }
+        }
+      }
+
+      if (remoteToken) {
+        log(chalk.yellow("  ⚠ Using non-JWT token from VM (may cause 401)"));
+        return { bearerToken: remoteToken, qaUser, qaPass, useTokenMode: true };
+      }
+    } catch (e) {
+      log(chalk.dim(`  SSH credential fetch failed: ${e.message}`));
+    }
+  }
+
+  return { bearerToken: "", qaUser, qaPass, useTokenMode: false };
+}
+
+/**
+ * Fetch landscape entities from a remote VM's Foundation API.
+ * Returns embeddable chunks tagged with the VM name.
+ */
+export async function fetchRemoteLandscape(vmName, vmUrl, ip, token, log) {
+  const hasDomain = vmUrl && !vmUrl.match(/^https?:\/\/\d+\.\d+\.\d+\.\d+/);
+  const apiBase = hasDomain ? `${vmUrl}/api` : `http://${ip}:9001/api`;
+  const headers = { Authorization: `Bearer ${token}`, "x-org": "root" };
+
+  let authRejected = 0;
+  const fetchJson = async (path) => {
+    const resp = await vmFetch(`${apiBase}${path}`, { headers, signal: AbortSignal.timeout(15_000) });
+    if (resp.status === 401 || resp.status === 403) { authRejected++; return null; }
+    if (!resp.ok) return null;
+    return resp.json();
+  };
+
+  const [meshRes, dpRes, dsRes, dSysRes] = await Promise.all([
+    fetchJson("/data/mesh/list?per_page=100").catch(() => null),
+    fetchJson("/data/data_product/list?per_page=200").catch(() => null),
+    fetchJson("/data/data_source/list?per_page=200").catch(() => null),
+    fetchJson("/data/data_system/list?per_page=100").catch(() => null),
+  ]);
+
+  if (authRejected >= 4) {
+    const err = new Error("token rejected by all endpoints");
+    err.code = "AUTH_REJECTED";
+    throw err;
+  }
+
+  const { parseListResponse } = await import("../fops-plugin-foundation/lib/api-spec.js");
+  const chunks = [];
+
+  for (const m of parseListResponse(meshRes)) {
+    const text = [`Mesh: ${m.name} (VM: ${vmName})`, m.label ? `Label: ${m.label}` : null, m.description ? `Description: ${m.description}` : null, m.purpose ? `Purpose: ${m.purpose}` : null, `Identifier: ${m.identifier}`, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/mesh/${m.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "mesh", vm: vmName, identifier: m.identifier, fileHash: hashContent(text) } });
+  }
+  for (const p of parseListResponse(dpRes)) {
+    const text = [`Data Product: ${p.name} (VM: ${vmName})`, p.label ? `Label: ${p.label}` : null, p.description ? `Description: ${p.description}` : null, p.data_product_type ? `Type: ${p.data_product_type}` : null, `Identifier: ${p.identifier}`, p.host_mesh_identifier ? `Mesh: ${p.host_mesh_identifier}` : null, p.state?.code ? `State: ${p.state.code} (healthy: ${p.state.healthy})` : null, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/product/${p.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_product", vm: vmName, identifier: p.identifier, fileHash: hashContent(text) } });
+  }
+  for (const s of parseListResponse(dsRes)) {
+    const text = [`Data Source: ${s.name} (VM: ${vmName})`, s.label ? `Label: ${s.label}` : null, s.description ? `Description: ${s.description}` : null, `Identifier: ${s.identifier}`, s.state?.code ? `State: ${s.state.code} (healthy: ${s.state.healthy})` : null, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/source/${s.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_source", vm: vmName, identifier: s.identifier, fileHash: hashContent(text) } });
+  }
+  for (const sys of parseListResponse(dSysRes)) {
+    const text = [`Data System: ${sys.name} (VM: ${vmName})`, sys.label ? `Label: ${sys.label}` : null, sys.description ? `Description: ${sys.description}` : null, `Identifier: ${sys.identifier}`, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/system/${sys.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_system", vm: vmName, identifier: sys.identifier, fileHash: hashContent(text) } });
+  }
+
+  const meshCount = parseListResponse(meshRes).length;
+  const prodCount = parseListResponse(dpRes).length;
+  const srcCount = parseListResponse(dsRes).length;
+  const sysCount = parseListResponse(dSysRes).length;
+  log(`    azure/${vmName}: ${meshCount} meshes, ${prodCount} products, ${srcCount} sources, ${sysCount} systems`);
+
+  return chunks;
+}
+
+/**
+ * Fetch remote landscape by SSHing into the VM and curling localhost:9001 directly,
+ * bypassing Traefik/OAuth. Used when the HTTPS endpoint rejects credentials.
+ */
+export async function fetchLandscapeViaSsh(vmName, ip, creds, log) {
+  const { lazyExeca, sshCmd, knockForVm, DEFAULTS } = await import("./azure.js");
+  const execa = await lazyExeca();
+  const user = DEFAULTS.adminUser;
+
+  try { await knockForVm({ publicIp: ip, vmName }); } catch {}
+
+  const ssh = (cmd, timeout = 15000) => sshCmd(execa, ip, user, cmd, timeout);
+
+  const loginPayload = JSON.stringify({ user: creds.user, password: creds.password });
+  const { stdout: loginOut, exitCode: loginExit } = await ssh(
+    `curl -sf -X POST http://localhost:9001/api/iam/login -H 'Content-Type: application/json' -d '${loginPayload.replace(/'/g, "'\\''")}'`,
+    15000,
+  );
+
+  if (loginExit !== 0 || !loginOut?.trim()) {
+    log(`    azure/${vmName}: SSH auth failed (exit ${loginExit})`);
+    return [];
+  }
+
+  let token;
+  try {
+    const data = JSON.parse(loginOut.trim());
+    token = data.access_token || data.token;
+  } catch {
+    log(`    azure/${vmName}: SSH auth returned invalid JSON`);
+    return [];
+  }
+  if (!token) {
+    log(`    azure/${vmName}: SSH auth returned no token`);
+    return [];
+  }
+
+  const authHeader = `Authorization: Bearer ${token}`;
+  const { stdout: dataOut, exitCode: dataExit } = await ssh(
+    [
+      `curl -sf -H '${authHeader}' -H 'x-org: root' 'http://localhost:9001/api/data/mesh/list?per_page=100'`,
+      `echo '___SEP___'`,
+      `curl -sf -H '${authHeader}' -H 'x-org: root' 'http://localhost:9001/api/data/data_product/list?per_page=200'`,
+      `echo '___SEP___'`,
+      `curl -sf -H '${authHeader}' -H 'x-org: root' 'http://localhost:9001/api/data/data_source/list?per_page=200'`,
+      `echo '___SEP___'`,
+      `curl -sf -H '${authHeader}' -H 'x-org: root' 'http://localhost:9001/api/data/data_system/list?per_page=100'`,
+    ].join(" && "),
+    30000,
+  );
+
+  if (dataExit !== 0 || !dataOut?.trim()) {
+    log(`    azure/${vmName}: SSH landscape fetch failed (exit ${dataExit})`);
+    return [];
+  }
+
+  const parts = dataOut.split("___SEP___");
+  const parseJson = (s) => { try { return JSON.parse(s.trim()); } catch { return null; } };
+  const meshRes = parseJson(parts[0] || "");
+  const dpRes = parseJson(parts[1] || "");
+  const dsRes = parseJson(parts[2] || "");
+  const dSysRes = parseJson(parts[3] || "");
+
+  const { parseListResponse } = await import("../fops-plugin-foundation/lib/api-spec.js");
+  const chunks = [];
+
+  for (const m of parseListResponse(meshRes)) {
+    const text = [`Mesh: ${m.name} (VM: ${vmName})`, m.label ? `Label: ${m.label}` : null, m.description ? `Description: ${m.description}` : null, m.purpose ? `Purpose: ${m.purpose}` : null, `Identifier: ${m.identifier}`, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/mesh/${m.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "mesh", vm: vmName, identifier: m.identifier, fileHash: hashContent(text) } });
+  }
+  for (const p of parseListResponse(dpRes)) {
+    const text = [`Data Product: ${p.name} (VM: ${vmName})`, p.label ? `Label: ${p.label}` : null, p.description ? `Description: ${p.description}` : null, p.data_product_type ? `Type: ${p.data_product_type}` : null, `Identifier: ${p.identifier}`, p.host_mesh_identifier ? `Mesh: ${p.host_mesh_identifier}` : null, p.state?.code ? `State: ${p.state.code} (healthy: ${p.state.healthy})` : null, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/product/${p.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_product", vm: vmName, identifier: p.identifier, fileHash: hashContent(text) } });
+  }
+  for (const s of parseListResponse(dsRes)) {
+    const text = [`Data Source: ${s.name} (VM: ${vmName})`, s.label ? `Label: ${s.label}` : null, s.description ? `Description: ${s.description}` : null, `Identifier: ${s.identifier}`, s.state?.code ? `State: ${s.state.code} (healthy: ${s.state.healthy})` : null, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/source/${s.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_source", vm: vmName, identifier: s.identifier, fileHash: hashContent(text) } });
+  }
+  for (const sys of parseListResponse(dSysRes)) {
+    const text = [`Data System: ${sys.name} (VM: ${vmName})`, sys.label ? `Label: ${sys.label}` : null, sys.description ? `Description: ${sys.description}` : null, `Identifier: ${sys.identifier}`, `VM: ${vmName}`].filter(Boolean).join("\n");
+    chunks.push({ title: `vm/${vmName}/system/${sys.name}`, content: text, metadata: { resourceType: "remote-landscape", entityType: "data_system", vm: vmName, identifier: sys.identifier, fileHash: hashContent(text) } });
+  }
+
+  const meshCount = parseListResponse(meshRes).length;
+  const prodCount = parseListResponse(dpRes).length;
+  const srcCount = parseListResponse(dsRes).length;
+  const sysCount = parseListResponse(dSysRes).length;
+  log(`    azure/${vmName}: ${meshCount} meshes, ${prodCount} products, ${srcCount} sources, ${sysCount} systems (via SSH)`);
+
+  return chunks;
+}

package/src/plugins/bundled/fops-plugin-azure/lib/azure-helpers.js

@@ -618,7 +618,9 @@ const FOPS_KNOCK_TAG = "fopsKnock";
 /** Persist knock sequence to VM tag so other machines can fetch it. */
 export async function setVmKnockTag(execa, rg, vmName, knockSequence, sub) {
   if (!knockSequence?.length) return;
-  const value = knockSequence.join(",");
+  // Use dash delimiter — Azure CLI --set treats commas as array separators
+  // which wraps the value in parens "(49198, 49200, 49180)" and breaks parsing.
+  const value = knockSequence.join("-");
   try {
     await execa("az", [
       "vm", "update", "--resource-group", rg, "--name", vmName,
@@ -629,23 +631,54 @@ export async function setVmKnockTag(execa, rg, vmName, knockSequence, sub) {
   } catch { /* non-fatal */ }
 }
 
+// Per-process cache: vmName → true, so we only fetch from Azure once per CLI invocation.
+const _vmStateSynced = new Set();
+
 /**
- * If local state has no knock sequence, try to load it from the VM's Azure tag
- * (set at provision time). Use when switching to another machine.
+ * Sync VM state (public IP + knock sequence) from Azure on the first call per process.
+ * Azure is the canonical source of truth — local state drifts when VMs are
+ * restarted/reallocated (new IP) or knock sequences change.
  */
 export async function ensureKnockSequence(state) {
-  if (state?.knockSequence?.length) return state;
   if (!state?.resourceGroup || !state?.vmName) return state;
+  if (_vmStateSynced.has(state.vmName)) return readVmState(state.vmName);
   const execa = await lazyExeca();
-  const { stdout, exitCode } = await execa("az", [
-    "vm", "show", "-g", state.resourceGroup, "-n", state.vmName,
-    "--query", `tags.${FOPS_KNOCK_TAG}`, "-o", "tsv",
-  ], { timeout: 15000, reject: false }).catch(() => ({ stdout: "", exitCode: 1 }));
-  const raw = (stdout || "").trim();
-  if (exitCode !== 0 || !raw) return state;
-  const knockSequence = raw.split(",").map((s) => parseInt(s, 10)).filter((n) => Number.isInteger(n) && n > 0);
-  if (knockSequence.length < 2) return state;
-  writeVmState(state.vmName, { knockSequence });
+  _vmStateSynced.add(state.vmName);
+
+  // Fetch public IP and knock sequence tag in parallel
+  const [ipResult, tagResult] = await Promise.all([
+    execa("az", [
+      "vm", "list-ip-addresses", "-g", state.resourceGroup, "-n", state.vmName, "--output", "json",
+    ], { timeout: 15000, reject: false }).catch(() => ({ stdout: "[]", exitCode: 1 })),
+    execa("az", [
+      "vm", "show", "-g", state.resourceGroup, "-n", state.vmName,
+      "--query", `tags.${FOPS_KNOCK_TAG}`, "-o", "tsv",
+    ], { timeout: 15000, reject: false }).catch(() => ({ stdout: "", exitCode: 1 })),
+  ]);
+
+  const patch = {};
+
+  // Sync public IP
+  try {
+    const ips = JSON.parse(ipResult.stdout || "[]");
+    const freshIp = ips?.[0]?.virtualMachine?.network?.publicIpAddresses?.[0]?.ipAddress || "";
+    if (freshIp && freshIp !== state.publicIp) patch.publicIp = freshIp;
+  } catch {}
+
+  // Sync knock sequence — tag value may use dash or comma delimiter;
+  // Azure CLI --set sometimes wraps comma values in parens like "(49198, 49200, 49180)"
+  const raw = (tagResult.stdout || "").trim().replace(/[()]/g, "");
+  if (tagResult.exitCode === 0 && raw) {
+    const knockSequence = raw.split(/[-,]/).map((s) => parseInt(s.trim(), 10)).filter((n) => Number.isInteger(n) && n > 0);
+    if (knockSequence.length >= 2) {
+      const local = state.knockSequence;
+      if (!local?.length || local.length !== knockSequence.length || local.some((v, i) => v !== knockSequence[i])) {
+        patch.knockSequence = knockSequence;
+      }
+    }
+  }
+
+  if (Object.keys(patch).length > 0) writeVmState(state.vmName, patch);
   return readVmState(state.vmName);
 }
 
@@ -656,6 +689,11 @@ export async function knockForVm(vmNameOrState) {
     ? readVmState(vmNameOrState) : vmNameOrState;
   state = await ensureKnockSequence(state);
   if (state?.knockSequence?.length && state.publicIp) {
+    // Close stale mux before knocking — ControlPersist=600 keeps the master alive for
+    // 10 min, but if the VM rebooted the underlying TCP is broken and all SSH commands
+    // through it will fail even after a successful knock.
+    const execa = await lazyExeca();
+    await closeMux(execa, state.publicIp, DEFAULTS.adminUser);
     await performKnock(state.publicIp, state.knockSequence, { quiet: true });
   }
 }