@meshxdata/fops 0.1.36 → 0.1.37

package/src/plugins/bundled/fops-plugin-azure/lib/azure-ops.js

@@ -8,7 +8,7 @@ import { readState, saveState, readVmState, writeVmState, listVms, requireVmStat
 import {
   DEFAULTS, DIM, OK, WARN, ERR, LABEL, ACCENT,
   banner, hint, kvLine, nameArg,
-  lazyExeca, ensureAzCli, ensureAzAuth, subArgs,
+  lazyExeca, ensureAzCli, ensureAzAuth, subArgs, fetchMyIp,
   resolvePublicIp, resolveGithubToken, verifyGithubToken,
   buildPublicUrl, buildDefaultUrl, resolveUniqueDomain,
   sshCmd, closeMux, MUX_OPTS, muxSocketPath, waitForSsh, knockForVm, fopsUpCmd, buildRemoteFopsUpArgs,
@@ -408,17 +408,25 @@ export async function azureVmCheck(opts = {}) {
 // ── ssh ─────────────────────────────────────────────────────────────────────
 
 export async function azureSsh(opts = {}) {
-  const state = requireVmState(opts.vmName);
-  const ip = state.publicIp;
+  const { knockForVm, ensureKnockSequence } = await import("./azure-helpers.js");
+  // Sync IP + knock sequence from Azure before using them
+  const freshState = await ensureKnockSequence(requireVmState(opts.vmName));
+  const ip = freshState?.publicIp;
   if (!ip) { console.error(chalk.red("\n  No IP address. Is the VM running? Try: fops azure start\n")); process.exit(1); }
 
-  await knockForVm(state);
+  await knockForVm(freshState);
 
   const { execa } = await import("execa");
-  await execa("ssh", [
+  const result = await execa("ssh", [
     "-o", "StrictHostKeyChecking=no",
+    "-o", "UserKnownHostsFile=/dev/null",
+    "-o", "ConnectTimeout=15",
     `${DEFAULTS.adminUser}@${ip}`,
   ], { stdio: "inherit", reject: false });
+  if (result.exitCode !== 0 && result.exitCode !== null) {
+    console.error(chalk.red(`\n  SSH failed (exit ${result.exitCode}). If port-knock is enabled, try: fops azure knock open ${freshState?.vmName}\n`));
+    process.exit(1);
+  }
 }
 
 // ── port forward ─────────────────────────────────────────────────────────────
@@ -1005,11 +1013,11 @@ export async function azureRunUp(opts = {}) {
       5000,
     );
     let npmCode = tryUserFirst.exitCode === 0
-      ? (await ssh('D="$(npm root -g 2>/dev/null)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest 2>&1', 300000)).exitCode
+      ? (await ssh('sudo rm -f /usr/bin/fops /usr/local/bin/fops 2>/dev/null; D="$(npm root -g 2>/dev/null)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest 2>&1', 300000)).exitCode
       : 1;
     if (npmCode !== 0) {
       npmCode = (await ssh(
-        "sudo -E bash -c 'D=\"$(npm root -g)/@meshxdata\"; rm -rf \"$D\" 2>/dev/null; npm install -g @meshxdata/fops@latest' 2>&1",
+        "sudo -E bash -c 'rm -f /usr/bin/fops /usr/local/bin/fops 2>/dev/null; D=\"$(npm root -g)/@meshxdata\"; rm -rf \"$D\" 2>/dev/null; npm install -g @meshxdata/fops@latest' 2>&1",
         300000,
       )).exitCode;
     }
@@ -1874,7 +1882,7 @@ export async function azureUpdate(opts = {}) {
       );
       if (tryUserFirst.exitCode === 0) {
         const userInstall = await ssh(
-          'D="$(npm root -g 2>/dev/null)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest 2>&1',
+          'sudo rm -f /usr/bin/fops /usr/local/bin/fops 2>/dev/null; D="$(npm root -g 2>/dev/null)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest 2>&1',
           300000,
         );
         npmCode = userInstall.exitCode;
@@ -1884,7 +1892,7 @@ export async function azureUpdate(opts = {}) {
       if (npmCode !== 0) {
         const sudoInstall = await ssh(
           "sudo -E bash -c '" +
-          'D="$(npm root -g)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest' +
+          'rm -f /usr/bin/fops /usr/local/bin/fops 2>/dev/null; D="$(npm root -g)/@meshxdata"; rm -rf "$D" 2>/dev/null; npm install -g @meshxdata/fops@latest' +
           "' 2>&1",
           300000,
         );
@@ -2775,8 +2783,10 @@ export async function azureApply(file, opts = {}) {
 // ── knock ────────────────────────────────────────────────────────────────────
 
 export async function azureKnock(opts = {}) {
-  const state = requireVmState(opts.vmName);
-  if (!state.knockSequence?.length) {
+  const { ensureKnockSequence } = await import("./azure-helpers.js");
+  // Sync IP + knock sequence from Azure before using them
+  const state = await ensureKnockSequence(requireVmState(opts.vmName));
+  if (!state?.knockSequence?.length) {
     console.error(ERR("\n  Port knocking is not configured on this VM."));
     hint(`Provision with:  fops azure up${nameArg(state?.vmName)}\n`);
     process.exit(1);
@@ -3096,7 +3106,7 @@ export async function azureKnockVerify(opts = {}) {
 
 export async function azureKnockFix(opts = {}) {
   const execa = await lazyExeca();
-  const { setupKnockd, generateKnockSequence } = await import("./port-knock.js");
+  const { generateKnockSequence } = await import("./port-knock.js");
   const { vms } = listVms();
   const targets = opts.vmName ? [opts.vmName] : Object.keys(vms);
 
@@ -3105,33 +3115,178 @@ export async function azureKnockFix(opts = {}) {
     return;
   }
 
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: opts.profile });
+
   banner("Knock Fix");
 
   for (const name of targets) {
     const state = vms[name];
-    if (!state?.publicIp) continue;
-    const ip = state.publicIp;
-
-    console.log(`\n  ${LABEL(name)} ${DIM(`(${ip})`)}`);
-
-    // Knock with existing sequence if available
-    if (state.knockSequence?.length) {
-      await performKnock(ip, state.knockSequence, { quiet: true });
+    if (!state?.resourceGroup) {
+      console.log(chalk.dim(`  ${name}: no resource group — skipped`));
+      continue;
     }
+    const { publicIp: ip, resourceGroup: rg } = state;
 
-    const ssh = (cmd) => sshCmd(execa, ip, DEFAULTS.adminUser, cmd, 30000);
+    console.log(`\n  ${LABEL(name)} ${DIM(`(${ip || "no IP"})`)}`);
 
-    // Generate new sequence and re-setup
     const knockSeq = state.knockSequence?.length ? state.knockSequence : generateKnockSequence();
     const appPort = DEFAULTS.publicPort || 443;
-    await setupKnockd(ssh, knockSeq, { appPort });
-    writeVmState(name, { knockSequence: knockSeq });
-    if (state.resourceGroup) await setVmKnockTag(execa, state.resourceGroup, name, knockSeq, opts.profile);
 
-    console.log(OK(`    ✓ Knock re-configured`));
+    // Build the setup script to run via az vm run-command (works even when SSH is locked)
+    const script = buildKnockdScript(knockSeq, appPort);
+
+    console.log(chalk.dim("  Running via az vm run-command (bypasses firewall)..."));
+    const { stdout, exitCode } = await execa("az", [
+      "vm", "run-command", "invoke",
+      "--resource-group", rg,
+      "--name", name,
+      "--command-id", "RunShellScript",
+      "--scripts", script,
+      "--output", "json",
+      ...subArgs(opts.profile),
+    ], { reject: false, timeout: 180000 });
+
+    if (exitCode !== 0) {
+      console.error(ERR(`    ✗ az run-command failed for ${name}`));
+      continue;
+    }
+
+    // Check script output for errors
+    try {
+      const result = JSON.parse(stdout);
+      const msg = result?.value?.[0]?.message || "";
+      if (msg.toLowerCase().includes("failed") || msg.toLowerCase().includes("error")) {
+        console.log(chalk.dim(`    Script output: ${msg.slice(0, 200)}`));
+      }
+    } catch { /* ignore parse errors */ }
 
-    await closeKnock(ssh, { quiet: true });
+    writeVmState(name, { knockSequence: knockSeq });
+    await setVmKnockTag(execa, rg, name, knockSeq, opts.profile);
+
+    console.log(OK(`    ✓ Knock re-configured (sequence: ${knockSeq.join(" → ")})`));
   }
 
   console.log(OK(`\n  ✓ Done — run fops azure knock verify to confirm.\n`));
 }
+
+/**
+ * Build a self-contained bash script that installs knockd and configures
+ * iptables. Designed to run via az vm run-command (no SSH needed).
+ */
+function buildKnockdScript(sequence, appPort) {
+  const ports = [22];
+  if (appPort && appPort !== 22) ports.push(Number(appPort));
+
+  const openCmds = ports.map(p => `/usr/sbin/iptables -I INPUT -s %IP% -p tcp --dport ${p} -j ACCEPT`).join(" && ");
+  const closeCmds = ports.map(p => `/usr/sbin/iptables -D INPUT -s %IP% -p tcp --dport ${p} -j ACCEPT`).join(" && ");
+  const KNOCK_CMD_TIMEOUT = 300;
+
+  // iptables rules: clean up stale entries, then insert DROP + ESTABLISHED
+  const iptRules = [];
+  for (const p of ports) {
+    iptRules.push(
+      `while iptables -D INPUT -p tcp --dport ${p} -j DROP 2>/dev/null; do :; done`,
+      `while iptables -D INPUT -p tcp --dport ${p} -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT 2>/dev/null; do :; done`,
+      `while iptables -D INPUT -p tcp --dport ${p} -j ACCEPT 2>/dev/null; do :; done`,
+    );
+  }
+  for (const p of ports) {
+    iptRules.push(
+      `iptables -I INPUT -p tcp --dport ${p} -j DROP`,
+      `iptables -I INPUT -p tcp --dport ${p} -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT`,
+    );
+  }
+
+  return [
+    "#!/bin/bash",
+    "set -e",
+    // Detect interface dynamically inside the script
+    "IFACE=$(ip route show default 2>/dev/null | awk '{print $5}' | head -1)",
+    "IFACE=${IFACE:-eth0}",
+    // Install knockd
+    "export DEBIAN_FRONTEND=noninteractive",
+    "apt-get update -qq",
+    "apt-get install -y -qq knockd iptables-persistent 2>/dev/null || true",
+    // Write knockd.conf using printf to avoid heredoc quoting issues
+    `printf '[options]\\n    UseSyslog\\n    Interface = %s\\n\\n[openPorts]\\n    sequence    = ${sequence.join(",")}\\n    seq_timeout = 15\\n    command     = ${openCmds}\\n    tcpflags    = syn\\n    cmd_timeout = ${KNOCK_CMD_TIMEOUT}\\n    stop_command = ${closeCmds}\\n' "$IFACE" > /etc/knockd.conf`,
+    // Enable knockd service
+    "sed -i 's/^START_KNOCKD=0/START_KNOCKD=1/' /etc/default/knockd 2>/dev/null || true",
+    `sed -i "s|^KNOCKD_OPTS=.*|KNOCKD_OPTS=\\"-i $IFACE\\"|" /etc/default/knockd 2>/dev/null || true`,
+    // Apply iptables rules
+    ...iptRules,
+    "netfilter-persistent save 2>/dev/null || true",
+    "systemctl enable knockd",
+    "systemctl restart knockd",
+    "echo 'knockd configured OK'",
+  ].join("\n");
+}
+
+// ── ssh whitelist-me ─────────────────────────────────────────────────────────
+
+export async function azureSshWhitelistMe(opts = {}) {
+  const execa = await lazyExeca();
+  const sub = opts.profile;
+  await ensureAzCli(execa);
+  await ensureAzAuth(execa, { subscription: sub });
+  const state = requireVmState(opts.vmName);
+  const { vmName, resourceGroup: rg } = state;
+
+  const myIp = await fetchMyIp();
+  if (!myIp) {
+    console.error(ERR("\n  Could not detect your public IP address.\n"));
+    process.exit(1);
+  }
+  const myCidr = `${myIp}/32`;
+
+  // Resolve NSG name from NIC
+  const nicName = `${vmName}VMNic`;
+  const { stdout: nicJson, exitCode: nicCode } = await execa("az", [
+    "network", "nic", "show", "-g", rg, "-n", nicName, "--output", "json",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  let nsgName = `${vmName}NSG`;
+  if (nicCode === 0) {
+    const nsgId = JSON.parse(nicJson).networkSecurityGroup?.id || "";
+    if (nsgId) nsgName = nsgId.split("/").pop();
+  }
+
+  // Fetch current SSH rule
+  const { stdout: rulesJson, exitCode: rulesCode } = await execa("az", [
+    "network", "nsg", "rule", "list", "-g", rg, "--nsg-name", nsgName, "--output", "json",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 15000 });
+  const rules = rulesCode === 0 ? JSON.parse(rulesJson || "[]") : [];
+  const inboundAllow = rules.filter(r => r.access === "Allow" && r.direction === "Inbound");
+  const sshRule = inboundAllow.find(r =>
+    r.destinationPortRange === "22" ||
+    (Array.isArray(r.destinationPortRanges) && r.destinationPortRanges.includes("22"))
+  );
+
+  const currentSources = [];
+  if (sshRule?.sourceAddressPrefix) currentSources.push(sshRule.sourceAddressPrefix);
+  if (Array.isArray(sshRule?.sourceAddressPrefixes)) currentSources.push(...sshRule.sourceAddressPrefixes);
+
+  if (currentSources.includes(myCidr)) {
+    console.log(OK(`\n  ✓ ${myCidr} is already whitelisted for SSH on ${vmName}\n`));
+    return;
+  }
+
+  const merged = [...new Set([...currentSources.filter(s => s && s !== "*" && s !== "Internet"), myCidr])];
+  console.log(chalk.yellow(`  ↻ Adding ${myCidr} to SSH rule on ${nsgName} (${currentSources.length} existing)...`));
+  const { exitCode: updateCode } = await execa("az", [
+    "network", "nsg", "rule", "create", "-g", rg, "--nsg-name", nsgName,
+    "-n", sshRule?.name || "allow-ssh", "--priority", String(sshRule?.priority || 1000),
+    "--destination-port-ranges", "22", "--access", "Allow",
+    "--source-address-prefixes", ...merged,
+    "--protocol", "Tcp", "--direction", "Inbound", "--output", "none",
+    ...subArgs(sub),
+  ], { reject: false, timeout: 30000 });
+
+  if (updateCode !== 0) {
+    console.error(ERR(`\n  Failed to update NSG rule on ${nsgName}\n`));
+    process.exit(1);
+  }
+  console.log(OK(`\n  ✓ SSH (22) whitelisted for ${myCidr} on ${vmName} (${nsgName})\n`));
+  console.log(`  Sources: ${merged.join(", ")}\n`);
+}

package/src/plugins/bundled/fops-plugin-azure/lib/azure-provision.js

@@ -8,7 +8,7 @@ import {
   DEFAULTS, DIM, OK, WARN, ERR,
   banner, hint, kvLine,
   resolvePublicIp, subArgs, buildTags, fetchMyIp,
-  sshCmd, waitForSsh, fopsUpCmd, buildPublicUrl,
+  sshCmd, waitForSsh, closeMux, fopsUpCmd, buildPublicUrl,
   runReconcilers, ensureOpenAiNetworkAccess,
   reconcileOk, RECONCILE_LABEL_WIDTH,
 } from "./azure-helpers.js";
@@ -448,7 +448,36 @@ async function vmReconcileNetworking(ctx) {
         }
       }
     } else {
-      console.log(chalk.yellow("  ⚠ No public IP attached to NIC"));
+      // No public IP attached — create one and attach it
+      const pipName = `${vmName}PublicIP`;
+      console.log(chalk.dim(`  ↻ ${"Public IP".padEnd(RECONCILE_LABEL_WIDTH)} — creating ${pipName}…`));
+      const { exitCode: createCode } = await execa("az", [
+        "network", "public-ip", "create", "-g", rg, "-n", pipName,
+        "--sku", "Standard", "--allocation-method", "Static", "--output", "none",
+        ...subArgs(sub),
+      ], { reject: false, timeout: 30000 });
+      if (createCode === 0) {
+        const { exitCode: attachCode } = await execa("az", [
+          "network", "nic", "ip-config", "update",
+          "-g", rg, "--nic-name", ctx.nicName, "-n", "ipconfig1",
+          "--public-ip-address", pipName, "--output", "none",
+          ...subArgs(sub),
+        ], { reject: false, timeout: 30000 });
+        if (attachCode === 0) {
+          const { stdout: newPipJson } = await execa("az", [
+            "network", "public-ip", "show", "-g", rg, "-n", pipName, "--output", "json",
+            ...subArgs(sub),
+          ], { reject: false, timeout: 15000 });
+          try {
+            ctx.ip = JSON.parse(newPipJson).ipAddress || "";
+          } catch {}
+          reconcileOk("Public IP", ctx.ip ? `${ctx.ip} (created)` : `${pipName} (created)`);
+        } else {
+          console.log(chalk.yellow(`  ⚠ Created ${pipName} but could not attach to NIC`));
+        }
+      } else {
+        console.log(chalk.yellow(`  ⚠ Could not create public IP — VM will have no public IP`));
+      }
     }
 
     const nsgId = nic.networkSecurityGroup?.id || "";
@@ -560,6 +589,7 @@ async function vmReconcileNsg(ctx) {
       ...subArgs(sub),
     ], { reject: false, timeout: 30000 });
     reconcileOk("SSH (22)", source);
+    ctx.operatorIpChanged = true;
   } else if (sshSource && !sshHasMyIp) {
     // Add our IP to existing admin IPs
     const merged = [...new Set([...sshCurrentSources, sshSource])];
@@ -573,6 +603,7 @@ async function vmReconcileNsg(ctx) {
       ...subArgs(sub),
     ], { reject: false, timeout: 30000 });
     reconcileOk("SSH (22)", merged.join(", "));
+    ctx.operatorIpChanged = true;
   } else {
     reconcileOk("SSH (22)", sshCurrentSources.join(", ") || "*");
   }
@@ -1028,12 +1059,19 @@ async function removeSshBypassViaRunCommand(execa, rg, vmName, sourceCidr, sub)
 async function vmReconcileSsh(ctx) {
   const { execa, ip, adminUser, knockSequence, port, desiredUrl, vmName, rg, sub } = ctx;
   console.log(chalk.dim("  Checking SSH..."));
-  const maxWaitFirst = 90000;
+  // Close any stale mux master before probing. ControlPersist=600 keeps the SSH master
+  // alive for 10 min, but if the VM rebooted its underlying TCP is broken — every probe
+  // through the stale mux fails even after a successful knock. A fresh connect fixes it.
+  await closeMux(execa, ip, adminUser);
+  // When Run Command is available (rg set), use a short knock probe then fall through to
+  // Run Command rather than burning 2+ minutes on knock retries.
+  const canRunCommand = knockSequence?.length && rg;
+  const maxWaitFirst = canRunCommand ? 20000 : 90000;
   if (knockSequence?.length) {
     await performKnock(ip, knockSequence, { quiet: true });
   }
   let sshReady = await waitForSsh(execa, ip, adminUser, maxWaitFirst);
-  if (!sshReady && knockSequence?.length) {
+  if (!sshReady && knockSequence?.length && !canRunCommand) {
     console.log(chalk.dim("  Re-sending knock and retrying SSH..."));
     await performKnock(ip, knockSequence, { quiet: true });
     await new Promise((r) => setTimeout(r, 5000));
@@ -1081,8 +1119,25 @@ async function vmReconcileKnock(ctx) {
   );
   const knockdRunning = knockdCheck?.trim() === "active";
 
-  if (knockdRunning && ctx.knockSequence?.length) {
-    reconcileOk("Port knocking", "active");
+  // Read the authoritative sequence from knockd.conf on the VM
+  const { stdout: knockConfOut } = await ssh(
+    "grep -m1 'sequence' /etc/knockd.conf 2>/dev/null | sed 's/.*sequence[[:space:]]*=[[:space:]]*//'", 8000,
+  );
+  const vmKnockSeq = (knockConfOut || "").trim()
+    .split(",").map((s) => parseInt(s.trim(), 10)).filter((n) => Number.isInteger(n) && n > 0);
+
+  if (knockdRunning && vmKnockSeq.length >= 2) {
+    // Reconcile: keep tag + local state in sync with knockd.conf
+    const local = ctx.knockSequence;
+    const seqDiffers = !local?.length || local.length !== vmKnockSeq.length || local.some((v, i) => v !== vmKnockSeq[i]);
+    if (seqDiffers) {
+      console.log(chalk.dim(`  ↻ ${"Port knocking".padEnd(RECONCILE_LABEL_WIDTH)} — sequence drift detected, syncing…`));
+      writeVmState(vmName, { knockSequence: vmKnockSeq });
+      const { setVmKnockTag } = await import("./azure-helpers.js");
+      await setVmKnockTag(execa, ctx.rg, vmName, vmKnockSeq, ctx.sub);
+      ctx.knockSequence = vmKnockSeq;
+    }
+    reconcileOk("Port knocking", `active [${vmKnockSeq.join(", ")}]`);
   } else if (!knockdRunning) {
     console.log(chalk.dim("  Setting up port knocking (after post-start + UI reachable)..."));
     const knockSeq = generateKnockSequence();
@@ -1092,8 +1147,7 @@ async function vmReconcileKnock(ctx) {
     await setVmKnockTag(execa, ctx.rg, vmName, knockSeq, ctx.sub);
     ctx.knockSequence = knockSeq;
   } else {
-    console.log(chalk.yellow("  ⚠ knockd active on VM but no local knock sequence stored"));
-    console.log(chalk.dim("    To re-create: fops azure knock disable && fops azure up"));
+    console.log(chalk.yellow("  ⚠ knockd active on VM but could not read sequence from knockd.conf"));
   }
 
   if (ctx.knockSequence?.length) {

package/src/plugins/bundled/fops-plugin-azure/lib/azure.js

@@ -27,7 +27,7 @@ export {
   managedImageId, resolvePublicIp,
   resolveGithubToken, verifyGithubToken,
   sshCmd, closeMux, MUX_OPTS, muxSocketPath, waitForSsh,
-  knockForVm, fopsUpCmd,
+  knockForVm, ensureKnockSequence, fopsUpCmd,
   runReconcilers, ensureOpenAiNetworkAccess,
 } from "./azure-helpers.js";
 
@@ -46,7 +46,7 @@ export {
 
 // ── VM operations ────────────────────────────────────────────────────────────
 export {
-  azureStatus, azureTrinoStatus, azureSsh, azurePortForward, azureSshAdminAdd, azureVmCheck, azureAgent, azureOpenAiDebugVm,
+  azureStatus, azureTrinoStatus, azureSsh, azureSshWhitelistMe, azurePortForward, azureSshAdminAdd, azureVmCheck, azureAgent, azureOpenAiDebugVm,
   azureDeploy, azurePull, azureDeployVersion, azureRunUp, azureConfig, azureConfigVersions, azureUpdate,
   azureLogs, azureGrantAdmin, azureContext,
   azureList, azureApply,