@meshxdata/fops 0.1.49 → 0.1.51

package/src/plugins/bundled/fops-plugin-azure/lib/azure-fleet-swarm.js

@@ -0,0 +1,936 @@
+/**
+ * azure-fleet-swarm.js
+ * Docker Swarm cluster management for Azure VMs.
+ * Extracted from azure-fleet.js for maintainability.
+ */
+
+import chalk from "chalk";
+import { performKnock, closeKnock, keepKnockAlive } from "./port-knock.js";
+import {
+  DEFAULTS, DIM, OK, WARN, ERR, LABEL, ACCENT,
+  banner, hint, kvLine,
+  lazyExeca,
+  listVms, readVmState, writeVmState,
+  sshCmd, closeMux,
+  knockForVm,
+  azureUp,
+} from "./azure.js";
+
+// ── swarm — Docker Swarm cluster management ─────────────────────────────────
+
+/**
+ * Initialize Docker Swarm on a VM (single-node manager).
+ * Optionally converts the running compose stack to a swarm stack deploy.
+ */
+export async function swarmInit(opts = {}) {
+  const execa = await lazyExeca();
+  const vmName = opts.vmName;
+  if (!vmName) {
+    console.error(ERR("\n  Usage: fops azure swarm init <vmName>\n"));
+    process.exit(1);
+  }
+
+  const vm = readVmState(vmName);
+  if (!vm) {
+    console.error(ERR(`\n  VM "${vmName}" not tracked. Run: fops azure list\n`));
+    process.exit(1);
+  }
+
+  banner("Swarm Init");
+  kvLine("VM", LABEL(vmName));
+  kvLine("IP", DIM(vm.publicIp));
+  console.log("");
+
+  if (vm.swarm?.role) {
+    console.log(WARN(`  VM "${vmName}" is already a swarm ${vm.swarm.role}.`));
+    hint("Use: fops azure swarm status " + vmName);
+    console.log("");
+    return;
+  }
+
+  await knockForVm(vm);
+  const stopKnockKeepAlive = keepKnockAlive(vm.publicIp, vm.knockSequence);
+  const user = DEFAULTS.adminUser;
+  const ssh = (cmd, t) => sshCmd(execa, vm.publicIp, user, cmd, t || 30000);
+
+  try {
+    // Get the private IP for the advertise address
+    hint("Detecting private IP…");
+    const { stdout: privateIp } = await ssh(
+      "hostname -I | awk '{print $1}'"
+    );
+    const advertiseAddr = (privateIp || "").trim();
+    if (!advertiseAddr) {
+      console.error(ERR("  Could not detect private IP"));
+      process.exit(1);
+    }
+    kvLine("Advertise", DIM(advertiseAddr));
+
+    // Check if already in a swarm
+    const { stdout: swarmStatus } = await ssh("sudo docker info --format '{{.Swarm.LocalNodeState}}'");
+    if ((swarmStatus || "").trim() === "active") {
+      console.log(WARN("\n  Docker Swarm is already active on this node."));
+      // Still save state if missing
+      const { stdout: nodeId } = await ssh("sudo docker info --format '{{.Swarm.NodeID}}'");
+      const { stdout: managerToken } = await ssh("sudo docker swarm join-token manager -q 2>/dev/null || echo ''");
+      const { stdout: workerToken } = await ssh("sudo docker swarm join-token worker -q 2>/dev/null || echo ''");
+
+      writeVmState(vmName, {
+        swarm: {
+          role: "manager",
+          nodeId: (nodeId || "").trim(),
+          advertiseAddr,
+          managerToken: (managerToken || "").trim(),
+          workerToken: (workerToken || "").trim(),
+          initAt: vm.swarm?.initAt || new Date().toISOString(),
+        },
+      });
+
+      if (opts.stack) {
+        hint("Deploying compose stack as swarm services…");
+        const nets = await genSwarmConfig(ssh);
+        await ensureOverlayNetworks(ssh, nets, hint);
+        const { exitCode: stackCode, stdout: stackOut } = await deployStackWithRetry(
+          ssh, nets, { hint, OK, WARN },
+        );
+        if (stackCode === 0) {
+          console.log(OK("  ✓ Stack deployed as swarm services"));
+        } else {
+          console.log(WARN(`  Stack deploy returned warnings:\n${(stackOut || "").trim()}`));
+        }
+      } else {
+        hint("State updated. Run: fops azure swarm status " + vmName);
+      }
+      console.log("");
+      return;
+    }
+
+    // Initialize swarm
+    hint("Initializing Docker Swarm…");
+    const { stdout: initOut, exitCode: initCode } = await ssh(
+      `sudo docker swarm init --advertise-addr ${advertiseAddr} 2>&1`,
+      60000,
+    );
+    if (initCode !== 0) {
+      console.error(ERR(`  Swarm init failed:\n${initOut}`));
+      process.exit(1);
+    }
+
+    // Collect tokens and node ID
+    const { stdout: nodeId } = await ssh("sudo docker info --format '{{.Swarm.NodeID}}'");
+    const { stdout: managerToken } = await ssh("sudo docker swarm join-token manager -q");
+    const { stdout: workerToken } = await ssh("sudo docker swarm join-token worker -q");
+
+    // Save to VM state
+    const swarmState = {
+      role: "manager",
+      nodeId: (nodeId || "").trim(),
+      advertiseAddr,
+      managerToken: (managerToken || "").trim(),
+      workerToken: (workerToken || "").trim(),
+      initAt: new Date().toISOString(),
+    };
+
+    writeVmState(vmName, { swarm: swarmState });
+
+    console.log(OK("\n  ✓ Docker Swarm initialized"));
+    kvLine("Node ID", DIM(swarmState.nodeId));
+    console.log("");
+
+    // Optionally convert to stack deploy
+    if (opts.stack) {
+      hint("Converting compose stack to swarm services…");
+      const nets = await genSwarmConfig(ssh);
+      await ensureOverlayNetworks(ssh, nets, hint);
+      const { exitCode: stackCode, stdout: stackOut } = await deployStackWithRetry(
+        ssh, nets, { hint, OK, WARN },
+      );
+      if (stackCode === 0) {
+        console.log(OK("  ✓ Stack deployed as swarm services"));
+      } else {
+        console.log(WARN(`  Stack deploy returned warnings:\n${(stackOut || "").trim()}`));
+      }
+      console.log("");
+    }
+
+    hint("Add workers:  fops azure swarm join <workerVm> --manager " + vmName);
+    hint("Check:        fops azure swarm status " + vmName);
+    if (!opts.stack) {
+      hint("Deploy stack: fops azure swarm deploy " + vmName);
+    }
+    console.log("");
+  } finally {
+    stopKnockKeepAlive();
+    if (vm.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, vm.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, vm.publicIp, user);
+  }
+}
+
+/**
+ * SSH commands to generate a swarm-compatible compose file.
+ * Fixes: top-level name, cpus (must be string), ports.published (must be int),
+ * depends_on (must be list not map).
+ * Non-external networks become stack-managed overlays so docker stack deploy
+ * creates them atomically with the services (no raft race).
+ * Truly external networks are collected so the deploy flow can pre-create them.
+ */
+const SWARM_FIXUP_PY = [
+  "import sys, yaml, json",
+  "doc = yaml.safe_load(sys.stdin)",
+  'doc.pop("name", None)',
+  "def fix(svc):",
+  '  for p in [("deploy","resources","limits"),("deploy","resources","reservations")]:',
+  "    n = svc",
+  "    for k in p: n = n.get(k, {}) if isinstance(n, dict) else {}",
+  '    if "cpus" in n and not isinstance(n["cpus"], str): n["cpus"] = str(n["cpus"])',
+  '  for port in svc.get("ports", []):',
+  "    if isinstance(port, dict):",
+  "      port.pop('host_ip', None)",
+  '      if "published" in port:',
+  '        try: port["published"] = int(port["published"])',
+  "        except (ValueError, TypeError): pass",
+  "  svc.pop('profiles', None)",
+  '  deps = svc.get("depends_on")',
+  "  if isinstance(deps, dict): svc['depends_on'] = list(deps.keys())",
+  '  for net_name, net_cfg in list(svc.get("networks", {}).items()):',
+  "    if isinstance(net_cfg, dict) and net_cfg.get('ipv4_address'):",
+  "      del net_cfg['ipv4_address']",
+  "    if isinstance(net_cfg, dict) and net_cfg.get('ipv6_address'):",
+  "      del net_cfg['ipv6_address']",
+  'for s in (doc.get("services") or {}).values(): fix(s)',
+  "# Non-external networks: let docker stack deploy manage them as overlays.",
+  "# Only truly external networks need pre-creation by the deploy flow.",
+  "net_names = []",
+  'for key, net in list((doc.get("networks") or {}).items()):',
+  "  if net is None:",
+  "    doc['networks'][key] = {'driver': 'overlay', 'attachable': True}",
+  "  elif isinstance(net, dict) and not net.get('external'):",
+  "    net.clear()",
+  "    net['driver'] = 'overlay'",
+  "    net['attachable'] = True",
+  "  elif isinstance(net, dict) and net.get('external'):",
+  "    net_names.append(net.get('name', key))",
+  "yaml.dump(doc, sys.stdout, default_flow_style=False, sort_keys=False)",
+  "with open('/tmp/_swarm_networks.json', 'w') as f: json.dump(net_names, f)",
+].join("\n");
+
+const SWARM_FIXUP_B64 = Buffer.from(SWARM_FIXUP_PY).toString("base64");
+
+/**
+ * Generate swarm-compatible config and return the list of network names
+ * that must be pre-created as overlay networks before deploying.
+ */
+async function genSwarmConfig(ssh) {
+  await ssh(
+    `echo '${SWARM_FIXUP_B64}' | base64 -d > /tmp/_swarm_fix.py`,
+    10000,
+  );
+  await ssh(
+    "cd /opt/foundation-compose && sudo docker compose config 2>/dev/null | python3 /tmp/_swarm_fix.py > /tmp/docker-compose-swarm.yaml",
+    60000,
+  );
+  const { stdout: netJson } = await ssh(
+    "cat /tmp/_swarm_networks.json 2>/dev/null || echo '[]'",
+  );
+  try { return JSON.parse(netJson?.trim() || "[]"); } catch { return []; }
+}
+
+/**
+ * Ensure all required overlay networks exist before deploying.
+ * Creates any missing networks and waits for raft propagation on multi-node swarms.
+ */
+async function ensureOverlayNetworks(ssh, networkNames, hint) {
+  if (!networkNames?.length) return;
+  for (const netName of networkNames) {
+    const { stdout: scopeOut } = await ssh(
+      `sudo docker network inspect ${netName} --format '{{.Scope}}' 2>/dev/null || echo missing`,
+    );
+    if ((scopeOut || "").trim() === "swarm") continue;
+    hint(`  Creating overlay network ${netName}…`);
+    await ssh(`sudo docker network create --driver overlay --attachable ${netName} 2>&1 || true`);
+  }
+}
+
+/**
+ * Deploy the swarm stack with retry logic for raft propagation delays.
+ * `docker stack deploy` queries the raft store for external networks, which can
+ * lag behind the local daemon. Retries on "network ... not found" errors.
+ */
+async function deployStackWithRetry(ssh, overlayNetworks, { hint, OK, WARN, maxRetries = 5, retryDelayMs = 15000 } = {}) {
+  let exitCode;
+  let stdout;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    const result = await ssh(
+      "cd /opt/foundation-compose && sudo docker stack deploy --with-registry-auth -c /tmp/docker-compose-swarm.yaml foundation 2>&1",
+      300000,
+    );
+    exitCode = result.exitCode;
+    stdout = result.stdout || "";
+
+    const isNetworkNotFound = /network\s+\S+\s+not found/i.test(stdout);
+
+    if (exitCode === 0 && !isNetworkNotFound) break;
+
+    if (!isNetworkNotFound || attempt === maxRetries) break;
+
+    if (WARN) console.log(WARN(`  Network not yet visible in raft store — retry ${attempt + 1}/${maxRetries}…`));
+
+    // Remove partially-created stack to avoid stale state on retry
+    await ssh("sudo docker stack rm foundation 2>&1 || true", 60000);
+    for (let i = 0; i < 10; i++) {
+      const { stdout: stackStill } = await ssh(
+        "sudo docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx foundation && echo yes || echo no",
+      );
+      if (stackStill?.trim() !== "yes") break;
+      await new Promise(r => setTimeout(r, 2000));
+    }
+
+    // Re-verify / recreate overlay networks
+    for (const netName of (overlayNetworks || [])) {
+      const { stdout: scopeCheck } = await ssh(
+        `sudo docker network inspect ${netName} --format '{{.Scope}}' 2>/dev/null || echo missing`,
+      );
+      if ((scopeCheck || "").trim() !== "swarm") {
+        await ssh(`sudo docker network create --driver overlay --attachable ${netName} 2>&1 || true`);
+      }
+    }
+    await new Promise(r => setTimeout(r, retryDelayMs));
+  }
+
+  return { exitCode, stdout };
+}
+
+/**
+ * Detect and remove broken overlay networks before a swarm deploy.
+ * Overlay corruption manifests as missing peers (fewer than node count),
+ * causing "no configured subnet" task rejections on worker nodes.
+ * Removing the stack lets docker stack deploy recreate fresh networks.
+ */
+async function reconcileOverlayNetworks(ssh, hint, OK, WARN) {
+  const { stdout: nodeCountOut } = await ssh(
+    "sudo docker node ls -q 2>/dev/null | wc -l",
+  );
+  const nodeCount = parseInt(nodeCountOut?.trim(), 10) || 1;
+  if (nodeCount <= 1) return;
+
+  const { stdout: overlayNets } = await ssh(
+    "sudo docker network ls --filter driver=overlay --format '{{.Name}}' 2>/dev/null | grep -E '^foundation[_-]' || true",
+  );
+  if (!overlayNets?.trim()) return;
+
+  const netNames = overlayNets.trim().split("\n").filter(Boolean);
+  let broken = false;
+  for (const netName of netNames) {
+    const { stdout: peerJson } = await ssh(
+      `sudo docker network inspect ${netName} --format '{{json .Peers}}' 2>/dev/null || echo '[]'`,
+    );
+    let peerCount = 0;
+    try { peerCount = JSON.parse(peerJson?.trim() || "[]").length; } catch {}
+
+    if (peerCount < nodeCount) {
+      hint(`Overlay "${netName}" has ${peerCount}/${nodeCount} peers — network is degraded`);
+      broken = true;
+    }
+  }
+
+  if (!broken) return;
+
+  // Remove the stack so docker stack deploy recreates fresh overlay networks.
+  const { stdout: hasStack } = await ssh(
+    "sudo docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx foundation && echo yes || echo no",
+  );
+  if (hasStack?.trim() === "yes") {
+    hint("Removing stack to recreate overlay networks…");
+    await ssh("sudo docker stack rm foundation 2>&1 || true", 60000);
+  }
+
+  for (let attempt = 0; attempt < 15; attempt++) {
+    const { stdout: stackStill } = await ssh(
+      "sudo docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx foundation && echo yes || echo no",
+    );
+    if (stackStill?.trim() !== "yes") break;
+    hint("Waiting for stack removal…");
+    await new Promise(r => setTimeout(r, 2000));
+  }
+
+  // Remove any leftover overlay networks (including legacy foundation-compose_ ones)
+  for (let attempt = 0; attempt < 20; attempt++) {
+    const { stdout: remaining } = await ssh(
+      "sudo docker network ls --filter driver=overlay --format '{{.Name}}' 2>/dev/null | grep -E '^foundation[_-]' || true",
+    );
+    if (!remaining?.trim()) break;
+    for (const net of remaining.trim().split("\n").filter(Boolean)) {
+      await ssh(`sudo docker network rm ${net} 2>&1 || true`);
+    }
+    await new Promise(r => setTimeout(r, 2000));
+  }
+
+  await new Promise(r => setTimeout(r, 5000));
+  console.log(OK("  ✓ Stale overlay networks cleared"));
+}
+
+/**
+ * After stack deploy, wait briefly for services to converge and detect
+ * overlay network issues early. If tasks are rejected with subnet errors,
+ * tear down and retry once — docker stack deploy recreates its own networks.
+ */
+async function waitForSwarmConvergence(ssh, hint, OK, WARN) {
+  hint("Waiting for services to converge…");
+  await new Promise(r => setTimeout(r, 15000));
+
+  // Show task errors for services that failed to start
+  const { stdout: failedTasks } = await ssh(
+    "sudo docker service ls --filter 'name=foundation_' --format '{{.Name}} {{.Replicas}}' 2>/dev/null " +
+    "| awk '$2 ~ /^0\\//' | while read svc rest; do " +
+    "  err=$(sudo docker service ps \"$svc\" --no-trunc --format '{{.Error}}' --filter 'desired-state=shutdown' 2>/dev/null | head -1); " +
+    "  [ -n \"$err\" ] && echo \"$svc: $err\"; " +
+    "done",
+    30000,
+  );
+  if (failedTasks?.trim()) {
+    console.log(WARN("\n  Tasks with errors:"));
+    for (const line of failedTasks.trim().split("\n").filter(Boolean)) {
+      console.log(WARN(`    ${line}`));
+    }
+  }
+
+  const { stdout: rejectedOut } = await ssh(
+    "sudo docker service ls -q --filter 'name=foundation_' 2>/dev/null " +
+    "| xargs -I{} sudo docker service ps {} --no-trunc --format '{{.Error}}' --filter 'desired-state=shutdown' 2>/dev/null " +
+    "| grep -c 'no configured subnet' || echo 0",
+    30000,
+  );
+  const rejectCount = parseInt(rejectedOut?.trim(), 10) || 0;
+  if (rejectCount === 0) return;
+
+  console.log(WARN(`\n  ⚠ ${rejectCount} task(s) rejected with overlay network errors — retrying deploy…`));
+
+  // Remove the stack (also removes its managed overlay networks)
+  await ssh("sudo docker stack rm foundation 2>&1 || true", 60000);
+  for (let i = 0; i < 15; i++) {
+    const { stdout: stackStill } = await ssh(
+      "sudo docker stack ls --format '{{.Name}}' 2>/dev/null | grep -qx foundation && echo yes || echo no",
+    );
+    if (stackStill?.trim() !== "yes") break;
+    await new Promise(r => setTimeout(r, 2000));
+  }
+
+  // Remove any lingering overlay networks that stack rm didn't clean up
+  const { stdout: netList } = await ssh(
+    "sudo docker network ls --filter driver=overlay --format '{{.Name}}' 2>/dev/null | grep -E '^foundation[_-]' || true",
+  );
+  for (const net of (netList?.trim() || "").split("\n").filter(Boolean)) {
+    await ssh(`sudo docker network rm ${net} 2>&1 || true`);
+  }
+  await new Promise(r => setTimeout(r, 5000));
+
+  hint("Redeploying stack…");
+  const { exitCode, stdout } = await deployStackWithRetry(
+    ssh, [], { hint, OK, WARN },
+  );
+  if (exitCode === 0) {
+    console.log(OK("  ✓ Stack redeployed successfully"));
+  } else {
+    console.log(WARN(`  Redeploy warnings:\n${(stdout || "").trim()}`));
+  }
+}
+
+/**
+ * Deploy (or update) the compose stack as swarm services on a manager node.
+ */
+export async function swarmDeploy(opts = {}) {
+  const execa = await lazyExeca();
+  const { activeVm, vms } = listVms();
+
+  const vmName = opts.vmName
+    || Object.keys(vms).find(n => vms[n].swarm?.role === "manager")
+    || activeVm;
+
+  if (!vmName || !vms[vmName]) {
+    console.error(ERR("\n  No swarm manager found. Initialize one: fops azure swarm init <vmName>\n"));
+    process.exit(1);
+  }
+
+  const vm = vms[vmName];
+  if (!vm.swarm?.role) {
+    console.error(ERR(`\n  "${vmName}" is not in a swarm. Run: fops azure swarm init ${vmName}\n`));
+    process.exit(1);
+  }
+  if (vm.swarm.role !== "manager") {
+    console.error(ERR(`\n  "${vmName}" is a worker, not a manager. Target the manager instead.\n`));
+    process.exit(1);
+  }
+
+  banner("Swarm Deploy");
+  kvLine("Manager", LABEL(vmName));
+  console.log("");
+
+  await knockForVm(vm);
+  const stopKnockKeepAlive = keepKnockAlive(vm.publicIp, vm.knockSequence);
+  const user = DEFAULTS.adminUser;
+  const ssh = (cmd, t) => sshCmd(execa, vm.publicIp, user, cmd, t || 30000);
+
+  try {
+    // Detect existing compose stack and migrate: stop compose, remove networks
+    // Detect existing compose stack or stale networks and clean up before swarm deploy.
+    // compose down alone may not remove containers with restart policies (e.g. traefik),
+    // so we kill first, then down, then force-remove any stragglers and networks.
+    const { stdout: composePsOut } = await ssh(
+      "cd /opt/foundation-compose && sudo docker compose ps -q 2>/dev/null | head -1",
+    );
+    if (composePsOut?.trim()) {
+      hint("Existing compose stack detected — stopping for swarm migration…");
+      await ssh("cd /opt/foundation-compose && sudo docker compose kill 2>&1 || true", 30000);
+      await ssh("cd /opt/foundation-compose && sudo docker compose down --remove-orphans 2>&1 || true", 120000);
+      // Force-remove any containers still attached to compose networks
+      const { stdout: stragglers } = await ssh(
+        "sudo docker ps -aq --filter 'network=foundation-compose_foundation-network' 2>/dev/null || true",
+      );
+      if (stragglers?.trim()) {
+        await ssh(`sudo docker rm -f ${stragglers.trim().split("\\n").join(" ")} 2>&1 || true`);
+      }
+      console.log(OK("  ✓ Compose stack stopped"));
+    }
+    // Remove any leftover compose bridge networks that would conflict with swarm overlay networks
+    const { stdout: staleNets } = await ssh(
+      "sudo docker network ls --filter 'driver=bridge' --format '{{.Name}}' 2>/dev/null | grep foundation-compose || true",
+    );
+    if (staleNets?.trim()) {
+      for (const net of staleNets.trim().split("\n").filter(Boolean)) {
+        hint(`Removing stale network ${net}…`);
+        await ssh(`sudo docker network rm ${net} 2>&1 || true`);
+      }
+      console.log(OK("  ✓ Stale networks removed"));
+    }
+
+    // Reconcile broken overlay networks before deploying.
+    await reconcileOverlayNetworks(ssh, hint, OK, WARN);
+
+    hint("Generating swarm-compatible config…");
+    const externalNetworks = await genSwarmConfig(ssh);
+
+    // Pre-create any truly external networks (not stack-managed).
+    // Stack-managed overlay networks are created atomically by docker stack deploy.
+    if (externalNetworks.length > 0) {
+      hint(`Ensuring ${externalNetworks.length} external network(s) exist…`);
+      for (const netName of externalNetworks) {
+        const { stdout: existsOut } = await ssh(
+          `sudo docker network inspect ${netName} --format '{{.Scope}}' 2>/dev/null || echo missing`,
+        );
+        if ((existsOut || "").trim() === "swarm") continue;
+        hint(`  Creating ${netName}…`);
+        await ssh(`sudo docker network create --driver overlay --attachable ${netName} 2>&1 || true`);
+      }
+      console.log(OK(`  ✓ ${externalNetworks.length} external network(s) ready`));
+    }
+
+    hint("Deploying compose stack as swarm services…");
+    const { exitCode: deployExitCode, stdout: deployStdout } = await deployStackWithRetry(
+      ssh, externalNetworks, { hint, OK, WARN },
+    );
+
+    if (deployExitCode === 0) {
+      console.log(OK("\n  ✓ Stack deployed as swarm services"));
+    } else {
+      console.log(WARN(`  Stack deploy warnings:\n${(deployStdout || "").trim()}`));
+    }
+
+    // Post-deploy: wait for services to converge, detect overlay issues early
+    await waitForSwarmConvergence(ssh, hint, OK, WARN);
+
+    // Show services
+    const { stdout: svcs } = await ssh(
+      "sudo docker stack services foundation --format '{{.Name}}\\t{{.Replicas}}\\t{{.Image}}' 2>/dev/null",
+    );
+    if (svcs?.trim()) {
+      console.log("");
+      hint("Services:");
+      for (const line of svcs.trim().split("\n")) {
+        console.log(`  ${line}`);
+      }
+    }
+
+    console.log("");
+    hint("Status:  fops azure swarm status " + vmName);
+    hint("Update:  fops azure swarm deploy " + vmName);
+    console.log("");
+  } finally {
+    stopKnockKeepAlive();
+    if (vm.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, vm.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, vm.publicIp, user);
+  }
+}
+
+/**
+ * Join a VM as a worker (or manager) to an existing swarm.
+ */
+export async function swarmJoin(opts = {}) {
+  const execa = await lazyExeca();
+  const vmName = opts.vmName;
+  const managerName = opts.manager;
+
+  if (!vmName || !managerName) {
+    console.error(ERR("\n  Usage: fops azure swarm join <vmName> --manager <managerVm>\n"));
+    process.exit(1);
+  }
+
+  const managerVm = readVmState(managerName);
+  if (!managerVm) {
+    console.error(ERR(`\n  Manager VM "${managerName}" not tracked.\n`));
+    process.exit(1);
+  }
+  if (!managerVm.swarm?.workerToken) {
+    console.error(ERR(`\n  "${managerName}" is not a swarm manager. Run: fops azure swarm init ${managerName}\n`));
+    process.exit(1);
+  }
+
+  let vm = readVmState(vmName);
+  if (!vm) {
+    hint(`VM "${vmName}" not found — creating it now…`);
+    await azureUp({
+      vmName,
+      location: opts.location || managerVm.location,
+      vmSize:   opts.vmSize,
+      image:    opts.image,
+      url:      opts.url,
+      profile:  opts.profile || managerVm.subscriptionId,
+    });
+    vm = readVmState(vmName);
+    if (!vm) {
+      console.error(ERR(`\n  Failed to create VM "${vmName}".\n`));
+      process.exit(1);
+    }
+  }
+
+  banner("Swarm Join");
+  kvLine("Worker", LABEL(vmName));
+  kvLine("Manager", LABEL(managerName));
+  console.log("");
+
+  if (vm.swarm?.role) {
+    console.log(WARN(`  "${vmName}" is already a swarm ${vm.swarm.role}.`));
+    hint("Leave first: fops azure swarm leave " + vmName);
+    console.log("");
+    return;
+  }
+
+  await knockForVm(vm);
+  const user = DEFAULTS.adminUser;
+  const ssh = (cmd, t) => sshCmd(execa, vm.publicIp, user, cmd, t || 30000);
+
+  try {
+    const token = opts.asManager ? managerVm.swarm.managerToken : managerVm.swarm.workerToken;
+    const joinAddr = `${managerVm.swarm.advertiseAddr}:2377`;
+    const role = opts.asManager ? "manager" : "worker";
+
+    hint(`Joining as ${role}…`);
+    const { stdout: joinOut, exitCode: joinCode } = await ssh(
+      `sudo docker swarm join --token ${token} ${joinAddr} 2>&1`,
+      60000,
+    );
+
+    if (joinCode !== 0) {
+      console.error(ERR(`  Join failed:\n${(joinOut || "").trim()}`));
+      process.exit(1);
+    }
+
+    const { stdout: nodeId } = await ssh("sudo docker info --format '{{.Swarm.NodeID}}'");
+
+    writeVmState(vmName, {
+      swarm: {
+        role,
+        nodeId: (nodeId || "").trim(),
+        manager: managerName,
+        joinedAt: new Date().toISOString(),
+      },
+    });
+
+    console.log(OK(`\n  ✓ ${vmName} joined swarm as ${role}`));
+    kvLine("Node ID", DIM((nodeId || "").trim()));
+    hint("Status: fops azure swarm status " + managerName);
+    console.log("");
+  } finally {
+    if (vm.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, vm.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, vm.publicIp, user);
+  }
+}
+
+/**
+ * Show swarm status for a VM (or the entire swarm from a manager node).
+ */
+export async function swarmStatus(opts = {}) {
+  const execa = await lazyExeca();
+  const { activeVm, vms } = listVms();
+
+  // Find the target — prefer explicit vmName, else find a manager, else active
+  const vmName = opts.vmName
+    || Object.keys(vms).find(n => vms[n].swarm?.role === "manager")
+    || activeVm;
+
+  if (!vmName || !vms[vmName]) {
+    console.error(ERR("\n  No swarm manager found. Initialize one: fops azure swarm init <vmName>\n"));
+    process.exit(1);
+  }
+
+  const vm = vms[vmName];
+
+  banner("Swarm Status");
+  kvLine("VM", LABEL(vmName));
+  kvLine("Role", vm.swarm?.role ? ACCENT(vm.swarm.role) : DIM("not in swarm"));
+
+  if (!vm.swarm?.role) {
+    console.log("");
+    hint("Initialize: fops azure swarm init " + vmName);
+    console.log("");
+    return;
+  }
+
+  await knockForVm(vm);
+  const user = DEFAULTS.adminUser;
+  const ssh = (cmd, t) => sshCmd(execa, vm.publicIp, user, cmd, t || 30000);
+
+  try {
+    // Node list (only works on managers)
+    const { stdout: nodesOut, exitCode: nodesCode } = await ssh(
+      "sudo docker node ls --format '{{.ID}}|{{.Hostname}}|{{.Status}}|{{.Availability}}|{{.ManagerStatus}}' 2>/dev/null"
+    );
+
+    if (nodesCode === 0 && nodesOut?.trim()) {
+      const nodes = nodesOut.trim().split("\n").filter(Boolean);
+      console.log("");
+
+      const parsed = nodes.map(line => {
+        const [id, hostname, status, availability, managerStatus] = line.split("|");
+        return { id, hostname, status, availability, managerStatus };
+      });
+
+      const maxHost = Math.max(...parsed.map(n => (n.hostname || "").length), 8);
+      const maxId = 12;
+
+      console.log(DIM(`  ${"NODE ID".padEnd(maxId)}  ${"HOSTNAME".padEnd(maxHost)}  ${"STATUS".padEnd(10)}  ${"AVAIL".padEnd(10)}  ROLE`));
+      console.log(DIM(`  ${"─".repeat(maxId)}  ${"─".repeat(maxHost)}  ${"─".repeat(10)}  ${"─".repeat(10)}  ${"─".repeat(10)}`));
+
+      for (const n of parsed) {
+        const statusColor = n.status === "Ready" ? OK : ERR;
+        const availColor = n.availability === "Active" ? OK : WARN;
+        const roleText = n.managerStatus
+          ? (n.managerStatus === "Leader" ? OK("Leader") : ACCENT(n.managerStatus))
+          : DIM("worker");
+
+        console.log(
+          `  ${DIM((n.id || "").slice(0, maxId).padEnd(maxId))}  ` +
+          `${chalk.bold.white((n.hostname || "").padEnd(maxHost))}  ` +
+          `${statusColor((n.status || "").padEnd(10))}  ` +
+          `${availColor((n.availability || "").padEnd(10))}  ` +
+          `${roleText}`
+        );
+      }
+
+      const managerCount = parsed.filter(n => n.managerStatus).length;
+      const readyCount = parsed.filter(n => n.status === "Ready").length;
+      console.log(DIM(`\n  ${parsed.length} node(s), ${managerCount} manager(s), ${readyCount} ready`));
+    }
+
+    // Service list (if any swarm services deployed)
+    const { stdout: svcsOut, exitCode: svcsCode } = await ssh(
+      "sudo docker service ls --format '{{.Name}}|{{.Replicas}}|{{.Image}}|{{.Ports}}' 2>/dev/null"
+    );
+
+    if (svcsCode === 0 && svcsOut?.trim()) {
+      const services = svcsOut.trim().split("\n").filter(Boolean);
+      console.log("");
+      console.log(DIM("  Services:"));
+
+      const maxSvc = Math.max(...services.map(s => s.split("|")[0]?.length || 0), 7);
+
+      console.log(DIM(`  ${"SERVICE".padEnd(maxSvc)}  ${"REPLICAS".padEnd(10)}  ${"IMAGE".padEnd(30)}  PORTS`));
+      console.log(DIM(`  ${"─".repeat(maxSvc)}  ${"─".repeat(10)}  ${"─".repeat(30)}  ${"─".repeat(20)}`));
+
+      for (const line of services) {
+        const [name, replicas, image, ports] = line.split("|");
+        const replicaParts = (replicas || "").split("/");
+        const running = parseInt(replicaParts[0]) || 0;
+        const desired = parseInt(replicaParts[1]) || 0;
+        const repColor = running === desired && running > 0 ? OK : running === 0 ? ERR : WARN;
+
+        console.log(
+          `  ${LABEL((name || "").padEnd(maxSvc))}  ` +
+          `${repColor((replicas || "").padEnd(10))}  ` +
+          `${DIM((image || "").slice(0, 30).padEnd(30))}  ` +
+          `${DIM(ports || "")}`
+        );
+      }
+    } else {
+      console.log(DIM("\n  No swarm services deployed."));
+      hint("Deploy: fops azure swarm deploy " + vmName);
+    }
+
+    console.log("");
+  } finally {
+    if (vm.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, vm.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, vm.publicIp, user);
+  }
+}
+
+/**
+ * Promote a worker node to manager.
+ */
+export async function swarmPromote(opts = {}) {
+  const execa = await lazyExeca();
+  const vmName = opts.vmName;
+  if (!vmName) {
+    console.error(ERR("\n  Usage: fops azure swarm promote <vmName>\n"));
+    process.exit(1);
+  }
+
+  const vm = readVmState(vmName);
+  if (!vm) {
+    console.error(ERR(`\n  VM "${vmName}" not tracked.\n`));
+    process.exit(1);
+  }
+
+  if (!vm.swarm?.nodeId) {
+    console.error(ERR(`\n  "${vmName}" is not in a swarm.\n`));
+    process.exit(1);
+  }
+
+  if (vm.swarm.role === "manager") {
+    console.log(WARN(`  "${vmName}" is already a manager.\n`));
+    return;
+  }
+
+  // Find a manager VM to run the promote command
+  const { vms } = listVms();
+  const managerEntry = Object.entries(vms).find(([, v]) => v.swarm?.role === "manager");
+  if (!managerEntry) {
+    console.error(ERR("\n  No swarm manager found in tracked VMs.\n"));
+    process.exit(1);
+  }
+
+  const [mgrName, mgr] = managerEntry;
+
+  banner("Swarm Promote");
+  kvLine("Promoting", LABEL(vmName));
+  kvLine("Via manager", DIM(mgrName));
+  console.log("");
+
+  await knockForVm(mgr);
+  const user = DEFAULTS.adminUser;
+
+  try {
+    const { exitCode, stdout } = await sshCmd(
+      execa, mgr.publicIp, user,
+      `sudo docker node promote ${vm.swarm.nodeId} 2>&1`,
+      30000,
+    );
+
+    if (exitCode !== 0) {
+      console.error(ERR(`  Promote failed: ${(stdout || "").trim()}`));
+      process.exit(1);
+    }
+
+    writeVmState(vmName, {
+      swarm: { ...vm.swarm, role: "manager", promotedAt: new Date().toISOString() },
+    });
+
+    console.log(OK(`  ✓ ${vmName} promoted to manager`));
+    console.log("");
+  } finally {
+    if (mgr.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, mgr.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, mgr.publicIp, user);
+  }
+}
+
+/**
+ * Remove a VM from the swarm.
+ */
+export async function swarmLeave(opts = {}) {
+  const execa = await lazyExeca();
+  const vmName = opts.vmName;
+  if (!vmName) {
+    console.error(ERR("\n  Usage: fops azure swarm leave <vmName>\n"));
+    process.exit(1);
+  }
+
+  const vm = readVmState(vmName);
+  if (!vm) {
+    console.error(ERR(`\n  VM "${vmName}" not tracked.\n`));
+    process.exit(1);
+  }
+
+  if (!vm.swarm?.role) {
+    console.log(DIM(`  "${vmName}" is not in a swarm.\n`));
+    return;
+  }
+
+  banner("Swarm Leave");
+  kvLine("VM", LABEL(vmName));
+  kvLine("Role", ACCENT(vm.swarm.role));
+  console.log("");
+
+  const isManager = vm.swarm.role === "manager";
+
+  if (isManager && !opts.force) {
+    console.log(WARN("  This is a manager node. Leaving will disrupt the swarm."));
+    hint("Use --force to confirm, or demote first.");
+    console.log("");
+    return;
+  }
+
+  await knockForVm(vm);
+  const user = DEFAULTS.adminUser;
+  const ssh = (cmd, t) => sshCmd(execa, vm.publicIp, user, cmd, t || 30000);
+
+  try {
+    const forceFlag = isManager ? " --force" : "";
+    const { exitCode, stdout: leaveOut } = await ssh(
+      `sudo docker swarm leave${forceFlag} 2>&1`,
+      60000,
+    );
+
+    if (exitCode !== 0) {
+      console.error(ERR(`  Leave failed: ${(leaveOut || "").trim()}`));
+      process.exit(1);
+    }
+
+    // Clear swarm state
+    writeVmState(vmName, { swarm: null });
+
+    console.log(OK(`  ✓ ${vmName} left the swarm`));
+
+    // If it was a manager, clean up worker references
+    if (isManager) {
+      const { vms } = listVms();
+      for (const [name, v] of Object.entries(vms)) {
+        if (v.swarm?.manager === vmName) {
+          writeVmState(name, { swarm: null });
+          console.log(DIM(`  Cleared swarm state for worker: ${name}`));
+        }
+      }
+    }
+
+    console.log("");
+  } finally {
+    if (vm.knockSequence?.length) {
+      const sshFn = (cmd) => sshCmd(execa, vm.publicIp, user, cmd);
+      await closeKnock(sshFn, { quiet: true }).catch(() => {});
+    }
+    await closeMux(execa, vm.publicIp, user);
+  }
+}