@meshxdata/fops 0.1.48 → 0.1.50

package/src/plugins/bundled/fops-plugin-azure/lib/azure-ops-ssh.js

@@ -0,0 +1,427 @@
+/**
+ * azure-ops-ssh.js
+ * SSH and remote agent operations for Azure VMs.
+ * Extracted from azure-ops.js for maintainability.
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import chalk from "chalk";
+import { readState, saveState, listVms, requireVmState } from "./azure-state.js";
+import {
+  DEFAULTS, DIM, OK, WARN, ERR,
+  banner, lazyExeca,
+  sshCmd, MUX_OPTS, waitForSsh, knockForVm,
+  ensureOpenAiNetworkAccess,
+} from "./azure-helpers.js";
+
+// ── ssh ─────────────────────────────────────────────────────────────────────
+
+export async function azureSsh(opts = {}) {
+  const { knockForVm, ensureKnockSequence } = await import("./azure-helpers.js");
+  // Sync IP + knock sequence from Azure before using them
+  const freshState = await ensureKnockSequence(requireVmState(opts.vmName));
+  const ip = freshState?.publicIp;
+  if (!ip) { console.error(chalk.red("\n  No IP address. Is the VM running? Try: fops azure start\n")); process.exit(1); }
+
+  const sshUser = opts.user || DEFAULTS.adminUser;
+  await knockForVm(freshState);
+
+  const { execa } = await import("execa");
+  const result = await execa("ssh", [
+    "-o", "StrictHostKeyChecking=no",
+    "-o", "UserKnownHostsFile=/dev/null",
+    "-o", "ConnectTimeout=15",
+    `${sshUser}@${ip}`,
+  ], { stdio: "inherit", reject: false });
+  if (result.exitCode !== 0 && result.exitCode !== null) {
+    console.error(chalk.red(`\n  SSH failed (exit ${result.exitCode}). If port-knock is enabled, try: fops azure knock open ${freshState?.vmName}\n`));
+    process.exit(1);
+  }
+}
+
+// ── port forward ─────────────────────────────────────────────────────────────
+
+export async function azurePortForward(opts = {}) {
+  const { remotePort, localPort } = opts;
+  const state = requireVmState(opts.vmName);
+  const ip = state.publicIp;
+  if (!ip) { console.error(chalk.red("\n  No IP address. Is the VM running? Try: fops azure start\n")); process.exit(1); }
+
+  await knockForVm(state);
+
+  const lp = localPort || remotePort;
+  console.log(chalk.cyan(`\n  Forwarding localhost:${lp} → ${ip}:${remotePort}`));
+  console.log(DIM("  Press Ctrl+C to stop\n"));
+
+  const { execa } = await import("execa");
+  await execa("ssh", [
+    ...MUX_OPTS(ip, DEFAULTS.adminUser),
+    "-N",
+    "-L", `${lp}:localhost:${remotePort}`,
+    `${DEFAULTS.adminUser}@${ip}`,
+  ], { stdio: "inherit", reject: false });
+}
+
+// ── ssh admin ────────────────────────────────────────────────────────────────
+
+export async function azureSshAdminAdd(opts = {}) {
+  const { pubKey } = opts;
+  if (!pubKey?.trim()) {
+    console.error(chalk.red("\n  ✗ No public key provided\n"));
+    process.exit(1);
+  }
+
+  const { vms } = listVms();
+  const vmNames = Object.keys(vms);
+  if (vmNames.length === 0) {
+    console.error(chalk.red("\n  ✗ No VMs tracked. Use: fops azure up <name>\n"));
+    process.exit(1);
+  }
+
+  const execa = await lazyExeca();
+  const adminUser = DEFAULTS.adminUser;
+  const keyLine = pubKey.trim();
+
+  banner("Adding SSH key to all VMs");
+  console.log(DIM(`  Key: ${keyLine.slice(0, 60)}...`));
+  console.log(DIM(`  VMs: ${vmNames.join(", ")}\n`));
+
+  let success = 0, failed = 0;
+
+  for (const vmName of vmNames) {
+    const vm = vms[vmName];
+    const ip = vm.publicIp;
+    if (!ip) {
+      console.log(WARN(`  ⚠ ${vmName}: no IP (VM stopped?) — skipped`));
+      failed++;
+      continue;
+    }
+
+    await knockForVm(vm);
+    const sshOk = await waitForSsh(execa, ip, adminUser, 15000);
+    if (!sshOk) {
+      console.log(ERR(`  ✗ ${vmName}: SSH unreachable — skipped`));
+      failed++;
+      continue;
+    }
+
+    const addKeyCmd = `grep -qxF '${keyLine}' ~/.ssh/authorized_keys 2>/dev/null || echo '${keyLine}' >> ~/.ssh/authorized_keys`;
+    const { exitCode } = await sshCmd(execa, ip, adminUser, addKeyCmd, 30000);
+    if (exitCode === 0) {
+      console.log(OK(`  ✓ ${vmName}: key added`));
+      success++;
+    } else {
+      console.log(ERR(`  ✗ ${vmName}: failed to add key`));
+      failed++;
+    }
+  }
+
+  console.log("");
+  if (failed === 0) {
+    console.log(OK(`  ✓ SSH key added to all ${success} VM(s)\n`));
+  } else {
+    console.log(WARN(`  Done: ${success} succeeded, ${failed} failed\n`));
+  }
+}
+
+// ── agent (remote TUI via SSH) ───────────────────────────────────────────────
+
+/** Shell-escape a value for single-quoted export (e.g. foo -> 'foo', a'b -> 'a'\''b'). */
+function shellExportValue(v) {
+  if (v == null || v === "") return "''";
+  const s = String(v);
+  return "'" + s.replace(/'/g, "'\\''") + "'";
+}
+
+/**
+ * Resolve Foundation project root for reading .env. Uses findProjectRoot() first, then
+ * readState().projectRoot so that keys always come from project .env even when cwd is not the repo.
+ * When we find root via cwd/FOUNDATION_ROOT, we persist it to state so running from ~ later still works.
+ */
+async function resolveProjectRootForEnv() {
+  const { findProjectRoot } = await import("./azure-openai.js");
+  let root = findProjectRoot();
+  if (root) {
+    root = path.resolve(root);
+    try {
+      const state = readState();
+      const existing = state?.projectRoot?.trim();
+      if (existing !== root) saveState({ ...state, projectRoot: root });
+    } catch {}
+    return root;
+  }
+  try {
+    const state = readState();
+    const candidate = state?.projectRoot?.trim();
+    if (candidate && fs.existsSync(candidate)) {
+      const compose = path.join(candidate, "docker-compose.yaml");
+      const composeAlt = path.join(candidate, "docker-compose.yml");
+      if (fs.existsSync(compose) || fs.existsSync(composeAlt)) return path.resolve(candidate);
+    }
+  } catch {}
+  return null;
+}
+
+/**
+ * Build Azure OpenAI env vars from local project .env for the remote session.
+ * Returns { exportPrefix, inlineEnv, envFileLines } where:
+ * - exportPrefix: "export VAR='val' ... ; " (for sourcing before other commands)
+ * - inlineEnv: "VAR='val' VAR2='val2' " for inline env (may be truncated by SSH argv limits)
+ * - envFileLines: array of "VAR=val" lines for writing to a file on the VM and sourcing (most reliable).
+ */
+async function buildAzureOpenAIEnvForRemote() {
+  const { readAzureOpenAIConfigFromEnv } = await import("./azure-openai.js");
+  const root = await resolveProjectRootForEnv();
+  if (!root) return { exportPrefix: "", inlineEnv: "", envFileLines: [] };
+  const envPath = path.join(root, ".env");
+  const cfg = readAzureOpenAIConfigFromEnv(envPath);
+  if (!cfg.endpoint || !cfg.key) return { exportPrefix: "", inlineEnv: "", envFileLines: [] };
+  const deployment = cfg.deployment || "gpt-4o";
+  const apiVersion = cfg.apiVersion || "2024-12-01-preview";
+  const lines = [
+    `AZURE_OPENAI_ENDPOINT=${cfg.endpoint}`,
+    `AZURE_OPENAI_API_KEY=${cfg.key}`,
+    `MX_OPENAI_API_KEY=${cfg.key}`,
+    `AZURE_OPENAI_DEPLOYMENT=${deployment}`,
+    `AZURE_OPENAI_API_VERSION=${apiVersion}`,
+  ];
+  if (cfg.fastDeployment) lines.push(`AZURE_OPENAI_FAST_DEPLOYMENT=${cfg.fastDeployment}`);
+  const parts = lines.map((l) => {
+    const eq = l.indexOf("=");
+    const k = l.slice(0, eq);
+    const v = l.slice(eq + 1);
+    return `${k}=${shellExportValue(v)}`;
+  });
+  const assign = parts.join(" ");
+  return {
+    exportPrefix: "export " + assign + "; ",
+    inlineEnv: assign + " ",
+    envFileLines: lines,
+  };
+}
+
+/** @deprecated Use buildAzureOpenAIEnvForRemote(). */
+async function buildAzureOpenAIExportPrefix() {
+  const { exportPrefix } = await buildAzureOpenAIEnvForRemote();
+  return exportPrefix;
+}
+
+/**
+ * Sync Azure OpenAI config from the local project .env to the VM's /opt/foundation-compose/.env
+ * when they differ. Source of truth is the project .env; we only push if the VM's Azure OpenAI
+ * vars don't match local .env. No-op if no project root, endpoint/key missing in .env, or VM
+ * already matches.
+ */
+async function syncOpenAIKeysToVm(execa, ip, adminUser, state) {
+  const {
+    readAzureOpenAIConfigFromEnv,
+    parseAzureOpenAIConfigFromContent,
+    azureOpenAIConfigEqual,
+  } = await import("./azure-openai.js");
+  const root = await resolveProjectRootForEnv();
+  if (!root) return;
+  const envPath = path.join(root, ".env");
+  const fromFile = readAzureOpenAIConfigFromEnv(envPath);
+  const endpoint = fromFile.endpoint;
+  const key = fromFile.key;
+  if (!endpoint || !key) return;
+
+  const deployment = fromFile.deployment || "gpt-4o";
+  const apiVersion = fromFile.apiVersion || "2024-12-01-preview";
+  const fastDeployment = fromFile.fastDeployment || "";
+
+  try {
+    const { stdout: remoteEnv } = await sshCmd(execa, ip, adminUser, "cat /opt/foundation-compose/.env 2>/dev/null || true", 10000);
+    const onVm = parseAzureOpenAIConfigFromContent(remoteEnv || "");
+    if (azureOpenAIConfigEqual(fromFile, onVm)) return;
+  } catch {
+    // If we can't read VM .env, sync anyway so VM gets local config
+  }
+  console.log(chalk.dim("  VM .env differs from local project .env; syncing Azure OpenAI config…"));
+
+  const tmpFile = path.join(os.tmpdir(), `fops-openai-env-${Date.now()}`);
+  const lines = [
+    `AZURE_OPENAI_ENDPOINT=${endpoint}`,
+    `AZURE_OPENAI_API_KEY=${key}`,
+    `MX_OPENAI_API_KEY=${key}`,
+    `AZURE_OPENAI_DEPLOYMENT=${deployment}`,
+    `AZURE_OPENAI_API_VERSION=${apiVersion}`,
+  ];
+  if (fastDeployment) lines.push(`AZURE_OPENAI_FAST_DEPLOYMENT=${fastDeployment}`);
+  fs.writeFileSync(tmpFile, lines.join("\n") + "\n", "utf8");
+  const remotePath = "/tmp/fops-openai-env-update";
+  try {
+    const { exitCode: scpCode } = await execa("scp", [
+      ...MUX_OPTS(ip, adminUser),
+      tmpFile,
+      `${adminUser}@${ip}:${remotePath}`,
+    ], { timeout: 15000, reject: false });
+    if (scpCode !== 0) {
+      console.log(chalk.yellow("  ⚠ SCP failed — could not upload env to VM"));
+      return;
+    }
+    // Loop body must use semicolons so "do" is not followed by "&&" (invalid on remote bash)
+    const mergeScript = [
+      "cd /opt/foundation-compose",
+      "touch .env",
+      "while IFS= read -r line; do [[ -z \"$line\" || \"$line\" =~ ^# ]] && continue; key=\"${line%%=*}\"; key=\"${key%% *}\"; [[ -z \"$key\" ]] && continue; (grep -v \"^$key=\" .env 2>/dev/null || true) > .env.tmp; echo \"$line\" >> .env.tmp; mv .env.tmp .env; done < " + remotePath,
+      "rm -f " + remotePath,
+    ].join(" && ");
+    const { exitCode: mergeCode } = await sshCmd(execa, ip, adminUser, mergeScript, 15000);
+    if (mergeCode === 0) {
+      console.log(chalk.dim("  ✓ Azure OpenAI keys synced to VM .env"));
+    } else {
+      console.log(chalk.yellow("  ⚠ Merge script failed on VM (exit " + mergeCode + ") — .env may not be updated"));
+    }
+  } finally {
+    try { fs.unlinkSync(tmpFile); } catch {}
+  }
+}
+
+export async function azureAgent(opts = {}) {
+  const execa = await lazyExeca();
+  const state = requireVmState(opts.vmName);
+  const ip = state.publicIp;
+  const adminUser = DEFAULTS.adminUser;
+  if (!ip) { console.error(chalk.red("\n  No IP address. Is the VM running? Try: fops azure start\n")); process.exit(1); }
+
+  await knockForVm(state);
+
+  // Whitelist VM's public IP on Azure OpenAI (if resource has firewall) so agent on VM can call the API
+  await ensureOpenAiNetworkAccess(execa, ip, opts.profile);
+
+  await syncOpenAIKeysToVm(execa, ip, adminUser, state);
+
+  const webPort = 3099;
+  const agentName = opts.agent || "foundation";
+  const vmName = state.vmName;
+  console.log(chalk.dim(`  VM: ${vmName} (${ip}) — agent: ${agentName}\n`));
+
+  // Sync local project root .env Azure OpenAI into the remote session (inline VAR=val before fops agent so the process gets them)
+  const { exportPrefix, inlineEnv } = await buildAzureOpenAIEnvForRemote();
+  if (!inlineEnv) {
+    const root = await resolveProjectRootForEnv();
+    const envPath = root ? path.join(root, ".env") : null;
+    console.log(chalk.yellow("  ⚠ Azure OpenAI keys are read from the project .env; none found."));
+    console.log(chalk.dim("     Agent on VM will show \"Connection error\" or \"No API key found\" unless keys are present."));
+    if (envPath) {
+      console.log(chalk.dim(`     Add AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY to ${envPath} then run this again.`));
+    } else {
+      console.log(chalk.dim(`     Project root not found (cwd: ${process.cwd()}). Set FOUNDATION_ROOT or projectRoot in ~/.fops.json to your foundation-compose directory, or run this from that directory. Then add AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY to that project's .env.`));
+    }
+    console.log(chalk.dim(""));
+  }
+  const envPrefix = "set -a && . /opt/foundation-compose/.env 2>/dev/null; set +a && cd /opt/foundation-compose && ";
+  const remotePrefix = exportPrefix + envPrefix;
+  const agentEnv = inlineEnv; // inline VAR=val before "fops agent" so the process inherits them (reliable with ssh sh -c)
+  const args = [
+    "-t",
+    "-L", `${webPort}:localhost:${webPort}`,
+    "-o", "StrictHostKeyChecking=no",
+    "-o", "ExitOnForwardFailure=no",
+    `${DEFAULTS.adminUser}@${ip}`,
+    remotePrefix + agentEnv + `fops agent ${agentName}`.trim(),
+  ];
+  if (opts.message) {
+    args.pop();
+    args.push(remotePrefix + agentEnv + `fops agent ${agentName} -m ${JSON.stringify(opts.message)}`.trim());
+  }
+  if (opts.classic) {
+    args[args.length - 1] += " --classic";
+  }
+  if (opts.model) {
+    args[args.length - 1] += ` --model ${opts.model}`;
+  }
+
+  console.log(chalk.cyan(`  Agent web UI: http://localhost:${webPort}`));
+  console.log(chalk.dim("  Wait for the TUI to appear on the VM before opening the URL (avoids \"channel 3: open failed: Connection refused\")."));
+  console.log(chalk.dim("  If the agent shows a connection error to Azure OpenAI, run: fops azure openai debug-vm " + (vmName && vmName !== "alessio" ? vmName : "") + "\n"));
+
+  const result = await execa("ssh", args, { stdio: "inherit", reject: false });
+  if (result?.exitCode !== 0 && result?.exitCode != null) {
+    console.log(chalk.yellow("\n  SSH exited with code " + result.exitCode + ". For Azure OpenAI connection errors run: fops azure openai debug-vm" + (vmName && vmName !== "alessio" ? " " + vmName : "") + "\n"));
+  }
+}
+
+/**
+ * Run a single agent request on the VM with DEBUG=1 and stream back stdout+stderr
+ * so we can see the real connection error (e.g. cause.message, cause.code).
+ */
+export async function azureOpenAiDebugVm(opts = {}) {
+  const execa = await lazyExeca();
+  const state = requireVmState(opts.vmName);
+  const ip = state.publicIp;
+  const adminUser = DEFAULTS.adminUser;
+  const remoteEnvPath = "/tmp/fops-azure-env." + process.pid;
+  if (!ip) {
+    console.error(chalk.red("\n  No IP address. Is the VM running? Try: fops azure start\n"));
+    process.exit(1);
+  }
+
+  await knockForVm(state);
+  await ensureOpenAiNetworkAccess(execa, ip, opts.profile);
+  await syncOpenAIKeysToVm(execa, ip, adminUser, state);
+
+  const agentName = opts.agent || "foundation";
+  const root = await resolveProjectRootForEnv();
+  const envPath = root ? path.join(root, ".env") : null;
+  const { envFileLines } = await buildAzureOpenAIEnvForRemote();
+
+  if (envFileLines.length) {
+    console.log(chalk.dim("  Pushing Azure OpenAI vars to VM .env, then running agent (sourcing .env so process gets keys)."));
+    const tmpFile = path.join(os.tmpdir(), `fops-azure-env-${process.pid}`);
+    fs.writeFileSync(tmpFile, envFileLines.join("\n") + "\n", "utf8");
+    try {
+      const { exitCode: scpCode } = await execa("scp", [
+        ...MUX_OPTS(ip, adminUser),
+        tmpFile,
+        `${adminUser}@${ip}:${remoteEnvPath}`,
+      ], { timeout: 15000, reject: false });
+      if (scpCode === 0) {
+        const mergeScript = [
+          "cd /opt/foundation-compose",
+          "touch .env",
+          "while IFS= read -r line; do [[ -z \"$line\" || \"$line\" =~ ^# ]] && continue; key=\"${line%%=*}\"; key=\"${key%% *}\"; [[ -z \"$key\" ]] && continue; (grep -v \"^$key=\" .env 2>/dev/null || true) > .env.tmp; echo \"$line\" >> .env.tmp; mv .env.tmp .env; done < " + remoteEnvPath,
+          "rm -f " + remoteEnvPath,
+        ].join(" && ");
+        const { exitCode: mergeCode } = await sshCmd(execa, ip, adminUser, mergeScript, 15000);
+        if (mergeCode === 0) {
+          const { stdout: check } = await sshCmd(execa, ip, adminUser, "grep -E '^AZURE_OPENAI_ENDPOINT=|^AZURE_OPENAI_API_KEY=' /opt/foundation-compose/.env 2>/dev/null | sed 's/=.*/=***/' || true", 5000);
+          if (check?.trim()) console.log(chalk.dim("  VM .env: " + check.trim().replace(/\n/g, " ")));
+        }
+      }
+    } finally {
+      try { fs.unlinkSync(tmpFile); } catch {}
+    }
+  } else {
+    console.log(chalk.yellow("  ⚠ Not injecting Azure OpenAI — remote session will have no keys from project .env."));
+    if (envPath) {
+      const { readAzureOpenAIConfigFromEnv } = await import("./azure-openai.js");
+      const cfg = fs.existsSync(envPath) ? readAzureOpenAIConfigFromEnv(envPath) : { endpoint: "", key: "" };
+      const missing = [];
+      if (!cfg.endpoint?.trim()) missing.push("AZURE_OPENAI_ENDPOINT");
+      if (!cfg.key?.trim()) missing.push("AZURE_OPENAI_API_KEY or MX_OPENAI_API_KEY");
+      if (missing.length) {
+        console.log(chalk.dim(`     Tried ${envPath} — missing: ${missing.join(", ")}.`));
+      } else {
+        console.log(chalk.dim(`     Project root not found. Tried cwd/FOUNDATION_ROOT/~/.fops.json projectRoot.`));
+      }
+      console.log(chalk.dim(`     Add the missing var(s) to ${envPath}, then run again.\n`));
+    } else {
+      console.log(chalk.dim(`     Project root not found (cwd: ${process.cwd()}). Set FOUNDATION_ROOT or projectRoot in ~/.fops.json, or run from foundation-compose. Then add AZURE_OPENAI_* to that project's .env.\n`));
+    }
+  }
+
+  // Source VM .env so AZURE_OPENAI_* (and MX_OPENAI_API_KEY) are in the environment, then run fops
+  const cmd =
+    "bash -c 'cd /opt/foundation-compose && set -a && . ./.env 2>/dev/null; set +a; exec env DEBUG=1 fops agent " + agentName + " -m \"hi\" 2>&1'";
+  console.log(chalk.cyan("\n  Running: DEBUG=1 fops agent " + agentName + " -m 'hi' (VM .env sourced so AZURE_OPENAI_* are exported)\n"));
+  const { stdout, stderr, exitCode } = await sshCmd(execa, ip, adminUser, cmd, 60000);
+  const out = [stdout, stderr].filter(Boolean).join("\n");
+  if (out) console.log(out);
+  if (exitCode !== 0) {
+    console.log(chalk.yellow("\n  Exit code: " + exitCode + " — see output above. If 'No API key': ensure VM .env has AZURE_OPENAI_* or run 'fops azure deploy' to update VM fops."));
+  }
+}