blitzwing 0.1.9 → 0.2.0

package/src/tunnel.js

@@ -0,0 +1,231 @@
+import { spawn, execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createWriteStream } from "node:fs";
+import { Readable } from "node:stream";
+import { HOME_DIR } from "./config.js";
+import { which } from "./net.js";
+
+const BIN_DIR = path.join(HOME_DIR, "bin");
+const URL_RE = /https:\/\/[a-z0-9-]+\.trycloudflare\.com/i;
+
+function platformAsset() {
+  const plat = process.platform;
+  const arch = process.arch;
+  if (plat === "linux" && arch === "x64") {
+    return { name: "cloudflared-linux-amd64", archive: false };
+  }
+  if (plat === "linux" && (arch === "arm64" || arch === "aarch64")) {
+    return { name: "cloudflared-linux-arm64", archive: false };
+  }
+  if (plat === "darwin" && arch === "arm64") {
+    return { name: "cloudflared-darwin-arm64.tgz", archive: true };
+  }
+  if (plat === "darwin" && arch === "x64") {
+    return { name: "cloudflared-darwin-amd64.tgz", archive: true };
+  }
+  if (plat === "win32" && arch === "x64") {
+    return { name: "cloudflared-windows-amd64.exe", archive: false, exe: true };
+  }
+  throw new Error(
+    `Unsupported platform for cloudflared auto-install: ${plat}/${arch}. Install cloudflared manually and retry.`
+  );
+}
+
+function localBinaryPath() {
+  const exe = process.platform === "win32" ? "cloudflared.exe" : "cloudflared";
+  return path.join(BIN_DIR, exe);
+}
+
+/**
+ * Resolve cloudflared binary: PATH first, then ~/.blitzwing/bin (download if missing).
+ * @param {{ onLog?: (msg: string) => void }} [opts]
+ */
+export async function ensureCloudflared({ onLog } = {}) {
+  const onPath = which("cloudflared");
+  if (onPath) return onPath;
+
+  const dest = localBinaryPath();
+  if (fs.existsSync(dest)) {
+    try {
+      fs.chmodSync(dest, 0o755);
+    } catch {
+      /* ignore */
+    }
+    return dest;
+  }
+
+  const asset = platformAsset();
+  const url = `https://github.com/cloudflare/cloudflared/releases/latest/download/${asset.name}`;
+  onLog?.(`Downloading cloudflared (${asset.name})…`);
+  fs.mkdirSync(BIN_DIR, { recursive: true });
+
+  const res = await fetch(url, { redirect: "follow" });
+  if (!res.ok || !res.body) {
+    throw new Error(`Failed to download cloudflared: HTTP ${res.status} from ${url}`);
+  }
+
+  if (asset.archive) {
+    const tgzPath = path.join(BIN_DIR, asset.name);
+    await pipeline(Readable.fromWeb(res.body), createWriteStream(tgzPath));
+    execFileSync("tar", ["-xzf", tgzPath, "-C", BIN_DIR], { stdio: "ignore" });
+    try {
+      fs.unlinkSync(tgzPath);
+    } catch {
+      /* ignore */
+    }
+    // tgz extracts a binary named cloudflared
+    const extracted = path.join(BIN_DIR, "cloudflared");
+    if (!fs.existsSync(extracted) && fs.existsSync(dest)) {
+      /* already named correctly */
+    } else if (fs.existsSync(extracted) && extracted !== dest) {
+      fs.renameSync(extracted, dest);
+    }
+  } else {
+    await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
+  }
+
+  try {
+    fs.chmodSync(dest, 0o755);
+  } catch {
+    /* windows */
+  }
+  if (!fs.existsSync(dest)) {
+    throw new Error(`cloudflared download finished but binary missing at ${dest}`);
+  }
+  onLog?.(`cloudflared ready at ${dest}`);
+  return dest;
+}
+
+/**
+ * Kill a process (and its group on Unix).
+ * @param {number} pid
+ */
+export function stopTunnelProcess(pid) {
+  if (!pid || !Number.isFinite(pid)) return;
+  try {
+    if (process.platform === "win32") {
+      execFileSync("taskkill", ["/PID", String(pid), "/T", "/F"], { stdio: "ignore" });
+    } else {
+      try {
+        process.kill(-pid, "SIGTERM");
+      } catch {
+        process.kill(pid, "SIGTERM");
+      }
+      setTimeout(() => {
+        try {
+          process.kill(-pid, "SIGKILL");
+        } catch {
+          try {
+            process.kill(pid, "SIGKILL");
+          } catch {
+            /* already gone */
+          }
+        }
+      }, 2000).unref?.();
+    }
+  } catch {
+    /* already gone */
+  }
+}
+
+/**
+ * Start a Cloudflare Quick Tunnel to http://127.0.0.1:port (no account/token).
+ * @param {{ port: number, binary?: string, logPath?: string, timeoutMs?: number }} opts
+ * @returns {Promise<{ url: string, pid: number, stop: () => void }>}
+ */
+export async function startQuickTunnel({
+  port,
+  binary,
+  logPath = path.join(HOME_DIR, "cloudflared.log"),
+  timeoutMs = 90_000,
+}) {
+  const bin = binary || (await ensureCloudflared());
+  fs.mkdirSync(path.dirname(logPath), { recursive: true });
+  const logFd = fs.openSync(logPath, "a");
+
+  const child = spawn(
+    bin,
+    ["tunnel", "--url", `http://127.0.0.1:${port}`, "--no-autoupdate"],
+    {
+      detached: process.platform !== "win32",
+      stdio: ["ignore", "pipe", "pipe"],
+      windowsHide: true,
+    }
+  );
+
+  let settled = false;
+  let buffer = "";
+
+  const appendLog = (chunk) => {
+    const text = chunk.toString();
+    buffer += text;
+    try {
+      fs.writeSync(logFd, text);
+    } catch {
+      /* ignore */
+    }
+  };
+
+  child.stdout?.on("data", appendLog);
+  child.stderr?.on("data", appendLog);
+
+  const stop = () => {
+    stopTunnelProcess(child.pid);
+    try {
+      fs.closeSync(logFd);
+    } catch {
+      /* ignore */
+    }
+  };
+
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      stop();
+      reject(
+        new Error(
+          `Timed out waiting for Cloudflare Quick Tunnel URL. See ${logPath}`
+        )
+      );
+    }, timeoutMs);
+
+    const tryMatch = () => {
+      const m = buffer.match(URL_RE);
+      if (!m || settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (process.platform !== "win32") {
+        child.unref();
+      }
+      resolve({
+        url: m[0].replace(/\/$/, ""),
+        pid: child.pid,
+        stop,
+      });
+    };
+
+    child.stdout?.on("data", tryMatch);
+    child.stderr?.on("data", tryMatch);
+
+    child.on("error", (err) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      reject(err);
+    });
+
+    child.on("exit", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      reject(
+        new Error(
+          `cloudflared exited early (code ${code}). See ${logPath}`
+        )
+      );
+    });
+  });
+}

package/src/wizard.js

@@ -1,23 +1,19 @@
 import * as p from "@clack/prompts";
 import color from "picocolors";
-import fs from "node:fs";
 import path from "node:path";
-import { spawn } from "node:child_process";
 import { DEFAULT_DISCOVERY_URL, HOME_DIR, SHARD_PORT, PETALS_PORT } from "./config.js";
 import { listMothers, motherHosts, joinHost, readyHost, leaveHost } from "./api.js";
-import {
-  detectPublicIp,
-  detectLocalIp,
-  discoverNgrokTcpAnnounce,
-  extractPeerMultiaddrFromLog,
-} from "./net.js";
+import { detectPublicIp, detectLocalIp } from "./net.js";
 import {
   ensureVenvAndPetals,
   startShardManagerProcess,
   waitForShardRunning,
+  syncRuntimeFiles,
   venvPython,
 } from "./install.js";
+import { ensureCloudflared, startQuickTunnel, stopTunnelProcess } from "./tunnel.js";
 import { saveState, loadState, clearState, ensureHome } from "./state.js";
+import { startContributorHeartbeatDaemon } from "./heartbeat.js";
 
 function parseArgs(argv) {
   const out = { cmd: null, discoveryUrl: process.env.BLITZWING_DISCOVERY_URL || DEFAULT_DISCOVERY_URL };
@@ -59,10 +55,9 @@ ${color.bold("blitzwing")} — join a Blitzwing mother swarm as a compute node
   blitzwing leave        Leave the swarm and reclaim your layers
 
 Env:
-  BLITZWING_DISCOVERY_URL   Override discovery service (default ${DEFAULT_DISCOVERY_URL})
-  BLITZWING_ANNOUNCE_MADDRS Explicit Petals announce multiaddr (VM with public IP)
-  BLITZWING_LOCAL_IP        Local IP for shard manager metadata
-  PETALS_USE_AUTO_RELAY=0   Disable libp2p auto-relay (use with public IP / port-forward)
+  BLITZWING_DISCOVERY_URL      Override discovery service (default ${DEFAULT_DISCOVERY_URL})
+  BLITZWING_HEDERA_ACCOUNT_ID  Prefill Hedera payout account (0.0.N)
+  BLITZWING_SHARD_PORT         Local shard HTTP port (default ${SHARD_PORT})
 `);
 }
 
@@ -70,6 +65,7 @@ async function wizard(args) {
   p.intro(color.bgCyan(color.black(" blitzwing ")));
   ensureHome();
 
+  const existing = loadState();
   const spin = p.spinner();
   spin.start("Loading network from Discovery Service…");
   let mothers;
@@ -137,74 +133,14 @@ async function wizard(args) {
   }
   const layersN = Number(layers);
 
-  const detected = await detectPublicIp();
-  const networkMode = await p.select({
-    message: "How will other peers reach your Petals node?",
-    options: [
-      {
-        value: "relay",
-        label: "Auto (recommended for home / NAT)",
-        hint: "Uses Petals libp2p relay — no port forwarding",
-      },
-      {
-        value: "public",
-        label: "Public IP / cloud VM",
-        hint: "TCP 31337 (and 8001) open on the internet",
-      },
-      {
-        value: "ngrok",
-        label: "Dev tunnel (ngrok TCP on :4040)",
-        hint: "Only if ngrok is already running",
-      },
-    ],
-    initialValue: "relay",
-  });
-  if (p.isCancel(networkMode)) {
-    p.cancel("Setup cancelled");
-    process.exit(0);
-  }
-
-  let publicIp = "";
-  let announceMaddrs = "";
-  let useAutoRelay = "1";
-
-  if (networkMode === "public") {
-    publicIp = await p.text({
-      message: "Public IPv4 (must be reachable on TCP " + PETALS_PORT + ")",
-      initialValue: detected || "",
-      validate(v) {
-        if (!v || !/^\d+\.\d+\.\d+\.\d+$/.test(String(v).trim())) {
-          return "Enter a valid public IPv4 address";
-        }
-      },
-    });
-    if (p.isCancel(publicIp)) {
-      p.cancel("Setup cancelled");
-      process.exit(0);
-    }
-    publicIp = String(publicIp).trim();
-    announceMaddrs = `/ip4/${publicIp}/tcp/${PETALS_PORT}`;
-    useAutoRelay = "0";
-  } else if (networkMode === "ngrok") {
-    const ngrok = await discoverNgrokTcpAnnounce(PETALS_PORT);
-    if (!ngrok) {
-      p.cancel("No ngrok TCP tunnel on http://127.0.0.1:4040. Start: ngrok tcp " + PETALS_PORT);
-      process.exit(1);
-    }
-    announceMaddrs = ngrok;
-    publicIp = ngrok.match(/dns4\/([^/]+)/)?.[1] || detected || "relay";
-    useAutoRelay = "0";
-    p.log.info(`Using ngrok announce ${announceMaddrs}`);
-  } else {
-    publicIp = detected || "relay";
-    p.log.info("Using Petals libp2p auto-relay (no inbound port forward required).");
-  }
-
-  const localIp = await detectLocalIp();
-
+  const hederaPrefill =
+    process.env.BLITZWING_HEDERA_ACCOUNT_ID ||
+    process.env.HEDERA_ACCOUNT_ID ||
+    existing?.hedera_account_id ||
+    "";
   const hederaAccount = await p.text({
     message: "Hedera account ID for layer payouts (e.g. 0.0.123456)",
-    initialValue: process.env.BLITZWING_HEDERA_ACCOUNT_ID || "",
+    initialValue: hederaPrefill,
     validate(v) {
       const s = String(v || "").trim();
       if (!/^0\.0\.\d+$/.test(s)) return "Enter a Hedera account like 0.0.123456";
@@ -217,7 +153,7 @@ async function wizard(args) {
   const hederaAccountId = String(hederaAccount).trim();
 
   const confirm = await p.confirm({
-    message: `Join ${selected.model} hosting ${layersN} layers (${networkMode}) paying to ${hederaAccountId}?`,
+    message: `Join ${selected.model} hosting ${layersN} layers, payouts to ${hederaAccountId}?`,
     initialValue: true,
   });
   if (p.isCancel(confirm) || !confirm) {
@@ -232,6 +168,7 @@ async function wizard(args) {
         spin.message(msg);
       },
     });
+    syncRuntimeFiles();
   } catch (err) {
     spin.stop("Install failed");
     p.cancel(err.message);
@@ -240,8 +177,28 @@ async function wizard(args) {
   const python = venvPython();
   spin.stop("Environment ready");
 
-  const shardManagerUrl = `http://${localIp}:${SHARD_PORT}`;
-  const petalsLogPath = path.join(HOME_DIR, "petals.log");
+  spin.start("Setting up Cloudflare Quick Tunnel…");
+  let tunnel;
+  try {
+    const cfBin = await ensureCloudflared({
+      onLog: (msg) => {
+        spin.message(msg);
+      },
+    });
+    tunnel = await startQuickTunnel({
+      port: SHARD_PORT,
+      binary: cfBin,
+      logPath: path.join(HOME_DIR, "cloudflared.log"),
+    });
+  } catch (err) {
+    spin.stop("Tunnel failed");
+    p.cancel(err.message);
+    process.exit(1);
+  }
+  spin.stop(`Public URL ${tunnel.url}`);
+
+  const localIp = await detectLocalIp();
+  const publicIp = (await detectPublicIp()) || localIp || "tunnel";
 
   spin.start("Requesting layer assignment from mother…");
   let assignment;
@@ -250,10 +207,11 @@ async function wizard(args) {
       model: selected.model,
       layers: layersN,
       public_ip: String(publicIp).trim(),
-      shard_manager_url: shardManagerUrl,
+      shard_manager_url: tunnel.url,
       hedera_account_id: hederaAccountId,
     });
   } catch (err) {
+    tunnel.stop();
     spin.stop("Join rejected");
     const max = err.body?.detail?.max_layers;
     p.cancel(`${err.message}${max != null ? ` (max available: ${max})` : ""}`);
@@ -264,19 +222,21 @@ async function wizard(args) {
   const logPath = path.join(HOME_DIR, "shard_manager.log");
   spin.start("Starting local Petals server…");
   const shardEnv = {
-      MODEL_NAME: selected.model,
-      PUBLIC_IP: announceMaddrs ? "" : String(publicIp).trim(),
-      BLOCK_INDICES: assignment.block_indices,
-      INITIAL_PEERS: (assignment.initial_peers || []).join(","),
-      PETALS_PORT: String(PETALS_PORT),
-      IDENTITY_PATH: path.join(HOME_DIR, "petals-identity"),
-      SHARD_AUTO_START: "1",
-      PETALS_USE_AUTO_RELAY: useAutoRelay,
-      PETALS_SKIP_REACHABILITY_CHECK: "1",
-    };
-  if (announceMaddrs) {
-    shardEnv.ANNOUNCE_MADDRS = announceMaddrs;
-  }
+    MODEL_NAME: selected.model,
+    PUBLIC_IP: String(publicIp).trim(),
+    BLOCK_INDICES: assignment.block_indices,
+    INITIAL_PEERS: "",
+    NEW_SWARM: "1",
+    BLITZWING_HTTP_ONLY: "1",
+    PETALS_PORT: String(PETALS_PORT),
+    IDENTITY_PATH: path.join(HOME_DIR, "petals-identity"),
+    SHARD_AUTO_START: "1",
+    PETALS_USE_AUTO_RELAY: "0",
+    PETALS_SKIP_REACHABILITY_CHECK: "1",
+    BLITZWING_MOTHER_URL: selected.mother_url,
+    MOTHER_PUBLIC_SHARD_URL: selected.mother_url,
+    BLITZWING_HOST_ID: assignment.host_id,
+  };
   const { pid } = startShardManagerProcess({
     python,
     logPath,
@@ -286,6 +246,7 @@ async function wizard(args) {
   try {
     await waitForShardRunning({ timeoutMs: 600000, statusHost: "127.0.0.1" });
   } catch (err) {
+    tunnel.stop();
     spin.stop("Petals did not become ready");
     p.cancel(`${err.message}. See ${logPath}`);
     process.exit(1);
@@ -293,13 +254,12 @@ async function wizard(args) {
   spin.stop("Petals is serving your layers");
 
   spin.start("Finalizing handoff with mother…");
-  const peerMultiaddr = extractPeerMultiaddrFromLog(petalsLogPath, fs);
   try {
     await readyHost(selected.mother_url, {
       host_id: assignment.host_id,
-      peer_multiaddr: peerMultiaddr || undefined,
     });
   } catch (err) {
+    tunnel.stop();
     spin.stop("Handoff failed");
     p.cancel(err.message);
     process.exit(1);
@@ -313,49 +273,28 @@ async function wizard(args) {
     block_indices: assignment.block_indices,
     layers_hosted: assignment.layers_hosted,
     public_ip: String(publicIp).trim(),
-    network_mode: networkMode,
-    announce_maddrs: announceMaddrs || null,
-    shard_manager_url: shardManagerUrl,
+    network_mode: "cloudflare",
+    shard_manager_url: tunnel.url,
+    tunnel_url: tunnel.url,
+    tunnel_pid: tunnel.pid,
     shard_pid: pid,
     discovery_url: args.discoveryUrl,
     hedera_account_id: hederaAccountId,
     joined_at: new Date().toISOString(),
   };
   saveState(state);
-  startHeartbeatDaemon(state);
+  startContributorHeartbeatDaemon({
+    motherUrl: state.mother_url,
+    hostId: state.host_id,
+  });
 
   p.outro(
     `${color.green("You are online.")} Hosting ${color.cyan(String(state.layers_hosted))} layers of ${color.cyan(state.model)} at ${state.block_indices}\n` +
+      `Tunnel: ${color.cyan(tunnel.url)}\n` +
       `Run ${color.bold("blitzwing status")} anytime, or ${color.bold("blitzwing leave")} to exit.`
   );
 }
 
-function startHeartbeatDaemon(state) {
-  const hbPath = path.join(HOME_DIR, "heartbeat.mjs");
-  fs.writeFileSync(
-    hbPath,
-    `const mother = ${JSON.stringify(state.mother_url)};
-const hostId = ${JSON.stringify(state.host_id)};
-async function beat() {
-  try {
-    await fetch(mother.replace(/\\/$/, "") + "/v1/hosts/heartbeat", {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ host_id: hostId }),
-    });
-  } catch {}
-}
-setInterval(beat, 60000);
-beat();
-`
-  );
-  const child = spawn(process.execPath, [hbPath], {
-    detached: true,
-    stdio: "ignore",
-  });
-  child.unref();
-}
-
 async function showStatus() {
   const state = loadState();
   if (!state) {
@@ -368,10 +307,10 @@ async function showStatus() {
   console.log(`  layers_hosted:  ${state.layers_hosted}`);
   console.log(`  block_indices:  ${state.block_indices}`);
   console.log(`  mother:         ${state.mother_url}`);
-  console.log(`  public_ip:      ${state.public_ip}`);
+  console.log(`  tunnel:         ${state.tunnel_url || state.shard_manager_url || "—"}`);
+  console.log(`  hedera:         ${state.hedera_account_id || "—"}`);
   try {
-    const statusHost = "127.0.0.1";
-    const res = await fetch(`http://${statusHost}:${SHARD_PORT}/status`);
+    const res = await fetch(`http://127.0.0.1:${SHARD_PORT}/status`);
     if (res.ok) {
       const st = await res.json();
       console.log(`  petals_running: ${st.running}`);
@@ -398,11 +337,13 @@ async function doLeave() {
     p.log.warn(`Mother leave call failed: ${err.message}`);
   }
   try {
-    const statusHost = "127.0.0.1";
-    await fetch(`http://${statusHost}:${SHARD_PORT}/stop`, { method: "POST" });
+    await fetch(`http://127.0.0.1:${SHARD_PORT}/stop`, { method: "POST" });
   } catch {
     /* ignore */
   }
+  if (state.tunnel_pid) {
+    stopTunnelProcess(state.tunnel_pid);
+  }
   clearState();
   spin.stop("Left swarm");
   p.outro("Your layers were reclaimed by the mother (when reachable).");