@iamken/cloudtunnel 0.10.4 → 0.11.0

package/dist/index.js

@@ -2,13 +2,13 @@
 import {
   listZones,
   resolveZone
-} from "./chunk-NB3OEY2P.js";
+} from "./chunk-OT3UCBBF.js";
 import {
   createCname,
   deleteDnsRecord,
   findCname,
   isManagedDns
-} from "./chunk-IDTF7SUY.js";
+} from "./chunk-ABLZ7XRZ.js";
 import {
   CliError,
   DEFAULT_API_BASE,
@@ -41,7 +41,7 @@ import {
   say,
   scanCacheFile,
   selectOne
-} from "./chunk-ZIUYW2HQ.js";
+} from "./chunk-7XACBYL7.js";
 
 // src/index.ts
 import { Command } from "commander";
@@ -143,18 +143,22 @@ function buildUpArgs(p) {
       "-d",
       p.zone,
       ...p.proto === "https" ? ["--proto", "https"] : [],
+      "--keep-alive",
       "-f",
       "-y"
     ];
   }
   const spec = formatTunnelSpec({ subdomain: p.subdomain, port: p.port, host: p.host });
+  const proto = p.protocol ?? "http2";
   return [
     "up",
     spec,
     "-d",
     p.zone,
     ...p.proto === "https" ? ["--proto", "https"] : [],
-    ...p.protocol ? ["--protocol", p.protocol] : [],
+    "--protocol",
+    proto,
+    "--keep-alive",
     "-f",
     "-y"
   ];
@@ -204,6 +208,7 @@ function buildUnit(d) {
     `Description=cloudtunnel ${d.fqdn} (Cloudflare Tunnel)`,
     "After=network-online.target",
     "Wants=network-online.target",
+    "StartLimitIntervalSec=0",
     "",
     "[Service]",
     "Type=simple",
@@ -211,7 +216,7 @@ function buildUnit(d) {
     `Environment=HOME=${d.home}`,
     `Environment=PATH=${nodeBin}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
     `ExecStart=${d.nodePath} ${d.scriptPath} ${d.argv.join(" ")}`,
-    "Restart=on-failure",
+    "Restart=always",
     "RestartSec=5",
     "",
     "[Install]",
@@ -381,7 +386,7 @@ function buildTaskXml(d) {
     "    <StopIfGoingOnBatteries>false</StopIfGoingOnBatteries>",
     "    <StartWhenAvailable>true</StartWhenAvailable>",
     "    <ExecutionTimeLimit>PT0S</ExecutionTimeLimit>",
-    "    <RestartOnFailure><Interval>PT1M</Interval><Count>3</Count></RestartOnFailure>",
+    "    <RestartOnFailure><Interval>PT1M</Interval><Count>999</Count></RestartOnFailure>",
     "    <Enabled>true</Enabled>",
     "  </Settings>",
     '  <Actions Context="Author">',
@@ -937,6 +942,7 @@ function extractTgz(bytes) {
 
 // src/core/up-runner.ts
 import { join as join7 } from "path";
+import { readFileSync as readFileSync6, existsSync as existsSync7 } from "fs";
 import * as clack2 from "@clack/prompts";
 
 // src/connector/process.ts
@@ -1063,7 +1069,7 @@ async function reconcile() {
 // src/connector/process.ts
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 function startConnector(opts) {
-  const args = ["tunnel", "run"];
+  const args = ["tunnel", "run", "--metrics", "127.0.0.1:0"];
   if (opts.protocol) args.push("--protocol", opts.protocol);
   const env = { ...process.env, TUNNEL_TOKEN: opts.token };
   const fd = openSync(opts.logFile, "a", 384);
@@ -1171,6 +1177,34 @@ async function waitHealthy(cf, tunnelId, opts = {}) {
   }
   return opts.signal?.aborted ? "dead" : "provisioning";
 }
+function extractMetricsPort(log) {
+  const matches = [...log.matchAll(/Starting metrics server on 127\.0\.0\.1:(\d+)\/metrics/g)];
+  if (!matches.length) return void 0;
+  const last = matches[matches.length - 1];
+  const port = Number(last?.[1]);
+  return Number.isInteger(port) && port > 0 && port <= 65535 ? port : void 0;
+}
+async function checkLocalReady(port, opts = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 2e3);
+  try {
+    const res = await fetch(`http://127.0.0.1:${port}/ready`, { signal: controller.signal });
+    if (!res.ok) return { healthy: false, readyConnections: 0, status: res.status };
+    const data = await res.json().catch(() => ({}));
+    const readyConnections = Number(data.readyConnections) || 0;
+    return { healthy: readyConnections > 0, readyConnections, status: res.status };
+  } catch {
+    return { healthy: false, readyConnections: 0 };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function probeEntryState(entry, gone) {
+  if (gone || entry.state !== "running" || !entry.pid) return "down";
+  if (!entry.metricsPort) return "up";
+  const ready = await checkLocalReady(entry.metricsPort, { timeoutMs: 300 });
+  return ready.healthy ? "up" : "reconnecting";
+}
 
 // src/core/orchestrator-create.ts
 import { randomInt as randomInt2 } from "crypto";
@@ -1432,8 +1466,8 @@ async function resolveRemoteTarget(cf, target) {
   }
   const tunnel = matches[0];
   if (!tunnel) return null;
-  const { listCargoCnames } = await import("./dns-2F5SYQNX.js");
-  const { listZones: listZones3 } = await import("./zones-PCUEPXHD.js");
+  const { listCargoCnames } = await import("./dns-DKPXBRCH.js");
+  const { listZones: listZones3 } = await import("./zones-XNCXYCB4.js");
   for (const zone of await listZones3(cf.token)) {
     const rec = (await listCargoCnames(cf.token, zone.id)).find((r) => tunnelIdFromCname2(r.content) === tunnel.id);
     if (rec) return { tunnel, fqdn: rec.name };
@@ -1454,24 +1488,25 @@ async function removeTunnelById(cf, tunnel, opts = {}) {
 async function listAll(cf, opts = {}) {
   const entries = await reconcile();
   const tunnels = new Map((await listTunnels(cf)).map((t) => [t.id, t]));
-  const rows = entries.map((e) => {
+  const rows = await Promise.all(entries.map(async (e) => {
     const fqdn = entryFqdn(e);
     const gone = e.tunnelId ? !tunnels.has(e.tunnelId) : false;
     const svc = serviceState(fqdn);
+    const state4 = await probeEntryState(e, gone);
     return {
       num: e.index ? String(e.index) : "-",
       url: `https://${fqdn}`,
       target: serviceUrl(e.proto, e.host ?? "localhost", e.port),
       protocol: e.protocol ?? "auto",
-      state: !gone && e.state === "running" ? "up" : "down",
+      state: state4,
       service: svc === "none" ? "-" : svc,
       pid: e.state === "running" && e.pid ? String(e.pid) : "-",
       managed: true
     };
-  });
+  }));
   if (opts.all) {
-    const { listCargoCnames } = await import("./dns-2F5SYQNX.js");
-    const { listZones: listZones3 } = await import("./zones-PCUEPXHD.js");
+    const { listCargoCnames } = await import("./dns-DKPXBRCH.js");
+    const { listZones: listZones3 } = await import("./zones-XNCXYCB4.js");
     const tracked = new Set(entries.map(entryFqdn));
     const unmanaged = [];
     for (const zone of await listZones3(cf.token)) {
@@ -1492,9 +1527,141 @@ async function listAll(cf, opts = {}) {
   return rows;
 }
 
+// src/core/watchdog.ts
+import { readFileSync as readFileSync5, existsSync as existsSync6 } from "fs";
+var activeWatchdogs = /* @__PURE__ */ new Map();
+var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
+function resolveMetricsPort(logFile, fallback) {
+  if (existsSync6(logFile)) {
+    try {
+      const port = extractMetricsPort(readFileSync5(logFile, "utf8"));
+      if (port) return port;
+    } catch {
+    }
+  }
+  return fallback;
+}
+async function verifyRemoteTunnel(cf, tunnelId) {
+  try {
+    await getTunnel(cf, tunnelId);
+    return true;
+  } catch (err) {
+    if (err instanceof CliError && err.status === 404) return false;
+    return true;
+  }
+}
+async function tier1Restart(ctx, state4) {
+  state4.lastRestartAt = Date.now();
+  state4.tier1Restarts++;
+  say.warn(`[watchdog] Health check failed for ${ctx.fqdn} -> restarting connector (Tier 1)...`);
+  await stopConnector({ pid: ctx.pid, bootId: currentBootId() });
+  const conn = startConnector({
+    bin: ctx.bin,
+    token: ctx.token,
+    detach: false,
+    logFile: ctx.logFile,
+    protocol: ctx.protocol,
+    onExit: () => {
+      void onConnectorExited(ctx.fqdn);
+    }
+  });
+  ctx.pid = conn.pid;
+  await sleep3(1e3);
+  ctx.metricsPort = resolveMetricsPort(ctx.logFile, ctx.metricsPort);
+  await patchEntry(ctx.fqdn, { pid: conn.pid, metricsPort: ctx.metricsPort, bootId: currentBootId() });
+  state4.consecutiveFailures = 0;
+}
+async function tier2Reprovision(ctx, state4) {
+  state4.lastRestartAt = Date.now();
+  state4.tier2Restarts++;
+  say.warn(`[watchdog] Tunnel missing or unrecoverable for ${ctx.fqdn} -> re-provisioning (Tier 2)...`);
+  await stopConnector({ pid: ctx.pid, bootId: currentBootId() });
+  const result = await createTunnelSubdomain(ctx.cf, { ...ctx.item, force: true, yes: true });
+  ctx.tunnelId = result.tunnelId;
+  ctx.token = result.token;
+  const conn = startConnector({
+    bin: ctx.bin,
+    token: ctx.token,
+    detach: false,
+    logFile: ctx.logFile,
+    protocol: ctx.protocol,
+    onExit: () => {
+      void onConnectorExited(ctx.fqdn);
+    }
+  });
+  ctx.pid = conn.pid;
+  await sleep3(1e3);
+  ctx.metricsPort = resolveMetricsPort(ctx.logFile, ctx.metricsPort);
+  await patchEntry(ctx.fqdn, { pid: conn.pid, metricsPort: ctx.metricsPort, bootId: currentBootId(), tunnelId: ctx.tunnelId });
+  state4.consecutiveFailures = 0;
+  state4.tier1Restarts = 0;
+}
+async function onConnectorExited(fqdn) {
+  const item = activeWatchdogs.get(fqdn);
+  if (!item) return;
+  const { ctx, state: state4 } = item;
+  if (Date.now() - state4.lastRestartAt < 5e3) return;
+  await tier1Restart(ctx, state4);
+}
+function startTunnelWatchdog(ctx, opts = {}) {
+  stopTunnelWatchdog(ctx.fqdn);
+  const intervalMs = opts.intervalMs ?? 15e3;
+  const cooldownMs = opts.cooldownMs ?? 5e3;
+  const state4 = { consecutiveFailures: 0, tier1Restarts: 0, tier2Restarts: 0, lastRestartAt: 0 };
+  const check = async () => {
+    if (!ctx.metricsPort) ctx.metricsPort = resolveMetricsPort(ctx.logFile, ctx.metricsPort);
+    let healthy = false;
+    if (ctx.metricsPort) {
+      const res = await checkLocalReady(ctx.metricsPort);
+      healthy = res.healthy;
+    }
+    if (healthy) {
+      state4.consecutiveFailures = 0;
+      return;
+    }
+    state4.consecutiveFailures++;
+    if (state4.consecutiveFailures >= 3 && Date.now() - state4.lastRestartAt >= cooldownMs) {
+      const remoteExists = await verifyRemoteTunnel(ctx.cf, ctx.tunnelId);
+      if (!remoteExists || state4.tier1Restarts >= 3) {
+        await tier2Reprovision(ctx, state4);
+      } else {
+        await tier1Restart(ctx, state4);
+      }
+    }
+  };
+  const timer = setInterval(() => {
+    void check();
+  }, intervalMs);
+  if (typeof timer.unref === "function") timer.unref();
+  activeWatchdogs.set(ctx.fqdn, { timer, ctx, state: state4 });
+}
+function stopTunnelWatchdog(fqdn) {
+  const item = activeWatchdogs.get(fqdn);
+  if (item) {
+    clearInterval(item.timer);
+    activeWatchdogs.delete(fqdn);
+  }
+}
+function stopAllWatchdogs() {
+  for (const [fqdn] of activeWatchdogs) stopTunnelWatchdog(fqdn);
+}
+
 // src/core/up-runner.ts
-function logFileFor(subdomain) {
-  return join7(logDir, `${subdomain === "@" ? "root" : subdomain}.log`);
+var sleep4 = (ms) => new Promise((r) => setTimeout(r, ms));
+function logFileFor(fqdn) {
+  return join7(logDir, `${serviceSlug(fqdn)}.log`);
+}
+async function createWithBootRetry(cf, item, keepAlive) {
+  const deadline = Date.now() + (keepAlive ? 3e4 : 0);
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await createTunnelSubdomain(cf, item);
+    } catch (err) {
+      if (!keepAlive || Date.now() >= deadline) throw err;
+      say.debug(`[boot-retry] Waiting for network to create ${item.name ?? "tunnel"}...`);
+      await sleep4(Math.min(2 ** attempt * 1e3, 5e3));
+    }
+  }
 }
 async function startTunnels(cf, bin, items, opts = {}) {
   const started = [];
@@ -1502,14 +1669,19 @@ async function startTunnels(cf, bin, items, opts = {}) {
   const teardownAll = async (code) => {
     if (tornDown) return;
     tornDown = true;
+    stopAllWatchdogs();
     try {
       for (const s of started) {
-        try {
-          await removeTunnelSubdomain(cf, s.fqdn, { force: true, quiet: true });
-        } catch {
+        if (opts.keepAlive) {
+          await stopConnector({ pid: s.pid, bootId: currentBootId() });
+        } else {
+          try {
+            await removeTunnelSubdomain(cf, s.fqdn, { force: true, quiet: true });
+          } catch {
+          }
         }
       }
-      if (process.stdout.isTTY) clack2.outro(`Stopped \xB7 released ${started.length} subdomain(s)`);
+      if (process.stdout.isTTY && !opts.keepAlive) clack2.outro(`Stopped \xB7 released ${started.length} subdomain(s)`);
     } catch (err) {
       reportError(err);
     } finally {
@@ -1520,9 +1692,9 @@ async function startTunnels(cf, bin, items, opts = {}) {
   spin.start(items.length > 1 ? "Creating tunnels\u2026" : "Creating tunnel\u2026");
   for (const item of items) {
     spin.message(`Creating ${item.name ?? "tunnel"} (:${item.port})\u2026`);
-    const result = await createTunnelSubdomain(cf, item);
+    const result = await createWithBootRetry(cf, item, opts.keepAlive);
     const fqdn = result.host.hostname;
-    const logFile = logFileFor(result.host.subdomain);
+    const logFile = logFileFor(fqdn);
     const conn = startConnector({
       bin,
       token: result.token,
@@ -1530,13 +1702,21 @@ async function startTunnels(cf, bin, items, opts = {}) {
       logFile,
       protocol: opts.protocol,
       onExit: opts.detach ? void 0 : (code) => {
-        if (!tornDown) {
+        if (!tornDown && !opts.keepAlive) {
           say.warn(`Connector for ${fqdn} exited.`);
           void teardownAll(code ?? 1);
         }
       }
     });
-    await patchEntry(fqdn, { pid: conn.pid, bootId: currentBootId(), logFile, protocol: opts.protocol });
+    await sleep4(600);
+    let metricsPort;
+    if (existsSync7(logFile)) {
+      try {
+        metricsPort = extractMetricsPort(readFileSync6(logFile, "utf8"));
+      } catch {
+      }
+    }
+    await patchEntry(fqdn, { pid: conn.pid, bootId: currentBootId(), logFile, metricsPort, protocol: opts.protocol });
     started.push({
       fqdn,
       subdomain: result.host.subdomain,
@@ -1544,6 +1724,20 @@ async function startTunnels(cf, bin, items, opts = {}) {
       target: serviceUrl(item.proto, item.host ?? "localhost", item.port),
       pid: conn.pid
     });
+    if (!opts.detach) {
+      startTunnelWatchdog({
+        cf,
+        bin,
+        fqdn,
+        tunnelId: result.tunnelId,
+        token: result.token,
+        logFile,
+        item,
+        pid: conn.pid,
+        metricsPort,
+        protocol: opts.protocol
+      });
+    }
   }
   if (opts.detach) {
     spin.stop(`${started.length} tunnel(s) started in the background`);
@@ -1561,7 +1755,7 @@ async function startTunnels(cf, bin, items, opts = {}) {
   spin.stop(`${started.length} tunnel(s) started`);
   const lines = started.map((s, i) => `${formatRoute(s.fqdn, s.target)}${healths[i] === "healthy" ? "" : dim(`  (${healths[i]})`)}`);
   clack2.note(lines.join("\n"), `${live}/${started.length} live`);
-  say.dim("Ctrl-C stops and releases them.");
+  say.dim(opts.keepAlive ? "Service running with auto-heal watchdog." : "Ctrl-C stops and releases them.");
 }
 
 // src/core/resolve-domain.ts
@@ -1642,7 +1836,7 @@ async function runUp(specArgs, opts) {
     await registerServices(cf, items, domain, opts.proto, protocol);
     return;
   }
-  await startTunnels(cf, bin, items, { detach: opts.detach, protocol });
+  await startTunnels(cf, bin, items, { detach: opts.detach, protocol, keepAlive: opts.keepAlive });
 }
 var SERVICE_UP_TIMEOUT_MS = 2e4;
 var delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -1688,7 +1882,7 @@ async function waitServicesUp(fqdns, timeoutMs) {
   return [...pending];
 }
 function registerUp(program) {
-  program.command("up", { isDefault: true }).argument("[specs...]", "tunnels to start: [subdomain:]port[@host]  (e.g. api:8080 web:8081@localhost)").description("Start one or more tunnels (also: `cloudtunnel 8080`)").option("-d, --domain <domain>", "domain for the subdomains (prompted from a list if unset)").option("--proto <proto>", "local service protocol: http | https", "http").option("--protocol <proto>", "cloudflared edge transport: auto | http2 | quic (http2 for UDP-hostile networks)").option("--detach", "run the connectors in the background").option("--service", "register each subdomain as a boot service (Linux systemd \xB7 macOS launchd \xB7 Windows Task Scheduler)").option("-f, --force", "replace a non-tunnel DNS record occupying the hostname").option("-y, --yes", "don't prompt; don't ask before replacing an existing record").action((specs, opts) => runUp(specs, opts));
+  program.command("up", { isDefault: true }).argument("[specs...]", "tunnels to start: [subdomain:]port[@host]  (e.g. api:8080 web:8081@localhost)").description("Start one or more tunnels (also: `cloudtunnel 8080`)").option("-d, --domain <domain>", "domain for the subdomains (prompted from a list if unset)").option("--proto <proto>", "local service protocol: http | https", "http").option("--protocol <proto>", "cloudflared edge transport: auto | http2 | quic (http2 for UDP-hostile networks)").option("--detach", "run the connectors in the background").option("--keep-alive", "supervised mode with auto-heal watchdog").option("--service", "register each subdomain as a boot service (Linux systemd \xB7 macOS launchd \xB7 Windows Task Scheduler)").option("-f, --force", "replace a non-tunnel DNS record occupying the hostname").option("-y, --yes", "don't prompt; don't ask before replacing an existing record").action((specs, opts) => runUp(specs, opts));
 }
 
 // src/commands/ls.ts
@@ -1763,9 +1957,9 @@ function registerDelete(program) {
 }
 
 // src/commands/logs.ts
-import { closeSync, existsSync as existsSync6, openSync as openSync2, readFileSync as readFileSync5, readSync, statSync, watch } from "fs";
+import { closeSync, existsSync as existsSync8, openSync as openSync2, readFileSync as readFileSync7, readSync, statSync, watch } from "fs";
 function printTail(file, n) {
-  const lines = readFileSync5(file, "utf8").split("\n");
+  const lines = readFileSync7(file, "utf8").split("\n");
   const tail = lines.slice(-n).join("\n");
   process.stdout.write(tail.endsWith("\n") ? tail : `${tail}
 `);
@@ -1797,7 +1991,7 @@ function follow(file, fromPos) {
 function registerLogs(program) {
   program.command("logs").argument("<target>", "subdomain name / hostname / id / #").description("Show the connector log for a subdomain (use -f to follow)").option("-f, --follow", "keep printing new log lines (like tail -f)").option("-n, --lines <n>", "number of lines to show", "50").action((name, opts) => {
     const { fqdn, entry } = resolveTarget(name);
-    if (!entry?.logFile || !existsSync6(entry.logFile)) {
+    if (!entry?.logFile || !existsSync8(entry.logFile)) {
       throw new CliError(`No logs for ${fqdn} yet.`, { hint: "start it with `cloudtunnel up` or `cloudtunnel run`" });
     }
     const n = Math.max(1, Number(opts.lines) || 50);