@mesh-tech/mesh-cli 0.1.0 → 0.12.0

package/dist/src/utils/tailscale.js

@@ -0,0 +1,1007 @@
+/**
+ * Userspace-Tailscale daemon management + tunnel-session state for
+ * `mesh vpn tunnel` (MESH-2040 / Slice 1 of MESH-2014).
+ *
+ * A per-tenant `tailscaled` runs in userspace-networking mode (brew formula,
+ * NOT the GUI app) exposing a SOCKS5 proxy; forwarders bridge `localhost:PORT`
+ * to VPC services through it. State lives under `~/.config/mesh/tailscale/<tenant>/`
+ * (r+w-granted under safehouse — see the MESH-2036 spike).
+ */
+import * as fs from "node:fs";
+import * as net from "node:net";
+import * as os from "node:os";
+import * as path from "node:path";
+import { spawn, execFileSync } from "node:child_process";
+import { CONFIG_DIR, atomicWriteFileSync } from "../commands/login.js";
+import { resolveStableMeshBin } from "./aws-auth.js";
+import { logInfo, logWarn } from "./log.js";
+import { MeshCliError, renderErrorBody } from "./errors.js";
+import { assertSignalablePid } from "./pid.js";
+import { getPlatformBastionInfo } from "./bastion.js";
+import { resolveTunnelTargets, resolveElbIp, tenantPortIndex, PORT_BLOCK } from "./tailscale-targets.js";
+import { createForwarder } from "./socks-forward.js";
+import { mintRunnerToken, readRunnerManifest, writeRunnerManifest, clearRunnerManifest, getProcessGroupId, discoverRunnerPids, startControlServer, verifyRunnerOwnership, pollOwnership, findSquattedPorts, buildSquatterError, } from "./tunnel-ownership.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+// ============================================================================
+// Paths
+// ============================================================================
+export function tenantStateDir(tenant) {
+    return path.join(CONFIG_DIR, "tailscale", tenant);
+}
+export function socketPath(tenant) {
+    return path.join(tenantStateDir(tenant), "tailscaled.sock");
+}
+export function logPath(tenant) {
+    return path.join(tenantStateDir(tenant), "tailscaled.log");
+}
+export function statePath(tenant) {
+    return path.join(tenantStateDir(tenant), "tunnel-state.json");
+}
+/**
+ * Daemon-scoped metadata, written by `ensureDaemon` when it actually spawns the
+ * daemon. Unlike `tunnel-state.json` (which `down` clears), this survives `down`
+ * so a later `up` reuses the RUNNING daemon's SOCKS port instead of minting a
+ * fresh one the daemon isn't listening on (which would leave every forwarder
+ * dialing a dead port). Also records the daemon pid so `stopDaemon` can kill it.
+ */
+export function daemonMetaPath(tenant) {
+    return path.join(tenantStateDir(tenant), "daemon.json");
+}
+/**
+ * Ownership manifest written by the RUNNER (the process that binds the
+ * forwarder ports) — pid, process-group id, per-run token, control port. The
+ * ownership handshake (MESH-2407) verifies against it; teardown signals the
+ * recorded process group.
+ */
+export function runnerManifestPath(tenant) {
+    return path.join(tenantStateDir(tenant), "runner.json");
+}
+export function readTunnelState(tenant) {
+    const p = statePath(tenant);
+    if (!fs.existsSync(p))
+        return null;
+    try {
+        return JSON.parse(fs.readFileSync(p, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+export function writeTunnelState(tenant, state) {
+    fs.mkdirSync(tenantStateDir(tenant), { recursive: true });
+    atomicWriteFileSync(statePath(tenant), JSON.stringify(state, null, 2), 0o600);
+}
+export function clearTunnelState(tenant) {
+    try {
+        fs.unlinkSync(statePath(tenant));
+    }
+    catch {
+        /* already gone */
+    }
+}
+export function readDaemonMeta(tenant) {
+    const p = daemonMetaPath(tenant);
+    if (!fs.existsSync(p))
+        return null;
+    try {
+        return JSON.parse(fs.readFileSync(p, "utf8"));
+    }
+    catch {
+        return null;
+    }
+}
+export function writeDaemonMeta(tenant, meta) {
+    fs.mkdirSync(tenantStateDir(tenant), { recursive: true });
+    atomicWriteFileSync(daemonMetaPath(tenant), JSON.stringify(meta, null, 2), 0o600);
+}
+export function clearDaemonMeta(tenant) {
+    try {
+        fs.unlinkSync(daemonMetaPath(tenant));
+    }
+    catch {
+        /* already gone */
+    }
+}
+// ============================================================================
+// Daemon status parsing
+// ============================================================================
+/** Parse `tailscale status --json` → the fields we care about. */
+export function parseDaemonState(statusJson) {
+    try {
+        const j = JSON.parse(statusJson);
+        return { backendState: j.BackendState ?? "Unknown", selfName: j.Self?.HostName };
+    }
+    catch {
+        return { backendState: "Unknown" };
+    }
+}
+// ============================================================================
+// Binary resolution (macOS / brew — Slice 1)
+// ============================================================================
+/** The brew `tailscaled` (NOT the GUI app, which ships no standalone daemon). */
+export function tailscaledBinPath() {
+    return resolveBrewBin("tailscaled");
+}
+/** The brew `tailscale` CLI (prefer it over the GUI shim, which aborts under sandbox). */
+export function tailscaleBinPath() {
+    return resolveBrewBin("tailscale");
+}
+function resolveBrewBin(name) {
+    const brew = `/opt/homebrew/bin/${name}`;
+    if (fs.existsSync(brew))
+        return brew;
+    try {
+        const p = execFileSync("which", [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
+        if (p)
+            return p;
+    }
+    catch {
+        /* fall through */
+    }
+    throw new Error(`${name} not found — install the standalone binary: brew install tailscale`);
+}
+// ============================================================================
+// Daemon lifecycle
+// ============================================================================
+/** Backend state via the tenant's control socket; "Down" when no daemon answers. */
+export function daemonState(tenant) {
+    if (!fs.existsSync(socketPath(tenant)))
+        return { backendState: "Down" };
+    try {
+        const out = execFileSync(tailscaleBinPath(), ["--socket", socketPath(tenant), "status", "--json"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+        return parseDaemonState(out);
+    }
+    catch {
+        return { backendState: "Down" };
+    }
+}
+/**
+ * Extract the `--socks5-server` port from a `tailscaled` command line. Pure so
+ * it's unit-testable. Accepts both `--socks5-server=127.0.0.1:PORT` and the
+ * space-separated form, and a `localhost` host literal. Returns null if absent
+ * or malformed.
+ */
+export function parseSocksPortFromArgv(argv) {
+    const m = argv.match(/--socks5-server[=\s]+(?:127\.0\.0\.1|localhost):(\d+)/);
+    if (!m)
+        return null;
+    const port = Number(m[1]);
+    return Number.isInteger(port) && port > 0 && port <= 65535 ? port : null;
+}
+/**
+ * Find the running userspace `tailscaled` for a tenant by its unique
+ * `--socket=<path>` argument, returning its pid and the SOCKS port it ACTUALLY
+ * bound (parsed from argv). Authoritative when `daemon.json` is missing or stale
+ * (e.g. a daemon left by an older build that predates the meta file). Returns
+ * null when no matching process is found or `ps` is unavailable.
+ */
+export function findRunningDaemon(tenant) {
+    const sock = socketPath(tenant);
+    let out;
+    try {
+        out = execFileSync("ps", ["ax", "-o", "pid=,command="], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        });
+    }
+    catch {
+        return null;
+    }
+    for (const line of out.split("\n")) {
+        // Match this tenant's daemon by its unique --socket path (skips the ps line,
+        // other tenants' daemons, and the GUI app).
+        if (!line.includes("tailscaled") || !line.includes(`--socket=${sock}`))
+            continue;
+        const m = line.trim().match(/^(\d+)\s+(.*)$/);
+        if (!m)
+            continue;
+        return { pid: Number(m[1]), socksPort: parseSocksPortFromArgv(m[2]) };
+    }
+    return null;
+}
+/** True if something accepts TCP on 127.0.0.1:port within `timeoutMs`. */
+export function portAccepts(port, timeoutMs = 400) {
+    return new Promise((resolve) => {
+        const s = new net.Socket();
+        let done = false;
+        const fin = (v) => {
+            if (done)
+                return;
+            done = true;
+            s.destroy();
+            resolve(v);
+        };
+        s.setTimeout(timeoutMs);
+        s.once("connect", () => fin(true));
+        s.once("timeout", () => fin(false));
+        s.once("error", () => fin(false));
+        s.connect(port, "127.0.0.1");
+    });
+}
+/**
+ * Ensure a userspace `tailscaled` is running for the tenant and return the SOCKS
+ * port the forwarders should dial. Idempotent — reuses a live daemon, but
+ * RECONCILES the port: a daemon that's up on a different port than requested
+ * (older build with no `daemon.json`, or stale meta) would otherwise leave every
+ * forwarder dialing a dead port (MESH-2169). Trust `opts.socksPort` only if it
+ * truly accepts; else adopt the running daemon's real port; else restart fresh.
+ */
+export async function ensureDaemon(tenant, opts) {
+    if (daemonState(tenant).backendState !== "Down") {
+        // The daemon's SOCKS proxy is only useful on the exact port the forwarders
+        // will dial. If the requested port already accepts, record it and use it.
+        if (await portAccepts(opts.socksPort)) {
+            writeDaemonMeta(tenant, {
+                socksPort: opts.socksPort,
+                pid: readDaemonMeta(tenant)?.pid ?? findRunningDaemon(tenant)?.pid ?? 0,
+            });
+            return opts.socksPort;
+        }
+        // Requested port is dead — the daemon is on a different one. Discover its
+        // real SOCKS port from argv and adopt it (no restart → no severing other
+        // sessions), rewriting daemon.json so future runs read the right port.
+        const running = findRunningDaemon(tenant);
+        if (running?.socksPort && (await portAccepts(running.socksPort))) {
+            writeDaemonMeta(tenant, { socksPort: running.socksPort, pid: running.pid });
+            return running.socksPort;
+        }
+        // Can't discover/verify a working SOCKS port — the daemon is unusable to us.
+        // Tear it down (state dir keeps the node identity → no re-login) and fall
+        // through to a fresh start on opts.socksPort.
+        stopDaemon(tenant);
+        for (let i = 0; i < 12 && daemonState(tenant).backendState !== "Down"; i++) {
+            await sleep(250);
+        }
+    }
+    const dir = tenantStateDir(tenant);
+    fs.mkdirSync(dir, { recursive: true });
+    const log = fs.openSync(logPath(tenant), "a");
+    try {
+        const child = spawn(tailscaledBinPath(), [
+            "--tun=userspace-networking",
+            `--socket=${socketPath(tenant)}`,
+            `--statedir=${dir}`,
+            `--socks5-server=127.0.0.1:${opts.socksPort}`,
+            "--port=0",
+        ], {
+            detached: true,
+            stdio: ["ignore", log, log],
+            env: { ...process.env, AWS_REGION: opts.region },
+        });
+        child.unref();
+        // Persist the socks port + pid so `down` → `up` reuses this daemon's real
+        // port (not a fresh one it isn't listening on) and `stopDaemon` can kill it.
+        if (child.pid)
+            writeDaemonMeta(tenant, { socksPort: opts.socksPort, pid: child.pid });
+    }
+    finally {
+        fs.closeSync(log);
+    }
+    for (let i = 0; i < 40; i++) {
+        await sleep(250);
+        if (daemonState(tenant).backendState !== "Down")
+            return opts.socksPort;
+    }
+    throw new Error(`tailscaled did not start for tenant '${tenant}' — see ${logPath(tenant)}`);
+}
+/**
+ * Stop the detached `tailscaled` for a tenant (SIGTERM its pid) and remove the
+ * socket + daemon metadata. Leaves the Headscale node identity in `tailscaled.state`
+ * so a later `up` re-spawns and rejoins without an interactive re-login — unless
+ * the caller logged out first.
+ *
+ * Best-effort, and that includes a corrupt `daemon.json`: an unsignalable
+ * recorded pid is REFUSED (never handed to `kill`) but only warns — the socket
+ * and meta are still cleared. `ensureDaemon` calls this on the UP path to
+ * recycle a daemon it can't verify, so throwing here would dead-end
+ * `mesh vpn tunnel up` / `mesh dev` on the very file this function deletes.
+ * Safe to call when nothing is running.
+ */
+export function stopDaemon(tenant) {
+    // Prefer the recorded pid; fall back to discovering it from argv so a daemon
+    // left without a daemon.json (older build / cleared meta) can still be killed.
+    const meta = readDaemonMeta(tenant) ?? (() => {
+        const running = findRunningDaemon(tenant);
+        return running ? { socksPort: running.socksPort ?? 0, pid: running.pid } : null;
+    })();
+    // `ensureDaemon` records `pid: 0` when it adopts a running daemon whose pid it
+    // could not attribute, so absent-or-0 is this file's "no pid known" sentinel —
+    // skip it, don't signal and don't fail. EVERY other unsignalable value (1, a
+    // negative, a fraction, a string from a truncated/hand-edited daemon.json) is
+    // corruption and must be refused rather than handed to kill. See utils/pid.ts.
+    if (meta?.pid) {
+        try {
+            assertSignalablePid(meta.pid, {
+                what: `the VPN daemon pid recorded for tenant '${tenant}'`,
+                file: daemonMetaPath(tenant),
+            });
+            process.kill(meta.pid, "SIGTERM");
+        }
+        catch (err) {
+            // Refusing to SIGNAL is the whole point; refusing to CLEAN UP is not.
+            // The unlink + clearDaemonMeta below are what make the corrupt file
+            // self-healing, and `ensureDaemon` (the UP path) depends on them. Warn
+            // with the `rm` remediation intact and fall through.
+            if (err instanceof MeshCliError)
+                logWarn(renderErrorBody(err));
+            /* else: ESRCH/EPERM from process.kill — already gone, as before */
+        }
+    }
+    for (const p of [socketPath(tenant)]) {
+        try {
+            fs.unlinkSync(p);
+        }
+        catch {
+            /* already gone */
+        }
+    }
+    clearDaemonMeta(tenant);
+}
+/**
+ * Build the `tailscale up` argument vector. Pure so it can be unit-tested and so
+ * the interactive (browser-register) and zero-touch (pre-auth key) forms differ
+ * only by the trailing `--authkey`.
+ */
+export function buildTailscaleUpArgs(opts) {
+    const args = [
+        "--socket", opts.socket, "up",
+        "--login-server", opts.loginServer,
+        "--accept-routes",
+        "--hostname", opts.hostname,
+    ];
+    if (opts.preAuthKey)
+        args.push("--authkey", opts.preAuthKey);
+    return args;
+}
+/**
+ * Join the tenant's Headscale tailnet via Zitadel OIDC if not already Running.
+ * Prints the device-flow registration URL and polls until registered. When a
+ * pre-auth key is supplied the join is non-interactive: no `/register/` URL is
+ * emitted, so the scrape/relay is skipped and we just wait for Running.
+ */
+export async function joinHeadscale(tenant, loginServer, opts = {}) {
+    if (daemonState(tenant).backendState === "Running")
+        return;
+    const hostname = `mesh-${tenant}-${os.hostname().split(".")[0]}`;
+    const child = spawn(tailscaleBinPath(), buildTailscaleUpArgs({ socket: socketPath(tenant), loginServer, hostname, preAuthKey: opts.preAuthKey }), { stdio: ["ignore", "pipe", "pipe"] });
+    if (!opts.preAuthKey) {
+        let buf = "";
+        let urlShown = false;
+        const capture = (d) => {
+            buf += d.toString();
+            if (urlShown)
+                return;
+            const m = buf.match(/https?:\/\/\S+\/register\/\S+/);
+            if (m) {
+                urlShown = true;
+                // Relay the registration URL for headless/SSH machines (there's no local
+                // browser to pop). One-time per machine — the node registration persists.
+                logInfo(`\n  This machine isn't registered with the platform VPN yet.\n` +
+                    `  Open this URL in a browser to authorize it (one-time, sign in with Zitadel):\n\n` +
+                    `      ${m[0]}\n\n` +
+                    `  Waiting for approval (up to 5 min)…`);
+            }
+        };
+        child.stdout.on("data", capture);
+        child.stderr.on("data", capture);
+    }
+    // Poll up to ~5 min — the join is an interactive browser approval.
+    for (let i = 0; i < 600; i++) {
+        await sleep(500);
+        if (daemonState(tenant).backendState === "Running") {
+            child.kill();
+            return;
+        }
+    }
+    child.kill();
+    throw new Error(`VPN join not completed for tenant '${tenant}' within 5 min. Re-run \`mesh vpn tunnel up\` to resume — the daemon is still up.`);
+}
+/** Log the tenant daemon out of Headscale (disconnect). Daemon process persists. */
+export function logout(tenant) {
+    try {
+        execFileSync(tailscaleBinPath(), ["--socket", socketPath(tenant), "logout"], { stdio: "ignore" });
+    }
+    catch {
+        /* best-effort */
+    }
+}
+// ============================================================================
+// Forwarder supervisor (shared per tenant) + devOutput rewrite
+// ============================================================================
+/**
+ * True iff two target sets are identical down to targetIp + remotePort. A live
+ * runner pins each forwarder's target at startup, so if the ELB IP rotated (or a
+ * port changed) the running forwarders point at a dead address and must be
+ * respawned, not reused. Pid-agnostic (a launchd runner records no supervisor pid).
+ */
+export function tunnelTargetsEqual(a, b) {
+    const key = (ts) => ts.map((t) => `${t.name}:${t.localPort}:${t.targetIp}:${t.remotePort}`).sort().join(",");
+    return key(a) === key(b);
+}
+/** True iff `state` has a supervisor pid and its target set equals `targets`. */
+export function supervisorTargetsMatch(state, targets) {
+    if (!state || !state.supervisorPid)
+        return false;
+    return tunnelTargetsEqual(state.tunnels, targets);
+}
+function pidAlive(pid) {
+    if (!pid)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * SIGTERM the supervisor's whole process GROUP, not the bare pid.
+ *
+ * `state.supervisorPid` is the pid of the detached spawn — which, under the
+ * canonical source launcher, is `bin/mesh.mjs`. That launcher blocks in
+ * `execFileSync` around the *real* runner (a `node --import tsx … mesh.ts`
+ * grandchild that actually binds the forwarder ports), and `execFileSync` does
+ * not forward signals. So signalling the bare pid kills only the launcher and
+ * ORPHANS the runner, which keeps its ports bound — the orphan then answers
+ * `allForwardersListening`'s ownership-less liveness dial and silently pins
+ * every tunnel to a stale target (the MESH-2169 / MESH-2014 failure class).
+ *
+ * The launcher is spawned `detached: true`, so it is a process-group leader
+ * (`pgid === pid`); `process.kill(-pid, …)` reaps the launcher and the runner
+ * together. Falls back to the bare pid for a legacy state file whose pid was
+ * not a group leader, and tolerates ESRCH (already gone).
+ *
+ * `pid` is always recovered — from `tunnel-state.json`, `runner.json`, or a
+ * `ps` scan — never constructed here, so it is guarded first. The guard rejects
+ * anything below 2, which is what makes the NEGATED form safe: `pid >= 2` ⇒
+ * `-pid <= -2`, a real process group, never `kill(0)` (this shell's own group)
+ * or `kill(-1)` (broadcast to everything the user can signal). See utils/pid.ts.
+ *
+ * @throws {MeshCliError} when `pid` is unsignalable. Callers reaping SEVERAL
+ * independently-recovered pids must catch per-pid (see `stopTunnelRunners`) —
+ * letting one refusal propagate would skip the other pids and leave a live
+ * runner holding ports.
+ */
+export function killSupervisor(pid, source) {
+    // MUST be outside the try — both signals below are wrapped in catch-alls, so a
+    // guard placed inside would have its refusal swallowed and fall through to the
+    // bare-pid kill it was meant to prevent.
+    assertSignalablePid(pid, source ?? { what: "tunnel supervisor pid" });
+    try {
+        process.kill(-pid, "SIGTERM"); // detached spawn ⇒ pid is the group leader
+    }
+    catch {
+        try {
+            process.kill(pid, "SIGTERM"); // not a group leader (old state file) — best effort
+        }
+        catch {
+            /* already gone (ESRCH) */
+        }
+    }
+}
+/**
+ * The ownership handshake for a tenant's tunnel runner (MESH-2407): manifest →
+ * live pid → control endpoint → token + SOCKS port + port coverage. This is
+ * the ONLY readiness/reuse signal — a bare TCP dial proves a listener exists,
+ * not that it is ours, and adopting a foreign listener is exactly the
+ * silent-dead-tunnel class (MESH-2169 / MESH-2014 / #2991). As a bonus the
+ * handshake reads listening state from the runner's in-process servers, so it
+ * opens no SOCKS→VPC data connection the way port dials do.
+ */
+export async function verifyTunnelOwnership(tenant, expected) {
+    return verifyRunnerOwnership({
+        manifest: readRunnerManifest(runnerManifestPath(tenant)),
+        expectedSocksPort: expected.socksPort,
+        expectedPorts: expected.ports,
+    });
+}
+/**
+ * Poll the ownership handshake until it passes or `deadlineMs` of WALL CLOCK
+ * elapses (each check can cost up to the control-query timeout, so the bound
+ * must be time, not rounds); returns the last check.
+ */
+function awaitOwnership(tenant, expected, deadlineMs) {
+    return pollOwnership(() => verifyTunnelOwnership(tenant, expected), deadlineMs);
+}
+/**
+ * Tear down EVERY runner we can attribute to the tenant: the recorded
+ * supervisor pid (as a process group — see `killSupervisor`), the launchd
+ * agent, the manifest's process group, and any `__run`/`__supervise` process
+ * discovered from argv (a runner from an older build, or one whose state file
+ * was lost). This is the single teardown path — the #2991 lesson generalized:
+ * no kill path may leave a grandchild with ports still bound.
+ */
+export function stopTunnelRunners(tenant, prior) {
+    if (prior?.managed === "launchd")
+        uninstallLaunchAgent(tenant);
+    // Each pid below is RECOVERED (two state files and a ps scan), so each names
+    // its own provenance — the refusal has to point at the exact file to delete,
+    // or a corrupt state dir bricks the very command that would have cleared it.
+    //
+    // The three sources are INDEPENDENT, so a refusal on one must not skip the
+    // others or `clearRunnerManifest`. Aborting would let one corrupt integer in
+    // `tunnel-state.json` strand a live, uncorrupted runner from `runner.json`
+    // with its forwarder ports still bound — the silent-squatter class this
+    // teardown exists to prevent ("no kill path may leave a grandchild with ports
+    // still bound", MESH-2407). Refuse the bad pid, reap the rest;
+    // `assertTunnelPortsFree` is the backstop that still fails loudly if an
+    // un-reaped runner really is holding ports.
+    const killOrWarn = (pid, source) => {
+        try {
+            killSupervisor(pid, source);
+        }
+        catch (err) {
+            if (!(err instanceof MeshCliError))
+                throw err;
+            logWarn(renderErrorBody(err));
+        }
+    };
+    if (prior?.supervisorPid)
+        killOrWarn(prior.supervisorPid, {
+            what: `the tunnel supervisor pid recorded for tenant '${tenant}'`,
+            file: statePath(tenant),
+        });
+    const manifest = readRunnerManifest(runnerManifestPath(tenant));
+    if (manifest)
+        killOrWarn(manifest.pgid || manifest.pid, {
+            what: `the tunnel runner process group recorded for tenant '${tenant}'`,
+            file: runnerManifestPath(tenant),
+        });
+    for (const pid of discoverRunnerPids(tenant, process.pid))
+        killOrWarn(pid, { what: `a tunnel runner pid discovered from 'ps' for tenant '${tenant}'` });
+    clearRunnerManifest(runnerManifestPath(tenant));
+}
+/**
+ * After teardown, wait for the tenant's forwarder ports to actually free. Any
+ * port still accepting is held by a process we could not kill — a foreign
+ * squatter — and spawning a runner over it would produce forwarders that
+ * silently fail to bind (the false-green readiness gate). Fail LOUDLY instead.
+ */
+async function assertTunnelPortsFree(tenant, ports) {
+    let held = ports;
+    for (let i = 0; i < 8; i++) {
+        const stillHeld = [];
+        for (const p of held) {
+            if (await portAccepts(p, 250))
+                stillHeld.push(p);
+        }
+        held = stillHeld;
+        if (held.length === 0)
+            return;
+        await sleep(250);
+    }
+    const squatted = await findSquattedPorts(held, (p) => portAccepts(p, 250));
+    throw new Error(buildSquatterError(tenant, squatted, "still bound after teardown"));
+}
+/**
+ * Ensure the per-tenant forwarder supervisor is running for `state.tunnels`.
+ * Idempotent: reuses a live runner ONLY when it passes the ownership handshake
+ * (our manifest token, our SOCKS port, every target port listening in-process);
+ * otherwise tears down every attributable runner, verifies the ports actually
+ * freed (a still-held port is a foreign squatter → loud failure, never spawn
+ * over it), and (re)spawns a fresh runner — whose readiness is again the
+ * handshake, not a bare port dial. Persists the state and returns the pid.
+ */
+export async function ensureForwarders(state) {
+    const prior = readTunnelState(state.tenant);
+    const expected = { socksPort: state.socksPort, ports: state.tunnels.map((t) => t.localPort) };
+    // Reuse-if-healthy must be mode-aware. A launchd runner records no supervisorPid
+    // (KeepAlive owns liveness), so gating reuse on pidAlive would tear down + rebind
+    // the shared tunnels on EVERY `up`/`mesh dev` — severing other concurrent sessions,
+    // the exact failure MESH-2014 exists to kill. Treat the launchd agent as "alive"
+    // when its plist is installed; keep the pid check for the detached path.
+    const runnerAlive = prior?.managed === "launchd"
+        ? fs.existsSync(launchAgentPath(state.tenant))
+        : pidAlive(prior?.supervisorPid);
+    if (prior &&
+        runnerAlive &&
+        tunnelTargetsEqual(prior.tunnels, state.tunnels) &&
+        // A prior supervisor spawned for a DIFFERENT SOCKS port still binds the same
+        // local ports but dials a dead upstream — reusing it silently strands every
+        // tunnel (MESH-2169). The port must match, and the runner must PROVE it is
+        // ours via the ownership handshake (liveness ≠ ownership — a port answering
+        // a dial can be another session's SSM tunnel or an orphaned runner, #2991).
+        // Poll briefly rather than sample once: `ports-not-listening` during the
+        // runner's own rebind window and `no-control-answer` under load are
+        // transient, and a single miss here would tear down a healthy runner SHARED
+        // by every concurrent session on the tenant (MESH-2014). This branch is
+        // only reached when the runner is alive with matching targets, so the
+        // retry costs nothing on cold-start or genuinely-dead paths.
+        prior.socksPort === state.socksPort &&
+        (await awaitOwnership(state.tenant, expected, 1000)).ok) {
+        state.supervisorPid = prior.supervisorPid;
+        state.managed = prior.managed;
+        writeTunnelState(state.tenant, state);
+        return prior.supervisorPid ?? 0;
+    }
+    // Tear down every attributable runner (state pid group, launchd agent,
+    // manifest process group, argv-discovered strays), then require the ports to
+    // actually free — a still-held port is foreign and must fail loudly here,
+    // not surface later as a forwarder that silently failed to bind.
+    stopTunnelRunners(state.tenant, prior);
+    await assertTunnelPortsFree(state.tenant, expected.ports);
+    // Write state first so the runner reads the correct targets on start.
+    state.supervisorPid = null;
+    // Prefer a launchd-supervised runner (survives reboot/crash); fall back to a
+    // detached runner where launchd isn't usable (e.g. sandboxed shells). Both run
+    // the same self-healing `__run`.
+    // argv[1] under the canonical source launcher is bin/mesh.ts, which plain
+    // node cannot execute (same failure class as the credential_process fix in
+    // aws-auth.ts) — resolve to the stable launcher, or the bare `mesh` fallback.
+    const meshBin = resolveStableMeshBin(process.argv[1]);
+    if (launchdAvailable()) {
+        state.managed = "launchd";
+        writeTunnelState(state.tenant, state);
+        // launchd does no PATH lookup, so the bare `mesh` fallback can't back a
+        // LaunchAgent (it would become `node mesh …`, crash-looping under KeepAlive).
+        // Require an absolute path; otherwise fall through to the detached runner,
+        // where spawn *does* resolve `mesh` on PATH.
+        if (path.isAbsolute(meshBin) && installLaunchAgent(state.tenant, meshBin)) {
+            // Installing the agent proves nothing about the forwarders — wait for the
+            // runner's ownership handshake (this path used to return success blind).
+            if ((await awaitOwnership(state.tenant, expected, 8000)).ok)
+                return 0;
+            uninstallLaunchAgent(state.tenant);
+            // `launchctl bootout` signals the job but doesn't wait for it — spawning
+            // the detached runner immediately would race the dying launchd runner for
+            // the ports (EADDRINUSE → a transient turned into a hard failure). Every
+            // teardown path must be followed by the ports-actually-freed check.
+            await assertTunnelPortsFree(state.tenant, expected.ports);
+            logInfo("launchd runner did not take ownership of the tunnel ports — using a detached runner.");
+        }
+        else {
+            logInfo("launchd install failed — using a detached self-healing runner.");
+        }
+    }
+    state.managed = "detached";
+    writeTunnelState(state.tenant, state);
+    // meshBin is the stable launcher, or the bare `mesh` fallback which spawn
+    // resolves on PATH here (no PATH-lookup ternary needed — that was a no-op).
+    const child = spawn(meshBin, ["vpn", "tunnel", "__run", state.tenant], {
+        detached: true,
+        stdio: "ignore",
+    });
+    // ENOENT from the bare `mesh` fallback (mesh-cli not installed globally)
+    // arrives asynchronously — without this handler an unhandled 'error' throws
+    // an uncaught exception and takes down `mesh dev` instead of degrading.
+    child.on("error", (err) => logWarn(`tunnel supervisor failed to start (${err.message}) — falling back to SSM.`));
+    child.unref();
+    state.supervisorPid = child.pid ?? null;
+    writeTunnelState(state.tenant, state);
+    // Readiness = the ownership handshake, not a port dial: the fresh runner must
+    // present its manifest token and report every forwarder listening from its
+    // own in-process servers. A dial would also pass for another session's SSM
+    // tunnel on the same 2XXXX port — the false-green this contract kills.
+    // Throwing here lands in startServices' catch → clean SSM fallback.
+    const last = await awaitOwnership(state.tenant, expected, 5000);
+    if (last.ok)
+        return child.pid ?? -1;
+    // Diagnose before failing: a port that accepts TCP while ownership failed is
+    // held by someone else — name it and say exactly what to do.
+    const squatted = await findSquattedPorts(expected.ports, (p) => portAccepts(p, 250));
+    if (squatted.length > 0) {
+        throw new Error(buildSquatterError(state.tenant, squatted, `ownership check: ${last.reason}`));
+    }
+    const bindErrors = Object.entries(last.resp?.bindErrors ?? {})
+        .map(([port, code]) => `port ${port}: ${code}`)
+        .join(", ");
+    throw new Error(`Tailscale forwarders did not take ownership for tenant '${state.tenant}' ` +
+        `(${last.reason}${bindErrors ? `; ${bindErrors}` : ""}) — see ${logPath(state.tenant)}.`);
+}
+/**
+ * Rewrite `devOutput.tunnels` so each resolved target points at its local
+ * forwarder (`localhost:localPort`) — the same shape `startSsmTunnels`
+ * produces. Unmatched tunnels are left untouched. Pure (no mutation of input).
+ */
+export function rewriteTunnelsForTargets(devOutput, targets) {
+    const tunnels = { ...devOutput.tunnels };
+    for (const t of targets) {
+        tunnels[t.name] = { host: "localhost", port: t.localPort };
+    }
+    return { ...devOutput, tunnels };
+}
+/** True when the brew `tailscaled` binary is resolvable (Tailscale transport is usable). */
+export function tailscaleAvailable() {
+    try {
+        tailscaledBinPath();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ============================================================================
+// Shared engine: ensure/start Tailscale tunnels (used by the command + mesh dev)
+// ============================================================================
+/**
+ * Ensure the tenant's Tailscale tunnels are fully up: the shared per-tenant
+ * daemon (started + OIDC-joined if needed), the resolved VPC targets, and the
+ * forwarder supervisor. Idempotent (shared across worktrees/invocations of the
+ * same tenant). Returns the resolved targets. Used by both `mesh vpn tunnel up`
+ * and `mesh dev --transport=tailscale`.
+ */
+export async function ensureTailscaleTunnels(tenant, env, opts) {
+    // Reconcile the daemon's SOCKS port — an already-running daemon may be on a
+    // different port than requested; the forwarders MUST dial the real one (MESH-2169).
+    const socksPort = await ensureDaemon(tenant, { region: opts.region, socksPort: opts.socksPort });
+    await joinHeadscale(tenant, opts.loginServer, { preAuthKey: opts.preAuthKey });
+    // Pass the region explicitly rather than clobbering process.env.AWS_REGION —
+    // this runs mid `mesh dev` launch and must not mutate the global env.
+    const bastion = await getPlatformBastionInfo(tenant, env, opts.region);
+    if (!bastion) {
+        throw new Error(`Could not read the platform bastion for ${tenant}/${env} (check AWS credentials).`);
+    }
+    const portOffset = tenantPortIndex(tenant) * PORT_BLOCK;
+    const targets = resolveTunnelTargets(bastion.services, resolveElbIp, portOffset);
+    if (targets.length === 0) {
+        throw new Error(`No resolvable VPC tunnels found for ${tenant}/${env}.`);
+    }
+    const state = {
+        tenant,
+        env,
+        socksPort,
+        loginServer: opts.loginServer,
+        supervisorPid: null,
+        startedAt: new Date().toISOString(),
+        tunnels: targets,
+    };
+    await ensureForwarders(state);
+    return targets;
+}
+/**
+ * `startSsmTunnels` drop-in for `mesh dev`: ensure the tenant's Tailscale
+ * tunnels, then rewrite `devOutput.tunnels` to the local forwarder ports.
+ */
+export async function startTailscaleTunnels(devOutput, ctx) {
+    const targets = await ensureTailscaleTunnels(ctx.tenant, ctx.env, {
+        region: ctx.region,
+        loginServer: ctx.loginServer,
+        socksPort: ctx.socksPort,
+        preAuthKey: ctx.preAuthKey,
+    });
+    return rewriteTunnelsForTargets(devOutput, targets);
+}
+// ============================================================================
+// launchd supervision + self-heal (Slice 2a)
+// ============================================================================
+export function launchAgentLabel(tenant) {
+    return `tech.mesh.tunnel.${tenant}`;
+}
+/** Minimal XML-escape for plist string values (controlled inputs, but paths can vary). */
+function xmlEscape(s) {
+    return s
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;");
+}
+/** Render the per-tenant LaunchAgent plist (pure). */
+export function renderLaunchAgentPlist(args) {
+    const pathEnv = args.pathEnv ?? "/usr/bin:/bin:/usr/sbin:/sbin";
+    return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>${launchAgentLabel(args.tenant)}</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>${xmlEscape(args.nodeBin)}</string>
+    <string>${xmlEscape(args.meshBin)}</string>
+    <string>vpn</string>
+    <string>tunnel</string>
+    <string>__run</string>
+    <string>${xmlEscape(args.tenant)}</string>
+  </array>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>${xmlEscape(pathEnv)}</string>
+  </dict>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>KeepAlive</key>
+  <dict>
+    <key>SuccessfulExit</key>
+    <false/>
+  </dict>
+  <key>StandardOutPath</key>
+  <string>${xmlEscape(args.logPath)}</string>
+  <key>StandardErrorPath</key>
+  <string>${xmlEscape(args.logPath)}</string>
+</dict>
+</plist>
+`;
+}
+/** Targets whose local forwarder port isn't currently accepting connections. */
+export function forwardersNeedingRebind(targets, listening) {
+    return targets.filter((t) => !listening.has(t.localPort));
+}
+export function launchAgentPath(tenant) {
+    return path.join(os.homedir(), "Library", "LaunchAgents", `${launchAgentLabel(tenant)}.plist`);
+}
+function guiDomain() {
+    return `gui/${process.getuid?.() ?? 0}`;
+}
+let _launchdAvailable;
+/** True when launchctl works for this user (blocked under the seatbelt sandbox → false). */
+export function launchdAvailable() {
+    if (_launchdAvailable !== undefined)
+        return _launchdAvailable;
+    try {
+        execFileSync("which", ["launchctl"], { stdio: "ignore" });
+        execFileSync("launchctl", ["print", guiDomain()], { stdio: "ignore" });
+        _launchdAvailable = true;
+    }
+    catch {
+        _launchdAvailable = false;
+    }
+    return _launchdAvailable;
+}
+/**
+ * Write + (re)load the per-tenant LaunchAgent. Returns true iff it fully
+ * succeeded — any failure (plist write blocked by the sandbox, `launchctl`
+ * bootstrap denied, …) returns false so the caller falls back to a detached
+ * runner. Never throws.
+ */
+export function installLaunchAgent(tenant, meshBin) {
+    const plistPath = launchAgentPath(tenant);
+    try {
+        // Resolve symlinks so the plist embeds a stable path (a since-deleted
+        // worktree symlink would leave a KeepAlive agent respawning a dead binary).
+        let realMeshBin = meshBin;
+        try {
+            realMeshBin = fs.realpathSync(meshBin);
+        }
+        catch {
+            /* keep the raw path */
+        }
+        // Bake a PATH that includes the tailscaled dir so the runner's daemon reheal
+        // works under launchd's minimal PATH (Intel installs live in /usr/local/bin).
+        let pathEnv = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin";
+        try {
+            pathEnv = `${path.dirname(tailscaledBinPath())}:${pathEnv}`;
+        }
+        catch {
+            /* fall back to the default PATH */
+        }
+        fs.mkdirSync(path.dirname(plistPath), { recursive: true });
+        fs.writeFileSync(plistPath, renderLaunchAgentPlist({
+            tenant,
+            meshBin: realMeshBin,
+            nodeBin: process.execPath,
+            logPath: path.join(tenantStateDir(tenant), "runner.log"),
+            pathEnv,
+        }));
+        try {
+            execFileSync("launchctl", ["bootout", guiDomain(), plistPath], { stdio: "ignore" });
+        }
+        catch {
+            /* not loaded — fine */
+        }
+        execFileSync("launchctl", ["bootstrap", guiDomain(), plistPath], { stdio: "ignore" });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Unload + remove the per-tenant LaunchAgent (best-effort). */
+export function uninstallLaunchAgent(tenant) {
+    const plistPath = launchAgentPath(tenant);
+    try {
+        execFileSync("launchctl", ["bootout", guiDomain(), plistPath], { stdio: "ignore" });
+    }
+    catch {
+        /* not loaded */
+    }
+    try {
+        fs.unlinkSync(plistPath);
+    }
+    catch {
+        /* already gone */
+    }
+}
+// ============================================================================
+// The runner (__run): forwarders + continuous self-heal
+// ============================================================================
+/**
+ * Long-lived runner for a tenant's tunnels — used both under launchd and as the
+ * detached fallback. Runs a forwarder per target and, every ~5s, self-heals:
+ * respawns the daemon if its process died, and rebinds any forwarder whose port
+ * stopped listening. As the process that actually binds the ports, it is the
+ * sole author of the ownership manifest + control endpoint (MESH-2407) that
+ * callers verify against. Needs no AWS (targets come from tunnel-state). Never
+ * returns.
+ */
+export async function runTunnels(tenant) {
+    const st = readTunnelState(tenant);
+    if (!st) {
+        // State is gone (torn down). Under launchd's KeepAlive a non-zero exit would
+        // respawn every ~10s forever (surviving reboots) — so self-uninstall the
+        // agent (no-op when detached), drop any manifest a previous incarnation
+        // left (it describes a runner that no longer exists), and exit cleanly.
+        uninstallLaunchAgent(tenant);
+        clearRunnerManifest(runnerManifestPath(tenant));
+        process.exit(0);
+    }
+    const token = mintRunnerToken();
+    const servers = new Map();
+    // port → last bind error code. Surfaced through the control endpoint so a
+    // failed bind (EADDRINUSE from a squatted port) is a visible, attributable
+    // failure at the caller — never a silently-dead forwarder.
+    const bindErrors = new Map();
+    const spawnFor = (t) => {
+        servers.get(t.localPort)?.close();
+        bindErrors.delete(t.localPort);
+        const server = createForwarder({
+            listenPort: t.localPort,
+            socksPort: st.socksPort,
+            targetIp: t.targetIp,
+            targetPort: t.remotePort,
+            onError: (err) => {
+                bindErrors.set(t.localPort, err.code ?? err.message);
+                logWarn(`forwarder ${t.name} failed on 127.0.0.1:${t.localPort}: ${err.code ?? err.message}`);
+            },
+        });
+        server.on("listening", () => bindErrors.delete(t.localPort));
+        servers.set(t.localPort, server);
+    };
+    for (const t of st.tunnels)
+        spawnFor(t);
+    const control = await startControlServer(() => ({
+        token,
+        tenant,
+        pid: process.pid,
+        socksPort: st.socksPort,
+        listening: st.tunnels.filter((t) => servers.get(t.localPort)?.listening).map((t) => t.localPort),
+        bindErrors: Object.fromEntries([...bindErrors].map(([p, c]) => [String(p), c])),
+    }));
+    writeRunnerManifest(runnerManifestPath(tenant), {
+        tenant,
+        pid: process.pid,
+        pgid: getProcessGroupId(process.pid) ?? process.pid,
+        token,
+        controlPort: control.port,
+        socksPort: st.socksPort,
+        ports: st.tunnels.map((t) => t.localPort),
+        startedAt: new Date().toISOString(),
+    });
+    const shutdown = () => {
+        control.server.close();
+        for (const s of servers.values())
+            s.close();
+        // Token-guarded: never clobber the manifest of a newer runner that already
+        // replaced us (teardown kills groups, and signal delivery order is loose).
+        clearRunnerManifest(runnerManifestPath(tenant), token);
+        process.exit(0);
+    };
+    process.on("SIGTERM", shutdown);
+    process.on("SIGINT", shutdown);
+    for (;;) {
+        await sleep(5000);
+        try {
+            if (daemonState(tenant).backendState === "Down") {
+                await ensureDaemon(tenant, {
+                    region: process.env.AWS_REGION ?? "us-east-2",
+                    socksPort: st.socksPort,
+                });
+            }
+            // Check the in-process net.Server directly — do NOT dial the forwarder.
+            // A dial makes the forwarder open a real SOCKS→VPC-target connection, and
+            // the probe's clean FIN doesn't tear the upstream leg down, leaking a
+            // connection to Temporal/RDS/etc. every 5s per target.
+            const listening = new Set();
+            for (const t of st.tunnels) {
+                if (servers.get(t.localPort)?.listening)
+                    listening.add(t.localPort);
+            }
+            for (const t of forwardersNeedingRebind(st.tunnels, listening))
+                spawnFor(t);
+        }
+        catch {
+            /* keep the loop alive across transient errors */
+        }
+    }
+}
+//# sourceMappingURL=tailscale.js.map