heyspex 0.2.5 → 0.2.7

package/dev-factory/spexcrawl/firecrawl-cli.mjs

@@ -0,0 +1,755 @@
+#!/usr/bin/env node
+/**
+ * firecrawl-cli.mjs — CLI wrapper around firecrawl-adapter.mjs so a Claude subagent (Bash tool
+ * only, no JS imports) can use self-hosted Firecrawl for search/scrape.
+ *
+ * THE GAP THIS CLOSES: `.heyspex/config.yaml` (agent_behaviors.research.default_source) and
+ * specs/protocols/research-capture-protocol.md declare self-hosted Firecrawl on Night Action the
+ * default search/scrape backend for every rung of the confidence ladder. But the ladder workflows
+ * (.claude/workflows/deep-research-{quick,medium,thorough}.js) dispatch Claude *subagents* whose
+ * prompts said "use WebSearch" / "use WebFetch" — subagents can't `import` firecrawl-adapter.mjs,
+ * so the claimed default was silently bypassed by every real run. This CLI is the bridge: the
+ * subagent prompts now say "try this CLI first"; this file does the actual Firecrawl call.
+ *
+ * SHIPPED IN THE `heyspex` NPM PACKAGE (owner ruling 2026-10-17): in a customer repo
+ * `node node_modules/heyspex/dev-factory/spexcrawl/firecrawl-cli.mjs search|scrape …` finds the
+ * self-hosted Firecrawl in the user's LOCAL Docker with zero config — see loadPlatform() below and
+ * dev-factory/lib/local-firecrawl.mjs. No env var, no API key, never hosted.
+ *
+ * HARD RULES (see [[firecrawl-routing-self-hosted-night]] — do not relax these):
+ *   1. NEVER spend hosted api.firecrawl.dev credits from this CLI. If the endpoint
+ *      firecrawl-adapter.mjs's getBaseUrl() resolves is the hosted host, REFUSE outright — exit
+ *      non-zero before any network call. Only the owner decides hosted spend, interactively.
+ *   2. Ollama has priority on the Night Action mode lease — Firecrawl work WAITS, never evicts.
+ *      Before its first call this CLI asks for the lease via requestFirecrawlMode() — the SAME
+ *      function every other caller uses. That function already refuses to touch anything when
+ *      Ollama holds the box and only starts the stack over SSH when the box is genuinely idle.
+ *      This CLI does not duplicate or bypass that logic.
+ *   3. On refusal/unreachable, print a single machine-readable line `FIRECRAWL_UNAVAILABLE: <reason>`
+ *      and exit non-zero, so a calling ladder-workflow prompt ("if it prints FIRECRAWL_UNAVAILABLE
+ *      or fails, fall back to WebSearch/WebFetch") can react deterministically instead of hanging
+ *      or silently returning nothing. Every path is caught — never a raw stack trace.
+ *
+ * Usage:
+ *   node dev-factory/spexcrawl/firecrawl-cli.mjs search "<query>" [--limit N] [--json]
+ *   node dev-factory/spexcrawl/firecrawl-cli.mjs scrape "<url>" [--json]
+ *   node dev-factory/spexcrawl/firecrawl-cli.mjs --self-test
+ *
+ * Exit codes: 0 ok · 1 unexpected error (clean message, no stack) · 2 refused (hosted endpoint,
+ *             policy violation) · 3 FIRECRAWL_UNAVAILABLE (lease refused / unreachable / network
+ *             failure — the fallback-to-WebSearch/WebFetch signal) · 64 usage error (bad args).
+ */
+import { readFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { OLLAMA } from "../lib/model-catalog.mjs";
+import {
+  detectLocalFirecrawl,
+  isHostedFirecrawlUrl as isHostedFirecrawl,
+  localFirecrawlScrape,
+  localFirecrawlSearch,
+  LOCAL_FIRECRAWL_SETUP_HINT,
+} from "../lib/local-firecrawl.mjs";
+
+// ── two modes: platform checkout vs. the `heyspex` npm package ───────────────────────────────────
+/**
+ * This CLI ships in the `heyspex` npm package (owner ruling 2026-10-17: customer research finds
+ * URLs through a self-hosted Firecrawl in the user's LOCAL Docker, zero config). The SpexFleet
+ * routing it uses in a platform checkout — firecrawl-adapter.mjs (fleet candidates, cost ledger)
+ * and night-action-mode.mjs (the Ollama/Firecrawl mode lease) — is NOT in the package, so it is
+ * loaded with a dynamic import that a published install answers with "absent":
+ *
+ *   platform   both modules load → fleet candidates + lease, then this machine's Docker, then
+ *              the on-demand/remote autostarts
+ *   local      either is missing  → detectLocalFirecrawl() only (optional override env,
+ *              `docker ps`, localhost:3002), plain HTTP, no API key, never hosted
+ *
+ * Only a genuinely missing module selects local mode; any other load error is rethrown.
+ */
+let platformPromise = null;
+let loadedPlatform = null;
+export function loadPlatform() {
+  if (!platformPromise) {
+    platformPromise = Promise.all([import("./firecrawl-adapter.mjs"), import("../lib/night-action-mode.mjs")])
+      .then(([adapter, mode]) => (loadedPlatform = { adapter, mode }))
+      .catch((e) => {
+        if (e?.code === "ERR_MODULE_NOT_FOUND") return null;
+        throw e;
+      });
+  }
+  return platformPromise;
+}
+
+export const MAX_SCRAPE_CHARS = 20000;
+
+export const HOSTED_REFUSAL_MESSAGE =
+  "REFUSED: resolved Firecrawl endpoint is the hosted api.firecrawl.dev API. Agents must NEVER " +
+  "spend hosted credits — only the owner decides, and only interactively (see " +
+  "specs/memory/decisions/firecrawl-routing-self-hosted-night.md). This CLI only ever talks to a " +
+  "self-hosted Firecrawl endpoint.";
+
+// ── on-demand local stack (fleet-down fallback) ────────────────────────────────────────────────
+/**
+ * Does THIS machine declare an ON-DEMAND self-hosted Firecrawl? Returns its base URL, or null.
+ *
+ * On-demand hosts are normally STOPPED ([[workstation-firecrawl-is-the-fleet-down]]) — the owner
+ * works on that box and the stack idles ~6 containers. So a probe failing against it is the
+ * EXPECTED resting state, not a broken install, and it is worth starting before giving up.
+ * Opt-in is explicit: `services.firecrawl.onDemand === true` on the machine named by
+ * HEYSPEX_FLEET_MACHINE_KEY. Any other host is left alone.
+ */
+export function localOnDemandFirecrawl(env = process.env) {
+  try {
+    const key = String(env.HEYSPEX_FLEET_MACHINE_KEY || "").trim();
+    if (!key) return null;
+    const table = JSON.parse(readFileSync(fileURLToPath(new URL("../routing-table.json", import.meta.url)), "utf8"));
+    const svc = table?.machines?.[key]?.services?.firecrawl;
+    if (!svc || svc.onDemand !== true) return null;
+    const baseUrl = String(svc.localUrl || svc.url || "").trim().replace(/\/+$/, "");
+    if (!baseUrl || isHostedFirecrawl(baseUrl)) return null;
+    return baseUrl;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Start the local on-demand stack and return its base URL, or null if there is nothing to start
+ * or the start failed. NEVER throws — a failure here must degrade to the original
+ * FIRECRAWL_UNAVAILABLE, never to a crash.
+ */
+export async function autostartLocalFirecrawl({ env = process.env, spawn = spawnSync } = {}) {
+  const baseUrl = localOnDemandFirecrawl(env);
+  if (!baseUrl) return null;
+  try {
+    const script = fileURLToPath(new URL("../scripts/firecrawl-local.sh", import.meta.url));
+    const res = spawn("bash", [script, "up"], { timeout: 300000, stdio: "ignore", encoding: "utf8" });
+    return res && res.status === 0 ? baseUrl : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── remote start (fleet machines with a declared day-2 SSH script) ────────────────────────────
+/**
+ * Fleet machines other than "this one" that declare `services.firecrawl.remoteStart.script` —
+ * a repo-relative day-2 script (mirroring firecrawl-local.sh's {status|up|down} contract) that
+ * SSHes out and brings the stack up on THAT machine, e.g. dev-factory/scripts/firecrawl-specops.sh.
+ *
+ * WHY THIS EXISTS (HS-DEC firecrawl-any-machine-docker-autostart, 2026-08-25): before this,
+ * `ensureAvailable()`'s only autostart path was `autostartLocalFirecrawl()`, which only ever acts
+ * on the machine THIS process is running on (`HEYSPEX_FLEET_MACHINE_KEY`). A remote fleet host
+ * with its own working day-2 script (Spec-ops) still read as FIRECRAWL_UNAVAILABLE and required a
+ * human to notice the script existed and run it by hand — exactly the gap this closes.
+ *
+ * Ordered by FLEET_FIRECRAWL_PREFERENCE so Spec-ops (the fleet default, owner ruling 2026-08-25)
+ * is tried before Basement Geek. Basement Geek has no verified remote-start script yet (its SSH
+ * shell mixed bash/cmd.exe output on a first probe — do not fabricate one untested); it simply
+ * has no `remoteStart` entry, so it is skipped here, same as any machine without one.
+ */
+export function remoteStartCandidates(env = process.env, preference = loadedPlatform?.adapter?.FLEET_FIRECRAWL_PREFERENCE ?? []) {
+  const out = [];
+  try {
+    const key = String(env.HEYSPEX_FLEET_MACHINE_KEY || "").trim();
+    const table = JSON.parse(readFileSync(fileURLToPath(new URL("../routing-table.json", import.meta.url)), "utf8"));
+    const machines = table?.machines || {};
+    const seenScripts = new Set();
+    const order = [...preference, ...Object.keys(machines).filter((k) => k !== "_comment" && !preference.includes(k))];
+    for (const machineKey of order) {
+      if (machineKey === key) continue; // this machine uses autostartLocalFirecrawl instead
+      const svc = machines[machineKey]?.services?.firecrawl;
+      const scriptRel = svc?.remoteStart?.script;
+      if (!scriptRel || typeof scriptRel !== "string") continue;
+      if (seenScripts.has(scriptRel)) continue; // specops-gpu0/gpu1 share one script/stack
+      seenScripts.add(scriptRel);
+      const baseUrl = String(svc.url || "").trim().replace(/\/+$/, "");
+      if (!baseUrl || isHostedFirecrawl(baseUrl)) continue;
+      out.push({ machineKey, scriptPath: scriptRel, baseUrl });
+    }
+  } catch {
+    return [];
+  }
+  return out;
+}
+
+/**
+ * Try each declared remote-start script in turn (repo-relative, resolved from this file's
+ * location); return the first one that starts successfully, or null. NEVER throws — same
+ * fail-safe contract as autostartLocalFirecrawl.
+ */
+export async function autostartRemoteFirecrawl({ env = process.env, spawn = spawnSync, candidatesFn = remoteStartCandidates } = {}) {
+  const candidates = typeof candidatesFn === "function" ? candidatesFn(env) : [];
+  for (const { scriptPath, baseUrl } of candidates) {
+    try {
+      const script = fileURLToPath(new URL(`../../${scriptPath}`, import.meta.url));
+      const res = spawn("bash", [script, "up"], { timeout: 300000, stdio: "ignore", encoding: "utf8" });
+      if (res && res.status === 0) return baseUrl;
+    } catch {
+      // try the next candidate — never throw
+    }
+  }
+  return null;
+}
+
+// ── the single availability gate every subcommand runs first — never throws ─────────────────────
+/**
+ * @returns {Promise<
+ *   {ok:true, baseUrl:string} |
+ *   {ok:false, refusedHosted:true, message:string} |
+ *   {ok:false, unavailable:true, reason:string}
+ * >}
+ */
+export async function ensureAvailable(opts = {}) {
+  const platform = opts.platform !== undefined ? opts.platform : await loadPlatform();
+  return platform ? ensurePlatformAvailable(platform, opts) : ensureLocalAvailable(opts);
+}
+
+/**
+ * npm-package mode: the local Docker Firecrawl or nothing. No env var required, no API key, never
+ * hosted. Unavailable carries the plain setup steps (start Docker Desktop + the Firecrawl stack).
+ */
+export async function ensureLocalAvailable({ localDetectFn = detectLocalFirecrawl, modeOpts = {}, env = process.env } = {}) {
+  let det;
+  try {
+    det = await localDetectFn({ env, ...(modeOpts.httpFetch ? { httpFetch: modeOpts.httpFetch } : {}) });
+  } catch (e) {
+    det = { ok: false, reason: `local Firecrawl detection failed: ${e?.message || e}` };
+  }
+  if (det?.ok && det.baseUrl && !isHostedFirecrawl(det.baseUrl)) {
+    return { ok: true, baseUrl: det.baseUrl, mode: "local", source: det.source };
+  }
+  return { ok: false, unavailable: true, mode: "local", reason: `${det?.reason || "no local Firecrawl found"}. ${LOCAL_FIRECRAWL_SETUP_HINT}` };
+}
+
+/** Platform-mode lookup of this machine's own Docker; null when nothing local answers. Never throws. */
+async function tryLocalDocker(localDetectFn, modeOpts) {
+  if (typeof localDetectFn !== "function") return null;
+  try {
+    const det = await localDetectFn(modeOpts?.httpFetch ? { httpFetch: modeOpts.httpFetch } : {});
+    return det?.ok && det.baseUrl && !isHostedFirecrawl(det.baseUrl) ? det.baseUrl : null;
+  } catch {
+    return null;
+  }
+}
+
+async function ensurePlatformAvailable(platform, opts = {}) {
+  const {
+    getBaseUrlFn = platform.adapter.getBaseUrl,
+    modeOpts = {},
+    candidatesFn = platform.adapter.listSelfHostedFirecrawlCandidates,
+    autostartFn = autostartLocalFirecrawl,
+    remoteStartFn = autostartRemoteFirecrawl,
+    localDetectFn = detectLocalFirecrawl,
+  } = opts;
+  const { requestFirecrawlMode, resolveLocalFirecrawlProbeUrl } = platform.mode;
+  let primary;
+  try {
+    primary = getBaseUrlFn();
+  } catch (e) {
+    return { ok: false, unavailable: true, reason: `could not resolve Firecrawl base URL: ${e?.message || e}` };
+  }
+  if (isHostedFirecrawl(primary)) {
+    // Prefer a fleet self-hosted duplicate before refusing — Basement / Spec-ops may be up
+    // even when FIRECRAWL_BASE_URL still points at the hosted default.
+    const fleet = typeof candidatesFn === "function" ? candidatesFn() : [];
+    if (!fleet.length) {
+      // A Firecrawl in this machine's own Docker is self-hosted too — use it before refusing.
+      const local = await tryLocalDocker(localDetectFn, modeOpts);
+      if (local) return { ok: true, baseUrl: local, source: "local-docker" };
+      return { ok: false, refusedHosted: true, message: HOSTED_REFUSAL_MESSAGE };
+    }
+  }
+
+  const candidates = [];
+  const seen = new Set();
+  const push = (url) => {
+    const u = typeof url === "string" ? url.trim().replace(/\/+$/, "") : "";
+    if (!u || isHostedFirecrawl(u) || seen.has(u)) return;
+    seen.add(u);
+    candidates.push(u);
+  };
+  push(primary);
+  if (typeof candidatesFn === "function") {
+    for (const u of candidatesFn()) push(u);
+  }
+  if (!candidates.length) {
+    return { ok: false, refusedHosted: true, message: HOSTED_REFUSAL_MESSAGE };
+  }
+
+  // Try each self-hosted host (Basement OR Spec-ops OR Night Action). Unleased hosts skip the
+  // XOR lease and just probe; leased Night Action keeps the SSH/lease path.
+  const reasons = [];
+  for (const baseUrl of candidates) {
+    let lease;
+    try {
+      lease = await requestFirecrawlMode({
+        localProbeUrl: resolveLocalFirecrawlProbeUrl({ FIRECRAWL_BASE_URL: baseUrl }),
+        ...modeOpts,
+      });
+    } catch (e) {
+      reasons.push(`${baseUrl}: mode-lease request failed: ${e?.message || e}`);
+      continue;
+    }
+    if (lease.granted) return { ok: true, baseUrl };
+    reasons.push(`${baseUrl}: ${lease.reason}`);
+  }
+
+  // Every declared candidate is down — a Firecrawl in THIS machine's Docker (any compose project,
+  // any published port) is still self-hosted. Found by `docker ps` + probe, no config needed.
+  {
+    const local = await tryLocalDocker(localDetectFn, modeOpts);
+    if (local) return { ok: true, baseUrl: local, source: "local-docker" };
+  }
+
+  // LAST RESort — every candidate is down (the fleet is powered off). If THIS machine declares an
+  // on-demand stack, start it and retry once. Deliberately placed after the loop so it can only
+  // ever turn a failure into a success: on any error we fall through to the original unavailable
+  // result below, and the "stop, never substitute" contract is preserved (a started local stack is
+  // still a self-hosted endpoint — not a hosted-credit spend and not a WebSearch fallback).
+  if (typeof autostartFn === "function") {
+    let started = null;
+    try {
+      started = await autostartFn();
+    } catch {
+      started = null;
+    }
+    if (started) {
+      try {
+        const lease = await requestFirecrawlMode({
+          localProbeUrl: resolveLocalFirecrawlProbeUrl({ FIRECRAWL_BASE_URL: started }),
+          ...modeOpts,
+        });
+        if (lease.granted) return { ok: true, baseUrl: started, autostarted: true };
+        reasons.push(`${started}: autostarted but lease refused — ${lease.reason}`);
+      } catch (e) {
+        reasons.push(`${started}: autostarted but probe failed: ${e?.message || e}`);
+      }
+    }
+  }
+
+  // SECOND LAST RESORT — this machine has nothing on-demand of its own, but a fleet machine
+  // (Spec-ops) may declare a working day-2 SSH script to start ITS stack remotely. Same fail-safe
+  // shape as the local autostart above: only ever turns a failure into a success.
+  if (typeof remoteStartFn === "function") {
+    let started = null;
+    try {
+      started = await remoteStartFn();
+    } catch {
+      started = null;
+    }
+    if (started) {
+      try {
+        const lease = await requestFirecrawlMode({
+          localProbeUrl: resolveLocalFirecrawlProbeUrl({ FIRECRAWL_BASE_URL: started }),
+          ...modeOpts,
+        });
+        if (lease.granted) return { ok: true, baseUrl: started, autostarted: true };
+        reasons.push(`${started}: remote-started but lease refused — ${lease.reason}`);
+      } catch (e) {
+        reasons.push(`${started}: remote-started but probe failed: ${e?.message || e}`);
+      }
+    }
+  }
+
+  return { ok: false, unavailable: true, reason: reasons.join(" | ") || "no self-hosted Firecrawl endpoint available" };
+}
+
+// ── subcommand implementations (testable independent of the CLI/exit layer) ─────────────────────
+export async function runSearch(query, { limit = 6, httpFetch, availabilityOpts = {} } = {}) {
+  const platform = availabilityOpts.platform !== undefined ? availabilityOpts.platform : await loadPlatform();
+  const avail = await ensureAvailable({ ...availabilityOpts, platform });
+  if (!avail.ok) return avail;
+  if (!platform) {
+    const lr = await localFirecrawlSearch(avail.baseUrl, query, { limit, ...(httpFetch ? { httpFetch } : {}) });
+    if (!lr.ok) return { ok: false, unavailable: true, reason: `search request failed on ${avail.baseUrl}: ${lr.error}` };
+    return { ok: true, results: lr.results, servedBy: avail.baseUrl, autostarted: false };
+  }
+  let r;
+  try {
+    r = await platform.adapter.firecrawlSearch(query, { limit, baseUrl: avail.baseUrl, httpFetch });
+  } catch (e) {
+    return { ok: false, unavailable: true, reason: `search request failed: ${e?.message || e}` };
+  }
+  if (r.blocked) return { ok: false, unavailable: true, reason: r.reason };
+  if (r.error) return { ok: false, unavailable: true, reason: r.error };
+  // servedBy: which candidate ACTUALLY answered. A research doc's Method block should record the
+  // real backend, not the intended one — the failover walks several hosts and an on-demand local
+  // stack may be the one that served.
+  return { ok: true, results: r.results, servedBy: avail.baseUrl, autostarted: avail.autostarted === true };
+}
+
+export async function runScrape(url, { httpFetch, availabilityOpts = {}, maxChars = MAX_SCRAPE_CHARS } = {}) {
+  const platform = availabilityOpts.platform !== undefined ? availabilityOpts.platform : await loadPlatform();
+  const avail = await ensureAvailable({ ...availabilityOpts, platform });
+  if (!avail.ok) return avail;
+  let r;
+  try {
+    r = platform
+      ? await platform.adapter.firecrawlScrape(url, { baseUrl: avail.baseUrl, httpFetch })
+      : await localFirecrawlScrape(avail.baseUrl, url, httpFetch ? { httpFetch } : {});
+  } catch (e) {
+    return { ok: false, unavailable: true, reason: `scrape request failed: ${e?.message || e}` };
+  }
+  if (r.blocked) return { ok: false, unavailable: true, reason: r.reason };
+  if (!r.ok) return { ok: false, unavailable: true, reason: r.error || `HTTP ${r.status || 0}` };
+  const full = r.markdown || "";
+  const truncated = full.length > maxChars;
+  const markdown = truncated ? full.slice(0, maxChars) : full;
+  return { ok: true, url, title: r.title || "", markdown, truncated, fullLength: full.length, servedBy: avail.baseUrl, autostarted: avail.autostarted === true };
+}
+
+// ── output formatting ─────────────────────────────────────────────────────────────────────────
+function formatSearchList(results) {
+  if (results.length === 0) return "(no results)";
+  return results
+    .map((r, i) => {
+      const lines = [`${i + 1}. ${r.title || "(untitled)"}`, `   ${r.url}`];
+      if (r.snippet) lines.push(`   ${r.snippet}`);
+      return lines.join("\n");
+    })
+    .join("\n\n");
+}
+
+function formatScrapeText(result) {
+  let out = result.markdown;
+  if (result.truncated) {
+    out += `\n\n[...truncated — showing first ${MAX_SCRAPE_CHARS} of ${result.fullLength} chars...]`;
+  }
+  return out;
+}
+
+// ── CLI entry ─────────────────────────────────────────────────────────────────────────────────
+function printUsage() {
+  console.error('usage: firecrawl-cli.mjs search "<query>" [--limit N] [--json]');
+  console.error('       firecrawl-cli.mjs scrape "<url>" [--json]');
+  console.error("       firecrawl-cli.mjs --self-test");
+}
+
+/**
+ * Set the exit code and RETURN — never call process.exit() after doing network work.
+ *
+ * Forcing an exit while the HTTP client still has live handles aborts the process on Windows with
+ * `Assertion failed: !(handle->flags & UV_HANDLE_CLOSING), src\\win\\async.c:94` (uv_async_send on
+ * an already-closing handle). stdout had already flushed, so callers saw perfect JSON alongside
+ * exit code 127: harmless when a human reads stdout, fatal for any pipeline that checks exit
+ * status — and it poisoned callers that correctly treat non-zero as failure. Letting the event
+ * loop drain on its own exits cleanly and promptly (verified: the same work run to completion
+ * without process.exit() returns 0 with no assertion).
+ */
+function finish(code) {
+  process.exitCode = code;
+}
+
+async function exitOnResult(result, { kind, json }) {
+  if (!result.ok) {
+    if (result.refusedHosted) {
+      console.log(result.message);
+      return finish(2);
+    }
+    // Machine-readable, single-line, exact prefix the calling workflow prompts grep for.
+    console.log(`FIRECRAWL_UNAVAILABLE: ${result.reason}`);
+    return finish(3);
+  }
+  if (kind === "search") {
+    console.log(json ? JSON.stringify({ results: result.results, servedBy: result.servedBy, autostarted: result.autostarted }) : formatSearchList(result.results));
+  } else {
+    console.log(
+      json
+        ? JSON.stringify({ url: result.url, title: result.title, markdown: result.markdown, truncated: result.truncated, fullLength: result.fullLength, servedBy: result.servedBy, autostarted: result.autostarted })
+        : formatScrapeText(result),
+    );
+  }
+  return finish(0);
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  const cmd = argv[0];
+  const target = argv[1];
+  const json = argv.includes("--json");
+  const limitIdx = argv.indexOf("--limit");
+  const limit = limitIdx !== -1 ? Number(argv[limitIdx + 1]) || 6 : 6;
+
+  if (cmd !== "search" && cmd !== "scrape") {
+    printUsage();
+    process.exit(64);
+    return;
+  }
+  if (!target) {
+    console.error(`usage error: missing ${cmd === "search" ? "<query>" : "<url>"} argument`);
+    printUsage();
+    process.exit(64);
+    return;
+  }
+
+  try {
+    const result = cmd === "search" ? await runSearch(target, { limit }) : await runScrape(target, {});
+    await exitOnResult(result, { kind: cmd, json });
+  } catch (e) {
+    // Last-resort catch — every internal path above already returns a clean shape, but this
+    // guarantees the CLI NEVER surfaces a raw stack trace, per the hard requirement.
+    console.log(`FIRECRAWL_UNAVAILABLE: unexpected error — ${e?.message || String(e)}`);
+    return finish(3);
+  }
+}
+
+const isMain = import.meta.url.endsWith("/" + (process.argv[1]?.split(/[\\/]/).pop() ?? ""));
+
+if (isMain && !process.argv.includes("--self-test")) {
+  main();
+}
+
+// ── self-test (mocked — no real network, no real SSH) — MAIN-entry guarded ──────────────────────
+if (isMain && process.argv.includes("--self-test")) {
+  (async () => {
+    let ok = true;
+    const check = (n, c) => {
+      console.log(`  ${c ? "✓" : "✗"} ${n}`);
+      if (!c) ok = false;
+    };
+    console.log("== firecrawl-cli.mjs self-test (mock-only, no real network/SSH) ==\n");
+    const P = await loadPlatform();
+    if (!P) {
+      console.log("  (platform-mode self-test needs a HeySpex checkout; local mode: firecrawl-cli.local.test.mjs)");
+      return;
+    }
+    const { HOSTED_FIRECRAWL_BASE_URL } = P.adapter;
+
+    // A single mock httpFetch that branches on URL shape — reused for BOTH the mode-lease probes
+    // (":11434"/":3002" GETs) and the actual Firecrawl POST calls (/search, /scrape).
+    const makeMockFetch = ({ ollamaModels = [], firecrawlUp = true, searchResults = [{ title: "T", url: "http://example.com/x", description: "snip" }], scrapeMarkdown = "Hello **world**" } = {}) =>
+      async (url) => {
+        const u = String(url);
+        if (u.includes(":11434")) {
+          return { ok: true, status: 200, json: async () => ({ models: ollamaModels }) };
+        }
+        if (u.endsWith("/search")) {
+          return { ok: true, status: 200, json: async () => ({ success: true, data: { web: searchResults } }) };
+        }
+        if (u.endsWith("/scrape")) {
+          return { ok: true, status: 200, json: async () => ({ success: true, data: { markdown: scrapeMarkdown, metadata: { title: "Page Title", statusCode: 200 } } }) };
+        }
+        // bare-root Firecrawl reachability probe
+        if (firecrawlUp) return { ok: true, status: 200, json: async () => ({}) };
+        throw new Error("ECONNREFUSED");
+      };
+    const noSsh = async () => { throw new Error("self-test: SSH must never be called by these cases"); };
+
+    // Isolate lease cases from fleet failover (Basement / Spec-ops candidates) —
+    // each case under test pins a single endpoint via getBaseUrlFn.
+    // autostartFn is stubbed OFF for every case below: the real one shells out to
+    // firecrawl-local.sh (docker compose), which must never run from a self-test. Cases that
+    // expect `unavailable` DO reach the autostart step, so this stub is load-bearing.
+    const noFleet = { candidatesFn: () => [], autostartFn: async () => null, remoteStartFn: async () => null, localDetectFn: null };
+
+    // ── 1. hosted endpoint is refused BEFORE any network call ──
+    let networkCalled = false;
+    const trackedFetch = async (...a) => { networkCalled = true; throw new Error("must not be called"); };
+    const hostedAvail = await ensureAvailable({ getBaseUrlFn: () => HOSTED_FIRECRAWL_BASE_URL, modeOpts: { httpFetch: trackedFetch, runCmd: noSsh }, ...noFleet });
+    check("1 hosted endpoint -> refusedHosted:true", hostedAvail.ok === false && hostedAvail.refusedHosted === true);
+    check("1b hosted refusal never calls the network", networkCalled === false);
+
+    // ── 2. self-hosted + Ollama busy -> unavailable, no SSH call (never evicts Ollama) ──
+    let sshCalled = false;
+    const ollamaBusyFetch = makeMockFetch({ ollamaModels: [{ name: OLLAMA.SMALL }], firecrawlUp: false });
+    const busyAvail = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.252:3002/v2",
+      modeOpts: { httpFetch: ollamaBusyFetch, runCmd: async () => { sshCalled = true; } },
+      ...noFleet,
+    });
+    check("2 Ollama-busy -> unavailable:true", busyAvail.ok === false && busyAvail.unavailable === true);
+    check("2b reason names the Ollama lease", /ollama/i.test(busyAvail.reason));
+    check("2c never calls SSH (no eviction)", sshCalled === false);
+
+    // ── 3. self-hosted + box offline -> unavailable ──
+    const offlineFetch = async () => { throw new Error("ECONNREFUSED"); };
+    const offlineAvail = await ensureAvailable({ getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: offlineFetch, runCmd: noSsh }, ...noFleet });
+    check("3 offline box -> unavailable:true", offlineAvail.ok === false && offlineAvail.unavailable === true);
+
+    // ── 4. self-hosted + box idle -> lease requested and granted (compose up mocked, no real SSH) ──
+    sshCalled = false;
+    const idleFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: false });
+    const idleAvail = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.252:3002/v2",
+      modeOpts: { httpFetch: idleFetch, runCmd: async () => { sshCalled = true; } },
+      ...noFleet,
+    });
+    check("4 idle box -> lease granted", idleAvail.ok === true);
+    check("4b idle grant DOES start the stack (mocked SSH call made)", sshCalled === true);
+
+    // ── 5. self-hosted + Firecrawl already up -> granted directly, no SSH ──
+    sshCalled = false;
+    const upFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true });
+    const upAvail = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.252:3002/v2",
+      modeOpts: { httpFetch: upFetch, runCmd: async () => { sshCalled = true; } },
+      ...noFleet,
+    });
+    check("5 already-up -> lease granted with no SSH call", upAvail.ok === true && sshCalled === false);
+
+    // ── 5d. hosted primary + Spec-ops/Basement candidate up -> failover grant (duplicate option) ──
+    const fleetUpFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true });
+    const failoverAvail = await ensureAvailable({
+      getBaseUrlFn: () => HOSTED_FIRECRAWL_BASE_URL,
+      candidatesFn: () => ["http://192.168.1.98:3002/v2"],
+      modeOpts: { httpFetch: fleetUpFetch, runCmd: noSsh },
+    });
+    check("5d hosted primary fails over to Spec-ops candidate", failoverAvail.ok === true && failoverAvail.baseUrl === "http://192.168.1.98:3002/v2");
+
+    // ── 6. runSearch end-to-end (mocked): returns normalized {title,url,snippet}[] ──
+    const searchFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true });
+    const searchResult = await runSearch("test query", {
+      httpFetch: searchFetch,
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: searchFetch, runCmd: noSsh }, ...noFleet },
+    });
+    check("6 runSearch resolves ok:true with results", searchResult.ok === true && searchResult.results.length === 1 && searchResult.results[0].url === "http://example.com/x");
+    check("6b formatSearchList renders a readable list", formatSearchList(searchResult.results).includes("http://example.com/x"));
+
+    // ── 7. runSearch when unavailable (Ollama busy) -> clean unavailable shape, not a throw ──
+    const busySearchResult = await runSearch("q", {
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: ollamaBusyFetch, runCmd: noSsh }, ...noFleet },
+    });
+    check("7 runSearch under Ollama lease -> unavailable:true (not a throw)", busySearchResult.ok === false && busySearchResult.unavailable === true);
+
+    // ── 8. runScrape end-to-end (mocked): returns markdown + title ──
+    const scrapeFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true, scrapeMarkdown: "Hello **world**" });
+    const scrapeResult = await runScrape("http://example.com/x", {
+      httpFetch: scrapeFetch,
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: scrapeFetch, runCmd: noSsh }, ...noFleet },
+    });
+    check("8 runScrape resolves ok:true", scrapeResult.ok === true && scrapeResult.title === "Page Title");
+    check("8b scrape markdown content present", /Hello \*\*world\*\*/.test(scrapeResult.markdown));
+    check("8c not truncated (short body)", scrapeResult.truncated === false);
+
+    // ── 9. runScrape truncates at MAX_SCRAPE_CHARS and notes it ──
+    const longMarkdown = "x".repeat(MAX_SCRAPE_CHARS + 500);
+    const longFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true, scrapeMarkdown: longMarkdown });
+    const longResult = await runScrape("http://example.com/long", {
+      httpFetch: longFetch,
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: longFetch, runCmd: noSsh }, ...noFleet },
+    });
+    check("9 long page truncated to MAX_SCRAPE_CHARS", longResult.ok === true && longResult.markdown.length === MAX_SCRAPE_CHARS && longResult.truncated === true);
+    check("9b formatScrapeText appends a truncation note", formatScrapeText(longResult).includes("truncated"));
+
+    // ── 10. network/parse failure inside runSearch/runScrape never throws ──
+    const throwingFetch = async (url) => {
+      if (String(url).includes(":11434") || String(url).endsWith("/") === false && !String(url).endsWith("/search") && !String(url).endsWith("/scrape")) {
+        return { ok: true, status: 200, json: async () => ({ models: [] }) };
+      }
+      throw new Error("ECONNRESET");
+    };
+    const errSearch = await runSearch("q", {
+      httpFetch: throwingFetch,
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: makeMockFetch({ ollamaModels: [], firecrawlUp: true }), runCmd: noSsh }, ...noFleet },
+    });
+    check("10 network failure inside search -> unavailable:true, not a throw", errSearch.ok === false && errSearch.unavailable === true);
+
+    // ── 11. on-demand local autostart: fires ONLY after every candidate failed, and is fail-safe ──
+    // The workstation stack is normally stopped ([[workstation-firecrawl-is-the-fleet-down]]), so a
+    // fleet-down run must be able to start it and retry — without ever weakening the refusal path.
+    const localUpFetch = async (url) => {
+      if (String(url).includes("127.0.0.1")) return { ok: true, status: 200, json: async () => ({}) };
+      throw new Error("ECONNREFUSED");
+    };
+    const autoAvail = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.98:3002/v2",
+      modeOpts: { httpFetch: localUpFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => "http://127.0.0.1:3002/v2",
+      remoteStartFn: async () => null,
+    });
+    check("11 all candidates down -> autostart rescues the run", autoAvail.ok === true && autoAvail.baseUrl === "http://127.0.0.1:3002/v2");
+    check("11b rescued result is flagged autostarted:true", autoAvail.autostarted === true);
+
+    // a failed start must degrade to the ORIGINAL unavailable, never to a crash
+    const autoFail = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.98:3002/v2",
+      modeOpts: { httpFetch: offlineFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => null,
+      remoteStartFn: async () => null,
+    });
+    check("11c failed autostart -> still unavailable:true", autoFail.ok === false && autoFail.unavailable === true);
+
+    const autoThrow = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.98:3002/v2",
+      modeOpts: { httpFetch: offlineFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => { throw new Error("docker exploded"); },
+      remoteStartFn: async () => null,
+    });
+    check("11d throwing autostart -> unavailable:true, not a throw", autoThrow.ok === false && autoThrow.unavailable === true);
+
+    // never start anything when a candidate already answered
+    let autostartCalled = false;
+    const noStartNeeded = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.252:3002/v2",
+      modeOpts: { httpFetch: upFetch, runCmd: async () => {} },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => { autostartCalled = true; return null; },
+      remoteStartFn: async () => null,
+    });
+    check("11e a healthy candidate never triggers autostart", noStartNeeded.ok === true && autostartCalled === false);
+
+    // the hosted refusal must still short-circuit BEFORE any autostart attempt
+    let hostedAutostartCalled = false;
+    const hostedStill = await ensureAvailable({
+      getBaseUrlFn: () => HOSTED_FIRECRAWL_BASE_URL,
+      modeOpts: { httpFetch: trackedFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => { hostedAutostartCalled = true; return "http://127.0.0.1:3002/v2"; },
+      remoteStartFn: async () => null,
+    });
+    check("11f hosted refusal still short-circuits, no autostart", hostedStill.refusedHosted === true && hostedAutostartCalled === false);
+
+    // ── 11g-11i. remote-start (Spec-ops SSH script) fires as the THIRD fallback, after local
+    // autostart also fails — and is itself fail-safe (never throws, degrades to unavailable). ──
+    let remoteStartCalled = false;
+    const remoteRescue = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.98:3002/v2",
+      modeOpts: { httpFetch: localUpFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => null,
+      remoteStartFn: async () => { remoteStartCalled = true; return "http://127.0.0.1:3002/v2"; },
+    });
+    check("11g local autostart failing falls through to remoteStartFn", remoteStartCalled === true);
+    check("11h remote-started result is flagged autostarted:true", remoteRescue.ok === true && remoteRescue.autostarted === true);
+
+    const remoteThrow = await ensureAvailable({
+      getBaseUrlFn: () => "http://192.168.1.98:3002/v2",
+      modeOpts: { httpFetch: offlineFetch, runCmd: noSsh },
+      candidatesFn: () => [],
+      localDetectFn: null,
+      autostartFn: async () => null,
+      remoteStartFn: async () => { throw new Error("ssh exploded"); },
+    });
+    check("11i throwing remoteStartFn -> unavailable:true, not a throw", remoteThrow.ok === false && remoteThrow.unavailable === true);
+
+    // ── 11j-11l. remoteStartCandidates(): excludes THIS machine, dedupes shared-stack GPUs,
+    // skips machines with no declared script. Reads the real routing-table.json by path (mirrors
+    // localOnDemandFirecrawl's shape, no table-injection param) — exercises the LIVE table's
+    // actual declarations (specops-gpu0/gpu1 -> firecrawl-specops.sh, basement-geek -> none). ──
+    const liveRemote = remoteStartCandidates({ HEYSPEX_FLEET_MACHINE_KEY: "coffee-geek" });
+    check("11j live routing-table declares Spec-ops' remote-start script, deduped to one entry", liveRemote.filter((c) => c.machineKey.startsWith("specops")).length === 1);
+    check("11k live routing-table excludes basement-geek (no remoteStart declared)", !liveRemote.some((c) => c.machineKey === "basement-geek"));
+    const selfExcluded = remoteStartCandidates({ HEYSPEX_FLEET_MACHINE_KEY: "specops-gpu0" });
+    check("11l remoteStartCandidates excludes the machine THIS process is on", !selfExcluded.some((c) => c.machineKey === "specops-gpu0"));
+
+    // ── 12. servedBy reports the candidate that ACTUALLY answered ──
+    const servedFetch = makeMockFetch({ ollamaModels: [], firecrawlUp: true });
+    const servedResult = await runSearch("q", {
+      httpFetch: servedFetch,
+      availabilityOpts: { getBaseUrlFn: () => "http://192.168.1.252:3002/v2", modeOpts: { httpFetch: servedFetch, runCmd: noSsh }, ...noFleet },
+    });
+    check("12 runSearch reports servedBy", servedResult.ok === true && servedResult.servedBy === "http://192.168.1.252:3002/v2");
+
+    console.log(`\n${ok ? "✓ self-test PASS" : "✗ self-test FAIL"}`);
+    process.exit(ok ? 0 : 1);
+  })();
+}