heyspex 0.2.5 → 0.2.6

package/dev-factory/spexcrawl/spexcrawl-cli.mjs

@@ -0,0 +1,734 @@
+#!/usr/bin/env node
+/**
+ * spexcrawl-cli.mjs — CLI bridge from a Claude subagent (Bash tool only, no JS imports) to
+ * SpexCrawl: search/gather through the user's HeySpex Companion, brokered by the spex-console.
+ *
+ * THE GAP THIS CLOSES ([[HS-AIV8-G805-DIAG-hs-res-declares-spexcrawl-primary-but]]): the
+ * 2026-08-14 owner ruling makes SpexCrawl-via-Companion the PRIMARY research gather method
+ * (`.heyspex/config.yaml` default_source: spexcrawl-companion), but until this CLI existed no
+ * agent-reachable bridge did — every /hs-res run fell straight to the permission-gated Firecrawl
+ * fallback. This is the SpexCrawl sibling of firecrawl-cli.mjs: same verbs, same --json shape
+ * intent, same machine-readable failure prefix + exit-code conventions, so the /hs-res skill
+ * swaps backends without reshaping its steps.
+ *
+ * HOW IT WORKS (asynchronous — results come from the user's own machine):
+ *   1. POST <console>/api/agent/gather-jobs queues the job as a launch intent.
+ *   2. The HeySpex Companion app polls intents, searches/fetches/cleans pages LOCALLY
+ *      (the user's browser context, robots-respecting), and uploads pages back to the console.
+ *   3. This CLI polls GET <console>/api/agent/gather-jobs/<jobId> every POLL_INTERVAL_MS until
+ *      status "complete" (or --timeout), then prints the stored cleaned pages.
+ *
+ * PREREQUISITE (stated in every failure line so nobody has to dig): the HeySpex Companion
+ * tray app must be running — it is the only thing that executes gather jobs. Since HS-V9WH
+ * its relay polls loopback consoles too, so a same-machine dev console needs no LAN bind;
+ * only a Companion on a DIFFERENT machine needs the console in LAN mode
+ * (`npm run console:log:lan`). Without a Companion, jobs queue forever and this CLI exits 3
+ * at the poll timeout.
+ *
+ * FAILURE CONTRACT (mirrors FIRECRAWL_UNAVAILABLE so calling skill prompts can grep one prefix):
+ *   - unreachable console, non-2xx that is not a 400, or poll timeout -> print a single line
+ *     starting `SPEXCRAWL_UNAVAILABLE: ...` (including the LAN-mode hint) and exit 3.
+ *     On the POLL leg only, a short streak of transient read failures is ridden out first
+ *     (MAX_TRANSIENT_POLL_FAILURES) — the job is already filed and the Companion keeps
+ *     working through a console blip, so one hiccup must not throw away a finished gather.
+ *   - a 400 from the API -> print the server's message and exit 1 (the request was wrong,
+ *     falling back to another backend would not fix it).
+ *
+ * Usage:
+ *   node dev-factory/spexcrawl/spexcrawl-cli.mjs search "<query>" [--json] [--timeout SECONDS]
+ *   node dev-factory/spexcrawl/spexcrawl-cli.mjs scrape "<url>" [<url>...] [--json] [--timeout SECONDS]
+ *   (`gather` is accepted as an alias of `scrape` — it is the vault-native verb for the same job.)
+ *   node dev-factory/spexcrawl/spexcrawl-cli.mjs crawl "<site>" [--max-pages N] [--max-depth N]
+ *       [--json] [--timeout SECONDS]
+ *   (walks a whole site — sitemap + same-origin links — instead of fetching known addresses;
+ *    default --timeout is longer (600s) than search/scrape/gather's 180s, since a walk is bigger
+ *    than a scrape. [[HS-QHES-90JL-TASK-crawl-reaches-agents-console-route]])
+ *
+ * Exit codes: 0 ok · 1 bad request / server-rejected input (server message printed) ·
+ *             3 SPEXCRAWL_UNAVAILABLE (unreachable / non-2xx / poll timeout — the
+ *             stop-and-ask-before-Firecrawl signal) · 64 usage error (bad args).
+ */
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// The ONE shared console-URL resolver ([[HS-GD45-MCPC-TASK-mcp-cloud-auth-shared-console-resolver]])
+// — env → ~/.heyspex/credentials.yaml console_url → mode-derived default, so this CLI flips
+// with `hs dev` / `hs login` like every other consumer instead of hand-rolling its own
+// env-only lookup.
+import { resolveConsoleUrl as resolveConsoleUrlShared } from "../../scripts/lib/console-url-resolver.mjs";
+import { parseRepoBinding } from "../../vendor/heyspex-mcp/vault-binding.mjs";
+
+export const DEFAULT_CONSOLE_URL = "http://localhost:3000";
+export const DEFAULT_TIMEOUT_MS = 180_000;
+/** `crawl`'s default poll timeout — a whole-site walk legitimately runs longer
+ * than a single search/scrape/gather round trip. */
+export const DEFAULT_CRAWL_TIMEOUT_MS = 600_000;
+export const POLL_INTERVAL_MS = 3_000;
+/**
+ * How many CONSECUTIVE transient read failures the poll loop rides out before declaring the
+ * console unavailable ([[HS-FBZ3-NWVU-DIAG-spexcrawl-companion-connectivity]]). Read failures
+ * are independent of the job: the Companion keeps working through a console blip. Bounded and
+ * still subject to the overall `--timeout`, so a console that is genuinely down costs at most
+ * this many extra poll intervals before the Firecrawl fallback fires. Any answered poll resets
+ * the streak. A 400 is NOT transient and is never retried.
+ */
+export const MAX_TRANSIENT_POLL_FAILURES = 3;
+
+/** Appended to every SPEXCRAWL_UNAVAILABLE line — the prerequisite must be stated by the
+ *  CLI's own error message, not left to archaeology (diagnosis AC3). Since HS-V9WH the
+ *  Companion polls loopback consoles too, so same-machine dev needs no LAN mode — LAN
+ *  bind only matters when the Companion runs on a different machine than the console. */
+export const LAN_HINT =
+  "if the Companion is not picking up jobs, check the HeySpex Companion tray app is running " +
+  "(rebuilt after 2026-08-21 — older builds ignore loopback consoles); a Companion on " +
+  "another machine additionally needs the console in LAN mode (npm run console:log:lan)";
+
+/** Appended instead of {@link LAN_HINT} when the console REFUSED the bearer (401/403). That is a
+ *  server-side auth verdict on this machine's token, not a Companion liveness problem — sending
+ *  the reader to check the tray app hid a wrong-route bug for a day
+ *  ([[HS-WN3F-LY85-DIAG-post-api-agent-gather-jobs-returns-401]] AC/fix 4). */
+export const AUTH_HINT =
+  "the console refused this machine's bearer (clerk.companion_token in ~/.heyspex/credentials.yaml, " +
+  "or HEYSPEX_COMPANION_TOKEN) — the Companion being up or down does not matter here; re-pair this " +
+  "machine to the console so a fresh, unrevoked token is written, then retry";
+
+/** Appended instead of {@link LAN_HINT} when the console refused the job because no Companion
+ *  polls its vault ([[HS-L4LC]]). Names the vault-mismatch case: the Companion may be running
+ *  fine, just bound (vaultRepoPath) to a different vault than the one this job named. */
+export function companionVaultHint(vaultId) {
+  const v = jobVault();
+  const run = v.runVaultId && v.runVaultId !== vaultId ? `; this run's own vault is ${v.runVaultId}` : "";
+  return (
+    `no Companion is polling vault ${vaultId || "(none)"} (resolved via ${v.source}${run}) — ` +
+    "a Companion claims jobs only for the ONE vault its companion.settings.json vaultRepoPath names: " +
+    "if the tray app is running it is bound to another vault (set HEYSPEX_SPEXCRAWL_VAULT_ID to that " +
+    "vault's id, or point vaultRepoPath here), otherwise start the HeySpex Companion tray app"
+  );
+}
+
+// ── console endpoint + auth resolution ─────────────────────────────────────────────────────────
+// Delegates to the shared resolver (env override still wins first); no `repoRoot` here, so the
+// admin-local discovery-file rung is never consulted by this CLI (unchanged from before).
+export function resolveConsoleUrl(env = process.env) {
+  return resolveConsoleUrlShared({ env });
+}
+
+/**
+ * Bearer token for /api/agent/*: env HEYSPEX_COMPANION_TOKEN first (LAN dev / CI), else the
+ * logged-in Companion session token (~/.heyspex/credentials.yaml, clerk.companion_token) — the
+ * SAME source the Companion itself reads (FileCompanionCredentialReader) and the same regex
+ * resolution as dev-factory/fleet/manager.mjs fleetToken(). Null = send no auth header, which
+ * mirrors the Companion's own optional-bearer behavior (the dev-loopback console accepts it).
+ */
+export function resolveConsoleToken(env = process.env, readFile = readFileSync) {
+  const fromEnv = String(env.HEYSPEX_COMPANION_TOKEN || "").trim();
+  if (fromEnv) return fromEnv;
+  try {
+    const yaml = readFile(join(homedir(), ".heyspex", "credentials.yaml"), "utf8");
+    const clerk = yaml.match(/^clerk:\s*$([\s\S]*?)(?=^\S|\s*$(?![\s\S]))/m)?.[1] ?? yaml;
+    const m = clerk.match(/^\s*companion_token:\s*["']?([^"'\r\n#]+)/m);
+    return m?.[1]?.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The vault this run is bound to, for `x-heyspex-vault-id`. The console refuses a bearer request
+ * that names no vault (409 VAULT_SCOPE_MISMATCH / vault-scope-required), so an unscoped POST to
+ * /api/agent/gather-jobs never reaches the Companion. Sources, first hit wins: env
+ * HEYSPEX_VAULT_ID; `vault.id` of a repo-binding heyspex.yaml (or legacy `vault_id:`) at the cwd (or an ancestor); then the vault root
+ * the repo's .mcp.json pins in mcpServers.heyspex.env.HEYSPEX_VAULT_ROOT (a code repo attached to a
+ * sibling vault). Null = send no header, which the console will refuse with its own hint.
+ */
+/** Vault id named by `<dir>/heyspex.yaml` — repo-binding `vault.id` first, legacy flat `vault_id:` last. */
+function vaultIdInDir(dir, readFile = readFileSync) {
+  try {
+    const text = readFile(join(dir, "heyspex.yaml"), "utf8");
+    return parseRepoBinding(text)?.id || text.match(/^vault_id:\s*["']?([^"'\s#]+)/m)?.[1] || null;
+  } catch {
+    return null;
+  }
+}
+
+export function resolveVaultId(env = process.env, readFile = readFileSync, cwd = process.cwd()) {
+  const fromEnv = String(env.HEYSPEX_VAULT_ID || "").trim();
+  if (fromEnv) return fromEnv;
+  const idIn = (dir) => vaultIdInDir(dir, readFile);
+  // HEYSPEX_VAULT_ROOT is the documented explicit override (owner ruling 2026-10-14 resolution
+  // order: --vault -> HEYSPEX_VAULT_ROOT -> walk up). It used to be ignored here, so a run aimed at
+  // another vault silently resolved the cwd's binding instead ([[HS-L4LC]] root cause 1).
+  const root = String(env.HEYSPEX_VAULT_ROOT || "").trim();
+  if (root && !root.includes("${")) {
+    const id = idIn(root);
+    if (id) return id;
+  }
+  for (let dir = cwd, prev = null; dir !== prev; prev = dir, dir = dirname(dir)) {
+    const id = idIn(dir);
+    if (id) return id;
+  }
+  try {
+    const pinned = JSON.parse(readFile(join(cwd, ".mcp.json"), "utf8"))?.mcpServers?.heyspex?.env?.HEYSPEX_VAULT_ROOT;
+    if (typeof pinned === "string" && pinned.trim() && !pinned.includes("${")) return idIn(pinned.trim());
+  } catch {
+    // no .mcp.json / unparseable: fall through to null
+  }
+  return null;
+}
+
+/**
+ * Where the local Companion keeps `companion.settings.json`, in the order its own
+ * CompanionPathLocator.SettingsPath() would find it: explicit env override, the source checkout
+ * this CLI lives in (`<repo>/companion/`, a dev-built Companion), then the per-user install dir.
+ */
+export function companionSettingsCandidates(env = process.env) {
+  const out = [];
+  const fromEnv = String(env.HEYSPEX_COMPANION_SETTINGS || "").trim();
+  if (fromEnv) out.push(fromEnv);
+  try {
+    const here = dirname(fileURLToPath(import.meta.url)); // <repo>/dev-factory/spexcrawl
+    out.push(join(here, "..", "..", "companion", "companion.settings.json"));
+  } catch {
+    // non-file URL (bundled): skip the checkout rung
+  }
+  const localAppData = String(env.LOCALAPPDATA || "").trim();
+  if (localAppData) out.push(join(localAppData, "HeySpex", "companion.settings.json"));
+  return out;
+}
+
+/**
+ * The vault the local Companion polls launch intents for — the ONLY vault whose gather jobs it
+ * will ever claim. The Companion stamps `x-heyspex-vault-id` from the `heyspex.yaml` under its
+ * single `vaultRepoPath` (RelayEnvironment.ResolveVaultId), and the console drains intents per
+ * (owner, vault), so a job filed under any other vault sits "pending" until the CLI times out
+ * ([[HS-L4LC]]: every HS-NPMN scrape for the MLC vault). Returns
+ * `{ vaultId, vaultRepoPath, settingsPath }` or null when no Companion settings are readable.
+ */
+export function resolveCompanionVault(env = process.env, readFile = readFileSync) {
+  for (const settingsPath of companionSettingsCandidates(env)) {
+    let text;
+    try {
+      text = readFile(settingsPath, "utf8");
+    } catch {
+      continue;
+    }
+    const raw = text.match(/"vaultRepoPath"\s*:\s*("(?:[^"\\]|\\.)*")/)?.[1];
+    let vaultRepoPath = null;
+    try {
+      vaultRepoPath = raw ? JSON.parse(raw).trim() : null;
+    } catch {
+      vaultRepoPath = null;
+    }
+    if (!vaultRepoPath) continue;
+    const vaultId = vaultIdInDir(vaultRepoPath, readFile);
+    if (vaultId) return { vaultId, vaultRepoPath, settingsPath };
+  }
+  return null;
+}
+
+/**
+ * The vault a SpexCrawl job is FILED under (`x-heyspex-vault-id` on the POST and the poll).
+ * A gather/research/crawl job is a work order for the user's own Companion, not vault content —
+ * the pages come straight back to this CLI, and whatever the caller saves lands in its own vault
+ * through its own write path. So the job is addressed to the vault the Companion actually
+ * listens on. Order: env HEYSPEX_SPEXCRAWL_VAULT_ID (explicit override) -> the local Companion's
+ * vault -> the run's own vault ({@link resolveVaultId}). `runVaultId` is reported so a mismatch
+ * is visible instead of silent.
+ */
+export function resolveJobVault(env = process.env, readFile = readFileSync, cwd = process.cwd()) {
+  const runVaultId = resolveVaultId(env, readFile, cwd);
+  const override = String(env.HEYSPEX_SPEXCRAWL_VAULT_ID || "").trim();
+  if (override) return { vaultId: override, source: "HEYSPEX_SPEXCRAWL_VAULT_ID", runVaultId };
+  const companion = resolveCompanionVault(env, readFile);
+  if (companion) {
+    return { vaultId: companion.vaultId, source: `companion (${companion.vaultRepoPath})`, runVaultId };
+  }
+  return { vaultId: runVaultId, source: "run vault (no Companion settings found)", runVaultId };
+}
+
+/** Resolved once per process: the vault every request of this run names. */
+let jobVaultMemo = null;
+function jobVault() {
+  jobVaultMemo ??= resolveJobVault();
+  return jobVaultMemo;
+}
+
+/** One stderr line naming the vault the job was queued under — stdout stays pure for --json. */
+function announceJobVault(posted) {
+  const v = jobVault();
+  const mismatch = v.runVaultId && v.vaultId && v.runVaultId !== v.vaultId ? ` (run vault ${v.runVaultId})` : "";
+  console.error(`spexcrawl: job ${posted.jobId} queued under vault ${v.vaultId || "(none)"} via ${v.source}${mismatch}`);
+  if (posted.companion?.likelyOffline) {
+    console.error(
+      `spexcrawl: WARNING no Companion has polled vault ${v.vaultId || "(none)"} recently ` +
+        `(last seen ${posted.companion.lastSeenAt || "never"}) — this job will likely sit pending until --timeout`,
+    );
+  }
+}
+
+function authHeaders(token) {
+  const vaultId = jobVault().vaultId;
+  return {
+    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    ...(vaultId ? { "x-heyspex-vault-id": vaultId } : {}),
+  };
+}
+
+// ── HTTP layer (every path returns a clean shape — never throws to the caller) ─────────────────
+/** Extract the server's own words from a 400 body (JSON `error`/`message` or raw text). */
+async function readErrorMessage(res) {
+  try {
+    const text = await res.text();
+    try {
+      const j = JSON.parse(text);
+      return String(j.error || j.message || text);
+    } catch {
+      return text || `HTTP ${res.status}`;
+    }
+  } catch {
+    return `HTTP ${res.status}`;
+  }
+}
+
+async function postJob(body, { baseUrl, token, httpFetch }) {
+  let res;
+  try {
+    res = await httpFetch(`${baseUrl}/api/agent/gather-jobs`, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...authHeaders(token) },
+      body: JSON.stringify(body),
+    });
+  } catch (e) {
+    return { ok: false, unavailable: true, reason: `console unreachable at ${baseUrl}: ${e?.message || e}` };
+  }
+  if (res.status === 400) {
+    return { ok: false, badRequest: true, message: await readErrorMessage(res) };
+  }
+  if (res.status === 401 || res.status === 403) {
+    // An auth refusal is a verdict on the TOKEN, not on whether the Companion is running —
+    // `renderFailure` swaps the Companion hint for the re-pair hint on this flag.
+    return {
+      ok: false,
+      unavailable: true,
+      authRefused: true,
+      reason: `POST /api/agent/gather-jobs -> HTTP ${res.status} (${await readErrorMessage(res)})`,
+    };
+  }
+  if (res.status === 503) {
+    // The console REFUSES (does not queue) a job when no Companion has polled the job's vault
+    // recently ([[HS-L4LC]] fix direction 2) — typed `companion_not_listening`. Fail fast with a
+    // vault-naming hint instead of the generic tray-app one; any other 503 stays generic.
+    let j = null;
+    try {
+      j = await res.json();
+    } catch {
+      j = null;
+    }
+    if (j?.error === "companion_not_listening") {
+      return {
+        ok: false,
+        unavailable: true,
+        companionNotListening: true,
+        vaultId: j.vaultId || jobVault().vaultId || null,
+        lastSeenAt: j.companion?.lastSeenAt ?? null,
+        reason: `POST /api/agent/gather-jobs -> HTTP 503 companion_not_listening (${j.message || "no Companion polls this vault"})`,
+      };
+    }
+    return { ok: false, unavailable: true, reason: `POST /api/agent/gather-jobs -> HTTP 503` };
+  }
+  if (!res.ok) {
+    return { ok: false, unavailable: true, reason: `POST /api/agent/gather-jobs -> HTTP ${res.status}` };
+  }
+  let parsed;
+  try {
+    parsed = await res.json();
+  } catch (e) {
+    return { ok: false, unavailable: true, reason: `POST /api/agent/gather-jobs returned invalid JSON: ${e?.message || e}` };
+  }
+  const jobId = parsed?.jobId;
+  if (!jobId) {
+    return { ok: false, unavailable: true, reason: "POST /api/agent/gather-jobs returned no jobId" };
+  }
+  return {
+    ok: true,
+    jobId,
+    intentId: parsed.intentId,
+    // Console-reported relay presence for the job's vault ([[HS-L4LC]]); absent on older consoles.
+    ...(parsed.companion && typeof parsed.companion === "object" ? { companion: parsed.companion } : {}),
+  };
+}
+
+/**
+ * Poll until the Companion has uploaded the pages (status "complete") or the deadline passes.
+ * The deadline path is the EXPECTED failure when the relay prerequisite is unmet (loopback
+ * console / Companion not running) — hence the LAN hint on the resulting unavailable line.
+ *
+ * TRANSIENT TOLERANCE ([[HS-FBZ3-NWVU-DIAG-spexcrawl-companion-connectivity]]): once the job is
+ * FILED, the Companion executes it whether or not this process can reach the console at that
+ * instant. A single hiccup on the read leg — the dev console self-restarting on its memory
+ * threshold under concurrent /hs-res load, a 4s-timeout reachability gap, a 5xx — therefore says
+ * nothing about the job. Measured live 2026-09-12: two pollers both took `GET
+ * /api/agent/gather-jobs/<id> 401` across one Next dev restart and gave up, while the Companion
+ * uploaded both jobs' pages seconds later. So up to {@link MAX_TRANSIENT_POLL_FAILURES}
+ * CONSECUTIVE transient read failures are retried at the normal poll cadence instead of ending
+ * the run. Nothing about the failure contract changes: a 400 still exits 1 immediately, and a
+ * streak longer than the bound (or the deadline, whichever comes first) still returns
+ * `unavailable` carrying the last real reason.
+ */
+export async function pollJob(jobId, { baseUrl, token, httpFetch, timeoutMs = DEFAULT_TIMEOUT_MS, pollIntervalMs = POLL_INTERVAL_MS, sleepFn = (ms) => new Promise((r) => setTimeout(r, ms)), nowFn = Date.now } = {}) {
+  const started = nowFn();
+  /** Consecutive transient read failures; reset by any answered poll. */
+  let transientStreak = 0;
+  const deadlinePassed = () => nowFn() - started >= timeoutMs;
+
+  /** Retry a transient read at the poll cadence, or give up with its reason. */
+  const tolerate = async (reason) => {
+    transientStreak += 1;
+    if (transientStreak > MAX_TRANSIENT_POLL_FAILURES || deadlinePassed()) {
+      return {
+        ok: false,
+        unavailable: true,
+        reason: `${reason} (${transientStreak} consecutive poll failure(s))`,
+      };
+    }
+    await sleepFn(pollIntervalMs);
+    return null; // keep polling
+  };
+
+  for (;;) {
+    let res;
+    try {
+      res = await httpFetch(`${baseUrl}/api/agent/gather-jobs/${encodeURIComponent(jobId)}`, {
+        headers: authHeaders(token),
+      });
+    } catch (e) {
+      const giveUp = await tolerate(`console unreachable at ${baseUrl} while polling job ${jobId}: ${e?.message || e}`);
+      if (giveUp) return giveUp;
+      continue;
+    }
+    if (res.status === 400) {
+      return { ok: false, badRequest: true, message: await readErrorMessage(res) };
+    }
+    if (!res.ok) {
+      const giveUp = await tolerate(`GET /api/agent/gather-jobs/${jobId} -> HTTP ${res.status}`);
+      if (giveUp) return giveUp;
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = await res.json();
+    } catch (e) {
+      const giveUp = await tolerate(`GET /api/agent/gather-jobs/${jobId} returned invalid JSON: ${e?.message || e}`);
+      if (giveUp) return giveUp;
+      continue;
+    }
+    transientStreak = 0;
+    if (parsed?.status === "complete") {
+      return {
+        ok: true,
+        jobId,
+        pages: Array.isArray(parsed.pages) ? parsed.pages : [],
+        // A research job's engine-level search account (which engines were
+        // tried, which answered, why the others did not) — present only when
+        // the Companion uploaded one. This is what makes a zero-page search
+        // explainable instead of silent (HS-RIZN).
+        ...(parsed.search !== undefined && parsed.search !== null ? { search: parsed.search } : {}),
+      };
+    }
+    if (deadlinePassed()) {
+      return {
+        ok: false,
+        unavailable: true,
+        reason: `job ${jobId} still "${parsed?.status || "pending"}" after ${Math.round(timeoutMs / 1000)}s — the Companion never uploaded results (job queued under vault ${jobVault().vaultId || "(none)"} via ${jobVault().source}; a Companion only claims jobs for the ONE vault its vaultRepoPath names — set HEYSPEX_SPEXCRAWL_VAULT_ID to that vault if they differ)`,
+      };
+    }
+    await sleepFn(pollIntervalMs);
+  }
+}
+
+// ── subcommand implementations (testable independent of the CLI/exit layer) ────────────────────
+/**
+ * `search`: ONE round trip covers the skill's search+scrape steps together — the Companion
+ * searches locally and gathers the top results in the same pass, so the completed job's pages
+ * are the CLEANED MARKDOWN of the top hits, not bare links.
+ */
+export async function runSearch(question, { httpFetch = (...a) => globalThis.fetch(...a), baseUrl = resolveConsoleUrl(), token = resolveConsoleToken(), timeoutMs = DEFAULT_TIMEOUT_MS, pollIntervalMs = POLL_INTERVAL_MS, sleepFn, nowFn } = {}) {
+  const posted = await postJob({ kind: "research", question }, { baseUrl, token, httpFetch });
+  if (!posted.ok) return posted;
+  announceJobVault(posted);
+  return pollJob(posted.jobId, { baseUrl, token, httpFetch, timeoutMs, pollIntervalMs, sleepFn, nowFn });
+}
+
+/** `scrape`/`gather`: fetch+clean the given URL(s) through the Companion. */
+export async function runGather(urls, { httpFetch = (...a) => globalThis.fetch(...a), baseUrl = resolveConsoleUrl(), token = resolveConsoleToken(), timeoutMs = DEFAULT_TIMEOUT_MS, pollIntervalMs = POLL_INTERVAL_MS, sleepFn, nowFn } = {}) {
+  const posted = await postJob({ kind: "gather", urls }, { baseUrl, token, httpFetch });
+  if (!posted.ok) return posted;
+  announceJobVault(posted);
+  return pollJob(posted.jobId, { baseUrl, token, httpFetch, timeoutMs, pollIntervalMs, sleepFn, nowFn });
+}
+
+/** `crawl`: walk a whole site (sitemap + same-origin links) through the Companion.
+ * `maxPages`/`maxDepth` are sent only when they are positive numbers — omitted
+ * entirely otherwise (never `null`, never `0`), matching the console route's
+ * validation and `emitCrawlIntent`'s envelope contract. */
+export async function runCrawl(site, { httpFetch = (...a) => globalThis.fetch(...a), baseUrl = resolveConsoleUrl(), token = resolveConsoleToken(), timeoutMs = DEFAULT_CRAWL_TIMEOUT_MS, pollIntervalMs = POLL_INTERVAL_MS, sleepFn, nowFn, maxPages, maxDepth } = {}) {
+  const body = { kind: "crawl", site };
+  if (typeof maxPages === "number" && maxPages > 0) body.max_pages = maxPages;
+  if (typeof maxDepth === "number" && maxDepth > 0) body.max_depth = maxDepth;
+  const posted = await postJob(body, { baseUrl, token, httpFetch });
+  if (!posted.ok) return posted;
+  announceJobVault(posted);
+  return pollJob(posted.jobId, { baseUrl, token, httpFetch, timeoutMs, pollIntervalMs, sleepFn, nowFn });
+}
+
+// ── output formatting + the failure contract (exact lines, exact codes) ────────────────────────
+/** @returns {{code:number, line:string}} what the CLI prints and the exit code it sets. */
+export function renderFailure(result) {
+  if (result.badRequest) {
+    return { code: 1, line: String(result.message || "bad request") };
+  }
+  const hint = result.authRefused
+    ? AUTH_HINT
+    : result.companionNotListening
+      ? companionVaultHint(result.vaultId)
+      : LAN_HINT;
+  return { code: 3, line: `SPEXCRAWL_UNAVAILABLE: ${result.reason} — ${hint}` };
+}
+
+/**
+ * Render a research job's search report: the one-line summary, then one row per
+ * engine — answered engines with their result count, failed engines with the
+ * engine's own reason. Empty string when the job carried no report (gather /
+ * crawl jobs, or a Companion built before the report existed).
+ */
+export function formatSearchReportText(search) {
+  if (!search || typeof search !== "object") return "";
+  const lines = [`── search: ${search.summary || "(no summary)"}`];
+  for (const e of Array.isArray(search.engines) ? search.engines : []) {
+    if (e.answered) {
+      lines.push(`   ${e.engine}: answered, ${e.results} result(s)`);
+    } else {
+      const status = e.httpStatus ? ` [HTTP ${e.httpStatus}]` : "";
+      lines.push(`   ${e.engine}: no answer — ${e.reason || "(no reason given)"}${status}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+/**
+ * The machine-readable failure tag for a page that was not cleaned, e.g.
+ * " [rate_limited, retry after 44s]". The Companion stamps `failureCode`
+ * (today only `rate_limited`: a request budget, ours or GitHub's, said no) and
+ * `retryAfterSeconds` when the wait is knowable (HS-L4LC). Empty for a page
+ * without a code, so older results print exactly as before. In --json both
+ * fields ride on each page verbatim.
+ */
+export function formatFailureTag(p) {
+  if (!p || typeof p.failureCode !== "string" || !p.failureCode) return "";
+  const wait =
+    Number.isFinite(p.retryAfterSeconds) && p.retryAfterSeconds > 0 ? `, retry after ${p.retryAfterSeconds}s` : "";
+  return ` [${p.failureCode}${wait}]`;
+}
+
+export function formatPagesText(pages) {
+  if (!pages.length) return "(no pages)";
+  return pages
+    .map((p, i) => {
+      const state = p.state ? ` [${p.state}]` : "";
+      const lines = [`── ${i + 1}. ${p.url || "(no url)"}${state}`];
+      if (p.finalUrl && p.finalUrl !== p.url) lines.push(`   finalUrl: ${p.finalUrl}`);
+      // The crawl identity this page was actually fetched under (preset id +
+      // robots.txt group token) — absent on pages fetched before the
+      // Companion stamped provenance, or when no identity was ever chosen.
+      if (p.identity && p.identity.presetId) {
+        lines.push(`   identity: ${p.identity.presetId} (robots: ${p.identity.robotsToken})`);
+      }
+      if (p.markdown) lines.push("", p.markdown);
+      else if (p.reason) lines.push(`   (no content: ${p.reason})${formatFailureTag(p)}`);
+      return lines.join("\n");
+    })
+    .join("\n\n");
+}
+
+/** `crawl`'s text rows: {title, url, markdown} per walked page — the shape a
+ * whole-site walk is expected to print (vs. `formatPagesText`'s state/finalUrl
+ * detail, which suits a known-address gather/scrape). */
+export function formatCrawlPagesText(pages) {
+  if (!pages.length) return "(no pages)";
+  return pages
+    .map((p, i) => {
+      const lines = [`── ${i + 1}. ${p.title || "(no title)"}`, `   ${p.url || "(no url)"}`];
+      if (p.markdown) lines.push("", p.markdown);
+      else if (p.reason) lines.push(`   (no content: ${p.reason})${formatFailureTag(p)}`);
+      return lines.join("\n");
+    })
+    .join("\n\n");
+}
+
+// ── CLI entry ──────────────────────────────────────────────────────────────────────────────────
+const HELP = `usage: spexcrawl-cli.mjs search "<query>" [--json] [--timeout SECONDS]
+       spexcrawl-cli.mjs scrape "<url>" [<url>...] [--json] [--timeout SECONDS]
+       (gather is an alias of scrape)
+       spexcrawl-cli.mjs crawl "<site>" [--max-pages N] [--max-depth N] [--json] [--timeout SECONDS]
+
+Runs SpexCrawl jobs through the HeySpex Companion via the spex-console: the job
+is queued as a launch intent, the user's Companion app searches/fetches/cleans
+on the local machine, and this CLI polls until the pages are uploaded.
+
+search NOTE: results are the CLEANED PAGES (markdown) of the top hits — the
+Companion searches and gathers in one pass, so one search call covers a research
+skill's search+scrape steps together.
+
+crawl NOTE: walks a whole site (sitemap + same-origin links) instead of fetching
+known addresses — prints {title, url, markdown} rows. Its default --timeout is
+longer (${DEFAULT_CRAWL_TIMEOUT_MS / 1000}) than search/scrape/gather's ${DEFAULT_TIMEOUT_MS / 1000}, since a walk is
+bigger than a scrape. --max-pages/--max-depth are omitted from the request
+entirely when not given (never sent as 0).
+
+--timeout SECONDS   give up polling after this long (default ${DEFAULT_TIMEOUT_MS / 1000}; crawl default ${DEFAULT_CRAWL_TIMEOUT_MS / 1000})
+--json              print { "jobId", "pages": [{ url, state, markdown, ... }],
+                    "search"? } — search jobs also carry the engine-level report
+                    { query, enginesAttempted, enginesAnswered, resultsFound,
+                    resultsGathered, summary, engines: [...] }, so a zero-page
+                    search says which engines were tried, which answered, and
+                    why the others did not. A page that was not cleaned may
+                    carry failureCode ("rate_limited") + retryAfterSeconds, so
+                    a caller can back off without parsing "reason"
+--max-pages N        crawl only: cap the number of pages walked (positive integer)
+--max-depth N        crawl only: cap the link depth walked (positive integer)
+
+Environment:
+  HEYSPEX_CONSOLE_URL      console base URL (default ${DEFAULT_CONSOLE_URL})
+  HEYSPEX_COMPANION_TOKEN  bearer token override; default reads
+                           clerk.companion_token from ~/.heyspex/credentials.yaml
+                           (no auth header is sent when neither resolves)
+
+Prerequisite: ${LAN_HINT}.
+
+Exit codes: 0 ok · 1 bad request (server message printed) · 3 SPEXCRAWL_UNAVAILABLE
+            (console unreachable / non-2xx / poll timeout) · 64 usage error`;
+
+function printUsage() {
+  console.error('usage: spexcrawl-cli.mjs search "<query>" [--json] [--timeout SECONDS]');
+  console.error('       spexcrawl-cli.mjs scrape "<url>" [<url>...] [--json] [--timeout SECONDS]   (alias: gather)');
+  console.error('       spexcrawl-cli.mjs crawl "<site>" [--max-pages N] [--max-depth N] [--json] [--timeout SECONDS]');
+  console.error("       spexcrawl-cli.mjs --help");
+}
+
+/**
+ * Set the exit code and RETURN — never call process.exit() after doing network work.
+ * (Same rule as firecrawl-cli.mjs: forcing an exit with live HTTP handles aborts the process on
+ * Windows with a libuv assertion AFTER stdout flushed — perfect JSON alongside a non-zero exit,
+ * which poisons any caller that checks exit status. Letting the loop drain exits cleanly.)
+ */
+function finish(code) {
+  process.exitCode = code;
+}
+
+function exitOnResult(result, { json, formatText = formatPagesText }) {
+  if (!result.ok) {
+    const { code, line } = renderFailure(result);
+    console.log(line);
+    return finish(code);
+  }
+  if (json) {
+    console.log(JSON.stringify({
+      jobId: result.jobId,
+      pages: result.pages,
+      ...(result.search !== undefined ? { search: result.search } : {}),
+    }));
+    return finish(0);
+  }
+  // The search report leads: it is short, and on a zero-page result it IS the
+  // answer — which engines were tried, which answered, why the others did not.
+  const searchText = result.search !== undefined ? formatSearchReportText(result.search) : "";
+  console.log([searchText, formatText(result.pages)].filter(Boolean).join("\n\n"));
+  return finish(0);
+}
+
+/** Flags that take a value — used to strip both the flag and its value out of
+ * the positional-args list, and (via indexOf) to parse the value itself. */
+const VALUE_FLAGS = ["--timeout", "--max-pages", "--max-depth"];
+
+async function main() {
+  const argv = process.argv.slice(2);
+  if (argv.includes("--help") || argv.includes("-h")) {
+    console.log(HELP);
+    process.exit(0);
+    return;
+  }
+
+  const cmd = argv[0];
+  const json = argv.includes("--json");
+
+  if (cmd !== "search" && cmd !== "scrape" && cmd !== "gather" && cmd !== "crawl") {
+    printUsage();
+    process.exit(64);
+    return;
+  }
+
+  const timeoutIdx = argv.indexOf("--timeout");
+  const timeoutSec = timeoutIdx !== -1 ? Number(argv[timeoutIdx + 1]) : NaN;
+  const defaultTimeoutMs = cmd === "crawl" ? DEFAULT_CRAWL_TIMEOUT_MS : DEFAULT_TIMEOUT_MS;
+  const timeoutMs = Number.isFinite(timeoutSec) && timeoutSec > 0 ? timeoutSec * 1000 : defaultTimeoutMs;
+
+  // positional args after the verb, minus --json and every value-flag + its value
+  const targets = argv
+    .slice(1)
+    .filter((a, i, arr) => a !== "--json" && !VALUE_FLAGS.includes(a) && !VALUE_FLAGS.includes(arr[i - 1]));
+
+  if (!targets.length) {
+    console.error(`usage error: missing ${cmd === "search" ? "<query>" : cmd === "crawl" ? "<site>" : "<url>"} argument`);
+    printUsage();
+    process.exit(64);
+    return;
+  }
+
+  let maxPages;
+  let maxDepth;
+  if (cmd === "crawl") {
+    const maxPagesIdx = argv.indexOf("--max-pages");
+    if (maxPagesIdx !== -1) {
+      maxPages = Number(argv[maxPagesIdx + 1]);
+      if (!Number.isInteger(maxPages) || maxPages <= 0) {
+        console.error(`usage error: --max-pages must be a positive integer, got ${JSON.stringify(argv[maxPagesIdx + 1])}`);
+        printUsage();
+        process.exit(64);
+        return;
+      }
+    }
+    const maxDepthIdx = argv.indexOf("--max-depth");
+    if (maxDepthIdx !== -1) {
+      maxDepth = Number(argv[maxDepthIdx + 1]);
+      if (!Number.isInteger(maxDepth) || maxDepth <= 0) {
+        console.error(`usage error: --max-depth must be a positive integer, got ${JSON.stringify(argv[maxDepthIdx + 1])}`);
+        printUsage();
+        process.exit(64);
+        return;
+      }
+    }
+  }
+
+  try {
+    const result =
+      cmd === "search"
+        ? await runSearch(targets.join(" "), { timeoutMs })
+        : cmd === "crawl"
+          ? await runCrawl(targets[0], { timeoutMs, maxPages, maxDepth })
+          : await runGather(targets, { timeoutMs });
+    exitOnResult(result, { json, formatText: cmd === "crawl" ? formatCrawlPagesText : formatPagesText });
+  } catch (e) {
+    // Last-resort catch — every internal path above already returns a clean shape, but this
+    // guarantees the CLI NEVER surfaces a raw stack trace, per the shared failure contract.
+    console.log(`SPEXCRAWL_UNAVAILABLE: unexpected error — ${e?.message || String(e)} — ${LAN_HINT}`);
+    return finish(3);
+  }
+}
+
+const isMain = import.meta.url.endsWith("/" + (process.argv[1]?.split(/[\\/]/).pop() ?? ""));
+
+if (isMain) {
+  main();
+}