@fangorn-network/westmarch 0.0.1-dev

package/consume/mcp.js

@@ -0,0 +1,381 @@
+#!/usr/bin/env node
+// Every Fangorn app's WebMCP tools, behind one stdio MCP server.
+//
+//   fangorn-mcp [--cdp <url>] [--headed] [--from-block <n>]
+//   claude mcp add fangorn -- fangorn-mcp
+//   claude mcp add fangorn -e FANGORN_MCP_CDP=ws://browser:9222 -- fangorn-mcp
+//
+// WebMCP tools live on `document.modelContext` inside a loaded tab. There is no
+// endpoint, so an agent outside the browser cannot reach them, and one MCP
+// server per app would mean one registration per app. Instead this server is
+// the directory and the browser both:
+//
+//   list-apps      apps read off the chain (AppAgentChanged → verified cards)
+//   open-app       verify one app's card, open its page in a tab, and add its
+//                  tools to this server as `<app>__<tool>` (tools/list_changed)
+//   call-app-tool  the same call by name, for clients that ignore list_changed
+//
+// One browser, one tab per opened app: a WebMCP browser at --cdp, or else a
+// local Chrome with WebMCP on, headless unless --headed. Calls
+// are relayed to the tab's own `modelContext.executeTool`, so the tools, their
+// schemas and their answers are the page's, and queries still run in the tab.
+// A local Chrome's profile persists (~/.cache/westmarch-mcp), so a page's
+// downloads, such as an embedding model, are fetched once. A tab or browser that
+// dies is reopened on the next call.
+
+import { spawn } from "node:child_process";
+import { mkdirSync, readFileSync, existsSync, rmSync } from "node:fs";
+import { homedir } from "node:os";
+import { createInterface } from "node:readline";
+import { APP_EXTENSION, listApps } from "./apps.js";
+
+// The current AppRegistry's first event on Arbitrum Sepolia. A reader's
+// default, not deployment config: pass --from-block for another deployment.
+const DEFAULT_FROM_BLOCK = 311637349n;
+const log = (...a) => console.error("[fangorn-mcp]", ...a);   // stdout is the protocol
+
+let client;
+async function fangorn() {
+    if (!client) {
+        const [{ Fangorn, FangornConfig }, { generatePrivateKey }] = await Promise.all([
+            import("@fangorn-network/sdk"), import("viem/accounts")]);
+        client = Fangorn.create({ privateKey: generatePrivateKey(), config: FangornConfig });   // reads only
+    }
+    return client;
+}
+
+/** A card URL → the verified card and the page to open. A plain A2A card (no
+ *  Fangorn extension) opens unverified: the caller asked for it by URL. */
+async function resolveCard(cardUrl) {
+    const card = await (await fetch(cardUrl)).json();
+    const fangornApp = card.capabilities?.extensions?.some((e) => e.uri === APP_EXTENSION);
+    if (fangornApp) await (await fangorn()).discoverApp(cardUrl);
+    const page = new URL(card.url);
+    if (page.protocol !== "https:" && page.protocol !== "http:") throw new Error(`card.url is not http(s): ${card.url}`);
+    return { card, page: page.toString(), verified: fangornApp };
+}
+
+// ── the browser ─────────────────────────────────────────────────────────────
+// Anything that speaks the Chrome DevTools Protocol and has WebMCP: a headless
+// Chromium service, a hosted browser, or the local Chrome. `--cdp` (or
+// FANGORN_MCP_CDP) names it as http(s)://host:port or a ws(s):// browser
+// endpoint. Without it, a local Chrome is reused if one is already serving
+// this profile (another session's), else launched.
+//
+// Everything goes over the one browser-level WebSocket (Target.* plus flat
+// sessions). That is the endpoint every CDP host exposes, where the per-tab
+// /json HTTP routes are not.
+
+const PROFILE = `${homedir()}/.cache/westmarch-mcp`;
+
+/** An http(s) DevTools address → its browser WebSocket, reached via the host we were given. */
+async function browserWs(http) {
+    const v = await (await fetch(new URL("/json/version", http), { signal: AbortSignal.timeout(3000) })).json();
+    const ws = new URL(v.webSocketDebuggerUrl), base = new URL(http);
+    ws.host = base.host;                                  // the service reports its own view of itself
+    if (base.protocol === "https:") ws.protocol = "wss:";
+    return ws.toString();
+}
+
+async function localChrome({ headed }) {
+    mkdirSync(PROFILE, { recursive: true });
+    const portFile = `${PROFILE}/DevToolsActivePort`;
+    if (existsSync(portFile)) {
+        const ws = await browserWs(`http://127.0.0.1:${readFileSync(portFile, "utf-8").split("\n")[0]}`).catch(() => null);
+        if (ws) { log("reusing the Chrome already serving this profile"); return { ws }; }
+        rmSync(portFile, { force: true });                // stale: that Chrome is gone
+    }
+    const bin = process.env.CHROME ?? "google-chrome";
+    const proc = spawn(bin, [
+        ...(headed ? [] : ["--headless=new"]),
+        "--enable-features=WebMCP", "--remote-debugging-port=0", "--no-first-run",
+        "--no-default-browser-check", `--user-data-dir=${PROFILE}`, "about:blank",
+    ], { stdio: "ignore" });
+    const failed = new Promise((_, j) => proc.on("error", (e) => j(new Error(
+        `no browser: cannot launch ${bin} (${e.code}). Point --cdp / FANGORN_MCP_CDP at a WebMCP browser, or set CHROME.`))));
+    for (let i = 0; i < 100; i++) {
+        await Promise.race([new Promise((r) => setTimeout(r, 100)), failed]);
+        if (existsSync(portFile)) return { ws: await browserWs(`http://127.0.0.1:${readFileSync(portFile, "utf-8").split("\n")[0]}`), proc };
+    }
+    proc.kill();
+    throw new Error(`${bin} did not start`);
+}
+
+let conn;
+/** The browser connection, opened once and reopened after it drops. */
+function browser(opts) {
+    return conn ??= (async () => {
+        const { cdp } = opts;
+        const { ws: url, proc } = !cdp ? await localChrome(opts)
+            : /^wss?:/.test(cdp) ? { ws: cdp } : { ws: await browserWs(cdp) };
+        const ws = new WebSocket(url);
+        await new Promise((r, j) => { ws.onopen = r; ws.onerror = () => j(new Error(`cannot connect to the browser at ${url}`)); });
+        let id = 0; const pending = new Map(), dead = new Set();
+        ws.onmessage = (m) => {
+            const d = JSON.parse(m.data);
+            if (d.method === "Target.detachedFromTarget") dead.add(d.params.sessionId);
+            if (d.id !== undefined) { pending.get(d.id)?.(d); pending.delete(d.id); }
+        };
+        ws.onclose = () => { conn = null; for (const r of pending.values()) r({ error: { message: "browser connection closed" } }); };
+        const send = (method, params = {}, sessionId) => new Promise((res, rej) => {
+            if (ws.readyState !== WebSocket.OPEN) return rej(new Error("browser connection closed"));
+            pending.set(++id, (d) => (d.error ? rej(new Error(d.error.message)) : res(d.result)));
+            ws.send(JSON.stringify({ id, method, params, ...(sessionId && { sessionId }) }));
+        });
+        const tabs = new Set();
+        // Close what we opened. A shared or remote browser outlives this process.
+        const bye = async () => {
+            await Promise.race([Promise.all([...tabs].map((t) => send("Target.closeTarget", { targetId: t }).catch(() => {}))),
+                                new Promise((r) => setTimeout(r, 500))]);
+            // A local Chrome may be serving another session: leave it running
+            // while anyone else has a tab open; the last one out closes it.
+            const others = await Promise.race([send("Target.getTargets").then((r) => r.targetInfos.filter((t) => t.type === "page" && t.url !== "about:blank").length).catch(() => 0),
+                                               new Promise((r) => setTimeout(() => r(0), 500))]);
+            // Never a --cdp browser: that one is somebody's service.
+            if (!cdp && !others) await Promise.race([send("Browser.close").catch(() => {}), new Promise((r) => setTimeout(r, 500))]);
+            proc?.kill();
+            process.exit(0);
+        };
+        process.once("SIGINT", bye); process.once("SIGTERM", bye); process.stdin.once("end", bye);
+        log(`browser: ${cdp ?? (proc ? "launched local Chrome" : "local Chrome")}`);
+        return { send, dead, tabs };
+    })().catch((e) => { conn = null; throw e; });
+}
+
+/** A new tab on `page`; returns `ev(expr)`, which evaluates in it. */
+async function openTab(page, opts) {
+    const { send, dead, tabs } = await browser(opts);
+    const { targetId } = await send("Target.createTarget", { url: page });
+    tabs.add(targetId);
+    const { sessionId } = await send("Target.attachToTarget", { targetId, flatten: true });
+    return async (expression) => {
+        if (dead.has(sessionId)) throw new Error(`tab closed: ${page}`);
+        const r = await send("Runtime.evaluate", { expression, awaitPromise: true, returnByValue: true }, sessionId)
+            .catch((e) => { throw new Error(`tab closed: ${page} (${e.message})`); });
+        if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description ?? r.exceptionDetails.text);
+        return r.result.value;
+    };
+}
+
+/** The tab's tools in MCP's shape. Waits for the page to register them. */
+async function tabTools(ev) {
+    let hasWebMCP = false;
+    for (let i = 0; i < 150; i++) {
+        const got = await ev(`(async () => {
+            const mc = document.modelContext;
+            if (!mc?.getTools) return { webmcp: false, loaded: location.protocol.startsWith("http") && document.readyState === "complete" };
+            return { webmcp: true, tools: (await mc.getTools()).map((t) => ({ name: t.name, description: t.description, inputSchema: t.inputSchema })) };
+        })()`).catch(() => null);
+        hasWebMCP ||= !!got?.webmcp;
+        // modelContext is there from the first script or never; no need to wait out the page.
+        if (got && !got.webmcp && got.loaded) break;
+        if (got?.tools?.length) return got.tools.map((t) => ({ ...t, inputSchema: t.inputSchema ? JSON.parse(t.inputSchema) : { type: "object" } }));
+        await new Promise((r) => setTimeout(r, 200));
+    }
+    throw new Error(hasWebMCP
+        ? "the page registered no WebMCP tools in 30s (is card.url the page that registers them?)"
+        : "this browser has no WebMCP (document.modelContext.getTools): it needs Chrome 150+ with --enable-features=WebMCP, or a Chromium built with it");
+}
+
+/** One call, relayed. WebMCP takes the arguments as a JSON string and returns one. */
+async function tabCall(ev, name, args) {
+    const out = await ev(`document.modelContext.getTools().then((ts) => {
+        const t = ts.find((t) => t.name === ${JSON.stringify(name)});
+        if (!t) throw new Error("no tool " + ${JSON.stringify(name)});
+        return document.modelContext.executeTool(t, ${JSON.stringify(JSON.stringify(args ?? {}))});
+    })`);
+    try { const r = JSON.parse(out); if (Array.isArray(r?.content)) return r; } catch {}
+    return { content: [{ type: "text", text: String(out) }] };
+}
+
+/** JSON-RPC over newline-delimited stdio. Returns `notify` for server → client messages. */
+export function serve(handlers, { input = process.stdin, output = process.stdout } = {}) {
+    const send = (m) => output.write(JSON.stringify({ jsonrpc: "2.0", ...m }) + "\n");
+    createInterface({ input }).on("line", async (line) => {
+        let msg; try { msg = JSON.parse(line); } catch { return; }
+        if (msg.id === undefined) return;   // notifications
+        const h = handlers[msg.method];
+        if (!h) return send({ id: msg.id, error: { code: -32601, message: `unknown method ${msg.method}` } });
+        try { send({ id: msg.id, result: await h(msg.params ?? {}) }); }
+        catch (e) { send({ id: msg.id, error: { code: -32000, message: e?.message ?? String(e) } }); }
+    });
+    return (method, params = {}) => send({ method, params });
+}
+
+const text = (o) => ({ content: [{ type: "text", text: typeof o === "string" ? o : JSON.stringify(o, null, 1) }] });
+// MCP tool names: [a-zA-Z0-9_-]{1,64}. `__` separates app from tool.
+const slug = (name) => String(name).toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-|-$/g, "").slice(0, 24) || "app";
+
+const META = [
+    { name: "list-apps",
+      description: "List the Fangorn apps registered on chain whose agent cards verify: name, description, card URL, and the WebMCP tools each page offers. Filter with `query`. Call open-app on one to use its tools.",
+      inputSchema: { type: "object", properties: { query: { type: "string", description: "Case-insensitive match on name or description" } } } },
+    { name: "open-app",
+      description: "Verify an app's card against the chain, open its page in a browser tab, and add its tools to this server as `<app>__<tool>`. Returns the tools and their input schemas. `app` is a name from list-apps, an appId, or a card URL.",
+      inputSchema: { type: "object", properties: { app: { type: "string" } }, required: ["app"] } },
+    { name: "call-app-tool",
+      description: "Call one tool of an opened app. Same as calling `<app>__<tool>` directly.",
+      inputSchema: { type: "object", properties: { app: { type: "string" }, tool: { type: "string" }, arguments: { type: "object" } }, required: ["app", "tool"] } },
+];
+
+/**
+ * The server. `deps` is the outside world, so the self-check can fake it:
+ *   findApps()          → { apps: [{ name, desc, card, appId }], rejected }
+ *   resolveCard(url)    → { card, page, verified }
+ *   openTab(page)       → ev
+ *   tabTools(ev), tabCall(ev, name, args)
+ *   notify(method)      set after serve()
+ */
+export function handlers(deps) {
+    const open = new Map();   // slug → { name, card, verified, ev, tools }
+
+    const findOpen = (app) => open.get(slug(app)) ?? [...open.values()].find((o) => o.card === app || o.appId === app);
+
+    async function openApp(app) {
+        const hit = findOpen(app);
+        if (hit) return hit;
+        let cardUrl = app, appId;
+        if (!/^https?:\/\//.test(app)) {
+            const { apps } = await deps.findApps();
+            const a = apps.find((x) => x.appId === app || slug(x.name) === slug(app));
+            if (!a) throw new Error(`no verified app named ${JSON.stringify(app)}; see list-apps`);
+            ({ card: cardUrl, appId } = a);
+        }
+        const { card, page, verified } = await deps.resolveCard(cardUrl);
+        let s = slug(card.name ?? app);
+        while (open.has(s)) s += "-";
+        const ev = await deps.openTab(page);
+        const entry = { slug: s, name: card.name, card: cardUrl, appId, verified, page, ev, tools: await deps.tabTools(ev) };
+        open.set(s, entry);
+        deps.notify?.("notifications/tools/list_changed");
+        return entry;
+    }
+
+    async function call(app, tool, args) {
+        const o = findOpen(app);
+        if (!o) throw new Error(`${app} is not open; call open-app first`);
+        try { return await deps.tabCall(o.ev, tool, args); }
+        catch (e) {
+            if (!/tab closed/.test(e.message)) throw e;
+            // The tab or the whole browser went away. Reopen once; the tools keep their names.
+            o.ev = await deps.openTab(o.page);
+            await deps.tabTools(o.ev);
+            return deps.tabCall(o.ev, tool, args);
+        }
+    }
+
+    const run = {
+        "list-apps": async ({ query }) => {
+            const { apps, rejected } = await deps.findApps();
+            const q = String(query ?? "").toLowerCase();
+            return text({
+                apps: apps.filter((a) => !q || `${a.name} ${a.desc}`.toLowerCase().includes(q)).map((a) => ({
+                    app: slug(a.name), name: a.name, description: a.desc, card: a.card, appId: a.appId,
+                    tools: a.tools, open: open.has(slug(a.name)),
+                })),
+                unverified: rejected.length,
+            });
+        },
+        "open-app": async ({ app }) => {
+            const o = await openApp(app);
+            return text({ app: o.slug, name: o.name, page: o.page, verified: o.verified,
+                          call_as: `${o.slug}__<tool>, or call-app-tool`, tools: o.tools });
+        },
+        "call-app-tool": async ({ app, tool, arguments: args }) => call(app, tool, args),
+    };
+
+    return {
+        initialize: async (p) => ({
+            protocolVersion: p.protocolVersion ?? "2025-06-18",
+            capabilities: { tools: { listChanged: true } },
+            serverInfo: { name: "fangorn-mcp", version: "0.1.0" },
+            instructions: "Fangorn apps are static sites whose tools run in the browser. list-apps to find one, open-app to load it, then call its tools as <app>__<tool>.",
+        }),
+        ping: async () => ({}),
+        "tools/list": async () => ({ tools: [
+            ...META,
+            ...[...open.values()].flatMap((o) => o.tools.map((t) => ({ ...t, name: `${o.slug}__${t.name}`, description: `[${o.name}] ${t.description ?? ""}` }))),
+        ] }),
+        "tools/call": async ({ name, arguments: args = {} }) => {
+            try {
+                if (run[name]) return await run[name](args);
+                const i = name.indexOf("__");
+                if (i > 0) return await call(name.slice(0, i), name.slice(i + 2), args);
+                throw new Error(`no tool ${name}`);
+            } catch (e) { return { isError: true, content: [{ type: "text", text: e?.message ?? String(e) }] }; }
+        },
+    };
+}
+
+// ── main / self-check ───────────────────────────────────────────────────────
+// A command, not a module: installed as a bin, argv[1] is the symlink, so there is no is-main check.
+if (process.argv[2] === "--selfcheck") {
+    const { PassThrough } = await import("node:stream");
+    const input = new PassThrough(), output = new PassThrough();
+    const replies = [];
+    createInterface({ input: output }).on("line", (l) => replies.push(JSON.parse(l)));
+    let opened = 0;
+    const deps = {
+        findApps: async () => ({ apps: [{ name: "Kings Foil", desc: "trials", card: "https://k.test/card", appId: "0xk", tools: ["greet"] }], rejected: [{}] }),
+        resolveCard: async () => ({ card: { name: "Kings Foil" }, page: "https://k.test/p", verified: true }),
+        openTab: async () => (opened++ ? "ev" : "dead"),   // the first tab dies
+        tabTools: async () => [{ name: "greet", description: "Hi.", inputSchema: { type: "object" } }],
+        tabCall: async (ev, name, args) => (name === "boom" ? Promise.reject(new Error("no tool boom"))
+            : ev === "dead" ? Promise.reject(new Error("tab closed: x")) : text({ ev, name, args })),
+    };
+    deps.notify = serve(handlers(deps), { input, output });
+    const rpc = (id, method, params) => input.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
+    const wait = () => new Promise((r) => setTimeout(r, 20));
+    const call = (id, name, args) => rpc(id, "tools/call", { name, arguments: args });
+    const res = (id) => replies.find((r) => r.id === id);
+    const body = (id) => JSON.parse(res(id).result.content[0].text);
+    const assert = (c, m) => { if (!c) throw new Error(m); };
+
+    rpc(1, "initialize", {}); rpc(2, "tools/list"); call(3, "list-apps", { query: "TRIAL" }); call(4, "kings-foil__greet"); await wait();
+    assert(res(1).result.capabilities.tools.listChanged, "advertises list_changed");
+    assert(res(2).result.tools.length === 3, "only the meta tools before anything is open");
+    assert(body(3).apps[0].app === "kings-foil" && body(3).unverified === 1, "list-apps filters and slugs");
+    assert(res(4).result.isError && /not open/.test(res(4).result.content[0].text), "an unopened app's tool is an error");
+
+    call(5, "open-app", { app: "Kings Foil" }); await wait();
+    call(6, "open-app", { app: "https://k.test/card" }); await wait();
+    assert(body(5).tools[0].name === "greet" && opened === 1, "open once, by name or by card URL");
+    call(50, "kings-foil__greet", { who: "b" }); await wait();
+    assert(body(50).ev === "ev" && opened === 2, "a dead tab is reopened and the call retried");
+    assert(replies.some((r) => r.method === "notifications/tools/list_changed"), "opening notifies");
+    rpc(7, "tools/list"); call(8, "kings-foil__greet", { who: "a" }); call(9, "call-app-tool", { app: "kings-foil", tool: "greet" });
+    call(10, "call-app-tool", { app: "kings-foil", tool: "boom" }); call(11, "open-app", { app: "nope" }); rpc(12, "nope"); await wait();
+    assert(res(7).result.tools.some((t) => t.name === "kings-foil__greet"), "opened tools are listed under the app");
+    assert(body(8).args.who === "a" && body(9).name === "greet", "both call paths reach the tab");
+    assert(res(10).result.isError && res(11).result.isError, "failures are tool errors");
+    assert(res(12).error.code === -32601, "unknown method");
+    console.log("mcp.js self-check ok — one server, apps opened on demand, their tools listed as <app>__<tool>");
+    process.exit(0);
+}
+
+const argv = process.argv.slice(2);
+const fb = argv.indexOf("--from-block");
+const fromBlock = fb >= 0 ? BigInt(argv[fb + 1]) : DEFAULT_FROM_BLOCK;
+const headed = argv.includes("--headed");
+const ci = argv.indexOf("--cdp");
+const cdp = ci >= 0 ? argv[ci + 1] : process.env.FANGORN_MCP_CDP || undefined;
+let found, foundAt = 0;
+const deps = {
+    // ponytail: cached for a minute. The scan is a few RPC calls plus a fetch per card.
+    findApps: async () => {
+        if (!found || Date.now() - foundAt > 60_000) {
+            const { apps, rejected } = await listApps(await fangorn(), { fromBlock });
+            found = { rejected, apps: await Promise.all(apps.map(async (a) => {
+                const card = await (await fetch(a.card)).json().catch(() => ({}));
+                return { ...a, tools: card.skills?.filter((s) => s.tags?.includes("webmcp")).map((s) => s.id) ?? [] };
+            })) };
+            foundAt = Date.now();
+        }
+        return found;
+    },
+    resolveCard,
+    openTab: (page) => openTab(page, { headed, cdp }),
+    tabTools, tabCall,
+};
+deps.notify = serve(handlers(deps));
+log(`ready: apps from block ${fromBlock}; browser ${cdp ?? "local Chrome"}, connected on the first open-app`);

package/consume/rank.js

@@ -0,0 +1,218 @@
+// Ranking a row against a query. Two modes, chosen per row, so search degrades
+// rather than breaks: cosine for rows that carry a vector, word-boundary lexical
+// for rows that don't (or when the embedder won't load).
+//
+// Nothing here knows what a row IS. A row is `{ text, vector, norm }` — whatever
+// the app's shard put in it. That is why this file is in the library and the
+// grouping rules that turn hits into results (series, episodes, cues) are not:
+// those are the app's semantics, this is arithmetic.
+
+/** Cosine of a row's vector against a query vector. `row.norm` is precomputed at
+ *  parse time (see shard.js `toRow`), `qn` by the caller once per query — the
+ *  scan runs per row, so neither belongs inside it.
+ *
+ *  The shorter of the two lengths wins: vectors are matryoshka prefixes, so a
+ *  256-d query and a 768-d row still line up on their leading components. */
+export const cosine = (row, qv, qn) => {
+    let d = 0;
+    for (let i = 0; i < qv.length && i < row.vector.length; i++) d += row.vector[i] * qv[i];
+    return d / (row.norm * qn);
+};
+
+/** L2 norm, for the query side. */
+export const norm = (v) => { let n = 0; for (const x of v) n += x * x; return Math.sqrt(n) || 1; };
+
+/**
+ * How far above the corpus mean a cosine has to sit to count as a hit.
+ *
+ * Raw nomic-256 cosine is offset per query, not per corpus: measured on the
+ * fangorn.tv catalog, EVERY row scores 0.38–0.65 against "panama", so `score > 0`
+ * admitted all twelve and the one right answer was buried in eleven near-misses
+ * that looked, by their printed scores, just as confident. Same measurement in z:
+ * the right row lands 2.4σ above that query's own mean, the plausible ones ~1.3σ,
+ * the rest below.
+ *
+ * Lexical scores keep a meaningful zero (a term either matched or it didn't), so
+ * this applies to the semantic half only.
+ */
+export const Z_FLOOR = 1;
+
+/** score ≥ mean + Z_FLOOR·σ, or -Infinity when there's no distribution to speak
+ *  of (too few rows, or every row scored the same). */
+export function zFloor(scores, z = Z_FLOOR) {
+    if (scores.length < 3) return -Infinity;
+    const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const sd = Math.sqrt(scores.reduce((s, x) => s + (x - mean) ** 2, 0) / scores.length);
+    return sd > 1e-6 ? mean + z * sd : -Infinity;
+}
+
+/**
+ * Maximal Marginal Relevance: pick the next-best row that is not already said.
+ *
+ * Ranking by score alone returns the same thing several times, because in an
+ * embedding space near-duplicates sit next to each other by construction. Ask
+ * for eight neighbours of Frostpunk and three of them are consecutive episodes
+ * of the same cartoon; build a taste from one film and five of the six
+ * recommendations are the same series. The list looks confident and says one
+ * thing.
+ *
+ * So: greedily take the row maximising `λ·score − (1−λ)·(closest already taken)`.
+ * λ=1 is the old behaviour. Lower trades a little relevance for a list that
+ * covers more ground, which for a recommendation IS the relevance.
+ *
+ * `ranked` is `[{ row, s }]` sorted desc, rows carrying `.vector` and `.norm`.
+ * Only the top `pool` are considered — beyond that the scores are noise and the
+ * arithmetic is O(limit · pool · dim), about 128k multiplies at the defaults.
+ */
+export function diversify(ranked, { limit = 10, lambda = 0.7, key = "s" } = {}) {
+    if (lambda >= 1 || ranked.length <= 1) return ranked.slice(0, limit);
+    const pool = ranked.slice(0, Math.max(limit * 6, 30));
+    const out = [];
+    while (out.length < limit && pool.length) {
+        let at = 0, best = -Infinity;
+        for (let i = 0; i < pool.length; i++) {
+            let sim = 0;
+            // `cosine(a, b.vector, b.norm)` is dot/(a.norm·b.norm) — the same
+            // arithmetic as a query, with the other row standing in for one.
+            for (const p of out) {
+                if (!pool[i].row?.vector || !p.row?.vector) continue;
+                const c = cosine(pool[i].row, p.row.vector, p.row.norm);
+                if (c > sim) sim = c;
+            }
+            const val = lambda * pool[i][key] - (1 - lambda) * sim;
+            if (val > best) { best = val; at = i; }
+        }
+        out.push(pool.splice(at, 1)[0]);
+    }
+    return out;
+}
+
+const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const wordRe = (s) => new RegExp(`\\b${esc(s)}\\b`);
+
+/**
+ * Word-boundary term scoring, normalized to 0..1 so it merges with cosine scores.
+ *
+ * This used to be a bare `text.includes(q)`, which meant a search for "cat"
+ * ranked "intoxi(cat)ed" and "edu(cat)ion" — every short query drowned in
+ * substring noise. Terms must now match whole words; the full phrase and an
+ * early position are bonuses on top.
+ *
+ * `q` is expected lowercased — the caller lowercases once per query, not per row.
+ */
+export function lexScore(row, q) {
+    const text = String(row.text ?? "").toLowerCase();
+    if (!text) return 0;
+    const terms = q.split(/\s+/).filter(Boolean);
+    if (!terms.length) return 0;
+
+    let hits = 0;
+    for (const t of terms) if (wordRe(t).test(text)) hits++;
+    if (!hits) return 0;
+
+    const at = text.indexOf(q);
+    return Math.min(1,
+        0.5 * (hits / terms.length)                       // how much of the query is present
+        + (wordRe(q).test(text) ? 0.35 : 0)               // the whole phrase, intact
+        + (at >= 0 ? 0.15 * (1 - at / text.length) : 0)); // earlier beats later
+}
+
+/** Of a list of timestamped passages, the one a hit should quote and seek to:
+ *  the line that best matches the query, or the first when nothing matches
+ *  lexically (a semantic hit on a whole transcript has no single line to point
+ *  at). Generic over `{ text }` — cues, chapters, log lines, commit messages. */
+export function bestPassage(passages, ql) {
+    let best = passages[0], score = 0;
+    for (const c of passages) {
+        const s = lexScore({ text: c.text }, ql);
+        if (s > score) { score = s; best = c; }
+    }
+    return best;
+}
+
+/**
+ * Rank catalog entries against a lookahead vector, WITHOUT downloading them.
+ *
+ * The point is choosing what to pull before the bytes move: a view that watches
+ * something big can't be downloaded whole, and the only thing available to decide
+ * with is the coverage centroids the bake writes into catalog.json.
+ *
+ * A domain baked before coverage existed gets `affinity: null` and sorts last —
+ * NOT 0, which would claim it was measured and found unrelated.
+ */
+export function rankDomains(domains = [], q) {
+    const qn = norm(q);
+    return domains
+        .map((d) => {
+            const c = d.coverage;
+            if (!c?.vectors?.length) return { ...d, affinity: null };
+            let best = -1;
+            for (const v of c.vectors) {
+                let dot = 0, vn = 0;
+                for (let i = 0; i < v.length && i < q.length; i++) { dot += v[i] * q[i]; vn += v[i] * v[i]; }
+                const score = dot / ((Math.sqrt(vn) || 1) * qn);
+                if (score > best) best = score;
+            }
+            return { ...d, affinity: best };
+        })
+        .sort((a, b) => (b.affinity ?? -Infinity) - (a.affinity ?? -Infinity));
+}
+
+// ── self-check: `node consume/rank.js` ──────────────────────────────────────
+if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
+    // Word boundaries: the bug this replaced ranked "intoxicated" for "cat".
+    if (lexScore({ text: "intoxicated education" }, "cat") !== 0) throw new Error("substring matched — word boundaries lost");
+    if (lexScore({ text: "a cat sat" }, "cat") <= 0) throw new Error("whole word must match");
+    // The intact phrase beats the same words scattered.
+    const intact = lexScore({ text: "cold war submarines" }, "cold war");
+    const apart = lexScore({ text: "cold nights, the war ended" }, "cold war");
+    if (!(intact > apart)) throw new Error(`phrase bonus lost: ${intact} vs ${apart}`);
+    // Regex metacharacters in a query are data, not syntax.
+    if (lexScore({ text: "what is c++" }, "c++") === undefined) throw new Error("must not throw on regex metachars");
+
+    // Cosine truncates to the shorter vector — matryoshka prefixes line up.
+    const row = { vector: [1, 0, 0, 0], norm: 1 };
+    if (Math.abs(cosine(row, [1, 0], 1) - 1) > 1e-9) throw new Error("cosine must truncate to the query");
+
+    // The floor is relative to the query's own distribution, which is the whole
+    // point — a corpus where everything scores 0.5 admits nothing.
+    if (zFloor([0.5, 0.5, 0.5, 0.5]) !== -Infinity) throw new Error("no spread must admit everything, not nothing");
+    if (zFloor([0.4, 0.4]) !== -Infinity) throw new Error("too few rows to have a distribution");
+    const f = zFloor([0.4, 0.41, 0.42, 0.9]);
+    if (!(0.9 >= f && 0.42 < f)) throw new Error(`floor did not separate the outlier: ${f}`);
+
+    // bestPassage quotes the matching line, and falls back to the first.
+    const cues = [{ text: "opening titles" }, { text: "the submarine dives" }];
+    if (bestPassage(cues, "submarine").text !== "the submarine dives") throw new Error("wrong passage quoted");
+    if (bestPassage(cues, "nothing here").text !== "opening titles") throw new Error("must fall back to the first passage");
+
+    // MMR: the same thing five times is one answer, not five.
+    {
+        const v = (x, y) => ({ vector: [x, y], norm: Math.sqrt(x * x + y * y) || 1 });
+        const ranked = [
+            { row: { id: "ep1", ...v(1, 0) }, s: 0.90 },
+            { row: { id: "ep2", ...v(1, 0.01) }, s: 0.89 },
+            { row: { id: "ep3", ...v(1, 0.02) }, s: 0.88 },
+            { row: { id: "other", ...v(0, 1) }, s: 0.70 },
+        ];
+        const plain = diversify(ranked, { limit: 2, lambda: 1 }).map((x) => x.row.id);
+        if (plain.join() !== "ep1,ep2") throw new Error(`lambda 1 must be the old behaviour: ${plain}`);
+        const mixed = diversify(ranked, { limit: 2, lambda: 0.7 }).map((x) => x.row.id);
+        if (mixed.join() !== "ep1,other") throw new Error(`MMR must not return the same episode twice: ${mixed}`);
+        // The best row is still first — diversity reorders the tail, never the head.
+        if (diversify(ranked, { limit: 4, lambda: 0.5 })[0].row.id !== "ep1") throw new Error("the top hit must survive");
+        // A row with no vector cannot be compared and must still be returnable.
+        if (diversify([{ row: { id: "a" }, s: 1 }], { limit: 3 }).length !== 1) throw new Error("vectorless rows must pass through");
+    }
+
+    // Uncovered domains sort last and say so — null, never 0.
+    const ranked = rankDomains([
+        { name: "old", coverage: null },
+        { name: "far", coverage: { vectors: [[0, 1]] } },
+        { name: "near", coverage: { vectors: [[1, 0]] } },
+    ], [1, 0]);
+    if (ranked.map((d) => d.name).join(",") !== "near,far,old") throw new Error(`domains misranked: ${ranked.map((d) => d.name)}`);
+    if (ranked[2].affinity !== null) throw new Error("an unmeasured domain must be null, not 0");
+
+    console.log("rank.js self-check ok — word boundaries, matryoshka cosine, z-floor, MMR drops near-duplicates without moving the top hit, passage pick, domain affinity");
+}