@fangorn-network/westmarch 0.0.4-dev → 0.0.6-dev

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fangorn-network/westmarch",
-  "version": "0.0.4-dev",
+  "version": "0.0.6-dev",
   "description": "Publisher and consumer surfaces for a Fangorn app: shape a graph, publish it, read it back as a searchable, agent-drivable catalog",
   "license": "MIT",
   "type": "module",
@@ -12,6 +12,7 @@
     "./embed": "./consume/embed.js",
     "./rank": "./consume/rank.js",
     "./apps": "./consume/apps.js",
+    "./explore": "./consume/explore.js",
     "./agent-card": "./publish/agent-card.js",
     "./view": "./publish/view.js",
     "./roles": "./consume/roles.js",
@@ -36,7 +37,7 @@
   "scripts": {
     "build": "tsc -p .",
     "prepare": "tsc -p .",
-    "test": "node publish/terms.js && node publish/reactions.js && node publish/cohort.js && node publish/vault.js && node publish/graph.js && node publish/envelope.js && node publish/demand.js && node publish/settle.js && node publish/manifest.js && node publish/agent-card.js && node publish/view.js && node publish/ship.js --selfcheck && node publish/eval.ts --selfcheck && node publish/cli.js --selfcheck && node publish/enrich.js && node consume/embed.js && node consume/rank.js && node consume/apps.js && node consume/mcp.js --selfcheck && node consume/roles.js && node consume/shard.js && node consume/directory.js && node consume/taste.js && node consume/steam.js && node consume/taste-doc.js && node consume/corpora.js && node consume/ui.js && node consume/lint.js && node consume/tools.js && node consume/x402.js && node site/feed.js && node site/store.js",
+    "test": "node publish/terms.js && node publish/reactions.js && node publish/cohort.js && node publish/vault.js && node publish/graph.js && node publish/envelope.js && node publish/demand.js && node publish/settle.js && node publish/manifest.js && node publish/agent-card.js && node publish/view.js && node publish/ship.js --selfcheck && node publish/eval.ts --selfcheck && node publish/cli.js --selfcheck && node publish/enrich.js && node consume/embed.js && node consume/rank.js && node consume/apps.js && node consume/explore.js && node consume/mcp.js --selfcheck && node consume/roles.js && node consume/shard.js && node consume/directory.js && node consume/taste.js && node consume/steam.js && node consume/taste-doc.js && node consume/corpora.js && node consume/ui.js && node consume/lint.js && node consume/tools.js && node consume/x402.js && node site/feed.js && node site/store.js",
     "lint": "node consume/lint.js"
   },
   "peerDependencies": {
@@ -62,7 +63,8 @@
     "fangorn-mcp": "./consume/mcp.js",
     "westmarch-view": "./publish/view-cli.js",
     "westmarch-ship": "./publish/ship.js",
-    "westmarch-eval": "./dist/ts/eval.js"
+    "westmarch-eval": "./dist/ts/eval.js",
+    "fangorn-fx": "./plugin/scripts/fx.mjs"
   },
   "files": [
     "consume",
@@ -73,7 +75,8 @@
     "site/feed.js",
     "site/worker.js",
     "site/store.js",
-    "dist/ts"
+    "dist/ts",
+    "plugin/scripts/fx.mjs"
   ],
   "engines": {
     "node": ">=20"

package/plugin/scripts/fx.mjs

@@ -0,0 +1,268 @@
+#!/usr/bin/env node
+// The Fangorn app index, headless. One command per verb, JSON on stdout.
+//
+// WHY THERE IS NO MCP SERVER HERE
+// -------------------------------
+// An index that POINTS AT apps rather than hosting them needs no runtime of its
+// own. Every verb below is a function `@fangorn/westmarch` already exports, and
+// the agent's shell is the transport. Nobody has to stand up a server to be in
+// the index — a publisher bakes a view, and that is the whole obligation.
+//
+// WHY THIS IS A PLUGIN AND NOT A PAGE
+// -----------------------------------
+// One thing here is not stateless: the taste kernel. It lives in
+// ~/.fangorn/taste.json — on the filesystem, between the human and the agent —
+// because it has to outlive any one app, any one tab and any one ORIGIN.
+// localStorage cannot do that: a taste learned in fangorn.tv is invisible to the
+// next app by construction, which is the whole reason apps keep rebuilding it.
+// A file the agent can read is the shortest thing that carries it across.
+//
+// ponytail: a plaintext file, no wallet, no sync. `publish/vault.js` is the
+// upgrade path — same object, sealed to a signature, kept in the graph — and it
+// is worth doing the day the kernel needs to travel between machines, not now.
+// ponytail: every command re-fetches its view. Fine at 917 rows, ~seconds at
+// 42k; add a disk cache keyed by view URL when a big corpus makes it hurt.
+
+import { homedir } from "node:os";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { configure, domainManifests, loadShard, trimView } from "../../consume/shard.js";
+import { findCorpora, sourcesFromChain } from "../../consume/directory.js";
+import { linkOf, rolesFrom, textOf, titleOf } from "../../consume/roles.js";
+import { exportTaste, recommend, taste } from "../../consume/taste.js";
+import { EMBED_MODEL, embedQuery, packVec, unpackVec } from "../../consume/embed.js";
+import { browse, describe, facet, getRow, neighbors, search } from "../../consume/tools.js";
+
+const HOME = process.env.FANGORN_HOME ?? join(homedir(), ".fangorn");
+const TASTE = join(HOME, "taste.json");
+
+// ── the kernel on disk ───────────────────────────────────────────────────────
+const store = () => { try { return JSON.parse(readFileSync(TASTE, "utf8")); } catch { return { model: EMBED_MODEL, likes: [], dislikes: [] }; } };
+const save = (s) => { mkdirSync(HOME, { recursive: true }); writeFileSync(TASTE, JSON.stringify(s, null, 1)); };
+/** Picks are stored PACKED (base64 int8, the shard encoding) and unpacked here.
+ *  Storing the picks rather than only the derived vector is what keeps the
+ *  kernel correctable: you can read what it was built from and drop one. */
+const picks = (l = []) => l.map((p) => ({ ...p, vector: unpackVec(p.v) })).filter((p) => p.vector);
+const kernel = (s = store()) => taste(picks(s.likes), picks(s.dislikes));
+
+// ── args ─────────────────────────────────────────────────────────────────────
+const argv = process.argv.slice(2);
+const [cmd, ...rest] = argv;
+const pos = rest.filter((a) => !a.startsWith("--"));
+const flag = (n, d) => { const a = rest.find((x) => x.startsWith(`--${n}=`)); return a === undefined ? d : a.slice(n.length + 3); };
+const num = (n, d) => { const v = flag(n); return v === undefined ? d : Number(v); };
+const pairs = (s) => s ? Object.fromEntries(String(s).split(",").map((p) => p.split("=").map((x) => x.trim()))) : undefined;
+const list = (s) => String(s ?? "").split(",").map((x) => x.trim()).filter(Boolean);
+const out = (o) => console.log(JSON.stringify(o, null, 1));
+const die = (m) => { console.error(m); process.exit(1); };
+
+// ── a view, loaded ───────────────────────────────────────────────────────────
+let roles = rolesFrom([], []);
+async function load(view) {
+    if (!view) die("need a view URL");
+    const base = trimView(view);
+    configure({
+        resolveView: () => base,
+        // Roles must be ready BEFORE the first row is parsed: rowText runs during
+        // parsing, so deriving them afterwards leaves every row with empty text
+        // and lexical search silently matching nothing.
+        onManifests: (ms) => { roles = rolesFrom(ms); },
+        rowText: (f) => textOf(f, roles),
+    });
+    const rows = await loadShard(base);
+    roles = rolesFrom(domainManifests(), rows);
+    let model = null;
+    try { model = (await (await fetch(`${base}/cdn/catalog`)).json()).embedding_model ?? null; } catch { /* unranked, not fatal */ }
+    return { rows, roles, base, model };
+}
+
+/** A query vector, or null with the reason. Never throws: every verb here
+ *  degrades to lexical and SAYS so rather than returning confident nonsense. */
+const qvOf = async (q) => { try { return { qv: await embedQuery(q), why: null }; } catch (e) { return { qv: null, why: e?.message ?? String(e) }; } };
+
+/** The corpus and the kernel must share an embedding space or cosine between
+ *  them is noise that still looks like a score. */
+const mismatch = (model, s) => model && s.model && model !== s.model
+    ? `taste was built with ${s.model}, this corpus is ${model} — scores would be noise` : null;
+
+const sources = async () => {
+    const f = flag("sources"); if (f) return list(f);
+    if (process.env.FANGORN_SOURCES) return list(process.env.FANGORN_SOURCES);
+    try { return JSON.parse(readFileSync(join(HOME, "sources.json"), "utf8")); } catch { /* none pinned */ }
+    return chainSources();
+};
+
+// Every app bound on chain, the scan fangorn-mcp's list-apps runs. From the
+// current AppRegistry's first event on Arbitrum Sepolia; --from-block overrides.
+async function chainSources() {
+    // The SDK scans 1,000 blocks per log request by default, which on the public
+    // RPC takes minutes; fangorn-mcp's install line sets the same window.
+    process.env.FANGORN_LOG_WINDOW ??= "100000";
+    const [{ Fangorn, FangornConfig }, { generatePrivateKey }] = await Promise.all([
+        import("@fangorn-network/sdk"), import("viem/accounts")]);
+    const fangorn = Fangorn.create({ privateKey: generatePrivateKey(), config: FangornConfig });   // reads only
+    const { sources } = await sourcesFromChain(fangorn, { fromBlock: BigInt(flag("from-block", "311637349")) });
+    return sources;
+}
+
+// ── verbs ────────────────────────────────────────────────────────────────────
+switch (cmd) {
+
+// WHICH app has this. Downloads no corpus — coverage centroids only, ~4KB each.
+case "find": {
+    const src = await sources();
+    if (!src.length) die("no apps found on chain; pass --sources=a,b or write ~/.fangorn/sources.json");
+    out(await findCorpora(pos.join(" "), { sources: src, embed: embedQuery, model: EMBED_MODEL, limit: num("limit", 10) }));
+    break;
+}
+
+case "describe": {
+    const { rows, roles: r, model } = await load(pos[0]);
+    out({
+        ...describe(rows), model,
+        shape: r.declared ? "declared" : "sniffed",
+        title: r.title, subtitle: r.subtitle, tags: r.tags, text: r.text,
+        entityTypes: r.entityTypes, gates: r.gates,
+        // Where the reader goes next. The index's job ends here.
+        launch: r.launch, externalUrl: r.externalUrl, actions: r.actions,
+    });
+    break;
+}
+
+case "browse": {
+    const { rows, roles: r } = await load(pos[0]);
+    out(browse(rows, r, { type: flag("type"), where: pairs(flag("where")), sort: flag("sort"),
+                          desc: flag("asc") === undefined, limit: num("limit", 20), offset: num("offset", 0), fields: list(flag("fields")) }));
+    break;
+}
+
+case "search": {
+    const { rows, roles: r } = await load(pos[0]);
+    const q = pos.slice(1).join(" ");
+    const { qv, why } = await qvOf(q);
+    out({ query: q, mode: qv ? "semantic where rows have vectors" : "lexical", ...(why ? { why } : {}),
+          hits: search(rows, q, r, { qv, limit: num("limit", 10), where: pairs(flag("where")), fields: list(flag("fields")) }) });
+    break;
+}
+
+case "facet": {
+    const { rows } = await load(pos[0]);
+    out(facet(rows, pos[1], { limit: num("limit", 20), where: pairs(flag("where")) }));
+    break;
+}
+
+case "row": {
+    const { rows, roles: r } = await load(pos[0]);
+    const g = getRow(rows, pos[1], r);
+    out(g ? { ...g, url: linkOf(rows.find((x) => x.id === g.id) ?? {}, r) } : null);
+    break;
+}
+
+case "similar": {
+    const { rows, roles: r } = await load(pos[0]);
+    out(neighbors(rows, pos[1], r, { limit: num("limit", 10), fields: list(flag("fields")) }));
+    break;
+}
+
+// ── the kernel ───────────────────────────────────────────────────────────────
+case "like": case "dislike": {
+    const { rows, roles: r, base, model } = await load(pos[0]);
+    const s = store();
+    const into = cmd === "like" ? s.likes : s.dislikes;
+    const added = [];
+    for (const id of pos.slice(1)) {
+        const row = rows.find((x) => x.id === id);
+        if (!row) { console.error(`no row ${id}`); continue; }
+        if (!row.vector) { console.error(`${id} has no vector — nothing to learn from`); continue; }
+        // Newest last: taste() weights by recency, so order in the file is signal.
+        const at = into.findIndex((p) => p.id === id);
+        if (at >= 0) into.splice(at, 1);
+        into.push({ id, title: titleOf(row, r), view: base, at: new Date().toISOString().slice(0, 10), v: packVec(Array.from(row.vector)) });
+        added.push(id);
+    }
+    s.model ??= model ?? EMBED_MODEL;
+    save(s);
+    const t = kernel(s);
+    out({ added, likes: s.likes.length, dislikes: s.dislikes.length, taste: t && exportTaste(t), file: TASTE });
+    break;
+}
+
+case "forget": {
+    const s = store();
+    const drop = new Set(pos);
+    const before = s.likes.length + s.dislikes.length;
+    s.likes = s.likes.filter((p) => !drop.has(p.id));
+    s.dislikes = s.dislikes.filter((p) => !drop.has(p.id));
+    save(s);
+    out({ removed: before - s.likes.length - s.dislikes.length, likes: s.likes.length, dislikes: s.dislikes.length });
+    break;
+}
+
+case "taste": {
+    const s = store();
+    const t = kernel(s);
+    out(t ? { model: s.model, n: t.n, from: t.from, rejected: t.rejected, drift: t.drift,
+              // Where each pick was learned. A kernel built in one publisher's
+              // corpus is meant to be pointed at another's — this is the record
+              // of whether it actually has been.
+              views: [...new Set([...s.likes, ...s.dislikes].map((p) => p.view))],
+              export: exportTaste(t), file: TASTE }
+            : { n: 0, file: TASTE, note: "nothing liked yet — `fx like <view> <id>`" });
+    break;
+}
+
+// The headline: rank a corpus by a taste learned somewhere else entirely.
+case "recommend": {
+    const { rows, roles: r, model } = await load(pos[0]);
+    const s = store();
+    const t = kernel(s);
+    if (!t) die("no taste yet — `fx like <view> <id>` first");
+    const warn = mismatch(model, s);
+    if (warn) die(warn);
+    const seen = new Set([...s.likes, ...s.dislikes].map((p) => p.id));
+    out({ from: t.from, against: pos[0],
+          hits: recommend(rows, t, { limit: num("limit", 10), exclude: seen })
+              .map(({ row, score }) => ({ id: row.id, title: titleOf(row, r), score, url: linkOf(row, r) })) });
+    break;
+}
+
+// ── the handoff. The index points; the app does the work. ────────────────────
+case "launch": {
+    const { rows, roles: r } = await load(pos[0]);
+    const id = flag("row");
+    if (id) {
+        const row = rows.find((x) => x.id === id);
+        if (!row) die(`no row ${id}`);
+        out({ url: linkOf(row, r), title: titleOf(row, r) });
+        break;
+    }
+    if (!r.launch) die("this corpus declares no presentation.launch — nowhere to hand off to");
+    const t = kernel();
+    const fill = { taste: t ? JSON.stringify(exportTaste(t)) : "", q: pos.slice(1).join(" ") };
+    const url = String(r.launch).replace(/\{(\w+)\}/g, (_, k) => encodeURIComponent(fill[k] ?? ""));
+    const u = new URL(url);
+    if (u.protocol !== "https:" && u.protocol !== "http:") die(`refusing a ${u.protocol} launch URL`);
+    // The kernel travels in the URL because that is the only channel a page on
+    // someone else's origin can receive it on. It is the app's copy from here.
+    out({ url: u.toString(), carries: t ? { n: t.n, from: t.from } : null });
+    break;
+}
+
+default:
+    console.log(`fx <verb>
+
+  find <query> [--sources=a,b|--from-block=N]       which app has this. No corpus downloaded.
+  describe <view>                                  rows, field coverage, roles, where it hands off
+  browse <view> [--type= --where=k=v --sort= --asc --limit= --offset=]
+  search <view> <query> [--where=k=v --limit=]     semantic where rows have vectors, else lexical
+  facet <view> <field> [--where=k=v --limit=]      count a field's values
+  row <view> <id>                                  one record, whole
+  similar <view> <id> [--limit=]                   nearest by vector
+
+  like|dislike <view> <id...>                      teach the kernel. Writes ${TASTE}
+  forget <id...>                                   drop picks from it
+  taste                                            what it is, and what it was built from
+  recommend <view> [--limit=]                      rank ANY corpus by it
+
+  launch <view> [query] | launch <view> --row=<id> the handoff URL, carrying the kernel`);
+}

package/publish/eval.ts

@@ -3,9 +3,10 @@
 //
 //   westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]
 //
-// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
 //   westmarch-eval --selfcheck
 //
+// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
+//
 // It grades the RECIPE — how records are shaped, which fields are embedded, how rows
 // rank — never the data. New records are facts, and a view is not worse for holding
 // them. So: build a base view (main's recipe) and a candidate (the change) from the
@@ -25,6 +26,16 @@
 //   {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ (St|Ave)\\b/"}},"max_pct":0.5,"hard":true}
 //       records: the % of rows that satisfy `expect`, lower is better; `hard`: any rise over the base fails.
 //       `of` (a predicate) narrows which rows count, e.g. only meeting items have a date to miss.
+//   {"id":"basin-towns","kind":"coverage","field":"city","values":["Plover","Stevens Point"],"min":0.9}
+//       coverage: the share of `values` that some row holds in `field`, and the missing ones.
+//       `values` may instead be a path (relative to the golden file) to a JSON list, or to an
+//       object {value: weight} such as each town's population: the share is then by weight, and
+//       the missing are listed heaviest first, so the report names the next places to add.
+//       With a base, any fall fails: a recipe that loses a town has dropped its records.
+//
+// Any check may carry a `goal`: the owner's bet it serves (eval/goals.md says what each one
+// is). The report scores each goal apart, so "trades 0.4, general 0.9" shows which bet the
+// app is losing, instead of one mean that hides it.
 //
 // A predicate is {where?, match?, not?}. `where` is fangorn's own whole-value,
 // case-insensitive match — the filter agents use. `match` maps a field (or several
@@ -37,17 +48,20 @@
 import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http";
 import { homedir } from "node:os";
+import { dirname, join } from "node:path";
 import { configure, loadShard, trimView } from "@fangorn-network/westmarch/shard";
-import { rolesFrom, textOf, titleOf, subtitleOf } from "@fangorn-network/westmarch/roles";
+import { rolesFrom, textOf, titleOf, subtitleOf, values } from "@fangorn-network/westmarch/roles";
 import { search, matches } from "@fangorn-network/westmarch/tools";
 import { EMBED_MODEL, embedQueryDirect } from "@fangorn-network/westmarch/embed";
 
 type Pred = { where?: Record<string, string>; match?: Record<string, string>; not?: Pred };
-type Check =
-    | { id: string; kind?: "search"; q: string; k?: number; expect: Pred; min?: number }
-    | { id: string; kind: "count"; where?: Record<string, string>; expect?: Pred; min?: number; max?: number }
-    | { id: string; kind: "records"; of?: Pred; expect: Pred; max_pct: number; hard?: boolean };
-type Result = { id: string; kind: "search" | "count" | "records" | "known"; score: number; pass: boolean; hard?: boolean; top?: string[] };
+type Check = { id: string; goal?: string } & (
+    | { kind?: "search"; q: string; k?: number; expect: Pred; min?: number }
+    | { kind: "count"; where?: Record<string, string>; expect?: Pred; min?: number; max?: number }
+    | { kind: "records"; of?: Pred; expect: Pred; max_pct: number; hard?: boolean }
+    | { kind: "coverage"; field: string; values: string[]; weights?: Record<string, number>; of?: Pred; min?: number });
+type Result = { id: string; kind: "search" | "count" | "records" | "coverage" | "known"; score: number; pass: boolean;
+                goal?: string; hard?: boolean; top?: string[] };
 type QueryVector = (q: string) => Promise<Float32Array | null>;
 
 /** A drop below this, in mean precision@k or known@10, is "worse". canton-corpus's number. */
@@ -74,19 +88,28 @@ async function grade(rows: Row[], roles: Roles, checks: Check[], qv: QueryVector
     const byId = new Map(rows.map((r) => [r.id, r]));
     const out: Result[] = [];
     for (const c of checks) {
+        const goal = c.goal ? { goal: c.goal } : {};
         if (c.kind === "count") {
             const ok = compile(c.expect);
             const n = rows.filter((r) => matches(r, c.where) && ok(r)).length;
-            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity) });
+            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity), ...goal });
         } else if (c.kind === "records") {
             const of = rows.filter(compile(c.of)), ok = compile(c.expect);
             const pct = of.length ? round((100 * of.filter(ok).length) / of.length) : 0;
-            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard });
+            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard, ...goal });
+        } else if (c.kind === "coverage") {
+            // The same whole-value, any-case reading `where` uses, so covered means an agent's filter finds it.
+            const held = new Set(rows.filter(compile(c.of)).flatMap((r) => values(r[c.field] ?? "").map((v) => v.toLowerCase())));
+            const w = (v: string) => c.weights?.[v] ?? 1;
+            const missing = c.values.filter((v) => !held.has(v.toLowerCase())).sort((a, b) => w(b) - w(a));
+            const total = c.values.reduce((a, v) => a + w(v), 0);
+            const score = total ? round(1 - missing.reduce((a, v) => a + w(v), 0) / total) : 1;
+            out.push({ id: c.id, kind: "coverage", score, pass: score >= (c.min ?? 0), top: missing.slice(0, 10), ...goal });
         } else {
             const k = c.k ?? 5, ok = compile(c.expect);
             const hits = search(rows, c.q, roles, { qv: await qv(c.q), limit: k }).map((h) => byId.get(h.id)!);
             const score = round(hits.filter(ok).length / k);
-            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)) });
+            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)), ...goal });
         }
     }
     // ponytail: the pool is each view's own rows, so base and candidate sample different
@@ -110,6 +133,19 @@ async function grade(rows: Row[], roles: Roles, checks: Check[], qv: QueryVector
     return out;
 }
 
+/** Each goal's checks, scored apart: how many pass, and the mean of its searches and coverages. */
+function byGoal(results: Result[]) {
+    const out: Record<string, { checks: number; passing: number; search: number | null; coverage: number | null }> = {};
+    const mean = (xs: number[]) => (xs.length ? round(xs.reduce((a, x) => a + x, 0) / xs.length) : null);
+    for (const g of new Set(results.map((r) => r.goal).filter((g): g is string => !!g))) {
+        const rs = results.filter((r) => r.goal === g);
+        out[g] = { checks: rs.length, passing: rs.filter((r) => r.pass).length,
+                   search: mean(rs.filter((r) => r.kind === "search").map((r) => r.score)),
+                   coverage: mean(rs.filter((r) => r.kind === "coverage").map((r) => r.score)) };
+    }
+    return out;
+}
+
 /** The candidate against the base. Worse on any one rule is worse. */
 function compare(cand: Result[], base: Result[]) {
     const was = new Map(base.map((r) => [r.id, r.score]));
@@ -120,6 +156,7 @@ function compare(cand: Result[], base: Result[]) {
         // The ratchet: a question the base answered at all, the candidate must too.
         if (r.kind === "search" && b !== undefined && b > 0 && r.score === 0) failures.push(`${r.id}: answered before, nothing now`);
         if (r.hard && b !== undefined && r.score > b) failures.push(`${r.id}: ${b}% → ${r.score}%, and it may not rise`);
+        if (r.kind === "coverage" && b !== undefined && r.score < b) failures.push(`${r.id}: coverage ${b} → ${r.score}; missing ${r.top?.join(", ")}`);
     }
     // Aggregates, over the checks both sides ran: one question moving a rank is noise.
     const both = (kind: Result["kind"]) => cand.filter((r) => (kind === "known" ? r.id === "known@10" : r.kind === kind) && was.has(r.id));
@@ -150,8 +187,14 @@ async function main() {
         console.error("usage: westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]");
         process.exit(flags.help ? 0 : 2);
     }
-    const checks: Check[] = readFileSync(String(flags.golden ?? "eval/golden.jsonl"), "utf8")
+    const golden = String(flags.golden ?? "eval/golden.jsonl");
+    const checks: Check[] = readFileSync(golden, "utf8")
         .split("\n").filter((l) => l.trim() && !l.startsWith("//")).map((l) => JSON.parse(l));
+    // A coverage list kept in its own file: a list, or {value: weight}.
+    for (const c of checks) if (c.kind === "coverage" && typeof c.values === "string") {
+        const list: string[] | Record<string, number> = JSON.parse(readFileSync(join(dirname(golden), c.values), "utf8"));
+        Object.assign(c, Array.isArray(list) ? { values: list } : { values: Object.keys(list), weights: list });
+    }
     const known = Number(flags.known ?? 200);
 
     const { env } = await import("@huggingface/transformers");
@@ -172,7 +215,7 @@ async function main() {
     const failures = cmp?.failures ?? cand.results.filter((r) => !r.pass).map((r) => `${r.id}: ${r.score} is outside its bounds`);
     // What reproduces this grade: the model, the runtime, and the views (whose manifests hash every shard).
     const report = { model: EMBED_MODEL, node: process.version, platform: `${process.platform}-${process.arch}`,
-                     candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
+                     goals: byGoal(cand.results), candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
 
     if (flags.report) writeFileSync(String(flags.report), JSON.stringify(report, null, 1));
     if (flags.json) console.log(JSON.stringify(report, null, 1));
@@ -184,6 +227,8 @@ async function main() {
             console.log(`${r.pass ? " " : "✗"} ${r.kind.padEnd(7)} ${r.id.padEnd(32)} ${b === undefined ? "" : `${b} → `}${r.score}`);
         }
         for (const d of report.deltas) console.log(`\n${d.kind}: ${d.base} → ${d.cand}`);
+        for (const [g, s] of Object.entries(report.goals))
+            console.log(`\ngoal ${g}: ${s.passing}/${s.checks} passing${s.search === null ? "" : `, search ${s.search}`}${s.coverage === null ? "" : `, coverage ${s.coverage}`}`);
         console.log(`\n${report.verdict ?? (report.pass ? "pass" : "fail")}${failures.length ? `\n  ${failures.join("\n  ")}` : ""}`);
     }
     process.exit(report.pass ? 0 : 1);
@@ -234,6 +279,15 @@ async function selfcheck() {
     assert(scoped.score === 50, `\`of\` narrows the rows a record check counts: ${scoped.score}`);
     assert(r["known@10"].score === 1, "every title finds its own row");
 
+    // Coverage, and goals scored apart.
+    const [cov] = await grade(rows, roles, [{ id: "towns", kind: "coverage", field: "city", values: ["plover", "Stevens Point", "Wausau"], min: 0.5, goal: "towns" }], async () => null, 0);
+    assert(cov.score === 0.667 && cov.pass && cov.top?.[0] === "Wausau", `coverage is the share present, the rest listed: ${JSON.stringify(cov)}`);
+    const [pop] = await grade(rows, roles, [{ id: "pop", kind: "coverage", field: "city", values: ["Plover", "Wausau", "Adell"],
+        weights: { Plover: 30, Wausau: 60, Adell: 10 } }], async () => null, 0);
+    assert(pop.score === 0.3 && pop.top?.join() === "Wausau,Adell", `weighted coverage, the heaviest gap first: ${JSON.stringify(pop)}`);
+    const goals = byGoal([cov, { ...r.rezoning, goal: "towns" }, r.plover]);
+    assert(goals.towns.checks === 2 && goals.towns.coverage === 0.667 && goals.towns.search === 1 && !("undefined" in goals), `goals: ${JSON.stringify(goals)}`);
+
     // The verdict.
     const res = (id: string, score: number, kind: Result["kind"] = "search", extra = {}): Result => ({ id, kind, score, pass: true, ...extra });
     const base = [res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })];
@@ -246,5 +300,6 @@ async function selfcheck() {
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.2, "records", { hard: true })], base).verdict === "worse", "a hard record may not rise");
     assert(compare([res("q1", 0.8, "search", { pass: false }), res("q2", 0.4)], base).verdict === "worse", "a floor fails whatever the base did");
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("q3", 0)], base).verdict === "unchanged", "a new question has no base to fall from");
+    assert(compare([res("cov", 0.9, "coverage")], [res("cov", 1, "coverage")]).verdict === "worse", "coverage may not fall");
     console.log("eval: selfcheck ok");
 }

package/publish/ship.js

@@ -235,7 +235,7 @@ export async function ship(configPath, { crawl = true, deploy = true, dry = fals
     if (!existsSync(join(site, "view/cdn/catalog")) && state.url && !dry) await mirrorView(`${state.url}/view`, join(site, "view"));
     const namespaces = [...new Set(cfg.sources.map((s) => s.namespace))];
     const report = dry ? {} : await view.publishView({ fangorn, app: cfg.app, namespaces, out: join(site, "view"),
-        fromBlock: BigInt(state.fromBlock), log });
+        fromBlock: BigInt(state.fromBlock), embed: view.cachedEmbed(join(dir, "vectors.ndjson")), log });
 
     // 6. the page, the card, the headers
     if (!existsSync(join(STOCK, "index.html"))) throw new Error(`no stock page at ${STOCK}; run \`npx vite build\` in westmarch/site`);

package/publish/view.js

@@ -201,7 +201,7 @@ export function liveFields(out) {
  * Returns per-domain counts of what changed.
  */
 export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0n, rebake = false,
-                                    embed = embedDocumentDirect, log = console.log }) {
+                                    embed = embedDocumentDirect, log = console.log, shardRows = 20000 }) {
     if (!namespaces?.length) throw new Error("publishView: name the data namespace(s) to publish");
     fangorn.setAppId(app);
     const appId = fangorn.getAppId();
@@ -222,7 +222,7 @@ export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0
         for (const { owner: publisher } of timelines) {
             const domain = domainFor(appId, publisher, ns);
             report[domain] = await publishDomain({
-                fangorn, schema, publisher, ns, domain, dir: `${out}/cdn/domains/${domain}`, rebake, embed, log,
+                fangorn, schema, publisher, ns, domain, dir: `${out}/cdn/domains/${domain}`, rebake, embed, log, shardRows,
             });
         }
     }
@@ -230,7 +230,7 @@ export async function publishView({ fangorn, app, namespaces, out, fromBlock = 0
     return report;
 }
 
-async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, rebake, embed, log }) {
+async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, rebake, embed, log, shardRows }) {
     const { contents } = await fangorn.readNamespace(publisher, ns);
     const records = new Map((contents.vertices ?? []).filter((v) => v.payload && typeof v.payload === "object")
         .map((v) => [v.cid, { ...v.payload, entityType: v.schemaId }]));
@@ -275,12 +275,17 @@ async function publishDomain({ fangorn, schema, publisher, ns, domain, dir, reba
     mkdirSync(`${dir}/shards`, { recursive: true });
     const manifest = had.manifest ?? { name: domain, description: "", count: 0, dim: EMBED_DIM, model: EMBED_MODEL,
         distance: "Cosine", embedder: EMBEDDER, filter: { owner: [publisher], namespace: [ns] }, shards: [], tombstones: [] };
-    const writeShard = (ls) => {
-        const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
-        const sha256 = createHash("sha256").update(gz).digest("hex");
-        const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${sha256.slice(0, 12)}.ndjson.gz`;
-        writeFileSync(`${dir}/shards/${file}`, gz);
-        manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256 });
+    // Pages serves files up to 25 MiB; 52k Steam games gzipped to 28 MiB in one shard.
+    // ponytail: split by row count (~550 B a row gzipped); split by bytes if rows get much fatter.
+    const writeShard = (all) => {
+        for (let i = 0; i < all.length; i += shardRows) {
+            const ls = all.slice(i, i + shardRows);
+            const gz = gzipSync(`${ls.join("\n")}\n`, { level: 9 });
+            const sha256 = createHash("sha256").update(gz).digest("hex");
+            const file = `shard-${String(manifest.shards.length).padStart(4, "0")}-${sha256.slice(0, 12)}.ndjson.gz`;
+            writeFileSync(`${dir}/shards/${file}`, gz);
+            manifest.shards.push({ file, count: ls.length, bytes: gz.length, sha256 });
+        }
     };
     if (lines.length) {
         writeShard(lines);
@@ -389,6 +394,11 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
     const run = () => publishView({ fangorn, app: APP, namespaces: ["shop"], out, embed, log: () => {} });
 
     const d = domainFor(APP, PUB, "shop");
+    const split = mkdtempSync(`${tmpdir()}/view-`);
+    await publishView({ fangorn, app: APP, namespaces: ["shop"], out: split, embed, log: () => {}, shardRows: 2 });
+    m0: { const m = readJson(`${split}/cdn/domains/${d}/manifest`);
+          eq([m.count, m.shards.map((s) => s.count)], [3, [2, 1]], "rows past shardRows go to the next shard (Pages caps a file at 25 MiB)"); }
+    embedded.length = 0;
     eq((await run())[d], { added: 3, removed: 0 }, "first run embeds everything");
     if (!embedded.includes("Title: Oak table. Tags: . a solid oak dining table")) throw new Error(`declared roles compose the text: ${embedded}`);
     let m = readJson(`${out}/cdn/domains/${d}/manifest`);

package/site/agent.js

@@ -1,6 +1,7 @@
 // The stock page's tools for agents in a browser: the same verbs as fangorn-mcp's data
 // tools, over the rows this page loaded. `ctx` holds them; ship's card captures these.
 import { describe, facet, getRow, search } from "../consume/tools.js";
+import { KNOBS, discover } from "../consume/taste.js";
 
 const ok = (o) => ({ content: [{ type: "text", text: JSON.stringify(o) }] });
 const obj = (properties, required = []) => ({ type: "object", properties, required });
@@ -41,4 +42,26 @@ export function registerAgent(ctx) {
         inputSchema: obj({}),
         execute: async () => ok(ctx.session?.() ?? { saved: [], searches: [], taste: null }),
     });
+    // Like / dislike and "For you": the same calls the page's 👍 👎 and knobs make (ctx.rate,
+    // ctx.votes, set by main.js), so what an agent records shows on the page.
+    mc.registerTool({
+        name: "rate",
+        description: "Record that the person liked or disliked a record (or clear it), by the id search returns. Builds the taste discover ranks with.",
+        inputSchema: obj({ id: { type: "string" }, verdict: { type: "string", enum: ["like", "dislike", "clear"] } }, ["id", "verdict"]),
+        execute: async ({ id, verdict }) => ok(ctx.rate?.(id, verdict) ?? { error: "this page keeps no taste" }),
+    });
+    mc.registerTool({
+        name: "discover",
+        description: "Records for a taste: a kernel over what was liked (newest last) and disliked, not a text search. Pass likes and dislikes (ids from search), or omit them to use the taste on this page. Knobs are 0–1.",
+        inputSchema: obj({ likes: { type: "array", items: { type: "string" } }, dislikes: { type: "array", items: { type: "string" } },
+            ...Object.fromEntries(Object.entries(KNOBS).map(([k, [d, what]]) => [k, { type: "number", minimum: 0, maximum: 1, default: d, description: what }])),
+            seed: { type: "integer", description: "Same seed, same shuffle" }, limit: { type: "integer", minimum: 1, maximum: 50, default: 10 } }),
+        execute: async ({ likes, dislikes, limit = 10, ...knobs }) => {
+            const byId = (ids) => ids.map((id) => ctx.rows.find((r) => r.id === getRow(ctx.rows, id, ctx.roles)?.id)).filter((r) => r?.vector);
+            const v = likes ? { likes: byId(likes), dislikes: byId(dislikes ?? []) } : ctx.votes?.() ?? { likes: [], dislikes: [] };
+            if (!v.likes.length) return ok({ error: "like at least one record first (ids from search, or rate)" });
+            const out = discover(ctx.rows, v.likes, v.dislikes, { ...knobs, limit: Math.min(50, limit) });
+            return ok({ ...out, picks: out.picks.map(({ row, score }) => ({ ...getRow(ctx.rows, row.id, ctx.roles), score })) });
+        },
+    });
 }