@fangorn-network/westmarch 0.0.4-dev → 0.0.5-dev

package/dist/ts/eval.js

@@ -3,9 +3,10 @@
 //
 //   westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]
 //
-// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
 //   westmarch-eval --selfcheck
 //
+// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
+//
 // It grades the RECIPE — how records are shaped, which fields are embedded, how rows
 // rank — never the data. New records are facts, and a view is not worse for holding
 // them. So: build a base view (main's recipe) and a candidate (the change) from the
@@ -25,6 +26,16 @@
 //   {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ (St|Ave)\\b/"}},"max_pct":0.5,"hard":true}
 //       records: the % of rows that satisfy `expect`, lower is better; `hard`: any rise over the base fails.
 //       `of` (a predicate) narrows which rows count, e.g. only meeting items have a date to miss.
+//   {"id":"basin-towns","kind":"coverage","field":"city","values":["Plover","Stevens Point"],"min":0.9}
+//       coverage: the share of `values` that some row holds in `field`, and the missing ones.
+//       `values` may instead be a path (relative to the golden file) to a JSON list, or to an
+//       object {value: weight} such as each town's population: the share is then by weight, and
+//       the missing are listed heaviest first, so the report names the next places to add.
+//       With a base, any fall fails: a recipe that loses a town has dropped its records.
+//
+// Any check may carry a `goal`: the owner's bet it serves (eval/goals.md says what each one
+// is). The report scores each goal apart, so "trades 0.4, general 0.9" shows which bet the
+// app is losing, instead of one mean that hides it.
 //
 // A predicate is {where?, match?, not?}. `where` is fangorn's own whole-value,
 // case-insensitive match — the filter agents use. `match` maps a field (or several
@@ -36,8 +47,9 @@
 import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http";
 import { homedir } from "node:os";
+import { dirname, join } from "node:path";
 import { configure, loadShard, trimView } from "@fangorn-network/westmarch/shard";
-import { rolesFrom, textOf, titleOf, subtitleOf } from "@fangorn-network/westmarch/roles";
+import { rolesFrom, textOf, titleOf, subtitleOf, values } from "@fangorn-network/westmarch/roles";
 import { search, matches } from "@fangorn-network/westmarch/tools";
 import { EMBED_MODEL, embedQueryDirect } from "@fangorn-network/westmarch/embed";
 /** A drop below this, in mean precision@k or known@10, is "worse". canton-corpus's number. */
@@ -60,21 +72,31 @@ async function grade(rows, roles, checks, qv, known) {
     const byId = new Map(rows.map((r) => [r.id, r]));
     const out = [];
     for (const c of checks) {
+        const goal = c.goal ? { goal: c.goal } : {};
         if (c.kind === "count") {
             const ok = compile(c.expect);
             const n = rows.filter((r) => matches(r, c.where) && ok(r)).length;
-            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity) });
+            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity), ...goal });
         }
         else if (c.kind === "records") {
             const of = rows.filter(compile(c.of)), ok = compile(c.expect);
             const pct = of.length ? round((100 * of.filter(ok).length) / of.length) : 0;
-            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard });
+            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard, ...goal });
+        }
+        else if (c.kind === "coverage") {
+            // The same whole-value, any-case reading `where` uses, so covered means an agent's filter finds it.
+            const held = new Set(rows.filter(compile(c.of)).flatMap((r) => values(r[c.field] ?? "").map((v) => v.toLowerCase())));
+            const w = (v) => c.weights?.[v] ?? 1;
+            const missing = c.values.filter((v) => !held.has(v.toLowerCase())).sort((a, b) => w(b) - w(a));
+            const total = c.values.reduce((a, v) => a + w(v), 0);
+            const score = total ? round(1 - missing.reduce((a, v) => a + w(v), 0) / total) : 1;
+            out.push({ id: c.id, kind: "coverage", score, pass: score >= (c.min ?? 0), top: missing.slice(0, 10), ...goal });
         }
         else {
             const k = c.k ?? 5, ok = compile(c.expect);
             const hits = search(rows, c.q, roles, { qv: await qv(c.q), limit: k }).map((h) => byId.get(h.id));
             const score = round(hits.filter(ok).length / k);
-            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)) });
+            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)), ...goal });
         }
     }
     // ponytail: the pool is each view's own rows, so base and candidate sample different
@@ -97,6 +119,18 @@ async function grade(rows, roles, checks, qv, known) {
     }
     return out;
 }
+/** Each goal's checks, scored apart: how many pass, and the mean of its searches and coverages. */
+function byGoal(results) {
+    const out = {};
+    const mean = (xs) => (xs.length ? round(xs.reduce((a, x) => a + x, 0) / xs.length) : null);
+    for (const g of new Set(results.map((r) => r.goal).filter((g) => !!g))) {
+        const rs = results.filter((r) => r.goal === g);
+        out[g] = { checks: rs.length, passing: rs.filter((r) => r.pass).length,
+            search: mean(rs.filter((r) => r.kind === "search").map((r) => r.score)),
+            coverage: mean(rs.filter((r) => r.kind === "coverage").map((r) => r.score)) };
+    }
+    return out;
+}
 /** The candidate against the base. Worse on any one rule is worse. */
 function compare(cand, base) {
     const was = new Map(base.map((r) => [r.id, r.score]));
@@ -110,6 +144,8 @@ function compare(cand, base) {
             failures.push(`${r.id}: answered before, nothing now`);
         if (r.hard && b !== undefined && r.score > b)
             failures.push(`${r.id}: ${b}% → ${r.score}%, and it may not rise`);
+        if (r.kind === "coverage" && b !== undefined && r.score < b)
+            failures.push(`${r.id}: coverage ${b} → ${r.score}; missing ${r.top?.join(", ")}`);
     }
     // Aggregates, over the checks both sides ran: one question moving a rank is noise.
     const both = (kind) => cand.filter((r) => (kind === "known" ? r.id === "known@10" : r.kind === kind) && was.has(r.id));
@@ -143,8 +179,15 @@ async function main() {
         console.error("usage: westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]");
         process.exit(flags.help ? 0 : 2);
     }
-    const checks = readFileSync(String(flags.golden ?? "eval/golden.jsonl"), "utf8")
+    const golden = String(flags.golden ?? "eval/golden.jsonl");
+    const checks = readFileSync(golden, "utf8")
         .split("\n").filter((l) => l.trim() && !l.startsWith("//")).map((l) => JSON.parse(l));
+    // A coverage list kept in its own file: a list, or {value: weight}.
+    for (const c of checks)
+        if (c.kind === "coverage" && typeof c.values === "string") {
+            const list = JSON.parse(readFileSync(join(dirname(golden), c.values), "utf8"));
+            Object.assign(c, Array.isArray(list) ? { values: list } : { values: Object.keys(list), weights: list });
+        }
     const known = Number(flags.known ?? 200);
     const { env } = await import("@huggingface/transformers");
     env.cacheDir = `${homedir()}/.cache/fangorn-mcp/models`; // shared with fangorn-mcp and westmarch-view
@@ -163,7 +206,7 @@ async function main() {
     const failures = cmp?.failures ?? cand.results.filter((r) => !r.pass).map((r) => `${r.id}: ${r.score} is outside its bounds`);
     // What reproduces this grade: the model, the runtime, and the views (whose manifests hash every shard).
     const report = { model: EMBED_MODEL, node: process.version, platform: `${process.platform}-${process.arch}`,
-        candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
+        goals: byGoal(cand.results), candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
     if (flags.report)
         writeFileSync(String(flags.report), JSON.stringify(report, null, 1));
     if (flags.json)
@@ -177,6 +220,8 @@ async function main() {
         }
         for (const d of report.deltas)
             console.log(`\n${d.kind}: ${d.base} → ${d.cand}`);
+        for (const [g, s] of Object.entries(report.goals))
+            console.log(`\ngoal ${g}: ${s.passing}/${s.checks} passing${s.search === null ? "" : `, search ${s.search}`}${s.coverage === null ? "" : `, coverage ${s.coverage}`}`);
         console.log(`\n${report.verdict ?? (report.pass ? "pass" : "fail")}${failures.length ? `\n  ${failures.join("\n  ")}` : ""}`);
     }
     process.exit(report.pass ? 0 : 1);
@@ -228,6 +273,14 @@ async function selfcheck() {
     const [scoped] = await grade(rows, roles, [{ id: "s", kind: "records", of: { where: { city: "Stevens Point" } }, expect: { match: { text: "Rd" } }, max_pct: 100 }], async () => null, 0);
     assert(scoped.score === 50, `\`of\` narrows the rows a record check counts: ${scoped.score}`);
     assert(r["known@10"].score === 1, "every title finds its own row");
+    // Coverage, and goals scored apart.
+    const [cov] = await grade(rows, roles, [{ id: "towns", kind: "coverage", field: "city", values: ["plover", "Stevens Point", "Wausau"], min: 0.5, goal: "towns" }], async () => null, 0);
+    assert(cov.score === 0.667 && cov.pass && cov.top?.[0] === "Wausau", `coverage is the share present, the rest listed: ${JSON.stringify(cov)}`);
+    const [pop] = await grade(rows, roles, [{ id: "pop", kind: "coverage", field: "city", values: ["Plover", "Wausau", "Adell"],
+            weights: { Plover: 30, Wausau: 60, Adell: 10 } }], async () => null, 0);
+    assert(pop.score === 0.3 && pop.top?.join() === "Wausau,Adell", `weighted coverage, the heaviest gap first: ${JSON.stringify(pop)}`);
+    const goals = byGoal([cov, { ...r.rezoning, goal: "towns" }, r.plover]);
+    assert(goals.towns.checks === 2 && goals.towns.coverage === 0.667 && goals.towns.search === 1 && !("undefined" in goals), `goals: ${JSON.stringify(goals)}`);
     // The verdict.
     const res = (id, score, kind = "search", extra = {}) => ({ id, kind, score, pass: true, ...extra });
     const base = [res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })];
@@ -240,5 +293,6 @@ async function selfcheck() {
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.2, "records", { hard: true })], base).verdict === "worse", "a hard record may not rise");
     assert(compare([res("q1", 0.8, "search", { pass: false }), res("q2", 0.4)], base).verdict === "worse", "a floor fails whatever the base did");
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("q3", 0)], base).verdict === "unchanged", "a new question has no base to fall from");
+    assert(compare([res("cov", 0.9, "coverage")], [res("cov", 1, "coverage")]).verdict === "worse", "coverage may not fall");
     console.log("eval: selfcheck ok");
 }

package/docs/app-to-agent.md

@@ -123,6 +123,13 @@ What is asked lives in `eval/golden.jsonl`, one check per line, as data rather t
 {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ St\\b/"}},"max_pct":0.5,"hard":true}
 ```
 
+Which questions go in is the owner's bet: who would pay for this, for which answers.
+Write it down in `eval/goals.md` (the customer, their questions, the coverage they need, a
+date, and what would count as failure), tag each check with the `goal` it serves, and the
+report scores each goal on its own. A `coverage` check (`{"kind":"coverage","field":"city",
+"values":[…]}`) measures how much of the list a goal needs is present at all; with a base,
+any fall fails, because a recipe that loses a town has dropped its records.
+
 `expect` is a predicate over a row: `where` (the whole-value match agents filter with),
 `match` (a field, or several joined by `|`, to a regex; case-insensitive unless written
 `/re/flags`), `not`. Relevance is a predicate and not a list of record ids, because ids

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fangorn-network/westmarch",
-  "version": "0.0.4-dev",
+  "version": "0.0.5-dev",
   "description": "Publisher and consumer surfaces for a Fangorn app: shape a graph, publish it, read it back as a searchable, agent-drivable catalog",
   "license": "MIT",
   "type": "module",

package/publish/eval.ts

@@ -3,9 +3,10 @@
 //
 //   westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]
 //
-// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
 //   westmarch-eval --selfcheck
 //
+// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
+//
 // It grades the RECIPE — how records are shaped, which fields are embedded, how rows
 // rank — never the data. New records are facts, and a view is not worse for holding
 // them. So: build a base view (main's recipe) and a candidate (the change) from the
@@ -25,6 +26,16 @@
 //   {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ (St|Ave)\\b/"}},"max_pct":0.5,"hard":true}
 //       records: the % of rows that satisfy `expect`, lower is better; `hard`: any rise over the base fails.
 //       `of` (a predicate) narrows which rows count, e.g. only meeting items have a date to miss.
+//   {"id":"basin-towns","kind":"coverage","field":"city","values":["Plover","Stevens Point"],"min":0.9}
+//       coverage: the share of `values` that some row holds in `field`, and the missing ones.
+//       `values` may instead be a path (relative to the golden file) to a JSON list, or to an
+//       object {value: weight} such as each town's population: the share is then by weight, and
+//       the missing are listed heaviest first, so the report names the next places to add.
+//       With a base, any fall fails: a recipe that loses a town has dropped its records.
+//
+// Any check may carry a `goal`: the owner's bet it serves (eval/goals.md says what each one
+// is). The report scores each goal apart, so "trades 0.4, general 0.9" shows which bet the
+// app is losing, instead of one mean that hides it.
 //
 // A predicate is {where?, match?, not?}. `where` is fangorn's own whole-value,
 // case-insensitive match — the filter agents use. `match` maps a field (or several
@@ -37,17 +48,20 @@
 import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http";
 import { homedir } from "node:os";
+import { dirname, join } from "node:path";
 import { configure, loadShard, trimView } from "@fangorn-network/westmarch/shard";
-import { rolesFrom, textOf, titleOf, subtitleOf } from "@fangorn-network/westmarch/roles";
+import { rolesFrom, textOf, titleOf, subtitleOf, values } from "@fangorn-network/westmarch/roles";
 import { search, matches } from "@fangorn-network/westmarch/tools";
 import { EMBED_MODEL, embedQueryDirect } from "@fangorn-network/westmarch/embed";
 
 type Pred = { where?: Record<string, string>; match?: Record<string, string>; not?: Pred };
-type Check =
-    | { id: string; kind?: "search"; q: string; k?: number; expect: Pred; min?: number }
-    | { id: string; kind: "count"; where?: Record<string, string>; expect?: Pred; min?: number; max?: number }
-    | { id: string; kind: "records"; of?: Pred; expect: Pred; max_pct: number; hard?: boolean };
-type Result = { id: string; kind: "search" | "count" | "records" | "known"; score: number; pass: boolean; hard?: boolean; top?: string[] };
+type Check = { id: string; goal?: string } & (
+    | { kind?: "search"; q: string; k?: number; expect: Pred; min?: number }
+    | { kind: "count"; where?: Record<string, string>; expect?: Pred; min?: number; max?: number }
+    | { kind: "records"; of?: Pred; expect: Pred; max_pct: number; hard?: boolean }
+    | { kind: "coverage"; field: string; values: string[]; weights?: Record<string, number>; of?: Pred; min?: number });
+type Result = { id: string; kind: "search" | "count" | "records" | "coverage" | "known"; score: number; pass: boolean;
+                goal?: string; hard?: boolean; top?: string[] };
 type QueryVector = (q: string) => Promise<Float32Array | null>;
 
 /** A drop below this, in mean precision@k or known@10, is "worse". canton-corpus's number. */
@@ -74,19 +88,28 @@ async function grade(rows: Row[], roles: Roles, checks: Check[], qv: QueryVector
     const byId = new Map(rows.map((r) => [r.id, r]));
     const out: Result[] = [];
     for (const c of checks) {
+        const goal = c.goal ? { goal: c.goal } : {};
         if (c.kind === "count") {
             const ok = compile(c.expect);
             const n = rows.filter((r) => matches(r, c.where) && ok(r)).length;
-            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity) });
+            out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity), ...goal });
         } else if (c.kind === "records") {
             const of = rows.filter(compile(c.of)), ok = compile(c.expect);
             const pct = of.length ? round((100 * of.filter(ok).length) / of.length) : 0;
-            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard });
+            out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard, ...goal });
+        } else if (c.kind === "coverage") {
+            // The same whole-value, any-case reading `where` uses, so covered means an agent's filter finds it.
+            const held = new Set(rows.filter(compile(c.of)).flatMap((r) => values(r[c.field] ?? "").map((v) => v.toLowerCase())));
+            const w = (v: string) => c.weights?.[v] ?? 1;
+            const missing = c.values.filter((v) => !held.has(v.toLowerCase())).sort((a, b) => w(b) - w(a));
+            const total = c.values.reduce((a, v) => a + w(v), 0);
+            const score = total ? round(1 - missing.reduce((a, v) => a + w(v), 0) / total) : 1;
+            out.push({ id: c.id, kind: "coverage", score, pass: score >= (c.min ?? 0), top: missing.slice(0, 10), ...goal });
         } else {
             const k = c.k ?? 5, ok = compile(c.expect);
             const hits = search(rows, c.q, roles, { qv: await qv(c.q), limit: k }).map((h) => byId.get(h.id)!);
             const score = round(hits.filter(ok).length / k);
-            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)) });
+            out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)), ...goal });
         }
     }
     // ponytail: the pool is each view's own rows, so base and candidate sample different
@@ -110,6 +133,19 @@ async function grade(rows: Row[], roles: Roles, checks: Check[], qv: QueryVector
     return out;
 }
 
+/** Each goal's checks, scored apart: how many pass, and the mean of its searches and coverages. */
+function byGoal(results: Result[]) {
+    const out: Record<string, { checks: number; passing: number; search: number | null; coverage: number | null }> = {};
+    const mean = (xs: number[]) => (xs.length ? round(xs.reduce((a, x) => a + x, 0) / xs.length) : null);
+    for (const g of new Set(results.map((r) => r.goal).filter((g): g is string => !!g))) {
+        const rs = results.filter((r) => r.goal === g);
+        out[g] = { checks: rs.length, passing: rs.filter((r) => r.pass).length,
+                   search: mean(rs.filter((r) => r.kind === "search").map((r) => r.score)),
+                   coverage: mean(rs.filter((r) => r.kind === "coverage").map((r) => r.score)) };
+    }
+    return out;
+}
+
 /** The candidate against the base. Worse on any one rule is worse. */
 function compare(cand: Result[], base: Result[]) {
     const was = new Map(base.map((r) => [r.id, r.score]));
@@ -120,6 +156,7 @@ function compare(cand: Result[], base: Result[]) {
         // The ratchet: a question the base answered at all, the candidate must too.
         if (r.kind === "search" && b !== undefined && b > 0 && r.score === 0) failures.push(`${r.id}: answered before, nothing now`);
         if (r.hard && b !== undefined && r.score > b) failures.push(`${r.id}: ${b}% → ${r.score}%, and it may not rise`);
+        if (r.kind === "coverage" && b !== undefined && r.score < b) failures.push(`${r.id}: coverage ${b} → ${r.score}; missing ${r.top?.join(", ")}`);
     }
     // Aggregates, over the checks both sides ran: one question moving a rank is noise.
     const both = (kind: Result["kind"]) => cand.filter((r) => (kind === "known" ? r.id === "known@10" : r.kind === kind) && was.has(r.id));
@@ -150,8 +187,14 @@ async function main() {
         console.error("usage: westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]");
         process.exit(flags.help ? 0 : 2);
     }
-    const checks: Check[] = readFileSync(String(flags.golden ?? "eval/golden.jsonl"), "utf8")
+    const golden = String(flags.golden ?? "eval/golden.jsonl");
+    const checks: Check[] = readFileSync(golden, "utf8")
         .split("\n").filter((l) => l.trim() && !l.startsWith("//")).map((l) => JSON.parse(l));
+    // A coverage list kept in its own file: a list, or {value: weight}.
+    for (const c of checks) if (c.kind === "coverage" && typeof c.values === "string") {
+        const list: string[] | Record<string, number> = JSON.parse(readFileSync(join(dirname(golden), c.values), "utf8"));
+        Object.assign(c, Array.isArray(list) ? { values: list } : { values: Object.keys(list), weights: list });
+    }
     const known = Number(flags.known ?? 200);
 
     const { env } = await import("@huggingface/transformers");
@@ -172,7 +215,7 @@ async function main() {
     const failures = cmp?.failures ?? cand.results.filter((r) => !r.pass).map((r) => `${r.id}: ${r.score} is outside its bounds`);
     // What reproduces this grade: the model, the runtime, and the views (whose manifests hash every shard).
     const report = { model: EMBED_MODEL, node: process.version, platform: `${process.platform}-${process.arch}`,
-                     candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
+                     goals: byGoal(cand.results), candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
 
     if (flags.report) writeFileSync(String(flags.report), JSON.stringify(report, null, 1));
     if (flags.json) console.log(JSON.stringify(report, null, 1));
@@ -184,6 +227,8 @@ async function main() {
             console.log(`${r.pass ? " " : "✗"} ${r.kind.padEnd(7)} ${r.id.padEnd(32)} ${b === undefined ? "" : `${b} → `}${r.score}`);
         }
         for (const d of report.deltas) console.log(`\n${d.kind}: ${d.base} → ${d.cand}`);
+        for (const [g, s] of Object.entries(report.goals))
+            console.log(`\ngoal ${g}: ${s.passing}/${s.checks} passing${s.search === null ? "" : `, search ${s.search}`}${s.coverage === null ? "" : `, coverage ${s.coverage}`}`);
         console.log(`\n${report.verdict ?? (report.pass ? "pass" : "fail")}${failures.length ? `\n  ${failures.join("\n  ")}` : ""}`);
     }
     process.exit(report.pass ? 0 : 1);
@@ -234,6 +279,15 @@ async function selfcheck() {
     assert(scoped.score === 50, `\`of\` narrows the rows a record check counts: ${scoped.score}`);
     assert(r["known@10"].score === 1, "every title finds its own row");
 
+    // Coverage, and goals scored apart.
+    const [cov] = await grade(rows, roles, [{ id: "towns", kind: "coverage", field: "city", values: ["plover", "Stevens Point", "Wausau"], min: 0.5, goal: "towns" }], async () => null, 0);
+    assert(cov.score === 0.667 && cov.pass && cov.top?.[0] === "Wausau", `coverage is the share present, the rest listed: ${JSON.stringify(cov)}`);
+    const [pop] = await grade(rows, roles, [{ id: "pop", kind: "coverage", field: "city", values: ["Plover", "Wausau", "Adell"],
+        weights: { Plover: 30, Wausau: 60, Adell: 10 } }], async () => null, 0);
+    assert(pop.score === 0.3 && pop.top?.join() === "Wausau,Adell", `weighted coverage, the heaviest gap first: ${JSON.stringify(pop)}`);
+    const goals = byGoal([cov, { ...r.rezoning, goal: "towns" }, r.plover]);
+    assert(goals.towns.checks === 2 && goals.towns.coverage === 0.667 && goals.towns.search === 1 && !("undefined" in goals), `goals: ${JSON.stringify(goals)}`);
+
     // The verdict.
     const res = (id: string, score: number, kind: Result["kind"] = "search", extra = {}): Result => ({ id, kind, score, pass: true, ...extra });
     const base = [res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })];
@@ -246,5 +300,6 @@ async function selfcheck() {
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.2, "records", { hard: true })], base).verdict === "worse", "a hard record may not rise");
     assert(compare([res("q1", 0.8, "search", { pass: false }), res("q2", 0.4)], base).verdict === "worse", "a floor fails whatever the base did");
     assert(compare([res("q1", 0.8), res("q2", 0.4), res("q3", 0)], base).verdict === "unchanged", "a new question has no base to fall from");
+    assert(compare([res("cov", 0.9, "coverage")], [res("cov", 1, "coverage")]).verdict === "worse", "coverage may not fall");
     console.log("eval: selfcheck ok");
 }