@fangorn-network/westmarch 0.0.1-dev → 0.0.3-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/consume/apps.js +4 -2
- package/consume/corpora.js +1 -1
- package/consume/directory.js +1 -1
- package/consume/embed.js +1 -1
- package/consume/lint.js +1 -1
- package/consume/mcp-bin.js +6 -0
- package/consume/mcp.js +193 -34
- package/consume/ort-preload.js +25 -0
- package/consume/rank.js +1 -1
- package/consume/roles.js +6 -4
- package/consume/shard.js +1 -1
- package/consume/steam.js +1 -1
- package/consume/taste-doc.js +1 -1
- package/consume/taste.js +1 -1
- package/consume/tools.js +28 -5
- package/consume/ui.js +1 -1
- package/consume/view-tools.js +202 -0
- package/consume/x402.js +94 -0
- package/dist/ts/eval.js +244 -0
- package/docs/app-to-agent.md +595 -185
- package/package.json +24 -10
- package/publish/agent-card.js +12 -2
- package/publish/cohort.js +1 -1
- package/publish/demand.js +1 -1
- package/publish/enrich.js +1 -1
- package/publish/envelope.js +1 -1
- package/publish/eval.ts +250 -0
- package/publish/graph.js +1 -1
- package/publish/manifest.js +1 -1
- package/publish/reactions.js +4 -2
- package/publish/settle.js +1 -1
- package/publish/ship.js +462 -0
- package/publish/terms.js +1 -1
- package/publish/vault.js +1 -1
- package/publish/view-cli.js +39 -0
- package/publish/view.js +434 -0
- package/publish/westmarch.d.ts +35 -0
- package/site/agent.js +44 -0
- package/site/dist/assets/__vite-browser-external-CKnVkMRn.js +1 -0
- package/site/dist/assets/embed.worker-CQqmvS52.js +1 -0
- package/site/dist/assets/index-CaHGhiKK.js +8 -0
- package/site/dist/assets/ort-wasm-simd-threaded.asyncify-DMmc6YqF.wasm +0 -0
- package/site/dist/assets/preload-helper-uBIymjUX.js +1 -0
- package/site/dist/assets/roles-C9V_uDr_.js +1 -0
- package/site/dist/assets/roles-YDRjv_PY.js +1 -0
- package/site/dist/assets/rolldown-runtime-Dd_uD5pT.js +1 -0
- package/site/dist/assets/transformers.web-BgUm7F_-.js +46 -0
- package/site/dist/assets/transformers.web-u9uSoh5E.js +37 -0
- package/site/dist/index.html +20 -0
- package/site/feed.js +62 -0
- package/site/store.js +87 -0
- package/site/worker.js +103 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fangorn-network/westmarch",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.3-dev",
|
|
4
4
|
"description": "Publisher and consumer surfaces for a Fangorn app: shape a graph, publish it, read it back as a searchable, agent-drivable catalog",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"./rank": "./consume/rank.js",
|
|
14
14
|
"./apps": "./consume/apps.js",
|
|
15
15
|
"./agent-card": "./publish/agent-card.js",
|
|
16
|
+
"./view": "./publish/view.js",
|
|
16
17
|
"./roles": "./consume/roles.js",
|
|
17
18
|
"./directory": "./consume/directory.js",
|
|
18
19
|
"./taste": "./consume/taste.js",
|
|
@@ -29,17 +30,20 @@
|
|
|
29
30
|
"./demand": "./publish/demand.js",
|
|
30
31
|
"./steam": "./consume/steam.js",
|
|
31
32
|
"./tools": "./consume/tools.js",
|
|
33
|
+
"./x402": "./consume/x402.js",
|
|
32
34
|
"./package.json": "./package.json"
|
|
33
35
|
},
|
|
34
36
|
"scripts": {
|
|
35
|
-
"
|
|
37
|
+
"build": "tsc -p .",
|
|
38
|
+
"prepare": "tsc -p .",
|
|
39
|
+
"test": "node publish/terms.js && node publish/reactions.js && node publish/cohort.js && node publish/vault.js && node publish/graph.js && node publish/envelope.js && node publish/demand.js && node publish/settle.js && node publish/manifest.js && node publish/agent-card.js && node publish/view.js && node publish/ship.js --selfcheck && node publish/eval.ts --selfcheck && node publish/cli.js --selfcheck && node publish/enrich.js && node consume/embed.js && node consume/rank.js && node consume/apps.js && node consume/mcp.js --selfcheck && node consume/roles.js && node consume/shard.js && node consume/directory.js && node consume/taste.js && node consume/steam.js && node consume/taste-doc.js && node consume/corpora.js && node consume/ui.js && node consume/lint.js && node consume/tools.js && node consume/x402.js && node site/feed.js && node site/store.js",
|
|
36
40
|
"lint": "node consume/lint.js"
|
|
37
41
|
},
|
|
38
42
|
"peerDependencies": {
|
|
39
|
-
"@huggingface/transformers": "^4.2.0",
|
|
40
|
-
"viem": "^2.55.1",
|
|
41
43
|
"@fangorn-network/sdk": "^2026.9.22-dev",
|
|
42
|
-
"@
|
|
44
|
+
"@huggingface/transformers": "^4.2.0",
|
|
45
|
+
"@noble/curves": "^1.9.1",
|
|
46
|
+
"viem": "^2.55.1"
|
|
43
47
|
},
|
|
44
48
|
"peerDependenciesMeta": {
|
|
45
49
|
"@huggingface/transformers": {
|
|
@@ -47,19 +51,29 @@
|
|
|
47
51
|
}
|
|
48
52
|
},
|
|
49
53
|
"devDependencies": {
|
|
50
|
-
"@huggingface/transformers": "^4.2.0",
|
|
51
|
-
"viem": "^2.56.3",
|
|
52
54
|
"@fangorn-network/sdk": "^2026.9.22-dev",
|
|
53
|
-
"@
|
|
55
|
+
"@huggingface/transformers": "^4.2.0",
|
|
56
|
+
"@noble/curves": "^1.9.1",
|
|
57
|
+
"typescript": "^5.9.3",
|
|
58
|
+
"viem": "^2.56.3"
|
|
54
59
|
},
|
|
55
60
|
"bin": {
|
|
56
61
|
"westmarch-publish": "./publish/cli.js",
|
|
57
|
-
"fangorn-mcp": "./consume/mcp.js"
|
|
62
|
+
"fangorn-mcp": "./consume/mcp.js",
|
|
63
|
+
"westmarch-view": "./publish/view-cli.js",
|
|
64
|
+
"westmarch-ship": "./publish/ship.js",
|
|
65
|
+
"westmarch-eval": "./dist/ts/eval.js"
|
|
58
66
|
},
|
|
59
67
|
"files": [
|
|
60
68
|
"consume",
|
|
61
69
|
"publish",
|
|
62
|
-
"docs/app-to-agent.md"
|
|
70
|
+
"docs/app-to-agent.md",
|
|
71
|
+
"site/agent.js",
|
|
72
|
+
"site/dist",
|
|
73
|
+
"site/feed.js",
|
|
74
|
+
"site/worker.js",
|
|
75
|
+
"site/store.js",
|
|
76
|
+
"dist/ts"
|
|
63
77
|
],
|
|
64
78
|
"engines": {
|
|
65
79
|
"node": ">=20"
|
package/publish/agent-card.js
CHANGED
|
@@ -73,7 +73,7 @@ export function agentCard({ name, description, url, version = "0.0.0", tools = [
|
|
|
73
73
|
|
|
74
74
|
/** The params `discoverApp` reads, validated the same way it validates them.
|
|
75
75
|
* A malformed card should fail the build, not the first reader. */
|
|
76
|
-
export function fangornExtension({ app, fromBlock, namespaces = [], views = [], config = FangornConfig }) {
|
|
76
|
+
export function fangornExtension({ app, fromBlock, namespaces = [], views = [], paid, config = FangornConfig }) {
|
|
77
77
|
if (!app) throw new Error("fangorn.app is required");
|
|
78
78
|
const id = HEX32.test(app) ? app : hashAppId(app);
|
|
79
79
|
const from = String(fromBlock ?? "");
|
|
@@ -83,6 +83,14 @@ export function fangornExtension({ app, fromBlock, namespaces = [], views = [],
|
|
|
83
83
|
const u = new URL(v); // throws on garbage
|
|
84
84
|
if (u.protocol !== "https:" && u.protocol !== "http:") throw new Error(`fangorn.views: not http(s): ${v}`);
|
|
85
85
|
}
|
|
86
|
+
// `paid`: records sold over x402. `url` has an `{id}` for the record's id; `price` is in
|
|
87
|
+
// the token's base units. Readers only offer to buy; the 402 itself is the quote.
|
|
88
|
+
if (paid) {
|
|
89
|
+
const u = new URL(paid.url.replace("{id}", "x"));
|
|
90
|
+
if (!/^https?:$/.test(u.protocol) || !paid.url.includes("{id}")) throw new Error(`fangorn.paid.url must be http(s) with {id}: ${paid.url}`);
|
|
91
|
+
if (!/^\d+$/.test(String(paid.price))) throw new Error(`fangorn.paid.price must be base units, got ${paid.price}`);
|
|
92
|
+
if (!/^0x[0-9a-fA-F]{40}$/.test(paid.asset ?? "")) throw new Error(`fangorn.paid.asset must be a token address`);
|
|
93
|
+
}
|
|
86
94
|
return {
|
|
87
95
|
uri: FANGORN_APP_EXTENSION,
|
|
88
96
|
description: "The Fangorn app this agent fronts.",
|
|
@@ -92,12 +100,14 @@ export function fangornExtension({ app, fromBlock, namespaces = [], views = [],
|
|
|
92
100
|
appRegistry: config.appRegistryContractAddress,
|
|
93
101
|
dataRegistry: config.dataRegistryContractAddress,
|
|
94
102
|
appId: id, fromBlock: from, namespaces, views,
|
|
103
|
+
...(paid ? { paid: { url: paid.url, price: String(paid.price), asset: paid.asset, network: paid.network,
|
|
104
|
+
symbol: paid.symbol ?? "USDC", decimals: paid.decimals ?? 6, description: paid.description ?? "" } } : {}),
|
|
95
105
|
},
|
|
96
106
|
};
|
|
97
107
|
}
|
|
98
108
|
|
|
99
109
|
// ── self-check: `node publish/agent-card.js` ────────────────────────────────
|
|
100
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
110
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/agent-card.js")) {
|
|
101
111
|
const eq = (a, b, m) => { if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error(`${m}: ${JSON.stringify(a)} != ${JSON.stringify(b)}`); };
|
|
102
112
|
const throws = (f, re, m) => { try { f(); } catch (e) { if (re.test(e.message)) return; throw e; } throw new Error(`must throw: ${m}`); };
|
|
103
113
|
|
package/publish/cohort.js
CHANGED
|
@@ -247,7 +247,7 @@ export function readout(total, cohort, { minSupport = MIN_SUPPORT } = {}) {
|
|
|
247
247
|
}
|
|
248
248
|
|
|
249
249
|
// ── self-check: `node publish/cohort.js` ──────────────────────────────────
|
|
250
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
250
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/cohort.js")) {
|
|
251
251
|
const vec = (a, b) => { const v = new Float32Array(STAT_DIM); v[0] = a; v[1] = b; return v; };
|
|
252
252
|
const log = (n, seed) => Array.from({ length: n }, (_, i) => ({
|
|
253
253
|
id: `row-${seed}-${i}`, corpus: "films", reaction: i % 4 === 3 ? "skip" : "like",
|
package/publish/demand.js
CHANGED
|
@@ -383,7 +383,7 @@ export function brief(rep, { title = "demand" } = {}) {
|
|
|
383
383
|
}
|
|
384
384
|
|
|
385
385
|
// ── self-check: `node publish/demand.js` ────────────────────────────────────
|
|
386
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
386
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/demand.js")) {
|
|
387
387
|
const { keypair, statistics, contribute, aggregate, readout } = await import("./cohort.js");
|
|
388
388
|
|
|
389
389
|
// A shelf with two genuine regions and a hole between them. 128 dims so the
|
package/publish/enrich.js
CHANGED
|
@@ -155,7 +155,7 @@ export function enrichText({ name, path, desc, mime, abs = null }) {
|
|
|
155
155
|
// in the same browser that already runs the embedder at publish time.
|
|
156
156
|
|
|
157
157
|
// ── self-check: `node server/enrich.js` — no files, no model ──────────────────
|
|
158
|
-
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
158
|
+
if (import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/enrich.js")) {
|
|
159
159
|
const { deflateSync } = await import("node:zlib");
|
|
160
160
|
const eq = (got, want, msg) => { if (got !== want) throw new Error(`${msg}: got ${JSON.stringify(got)}`); };
|
|
161
161
|
|
package/publish/envelope.js
CHANGED
|
@@ -157,7 +157,7 @@ export async function deleteResource({ workerUrl, uploadToken, resourceId, chunk
|
|
|
157
157
|
// every resourceId ever minted by fangorn.tv is derived by this function, and a
|
|
158
158
|
// change here that shifts one byte makes ciphertext already in R2 unreachable from
|
|
159
159
|
// the resource anyone paid for. If this fails, the extraction broke money.
|
|
160
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
160
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/envelope.js")) {
|
|
161
161
|
const owner = `0x${"11".repeat(20)}`;
|
|
162
162
|
const uid = "Show/S1/a.mp4";
|
|
163
163
|
|
package/publish/eval.ts
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// westmarch-eval: does a view answer what its app says it should?
|
|
3
|
+
//
|
|
4
|
+
// westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]
|
|
5
|
+
//
|
|
6
|
+
// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
|
|
7
|
+
// westmarch-eval --selfcheck
|
|
8
|
+
//
|
|
9
|
+
// It grades the RECIPE — how records are shaped, which fields are embedded, how rows
|
|
10
|
+
// rank — never the data. New records are facts, and a view is not worse for holding
|
|
11
|
+
// them. So: build a base view (main's recipe) and a candidate (the change) from the
|
|
12
|
+
// same data, on the same machine, and run both through here. The exit code is the gate.
|
|
13
|
+
//
|
|
14
|
+
// Every check runs the reader's own code (consume/tools.js `search`, the same model),
|
|
15
|
+
// so a score is what an agent calling fangorn-mcp would get, not what a test harness
|
|
16
|
+
// with its own ranker would.
|
|
17
|
+
//
|
|
18
|
+
// golden.jsonl holds one check per line. It is data, not code, so a change proposed
|
|
19
|
+
// by an agent cannot also edit how it is graded:
|
|
20
|
+
//
|
|
21
|
+
// {"id":"plover","q":"Plover village board","expect":{"where":{"city":"Plover"}},"min":0.8}
|
|
22
|
+
// search: the share of the top k (default 5) that satisfies `expect`
|
|
23
|
+
// {"id":"portage-solar","kind":"count","where":{"county":"Portage"},"expect":{"match":{"heading":"solar"}},"min":3}
|
|
24
|
+
// count: rows the agent's `where` reaches AND that satisfy `expect` — can a filter get to them at all
|
|
25
|
+
// {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ (St|Ave)\\b/"}},"max_pct":0.5,"hard":true}
|
|
26
|
+
// records: the % of rows that satisfy `expect`, lower is better; `hard`: any rise over the base fails.
|
|
27
|
+
// `of` (a predicate) narrows which rows count, e.g. only meeting items have a date to miss.
|
|
28
|
+
//
|
|
29
|
+
// A predicate is {where?, match?, not?}. `where` is fangorn's own whole-value,
|
|
30
|
+
// case-insensitive match — the filter agents use. `match` maps a field (or several
|
|
31
|
+
// joined by `|`) to a regex, case-insensitive unless written /like this/flags. `not`
|
|
32
|
+
// negates another predicate.
|
|
33
|
+
//
|
|
34
|
+
// Built in, with no labels: known items. N rows are sampled (seeded, so reruns agree)
|
|
35
|
+
// and searched by their own title; `known@1`/`known@10` are how often they come back.
|
|
36
|
+
|
|
37
|
+
import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
|
|
38
|
+
import { createServer } from "node:http";
|
|
39
|
+
import { homedir } from "node:os";
|
|
40
|
+
import { configure, loadShard, trimView } from "@fangorn-network/westmarch/shard";
|
|
41
|
+
import { rolesFrom, textOf, titleOf, subtitleOf } from "@fangorn-network/westmarch/roles";
|
|
42
|
+
import { search, matches } from "@fangorn-network/westmarch/tools";
|
|
43
|
+
import { EMBED_MODEL, embedQueryDirect } from "@fangorn-network/westmarch/embed";
|
|
44
|
+
|
|
45
|
+
type Pred = { where?: Record<string, string>; match?: Record<string, string>; not?: Pred };
|
|
46
|
+
type Check =
|
|
47
|
+
| { id: string; kind?: "search"; q: string; k?: number; expect: Pred; min?: number }
|
|
48
|
+
| { id: string; kind: "count"; where?: Record<string, string>; expect?: Pred; min?: number; max?: number }
|
|
49
|
+
| { id: string; kind: "records"; of?: Pred; expect: Pred; max_pct: number; hard?: boolean };
|
|
50
|
+
type Result = { id: string; kind: "search" | "count" | "records" | "known"; score: number; pass: boolean; hard?: boolean; top?: string[] };
|
|
51
|
+
type QueryVector = (q: string) => Promise<Float32Array | null>;
|
|
52
|
+
|
|
53
|
+
/** A drop below this, in mean precision@k or known@10, is "worse". canton-corpus's number. */
|
|
54
|
+
const TOLERANCE = 0.05;
|
|
55
|
+
|
|
56
|
+
const regex = (s: string) => {
|
|
57
|
+
const m = /^\/(.*)\/([a-z]*)$/s.exec(s);
|
|
58
|
+
return m ? new RegExp(m[1], m[2]) : new RegExp(s, "i");
|
|
59
|
+
};
|
|
60
|
+
const fieldText = (v: unknown) => (Array.isArray(v) ? v.join(" ") : String(v ?? ""));
|
|
61
|
+
|
|
62
|
+
/** A predicate, compiled once: it runs against every row. */
|
|
63
|
+
function compile(p: Pred = {}): (r: Row) => boolean {
|
|
64
|
+
const res = Object.entries(p.match ?? {}).map(([k, re]) => [k.split("|"), regex(re)] as const);
|
|
65
|
+
const not = p.not ? compile(p.not) : null;
|
|
66
|
+
return (r) => matches(r, p.where)
|
|
67
|
+
&& res.every(([fs, re]) => re.test(fs.map((f) => fieldText(r[f])).join(" ")))
|
|
68
|
+
&& !not?.(r);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const round = (x: number) => Math.round(x * 1000) / 1000;
|
|
72
|
+
|
|
73
|
+
async function grade(rows: Row[], roles: Roles, checks: Check[], qv: QueryVector, known: number): Promise<Result[]> {
|
|
74
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
75
|
+
const out: Result[] = [];
|
|
76
|
+
for (const c of checks) {
|
|
77
|
+
if (c.kind === "count") {
|
|
78
|
+
const ok = compile(c.expect);
|
|
79
|
+
const n = rows.filter((r) => matches(r, c.where) && ok(r)).length;
|
|
80
|
+
out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity) });
|
|
81
|
+
} else if (c.kind === "records") {
|
|
82
|
+
const of = rows.filter(compile(c.of)), ok = compile(c.expect);
|
|
83
|
+
const pct = of.length ? round((100 * of.filter(ok).length) / of.length) : 0;
|
|
84
|
+
out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard });
|
|
85
|
+
} else {
|
|
86
|
+
const k = c.k ?? 5, ok = compile(c.expect);
|
|
87
|
+
const hits = search(rows, c.q, roles, { qv: await qv(c.q), limit: k }).map((h) => byId.get(h.id)!);
|
|
88
|
+
const score = round(hits.filter(ok).length / k);
|
|
89
|
+
out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)) });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// ponytail: the pool is each view's own rows, so base and candidate sample different
|
|
93
|
+
// rows once their records differ. TOLERANCE absorbs it; sample ids from the base if it doesn't.
|
|
94
|
+
const pool = rows.filter((r) => titleOf(r, roles).split(/\s+/).length >= 4);
|
|
95
|
+
const n = Math.min(known, pool.length);
|
|
96
|
+
if (n) {
|
|
97
|
+
let seed = 42;
|
|
98
|
+
const rand = () => (seed = (Math.imul(seed, 1103515245) + 12345) >>> 0) / 2 ** 32;
|
|
99
|
+
// Many rows share a title ("Approve the minutes"): the same title and subtitle is a find.
|
|
100
|
+
const same = (a: Row, b: Row) => a.id === b.id || (titleOf(a, roles) === titleOf(b, roles) && subtitleOf(a, roles) === subtitleOf(b, roles));
|
|
101
|
+
let at1 = 0, at10 = 0;
|
|
102
|
+
for (let i = 0; i < n; i++) {
|
|
103
|
+
const r = pool[Math.floor(rand() * pool.length)], t = titleOf(r, roles);
|
|
104
|
+
const at = search(rows, t, roles, { qv: await qv(t), limit: 10 }).findIndex((h) => same(byId.get(h.id)!, r));
|
|
105
|
+
at1 += +(at === 0); at10 += +(at >= 0);
|
|
106
|
+
}
|
|
107
|
+
out.push({ id: "known@1", kind: "known", score: round(at1 / n), pass: true },
|
|
108
|
+
{ id: "known@10", kind: "known", score: round(at10 / n), pass: true });
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** The candidate against the base. Worse on any one rule is worse. */
|
|
114
|
+
function compare(cand: Result[], base: Result[]) {
|
|
115
|
+
const was = new Map(base.map((r) => [r.id, r.score]));
|
|
116
|
+
const failures: string[] = [];
|
|
117
|
+
for (const r of cand) {
|
|
118
|
+
const b = was.get(r.id);
|
|
119
|
+
if (!r.pass) failures.push(`${r.id}: ${r.score} is outside its bounds`);
|
|
120
|
+
// The ratchet: a question the base answered at all, the candidate must too.
|
|
121
|
+
if (r.kind === "search" && b !== undefined && b > 0 && r.score === 0) failures.push(`${r.id}: answered before, nothing now`);
|
|
122
|
+
if (r.hard && b !== undefined && r.score > b) failures.push(`${r.id}: ${b}% → ${r.score}%, and it may not rise`);
|
|
123
|
+
}
|
|
124
|
+
// Aggregates, over the checks both sides ran: one question moving a rank is noise.
|
|
125
|
+
const both = (kind: Result["kind"]) => cand.filter((r) => (kind === "known" ? r.id === "known@10" : r.kind === kind) && was.has(r.id));
|
|
126
|
+
const mean = (rs: Result[], f: (r: Result) => number) => rs.reduce((a, r) => a + f(r), 0) / rs.length;
|
|
127
|
+
const deltas = (["search", "known"] as const).flatMap((kind) => {
|
|
128
|
+
const rs = both(kind);
|
|
129
|
+
return rs.length ? [{ kind, cand: round(mean(rs, (r) => r.score)), base: round(mean(rs, (r) => was.get(r.id)!)) }] : [];
|
|
130
|
+
});
|
|
131
|
+
for (const d of deltas) if (d.cand < d.base - TOLERANCE) failures.push(`${d.kind}: ${d.base} → ${d.cand}`);
|
|
132
|
+
const verdict = failures.length ? "worse" : deltas.some((d) => d.cand > d.base + TOLERANCE) ? "better" : "unchanged";
|
|
133
|
+
return { verdict, deltas, failures };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ── CLI ───────────────────────────────────────────────────────────────────────
|
|
137
|
+
|
|
138
|
+
if (process.argv[2] === "--selfcheck") await selfcheck();
|
|
139
|
+
else if (realpathSync(process.argv[1]).match(/\/eval\.[jt]s$/)) await main();
|
|
140
|
+
|
|
141
|
+
async function main() {
|
|
142
|
+
const flags: Record<string, string | boolean> = {};
|
|
143
|
+
const pos: string[] = [];
|
|
144
|
+
for (let i = 2; i < process.argv.length; i++) {
|
|
145
|
+
const m = /^--([^=]+)(?:=(.*))?$/.exec(process.argv[i]);
|
|
146
|
+
if (!m) pos.push(process.argv[i]);
|
|
147
|
+
else flags[m[1]] = m[2] ?? (["json", "help"].includes(m[1]) ? true : process.argv[++i]);
|
|
148
|
+
}
|
|
149
|
+
if (flags.help || pos.length !== 1) {
|
|
150
|
+
console.error("usage: westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]");
|
|
151
|
+
process.exit(flags.help ? 0 : 2);
|
|
152
|
+
}
|
|
153
|
+
const checks: Check[] = readFileSync(String(flags.golden ?? "eval/golden.jsonl"), "utf8")
|
|
154
|
+
.split("\n").filter((l) => l.trim() && !l.startsWith("//")).map((l) => JSON.parse(l));
|
|
155
|
+
const known = Number(flags.known ?? 200);
|
|
156
|
+
|
|
157
|
+
const { env } = await import("@huggingface/transformers");
|
|
158
|
+
env.cacheDir = `${homedir()}/.cache/fangorn-mcp/models`; // shared with fangorn-mcp and westmarch-view
|
|
159
|
+
const roles = new Map<string, Roles>();
|
|
160
|
+
configure({ onManifests: (ms, v) => { roles.set(v, rolesFrom(ms)); }, rowText: (f, v) => textOf(f, roles.get(v)!) });
|
|
161
|
+
// No fallback to word matching: a grade the reader would not get is not a grade.
|
|
162
|
+
const vectors = new Map<string, Promise<Float32Array>>();
|
|
163
|
+
const qv = (q: string) => vectors.get(q) ?? vectors.set(q, embedQueryDirect(q)).get(q)!;
|
|
164
|
+
const run = async (at: string) => {
|
|
165
|
+
const v = existsSync(at) ? await serve(at) : trimView(at), rows = await loadShard(v);
|
|
166
|
+
return { view: at, rows: rows.length, results: await grade(rows, roles.get(v)!, checks, qv, known) };
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const cand = await run(pos[0]);
|
|
170
|
+
const base = flags.base ? await run(String(flags.base)) : null;
|
|
171
|
+
const cmp = base ? compare(cand.results, base.results) : null;
|
|
172
|
+
const failures = cmp?.failures ?? cand.results.filter((r) => !r.pass).map((r) => `${r.id}: ${r.score} is outside its bounds`);
|
|
173
|
+
// What reproduces this grade: the model, the runtime, and the views (whose manifests hash every shard).
|
|
174
|
+
const report = { model: EMBED_MODEL, node: process.version, platform: `${process.platform}-${process.arch}`,
|
|
175
|
+
candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
|
|
176
|
+
|
|
177
|
+
if (flags.report) writeFileSync(String(flags.report), JSON.stringify(report, null, 1));
|
|
178
|
+
if (flags.json) console.log(JSON.stringify(report, null, 1));
|
|
179
|
+
else {
|
|
180
|
+
const was = new Map(base?.results.map((r) => [r.id, r.score]));
|
|
181
|
+
console.log(`${cand.view}: ${cand.rows} rows${base ? ` (base ${base.view}: ${base.rows} rows)` : ""}\n`);
|
|
182
|
+
for (const r of cand.results) {
|
|
183
|
+
const b = was.get(r.id);
|
|
184
|
+
console.log(`${r.pass ? " " : "✗"} ${r.kind.padEnd(7)} ${r.id.padEnd(32)} ${b === undefined ? "" : `${b} → `}${r.score}`);
|
|
185
|
+
}
|
|
186
|
+
for (const d of report.deltas) console.log(`\n${d.kind}: ${d.base} → ${d.cand}`);
|
|
187
|
+
console.log(`\n${report.verdict ?? (report.pass ? "pass" : "fail")}${failures.length ? `\n ${failures.join("\n ")}` : ""}`);
|
|
188
|
+
}
|
|
189
|
+
process.exit(report.pass ? 0 : 1);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** A view directory over HTTP on a free port, for this process's lifetime. */
|
|
193
|
+
async function serve(dir: string): Promise<string> {
|
|
194
|
+
const srv = createServer((q, s) => {
|
|
195
|
+
try { s.end(readFileSync(`${dir}${decodeURIComponent(new URL(q.url ?? "/", "http://x").pathname)}`)); }
|
|
196
|
+
catch { s.statusCode = 404; s.end(); }
|
|
197
|
+
});
|
|
198
|
+
await new Promise<void>((ok) => srv.listen(0, "127.0.0.1", ok));
|
|
199
|
+
srv.unref();
|
|
200
|
+
const a = srv.address();
|
|
201
|
+
return `http://127.0.0.1:${typeof a === "object" && a ? a.port : 0}`;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function selfcheck() {
|
|
205
|
+
const assert = (ok: unknown, msg: string) => { if (!ok) throw new Error(msg); };
|
|
206
|
+
const roles = rolesFrom([{ role_map: { title: "heading", tags: ["city"], text: ["text"] } }]);
|
|
207
|
+
const rows: Row[] = [
|
|
208
|
+
{ id: "a", heading: "Rezoning of the lot on Main", city: "Plover", text: "Rezoning of the lot on Main. Approved 5-0." },
|
|
209
|
+
{ id: "b", heading: "Liquor license for the Brewing Company", city: "Plover", text: "Liquor license for the Brewing Company." },
|
|
210
|
+
{ id: "c", heading: "Water main replacement on Post Road", city: "Stevens Point", county: ["Portage", "Wood"], text: "Water main replacement on Post Road, 1200 Post Rd." },
|
|
211
|
+
{ id: "d", heading: "Snow removal contract for the winter", city: "Stevens Point", text: "Snow removal contract for the winter season." },
|
|
212
|
+
];
|
|
213
|
+
|
|
214
|
+
// Predicates: fangorn's where (whole value, any case, arrays), field-joined regex, /flags/, not.
|
|
215
|
+
assert(compile({ where: { city: "plover" } })(rows[0]), "where matches case-insensitively");
|
|
216
|
+
assert(!compile({ where: { city: "Plov" } })(rows[0]), "where matches whole values only");
|
|
217
|
+
assert(compile({ where: { county: "wood" } })(rows[2]), "where matches inside an array");
|
|
218
|
+
assert(compile({ match: { "heading|text": "approved" } })(rows[0]), "match joins the fields it names");
|
|
219
|
+
assert(!compile({ match: { heading: "/rezoning/" } })(rows[0]), "/re/ is case-sensitive");
|
|
220
|
+
assert(!compile({ match: { heading: "rezon" }, not: { where: { city: "Plover" } } })(rows[0]), "not negates");
|
|
221
|
+
|
|
222
|
+
const checks: Check[] = [
|
|
223
|
+
{ id: "rezoning", q: "rezoning", expect: { match: { heading: "rezon" } }, k: 1, min: 1 },
|
|
224
|
+
{ id: "plover", kind: "count", where: { city: "Plover" }, min: 2, max: 2 },
|
|
225
|
+
{ id: "wells", kind: "count", where: { city: "Plover" }, expect: { match: { heading: "well" } } },
|
|
226
|
+
{ id: "addresses", kind: "records", expect: { match: { text: "/\\b\\d{2,6} [A-Z]\\w+ (Rd|St)\\b/" } }, max_pct: 30, hard: true },
|
|
227
|
+
];
|
|
228
|
+
const r = Object.fromEntries((await grade(rows, roles, checks, async () => null, 2)).map((x) => [x.id, x]));
|
|
229
|
+
assert(r.rezoning.score === 1 && r.rezoning.pass, `lexical search finds the rezoning: ${JSON.stringify(r.rezoning)}`);
|
|
230
|
+
assert(r.plover.score === 2 && r.plover.pass, "count reaches both Plover rows");
|
|
231
|
+
assert(r.wells.score === 0 && !r.wells.pass, "a count with nothing to reach fails (min defaults to 1)");
|
|
232
|
+
assert(r.addresses.score === 25 && r.addresses.pass, `records is a percentage: ${r.addresses.score}`);
|
|
233
|
+
const [scoped] = await grade(rows, roles, [{ id: "s", kind: "records", of: { where: { city: "Stevens Point" } }, expect: { match: { text: "Rd" } }, max_pct: 100 }], async () => null, 0);
|
|
234
|
+
assert(scoped.score === 50, `\`of\` narrows the rows a record check counts: ${scoped.score}`);
|
|
235
|
+
assert(r["known@10"].score === 1, "every title finds its own row");
|
|
236
|
+
|
|
237
|
+
// The verdict.
|
|
238
|
+
const res = (id: string, score: number, kind: Result["kind"] = "search", extra = {}): Result => ({ id, kind, score, pass: true, ...extra });
|
|
239
|
+
const base = [res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })];
|
|
240
|
+
assert(compare(base, base).verdict === "unchanged", "same is unchanged");
|
|
241
|
+
assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.93, "known"), res("pii", 0.1, "records", { hard: true })], base).verdict === "unchanged", "within tolerance is unchanged");
|
|
242
|
+
assert(compare([res("q1", 1), res("q2", 0.6), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })], base).verdict === "better", "a mean rise is better");
|
|
243
|
+
assert(compare([res("q1", 0.6), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })], base).verdict === "worse", "a mean drop is worse");
|
|
244
|
+
const ratchet = compare([res("q1", 1), res("q2", 0), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })], base);
|
|
245
|
+
assert(ratchet.verdict === "worse" && /answered before/.test(ratchet.failures[0]), "the ratchet holds even when the mean does not drop");
|
|
246
|
+
assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.2, "records", { hard: true })], base).verdict === "worse", "a hard record may not rise");
|
|
247
|
+
assert(compare([res("q1", 0.8, "search", { pass: false }), res("q2", 0.4)], base).verdict === "worse", "a floor fails whatever the base did");
|
|
248
|
+
assert(compare([res("q1", 0.8), res("q2", 0.4), res("q3", 0)], base).verdict === "unchanged", "a new question has no base to fall from");
|
|
249
|
+
console.log("eval: selfcheck ok");
|
|
250
|
+
}
|
package/publish/graph.js
CHANGED
|
@@ -183,7 +183,7 @@ export function treeFromGraph(contents) {
|
|
|
183
183
|
// `process` doesn't exist in the browser, and consumers import nest() from here
|
|
184
184
|
// to build a viewer tree — so this guard has to survive being evaluated by a
|
|
185
185
|
// browser.
|
|
186
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
186
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/graph.js")) {
|
|
187
187
|
// sond3r's pinned names, which is also the case with every option exercised.
|
|
188
188
|
const SOND3R = {
|
|
189
189
|
file: "video",
|
package/publish/manifest.js
CHANGED
|
@@ -90,7 +90,7 @@ export function manifestFromTree(files, { defaultPrice = "1000", newUid: uid = n
|
|
|
90
90
|
}
|
|
91
91
|
|
|
92
92
|
// ── self-check: `node publish/manifest.js` ───────────────────────────────────
|
|
93
|
-
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
93
|
+
if (import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/manifest.js")) {
|
|
94
94
|
const assert = (c, m) => { if (!c) throw new Error(m); };
|
|
95
95
|
|
|
96
96
|
// PINNED. sond3r's manifests are already in R2 under this exact key; if this
|
package/publish/reactions.js
CHANGED
|
@@ -53,7 +53,9 @@
|
|
|
53
53
|
import { packVec } from "../consume/embed.js";
|
|
54
54
|
import { lineage } from "./terms.js";
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
// 32, not 8: on Kingsfoil the top 3 of 23 views held 92% of the true top 10 with 32
|
|
57
|
+
// centroids and 86% with 8, and 8 missed whole topics (ALS → neurology).
|
|
58
|
+
export const COVERAGE_K = 32;
|
|
57
59
|
export const COVERAGE_DIM = 128;
|
|
58
60
|
/** Fewest reactions that may stand behind one published centroid. */
|
|
59
61
|
export const MIN_PER_CENTROID = 8;
|
|
@@ -235,7 +237,7 @@ export function reactionCorpus(events = [], {
|
|
|
235
237
|
}
|
|
236
238
|
|
|
237
239
|
// ── self-check: `node publish/reactions.js` ────────────────────────────────
|
|
238
|
-
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}`) {
|
|
240
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/reactions.js")) {
|
|
239
241
|
const { splitPayment } = await import("./terms.js");
|
|
240
242
|
const A = (n) => `0x${String(n).repeat(40).slice(0, 40)}`;
|
|
241
243
|
const vec = (a, b) => { const v = new Float32Array(256); v[0] = a; v[1] = b; return v; };
|
package/publish/settle.js
CHANGED
|
@@ -70,7 +70,7 @@ export async function encryptAndUpload({ file, resourceId, workerUrl, uploadToke
|
|
|
70
70
|
// under test here is the ENCRYPTOR: chunk addressing, the AAD index binding, the
|
|
71
71
|
// ragged tail, and that pack/unpack round trip. An app's full publisher→buyer
|
|
72
72
|
// round trip belongs in that app, against its own buyer.
|
|
73
|
-
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
73
|
+
if (import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/settle.js")) {
|
|
74
74
|
const { writeFileSync, rmSync } = await import("node:fs");
|
|
75
75
|
const { unseal } = await import("@fangorn-network/sdk");
|
|
76
76
|
const { x25519 } = await import("@noble/curves/ed25519");
|