@fangorn-network/westmarch 0.0.4-dev → 0.0.6-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/consume/explore.js +243 -0
- package/consume/mcp.js +30 -2
- package/consume/taste.js +51 -1
- package/consume/view-tools.js +1 -1
- package/dist/ts/eval.js +61 -7
- package/docs/app-to-agent.md +15 -17
- package/package.json +7 -4
- package/plugin/scripts/fx.mjs +268 -0
- package/publish/eval.ts +67 -12
- package/publish/ship.js +1 -1
- package/publish/view.js +19 -9
- package/site/agent.js +23 -0
- package/site/dist/assets/index-DxAeTEch.js +8 -0
- package/site/dist/index.html +5 -4
- package/site/dist/theme.css +7 -0
- package/site/store.js +10 -3
- package/site/dist/assets/index-CaHGhiKK.js +0 -8
package/README.md
CHANGED
|
@@ -5,6 +5,9 @@ Everything you need to get started building apps with Fangorn.
|
|
|
5
5
|
**Start here:** [docs/app-to-agent.md](docs/app-to-agent.md) takes a static site from nothing
|
|
6
6
|
to a Fangorn app and ERC-8004 agent that any agent can discover, verify and drive.
|
|
7
7
|
|
|
8
|
+
**Just want to query apps from Claude or another MCP client?** [docs/mcp.md](docs/mcp.md)
|
|
9
|
+
installs `fangorn-mcp` in one command.
|
|
10
|
+
|
|
8
11
|
|
|
9
12
|
**For Publishers** — shape a tree into a commit graph, describe what's in it, seal and upload the bytes. Needs `fangorn`. Runs anywhere: a script, a server, a browser.
|
|
10
13
|
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// The Fangorn explorer: every verified app, what it is about, and what the
|
|
2
|
+
// ERC-8004 registries say about it — read live, with no bake of its own.
|
|
3
|
+
//
|
|
4
|
+
// Everything here is already published by someone else:
|
|
5
|
+
// AppRegistry `AppAgentChanged` which apps exist and where their cards are
|
|
6
|
+
// each card name, tools, views; checked against the chain
|
|
7
|
+
// each view's catalog rows, domains, and coverage centroids (what it is about)
|
|
8
|
+
// ERC-8004 identity `Registered` the agent minted for the app
|
|
9
|
+
// ERC-8004 reputation feedback count and average per agent
|
|
10
|
+
//
|
|
11
|
+
// Both registries answer the whole range in one `eth_getLogs` on Arbitrum
|
|
12
|
+
// Sepolia (39 and ~1.5k logs today), so the page and the MCP tool read the same
|
|
13
|
+
// chain the same way, and a new app shows up on the next read.
|
|
14
|
+
//
|
|
15
|
+
// `client` is a viem PublicClient (getLogs, readContract): passed in, so this
|
|
16
|
+
// runs in a tab and in node, and the self-check needs no chain.
|
|
17
|
+
//
|
|
18
|
+
// ponytail: one getLogs per registry from block 0. An RPC that caps log ranges
|
|
19
|
+
// will refuse it; window the scan then (the SDK's getLogsInWindows).
|
|
20
|
+
|
|
21
|
+
import { parseAbi, parseAbiItem } from "viem";
|
|
22
|
+
import { APP_EXTENSION } from "./apps.js";
|
|
23
|
+
import { survey, comparable } from "./directory.js";
|
|
24
|
+
import { rankDomains, zFloor, lexScore } from "./rank.js";
|
|
25
|
+
|
|
26
|
+
export const CHAIN = {
|
|
27
|
+
chainId: 421614,
|
|
28
|
+
appRegistry: "0x11d228c4774af3d9cae3b4b6874a12576a1a83ec",
|
|
29
|
+
identity: "0x8004A818BFB912233c491871b3d84c89A494BD9e",
|
|
30
|
+
reputation: "0x8004B663056A597Dffe9eCcC1965A193B7388713",
|
|
31
|
+
scan: (agentId) => `https://8004scan.io/agents/arbitrum-sepolia/${agentId}`,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const APP_AGENT_CHANGED = parseAbiItem("event AppAgentChanged(bytes32 indexed app_id, string agent_uri)");
|
|
35
|
+
const REGISTERED = parseAbiItem("event Registered(uint256 indexed agentId, string agentURI, address indexed owner)");
|
|
36
|
+
const APPS = parseAbi(["function getAppOwner(bytes32) view returns (address)"]);
|
|
37
|
+
const REP = parseAbi([
|
|
38
|
+
"function getClients(uint256) view returns (address[])",
|
|
39
|
+
"function getSummary(uint256, address[], string, string) view returns (uint64, int128, uint8)",
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
/** A link someone else wrote, shown to a reader: https or nothing. */
|
|
43
|
+
const https = (u) => { try { const x = new URL(String(u)); return x.protocol === "https:" ? x.href : null; } catch { return null; } };
|
|
44
|
+
const slug = (name) => String(name).toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-|-$/g, "").slice(0, 24) || "app";
|
|
45
|
+
|
|
46
|
+
/** A card is listed only if it names this chain's AppRegistry and the app that
|
|
47
|
+
* bound it — the checks `discoverApp` makes. The binding itself is the log we
|
|
48
|
+
* read it from: we scanned to `latest`, so the last write per app is current. */
|
|
49
|
+
export function verifyCard(card, appId, chain = CHAIN) {
|
|
50
|
+
const p = card?.capabilities?.extensions?.find((e) => e.uri === APP_EXTENSION)?.params;
|
|
51
|
+
if (!p) return "no fangorn app extension";
|
|
52
|
+
if (String(p.appId).toLowerCase() !== appId.toLowerCase()) return "card names another app";
|
|
53
|
+
if (p.chainId !== chain.chainId || String(p.appRegistry).toLowerCase() !== chain.appRegistry.toLowerCase()) return "card is for another registry";
|
|
54
|
+
if (!/^\d+$/.test(String(p.fromBlock))) return "bad fromBlock";
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The agent an app registered. `fangorn app agent` mints it and then binds the
|
|
60
|
+
* card, from the owner's wallet, so it is the owner's latest `Registered` at or
|
|
61
|
+
* before the binding.
|
|
62
|
+
*
|
|
63
|
+
* ponytail: paired by owner and block, not by reading the registration file
|
|
64
|
+
* (IPFS gateways are too slow and flaky for a page load). An owner that
|
|
65
|
+
* rebinds with --skip-register after minting another app's agent gets that
|
|
66
|
+
* agent; fetch tokenURI and match its A2A endpoint if that starts happening.
|
|
67
|
+
*/
|
|
68
|
+
export function pairAgent(registered, owner, bindBlock) {
|
|
69
|
+
let best = null;
|
|
70
|
+
for (const r of registered) {
|
|
71
|
+
if (r.owner.toLowerCase() !== owner.toLowerCase() || r.block > bindBlock) continue;
|
|
72
|
+
if (!best || r.block > best.block || (r.block === best.block && r.agentId > best.agentId)) best = r;
|
|
73
|
+
}
|
|
74
|
+
return best?.agentId ?? null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Retried: dropping a verified app over a blip hides it from everyone who
|
|
78
|
+
* loads the page in that second. Seen 2026-09-28: headless Chrome in WSL got
|
|
79
|
+
* intermittent net::ERR_SSL_PROTOCOL_ERROR from one pages.dev host (curl never
|
|
80
|
+
* did); three tries do not always get past that one. */
|
|
81
|
+
async function json(url, timeoutMs = 8000, tries = 3) {
|
|
82
|
+
try {
|
|
83
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
84
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
85
|
+
return await res.json();
|
|
86
|
+
} catch (e) {
|
|
87
|
+
if (tries <= 1) throw e;
|
|
88
|
+
await new Promise((ok) => setTimeout(ok, 600 * (4 - tries))); // 600ms, then 1.2s
|
|
89
|
+
return json(url, timeoutMs, tries - 1);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function reputation(client, agentId, chain) {
|
|
94
|
+
try {
|
|
95
|
+
const clients = await client.readContract({ address: chain.reputation, abi: REP, functionName: "getClients", args: [agentId] });
|
|
96
|
+
if (!clients.length) return { feedback: 0, average: null };
|
|
97
|
+
const [count, value, decimals] = await client.readContract({ address: chain.reputation, abi: REP, functionName: "getSummary", args: [agentId, clients, "", ""] });
|
|
98
|
+
return { feedback: Number(count), average: Number(value) / 10 ** decimals, clients: clients.length };
|
|
99
|
+
} catch (e) {
|
|
100
|
+
return { error: e?.shortMessage ?? e?.message ?? String(e) };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Every verified app, with its 8004 agent, reputation, and catalog.
|
|
106
|
+
* Unverified apps are dropped, not listed; `dropped` says why, for debugging.
|
|
107
|
+
*/
|
|
108
|
+
export async function readDirectory(client, { chain = CHAIN } = {}) {
|
|
109
|
+
const [bindLogs, regLogs] = await Promise.all([
|
|
110
|
+
client.getLogs({ address: chain.appRegistry, event: APP_AGENT_CHANGED, fromBlock: 0n, toBlock: "latest" }),
|
|
111
|
+
client.getLogs({ address: chain.identity, event: REGISTERED, fromBlock: 0n, toBlock: "latest" }),
|
|
112
|
+
]);
|
|
113
|
+
const registered = regLogs.map((l) => ({ agentId: l.args.agentId, owner: l.args.owner, block: l.blockNumber }));
|
|
114
|
+
const latest = new Map(); // logs come oldest first: the last write wins
|
|
115
|
+
for (const l of bindLogs) latest.set(l.args.app_id, { uri: l.args.agent_uri, block: l.blockNumber });
|
|
116
|
+
|
|
117
|
+
// Two passes: verify every card (small), then read catalogs (Quorum's is
|
|
118
|
+
// 1.7 MB, Kingsfoil has 23). Only a card failure drops an app; an owner or
|
|
119
|
+
// catalog that fails leaves it listed with less to say about it.
|
|
120
|
+
const dropped = [];
|
|
121
|
+
const verified = (await Promise.all([...latest].map(async ([appId, { uri, block }]) => {
|
|
122
|
+
if (!uri) return null; // unbound on purpose
|
|
123
|
+
try {
|
|
124
|
+
if (!/^https:\/\//.test(uri)) throw new Error("card is not https");
|
|
125
|
+
const card = await json(uri);
|
|
126
|
+
const why = verifyCard(card, appId, chain);
|
|
127
|
+
if (why) throw new Error(why);
|
|
128
|
+
return { appId, uri, block, card };
|
|
129
|
+
} catch (e) { dropped.push({ card: uri, why: e?.message ?? String(e) }); return null; }
|
|
130
|
+
}))).filter(Boolean);
|
|
131
|
+
|
|
132
|
+
const apps = await Promise.all(verified.map(async ({ appId, uri, block, card }) => {
|
|
133
|
+
const p = card.capabilities.extensions.find((e) => e.uri === APP_EXTENSION).params;
|
|
134
|
+
const owner = await client.readContract({ address: chain.appRegistry, abi: APPS, functionName: "getAppOwner", args: [appId] }).catch(() => null);
|
|
135
|
+
const agentId = owner && pairAgent(registered, owner, block);
|
|
136
|
+
const { corpora, unreachable } = await survey((p.views ?? []).filter((v) => /^https:\/\//.test(v)));
|
|
137
|
+
const name = typeof card.name === "string" && card.name.trim() ? card.name.trim() : `App ${appId.slice(0, 10)}`;
|
|
138
|
+
return {
|
|
139
|
+
app: slug(name), name, description: typeof card.description === "string" ? card.description : "",
|
|
140
|
+
site: https(card.url), card: uri, appId, owner,
|
|
141
|
+
boundAt: Number(block), fromBlock: Number(p.fromBlock),
|
|
142
|
+
tools: (card.skills ?? []).map((s) => s.id),
|
|
143
|
+
agent: agentId == null ? null : { id: Number(agentId), scan: chain.scan(agentId) },
|
|
144
|
+
reputation: agentId == null ? null : await reputation(client, agentId, chain),
|
|
145
|
+
// null, not 0, when no catalog answered: "0 records" would claim it was counted.
|
|
146
|
+
rows: corpora.length ? corpora.reduce((n, c) => n + (c.rows || 0), 0) : null,
|
|
147
|
+
unreachable: unreachable.length,
|
|
148
|
+
domains: corpora.map((c) => ({ name: c.domain, description: c.description, rows: c.rows, coverage: c.coverage, model: c.model })),
|
|
149
|
+
};
|
|
150
|
+
}));
|
|
151
|
+
apps.sort((a, b) => a.name.localeCompare(b.name));
|
|
152
|
+
return { apps, dropped };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** A calibration knob, not a law: nomic-embed-text-v1.5 at 256d, 5 apps
|
|
156
|
+
* (2026-09-28). Every on-topic top app scored >= 0.545 ("a cozy game" ->
|
|
157
|
+
* Sidequest), every off-topic top <= 0.52 ("bitcoin" -> Audius). Re-measure
|
|
158
|
+
* with the explorer's check when the model or the network changes much. */
|
|
159
|
+
export const MIN_AFFINITY = 0.53;
|
|
160
|
+
|
|
161
|
+
// ponytail: a short English list, enough for questions put to a directory.
|
|
162
|
+
const FILLER = new Set("the and for are was were has have had not but with from that this these those what which where when who whom whose why how can could would should will shall may might must about into onto over under than then there their them they you your our any all some more most much many app apps find show tell learn know want need good best reputable".split(" "));
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Rank a directory against a question. An app scores as its best domain's
|
|
166
|
+
* coverage affinity (what its data is about, not what its blurb says). With no
|
|
167
|
+
* `qv` (no embedder yet), name and description are matched by words.
|
|
168
|
+
*
|
|
169
|
+
* `relevant`: mean + 1σ of this query's scores, and at least MIN_AFFINITY.
|
|
170
|
+
* Unlike findCorpora the top app is NOT always relevant: "which app has stock
|
|
171
|
+
* prices" must be able to answer "none", or the explorer vouches for a
|
|
172
|
+
* government-records app on a finance question.
|
|
173
|
+
*/
|
|
174
|
+
export function rankApps(apps, query, { qv = null, model = null } = {}) {
|
|
175
|
+
const q = String(query ?? "").trim().toLowerCase();
|
|
176
|
+
if (!q) return apps.map((a) => ({ ...a, score: null, relevant: null, ranked_by: null }));
|
|
177
|
+
// Filler dropped, or "where can I learn about X" matches every blurb with "about" in it.
|
|
178
|
+
const content = q.split(/[^\p{L}\p{N}]+/u).filter((w) => w.length > 2 && !FILLER.has(w)).join(" ");
|
|
179
|
+
const words = (a) => (content ? lexScore({ text: `${a.name} ${a.description} ${a.domains.map((d) => d.description ?? "").join(" ")}` }, content) : 0);
|
|
180
|
+
let scored;
|
|
181
|
+
if (qv) {
|
|
182
|
+
scored = apps.map((a) => {
|
|
183
|
+
const doms = rankDomains(a.domains.filter((d) => comparable(d, model)), qv);
|
|
184
|
+
const best = doms.find((d) => d.affinity != null);
|
|
185
|
+
// No coverage (its catalog did not answer, or predates coverage):
|
|
186
|
+
// its own blurb, by words, rather than silently ranking it last.
|
|
187
|
+
if (!best) return { ...a, score: null, matched: null, ranked_by: "words", wordHit: words(a) > 0 };
|
|
188
|
+
return { ...a, score: best.affinity, matched: best.name, ranked_by: "meaning" };
|
|
189
|
+
});
|
|
190
|
+
} else {
|
|
191
|
+
scored = apps.map((a) => ({ ...a, score: words(a), ranked_by: "words" }));
|
|
192
|
+
}
|
|
193
|
+
scored.sort((a, b) => (b.score ?? -Infinity) - (a.score ?? -Infinity));
|
|
194
|
+
const s = scored.filter((a) => a.ranked_by === "meaning").map((a) => a.score);
|
|
195
|
+
const floor = qv ? Math.max(zFloor(s), MIN_AFFINITY) : 1e-9;
|
|
196
|
+
return scored.map(({ wordHit, ...a }) => ({ ...a, relevant: wordHit ?? (a.score != null && a.score >= floor) }));
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** What a caller sees: the ranking minus the centroids. Quorum alone has 29
|
|
200
|
+
* domains, so an app reports how many, and which one matched. */
|
|
201
|
+
export const brief = ({ domains, ...a }) => ({ ...a, domains: domains.length });
|
|
202
|
+
|
|
203
|
+
// ── self-check: `node consume/explore.js` ──────────────────────────────────
|
|
204
|
+
if (typeof process !== "undefined" && import.meta.url === `file://${process.argv[1]}` && process.argv[1].endsWith("/explore.js")) {
|
|
205
|
+
const j = (x) => JSON.stringify(x, (_, v) => (typeof v === "bigint" ? `${v}n` : v));
|
|
206
|
+
const eq = (a, b, m) => { if (j(a) !== j(b)) throw new Error(`${m}: ${j(a)} != ${j(b)}`); };
|
|
207
|
+
const O = "0x00000000000000000000000000000000000000aa", P = "0x00000000000000000000000000000000000000bb";
|
|
208
|
+
const reg = [
|
|
209
|
+
{ agentId: 1n, owner: O, block: 10n }, { agentId: 2n, owner: P, block: 11n },
|
|
210
|
+
{ agentId: 3n, owner: O, block: 20n }, { agentId: 4n, owner: O, block: 99n },
|
|
211
|
+
];
|
|
212
|
+
eq(pairAgent(reg, O, 21n), 3n, "owner's latest agent at or before the binding");
|
|
213
|
+
eq(pairAgent(reg, P.toUpperCase().replace("0X", "0x"), 50n), 2n, "owner compared case-insensitively");
|
|
214
|
+
eq(pairAgent(reg, O, 5n), null, "no agent before the binding");
|
|
215
|
+
|
|
216
|
+
const A = "0x" + "a".repeat(64);
|
|
217
|
+
const card = (params) => ({ capabilities: { extensions: [{ uri: APP_EXTENSION, params }] } });
|
|
218
|
+
const good = { appId: A, chainId: CHAIN.chainId, appRegistry: CHAIN.appRegistry.toUpperCase().replace("0X", "0x"), fromBlock: "7" };
|
|
219
|
+
eq(verifyCard(card(good), A), null, "a good card verifies");
|
|
220
|
+
eq(verifyCard(card({ ...good, appId: "0x" + "b".repeat(64) }), A), "card names another app", "wrong app");
|
|
221
|
+
eq(verifyCard(card({ ...good, chainId: 1 }), A), "card is for another registry", "wrong chain");
|
|
222
|
+
eq(verifyCard({}, A), "no fangorn app extension", "no extension");
|
|
223
|
+
|
|
224
|
+
const cov = (v) => ({ vectors: [v] });
|
|
225
|
+
const apps = [
|
|
226
|
+
{ name: "Tunes", description: "music tracks", domains: [{ name: "tracks", coverage: cov([1, 0, 0]) }] },
|
|
227
|
+
{ name: "Sky", description: "weather alerts", domains: [{ name: "alerts", coverage: cov([0, 1, 0]) }, { name: "old", coverage: null }] },
|
|
228
|
+
{ name: "Law", description: "meetings", domains: [{ name: "minutes", coverage: cov([0, 0, 1]) }] },
|
|
229
|
+
{ name: "Empty", description: "", domains: [] },
|
|
230
|
+
];
|
|
231
|
+
const r = rankApps(apps, "rain", { qv: [0.1, 1, 0] });
|
|
232
|
+
eq(r.map((a) => a.name), ["Sky", "Tunes", "Law", "Empty"], "ranked by best domain affinity, unscored last");
|
|
233
|
+
eq([r[0].relevant, r[0].matched, r[3].relevant], [true, "alerts", false], "top is relevant, unscored is not");
|
|
234
|
+
const blind = rankApps([{ ...apps[1], domains: [] }, apps[0], apps[2]], "weather alerts", { qv: [0.1, 1, 0] });
|
|
235
|
+
eq(blind.map((a) => [a.name, a.relevant, a.ranked_by]).at(-1), ["Sky", true, "words"], "an app with no coverage still matches on its own words");
|
|
236
|
+
eq(rankApps([{ ...apps[1], description: "all about the sky", domains: [] }], "where can I learn about music", { qv: [1, 0, 0] })[0].relevant, false, "filler words never match");
|
|
237
|
+
eq(rankApps(apps, "stock prices", { qv: [-1, -1, -1] }).filter((a) => a.relevant).length, 0, "a question no app covers matches none");
|
|
238
|
+
eq(rankApps(apps, "music").map((a) => [a.name, a.relevant]).slice(0, 2), [["Tunes", true], ["Sky", false]], "word fallback");
|
|
239
|
+
eq(rankApps(apps, "").map((a) => a.score), [null, null, null, null], "no query: listed, not ranked");
|
|
240
|
+
eq([https("javascript:alert(1)"), https("http://x.test"), https("https://x.test/a")], [null, null, "https://x.test/a"], "only https links reach a reader");
|
|
241
|
+
eq(brief({ name: "x", domains: [{ name: "d", coverage: 1 }] }), { name: "x", domains: 1 }, "brief drops centroids");
|
|
242
|
+
console.log("explore.js self-check ok — cards verified, agents paired by owner and block, apps ranked by coverage or words");
|
|
243
|
+
}
|
package/consume/mcp.js
CHANGED
|
@@ -284,6 +284,11 @@ const META = [
|
|
|
284
284
|
{ name: "list-apps",
|
|
285
285
|
description: "List the Fangorn apps registered on chain whose agent cards verify: name, description, card URL, and the WebMCP tools each page offers. Filter with `query`. Call open-app on one to use its tools.",
|
|
286
286
|
inputSchema: { type: "object", properties: { query: { type: "string", description: "Case-insensitive match on name or description" } } } },
|
|
287
|
+
{ name: "find-apps",
|
|
288
|
+
description: "Find Fangorn apps for a question, ranked by what their data is about (e.g. \"where can I learn about weather?\", \"music\"). Each app carries its site, agent card, ERC-8004 agent with an 8004scan link, ERC-8004 reputation (feedback count and average), record count and bound block. Only apps whose cards verify on chain. `relevant: true` marks the ones worth opening; the rest are sorted, not relevant. With no query, lists every app. Then open-app.",
|
|
289
|
+
inputSchema: { type: "object", properties: {
|
|
290
|
+
query: { type: "string", description: "What the person is looking for, in their words" },
|
|
291
|
+
limit: { type: "integer", minimum: 1, maximum: 50, default: 10 } } } },
|
|
287
292
|
{ name: "open-app",
|
|
288
293
|
description: "Verify an app's card against the chain and add its tools to this server as `<app>__<tool>`. An app that publishes views gets fast data tools (describe, search, get, similar, count, browse) with no browser. `page: true` also opens the app's page in a browser for its own WebMCP tools (slower). `app` is a name from list-apps, an appId, or a card URL.",
|
|
289
294
|
inputSchema: { type: "object", properties: { app: { type: "string" }, page: { type: "boolean", description: "Also load the page's own WebMCP tools in a browser" } }, required: ["app"] } },
|
|
@@ -295,6 +300,7 @@ const META = [
|
|
|
295
300
|
/**
|
|
296
301
|
* The server. `deps` is the outside world, so the self-check can fake it:
|
|
297
302
|
* findApps() → { apps: [{ name, desc, card, appId }], rejected }
|
|
303
|
+
* exploreApps(query) → { apps: ranked explore.js entries, ranked_by }
|
|
298
304
|
* findByName(name) → { card, appId } | null (optional: a direct registry read)
|
|
299
305
|
* resolveCard(url) → { card, page, verified, views }
|
|
300
306
|
* viewTools(app) → { tools, call(name, args) } | null (no browser)
|
|
@@ -377,6 +383,11 @@ export function handlers(deps) {
|
|
|
377
383
|
unverified: rejected.length,
|
|
378
384
|
});
|
|
379
385
|
},
|
|
386
|
+
"find-apps": async ({ query, limit = 10 }) => {
|
|
387
|
+
const { apps, ranked_by } = await deps.exploreApps(query);
|
|
388
|
+
return text({ query: query ?? null, ranked_by, apps: apps.slice(0, Math.min(50, Math.max(1, limit)))
|
|
389
|
+
.map((a) => ({ ...a, open: open.has(a.app) })) });
|
|
390
|
+
},
|
|
380
391
|
"open-app": async ({ app, page }) => {
|
|
381
392
|
const o = await openApp(app, { page });
|
|
382
393
|
return text({ app: o.slug, name: o.name, verified: o.verified,
|
|
@@ -419,6 +430,7 @@ if (process.argv[2] === "--selfcheck") {
|
|
|
419
430
|
let opened = 0;
|
|
420
431
|
const deps = {
|
|
421
432
|
findApps: async () => ({ apps: [{ name: "Kings Foil", desc: "trials", card: "https://k.test/card", appId: "0xk", tools: ["greet"] }], rejected: [{}] }),
|
|
433
|
+
exploreApps: async (q) => ({ ranked_by: q ? "meaning" : null, apps: [{ app: "kings-foil", name: "Kings Foil", relevant: true }, { app: "b", name: "B", relevant: false }] }),
|
|
422
434
|
resolveCard: async () => ({ card: { name: "Kings Foil" }, page: "https://k.test/p", verified: true }),
|
|
423
435
|
openTab: async () => (opened++ ? "ev" : "dead"), // the first tab dies
|
|
424
436
|
tabTools: async () => [{ name: "greet", description: "Hi.", inputSchema: { type: "object" } }],
|
|
@@ -435,10 +447,13 @@ if (process.argv[2] === "--selfcheck") {
|
|
|
435
447
|
|
|
436
448
|
rpc(1, "initialize", {}); rpc(2, "tools/list"); call(3, "list-apps", { query: "TRIAL" }); call(4, "kings-foil__greet"); await wait();
|
|
437
449
|
assert(res(1).result.capabilities.tools.listChanged, "advertises list_changed");
|
|
438
|
-
assert(res(2).result.tools.length ===
|
|
450
|
+
assert(res(2).result.tools.length === 4, "only the meta tools before anything is open");
|
|
439
451
|
assert(body(3).apps[0].app === "kings-foil" && body(3).unverified === 1, "list-apps filters and slugs");
|
|
440
452
|
assert(res(4).result.isError && /not open/.test(res(4).result.content[0].text), "an unopened app's tool is an error");
|
|
441
453
|
|
|
454
|
+
call(30, "find-apps", { query: "trials", limit: 1 }); await wait();
|
|
455
|
+
assert(body(30).ranked_by === "meaning" && body(30).apps.length === 1 && body(30).apps[0].open === false, "find-apps ranks, caps, and says what is open");
|
|
456
|
+
|
|
442
457
|
call(5, "open-app", { app: "Kings Foil" }); await wait();
|
|
443
458
|
call(6, "open-app", { app: "https://k.test/card" }); await wait();
|
|
444
459
|
assert(body(5).tools[0].name === "greet" && opened === 1, "open once, by name or by card URL");
|
|
@@ -490,7 +505,7 @@ const fromBlock = fb >= 0 ? BigInt(argv[fb + 1]) : DEFAULT_FROM_BLOCK;
|
|
|
490
505
|
const headed = argv.includes("--headed");
|
|
491
506
|
const ci = argv.indexOf("--cdp");
|
|
492
507
|
const cdp = ci >= 0 ? argv[ci + 1] : process.env.FANGORN_MCP_CDP || undefined;
|
|
493
|
-
let found, foundAt = 0;
|
|
508
|
+
let found, foundAt = 0, explored, exploredAt = 0;
|
|
494
509
|
const deps = {
|
|
495
510
|
// ponytail: cached for a minute. The scan is a few RPC calls plus a fetch per card.
|
|
496
511
|
findApps: async () => {
|
|
@@ -504,6 +519,19 @@ const deps = {
|
|
|
504
519
|
}
|
|
505
520
|
return found;
|
|
506
521
|
},
|
|
522
|
+
// ponytail: cached for a minute, like findApps. Two getLogs, a card and catalog per app.
|
|
523
|
+
exploreApps: async (query) => {
|
|
524
|
+
if (!explored || Date.now() - exploredAt > 60_000) {
|
|
525
|
+
const [ex, { createPublicClient, http }, { arbitrumSepolia }] = await Promise.all([import("./explore.js"), import("viem"), import("viem/chains")]);
|
|
526
|
+
explored = { ex, dir: await ex.readDirectory(createPublicClient({ chain: arbitrumSepolia, transport: http() })) };
|
|
527
|
+
exploredAt = Date.now();
|
|
528
|
+
}
|
|
529
|
+
const { ex, dir } = explored;
|
|
530
|
+
const { EMBED_MODEL } = await import("./embed.js");
|
|
531
|
+
const qv = query?.trim() ? await vt.queryVector(query) : null;
|
|
532
|
+
const apps = ex.rankApps(dir.apps, query, { qv, model: EMBED_MODEL }).map(ex.brief);
|
|
533
|
+
return { apps, ranked_by: apps[0]?.ranked_by ?? null };
|
|
534
|
+
},
|
|
507
535
|
findByName: async (name) => {
|
|
508
536
|
const [f, { appId }] = await Promise.all([fangorn(), import("@fangorn-network/sdk")]);
|
|
509
537
|
const reg = f.getAppRegistry();
|
package/consume/taste.js
CHANGED
|
@@ -143,6 +143,42 @@ export function recommend(rows, t, { limit = 10, exclude = new Set(), lambda = 0
|
|
|
143
143
|
.map(({ row, s }) => ({ row, score: Number(s.toFixed(4)) }));
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
+
/** The four knobs a person turns on "For you", each 0–1: [default, what it means].
|
|
147
|
+
* Tools publish these descriptions as-is, so a person and an agent turn the same dials. */
|
|
148
|
+
export const KNOBS = {
|
|
149
|
+
lookahead: [0.35, "how far past what you like, toward where your recent likes are heading (the kernel's drift)"],
|
|
150
|
+
variety: [0.3, "how different the picks are from each other"],
|
|
151
|
+
surprise: [0, "how much chance reorders the list; 0 is strict best-first"],
|
|
152
|
+
reach: [0, "how far out from the closest matches to start: 0 is nearest, 1 skips the nearest ~400"],
|
|
153
|
+
};
|
|
154
|
+
const dial = (x, d) => Math.min(1, Math.max(0, Number.isFinite(Number(x)) ? Number(x) : d));
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* "For you" with the knobs: the kernel over likes and dislikes (`{ id, title, vector }`,
|
|
158
|
+
* newest last), then `reach` skips the nearest, `surprise` reorders with seeded Gumbel noise
|
|
159
|
+
* (same seed, same list), and `variety` sets how hard MMR spreads the picks. What Sidequest
|
|
160
|
+
* shipped, lifted so every page and its `discover` tool rank the same way.
|
|
161
|
+
*/
|
|
162
|
+
export function discover(rows, likes = [], dislikes = [], { limit = 20, seed = 1, exclude = new Set(), ...k } = {}) {
|
|
163
|
+
const knobs = Object.fromEntries(Object.entries(KNOBS).map(([n, [d]]) => [n, dial(k[n], d)]));
|
|
164
|
+
const t = taste(likes, dislikes, { drift: knobs.lookahead });
|
|
165
|
+
if (!t) return { taste: null, knobs: { ...knobs, seed }, picks: [] };
|
|
166
|
+
const skip = new Set([...exclude, ...likes.map((x) => x.id), ...dislikes.map((x) => x.id)]);
|
|
167
|
+
let ranked = rows.filter((r) => r.vector && !skip.has(r.id)).map((row) => ({ row, s: score(row, t) })).sort((a, b) => b.s - a.s);
|
|
168
|
+
const start = Math.round(knobs.reach * Math.min(400, ranked.length / 4));
|
|
169
|
+
ranked = ranked.slice(start, start + 300);
|
|
170
|
+
if (knobs.surprise) {
|
|
171
|
+
let a = (Number(seed) >>> 0) || 1;
|
|
172
|
+
const rand = () => { a = (a + 0x6d2b79f5) >>> 0; let x = Math.imul(a ^ (a >>> 15), 1 | a); x ^= x + Math.imul(x ^ (x >>> 7), 61 | x); return ((x ^ (x >>> 14)) >>> 0) / 4294967296; };
|
|
173
|
+
ranked = ranked.map((p) => ({ ...p, s: p.s + 0.04 * knobs.surprise * -Math.log(-Math.log(rand() * 0.999998 + 1e-6)) })).sort((x, y) => y.s - x.s);
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
taste: { from: t.from, rejected: t.rejected, heading: !!t.v, n: t.n },
|
|
177
|
+
knobs: { ...knobs, seed },
|
|
178
|
+
picks: diversify(ranked, { limit, lambda: 1 - 0.6 * knobs.variety }).map(({ row, s }) => ({ row, score: Number(s.toFixed(4)) })),
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
146
182
|
/**
|
|
147
183
|
* A taste, small enough to paste.
|
|
148
184
|
*
|
|
@@ -346,5 +382,19 @@ if (typeof process !== "undefined" && import.meta.url === `file://${process.argv
|
|
|
346
382
|
if (twice.n !== 3 || twice.from.join() !== kept.from.join()) throw new Error("a second reload must change nothing");
|
|
347
383
|
}
|
|
348
384
|
|
|
349
|
-
|
|
385
|
+
// discover: the knobs move the list, a seed pins it, and nothing voted on comes back.
|
|
386
|
+
{
|
|
387
|
+
const pool = Array.from({ length: 40 }, (_, i) => { const a = (i * Math.PI) / 80; return { id: `g${i}`, title: `g${i}`, vector: [Math.cos(a), Math.sin(a)], norm: 1 }; });
|
|
388
|
+
const likes = [pool[0]], dislikes = [pool[39]];
|
|
389
|
+
const ids = (o) => o.picks.map((p) => p.row.id).join();
|
|
390
|
+
const base = discover(pool, likes, dislikes, { limit: 5 });
|
|
391
|
+
if (base.picks[0].row.id !== "g1") throw new Error(`nearest first by default: ${ids(base)}`);
|
|
392
|
+
if (base.picks.some((p) => p.row.id === "g0" || p.row.id === "g39")) throw new Error("discover must not return what was voted on");
|
|
393
|
+
if (discover(pool, likes, [], { limit: 5, reach: 1 }).picks[0].row.id === "g1") throw new Error("reach must skip the nearest");
|
|
394
|
+
const s1 = discover(pool, likes, [], { limit: 5, surprise: 1, seed: 7 });
|
|
395
|
+
if (ids(s1) !== ids(discover(pool, likes, [], { limit: 5, surprise: 1, seed: 7 }))) throw new Error("the same seed must give the same list");
|
|
396
|
+
if (discover(pool, [], [], {}).picks.length) throw new Error("no likes, no picks");
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
console.log(`taste.js self-check ok — discover's knobs, mu/v/q, drift leans forward, rejection steers, portable in ${size}B, ranks a pool it never saw, MMR keeps a run of near-duplicates from filling the list, provenance and heading survive the restore collapse and do not drift on reload`);
|
|
350
400
|
}
|
package/consume/view-tools.js
CHANGED
|
@@ -51,7 +51,7 @@ export const warm = ({ cacheDir } = {}) => (embedder ??= (async () => {
|
|
|
51
51
|
export let warmError = null;
|
|
52
52
|
// A model on disk loads in ~0.5s and is worth the wait. A download is 131MB, so
|
|
53
53
|
// until it lands, search ranks by words and says so, rather than stall ~10s.
|
|
54
|
-
const queryVector = async (q) => {
|
|
54
|
+
export const queryVector = async (q) => {
|
|
55
55
|
if (!ready && downloading) return null;
|
|
56
56
|
return (await warm()) ? embedQueryDirect(q).catch(() => null) : null;
|
|
57
57
|
};
|
package/dist/ts/eval.js
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
//
|
|
4
4
|
// westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]
|
|
5
5
|
//
|
|
6
|
-
// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
|
|
7
6
|
// westmarch-eval --selfcheck
|
|
8
7
|
//
|
|
8
|
+
// A view is a URL or a directory (westmarch-ship --local writes one), served here as a reader fetches it.
|
|
9
|
+
//
|
|
9
10
|
// It grades the RECIPE — how records are shaped, which fields are embedded, how rows
|
|
10
11
|
// rank — never the data. New records are facts, and a view is not worse for holding
|
|
11
12
|
// them. So: build a base view (main's recipe) and a candidate (the change) from the
|
|
@@ -25,6 +26,16 @@
|
|
|
25
26
|
// {"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ (St|Ave)\\b/"}},"max_pct":0.5,"hard":true}
|
|
26
27
|
// records: the % of rows that satisfy `expect`, lower is better; `hard`: any rise over the base fails.
|
|
27
28
|
// `of` (a predicate) narrows which rows count, e.g. only meeting items have a date to miss.
|
|
29
|
+
// {"id":"basin-towns","kind":"coverage","field":"city","values":["Plover","Stevens Point"],"min":0.9}
|
|
30
|
+
// coverage: the share of `values` that some row holds in `field`, and the missing ones.
|
|
31
|
+
// `values` may instead be a path (relative to the golden file) to a JSON list, or to an
|
|
32
|
+
// object {value: weight} such as each town's population: the share is then by weight, and
|
|
33
|
+
// the missing are listed heaviest first, so the report names the next places to add.
|
|
34
|
+
// With a base, any fall fails: a recipe that loses a town has dropped its records.
|
|
35
|
+
//
|
|
36
|
+
// Any check may carry a `goal`: the owner's bet it serves (eval/goals.md says what each one
|
|
37
|
+
// is). The report scores each goal apart, so "trades 0.4, general 0.9" shows which bet the
|
|
38
|
+
// app is losing, instead of one mean that hides it.
|
|
28
39
|
//
|
|
29
40
|
// A predicate is {where?, match?, not?}. `where` is fangorn's own whole-value,
|
|
30
41
|
// case-insensitive match — the filter agents use. `match` maps a field (or several
|
|
@@ -36,8 +47,9 @@
|
|
|
36
47
|
import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
|
|
37
48
|
import { createServer } from "node:http";
|
|
38
49
|
import { homedir } from "node:os";
|
|
50
|
+
import { dirname, join } from "node:path";
|
|
39
51
|
import { configure, loadShard, trimView } from "@fangorn-network/westmarch/shard";
|
|
40
|
-
import { rolesFrom, textOf, titleOf, subtitleOf } from "@fangorn-network/westmarch/roles";
|
|
52
|
+
import { rolesFrom, textOf, titleOf, subtitleOf, values } from "@fangorn-network/westmarch/roles";
|
|
41
53
|
import { search, matches } from "@fangorn-network/westmarch/tools";
|
|
42
54
|
import { EMBED_MODEL, embedQueryDirect } from "@fangorn-network/westmarch/embed";
|
|
43
55
|
/** A drop below this, in mean precision@k or known@10, is "worse". canton-corpus's number. */
|
|
@@ -60,21 +72,31 @@ async function grade(rows, roles, checks, qv, known) {
|
|
|
60
72
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
61
73
|
const out = [];
|
|
62
74
|
for (const c of checks) {
|
|
75
|
+
const goal = c.goal ? { goal: c.goal } : {};
|
|
63
76
|
if (c.kind === "count") {
|
|
64
77
|
const ok = compile(c.expect);
|
|
65
78
|
const n = rows.filter((r) => matches(r, c.where) && ok(r)).length;
|
|
66
|
-
out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity) });
|
|
79
|
+
out.push({ id: c.id, kind: "count", score: n, pass: n >= (c.min ?? 1) && n <= (c.max ?? Infinity), ...goal });
|
|
67
80
|
}
|
|
68
81
|
else if (c.kind === "records") {
|
|
69
82
|
const of = rows.filter(compile(c.of)), ok = compile(c.expect);
|
|
70
83
|
const pct = of.length ? round((100 * of.filter(ok).length) / of.length) : 0;
|
|
71
|
-
out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard });
|
|
84
|
+
out.push({ id: c.id, kind: "records", score: pct, pass: pct <= c.max_pct, hard: c.hard, ...goal });
|
|
85
|
+
}
|
|
86
|
+
else if (c.kind === "coverage") {
|
|
87
|
+
// The same whole-value, any-case reading `where` uses, so covered means an agent's filter finds it.
|
|
88
|
+
const held = new Set(rows.filter(compile(c.of)).flatMap((r) => values(r[c.field] ?? "").map((v) => v.toLowerCase())));
|
|
89
|
+
const w = (v) => c.weights?.[v] ?? 1;
|
|
90
|
+
const missing = c.values.filter((v) => !held.has(v.toLowerCase())).sort((a, b) => w(b) - w(a));
|
|
91
|
+
const total = c.values.reduce((a, v) => a + w(v), 0);
|
|
92
|
+
const score = total ? round(1 - missing.reduce((a, v) => a + w(v), 0) / total) : 1;
|
|
93
|
+
out.push({ id: c.id, kind: "coverage", score, pass: score >= (c.min ?? 0), top: missing.slice(0, 10), ...goal });
|
|
72
94
|
}
|
|
73
95
|
else {
|
|
74
96
|
const k = c.k ?? 5, ok = compile(c.expect);
|
|
75
97
|
const hits = search(rows, c.q, roles, { qv: await qv(c.q), limit: k }).map((h) => byId.get(h.id));
|
|
76
98
|
const score = round(hits.filter(ok).length / k);
|
|
77
|
-
out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)) });
|
|
99
|
+
out.push({ id: c.id, kind: "search", score, pass: score >= (c.min ?? 0), top: hits.slice(0, 3).map((r) => titleOf(r, roles).slice(0, 90)), ...goal });
|
|
78
100
|
}
|
|
79
101
|
}
|
|
80
102
|
// ponytail: the pool is each view's own rows, so base and candidate sample different
|
|
@@ -97,6 +119,18 @@ async function grade(rows, roles, checks, qv, known) {
|
|
|
97
119
|
}
|
|
98
120
|
return out;
|
|
99
121
|
}
|
|
122
|
+
/** Each goal's checks, scored apart: how many pass, and the mean of its searches and coverages. */
|
|
123
|
+
function byGoal(results) {
|
|
124
|
+
const out = {};
|
|
125
|
+
const mean = (xs) => (xs.length ? round(xs.reduce((a, x) => a + x, 0) / xs.length) : null);
|
|
126
|
+
for (const g of new Set(results.map((r) => r.goal).filter((g) => !!g))) {
|
|
127
|
+
const rs = results.filter((r) => r.goal === g);
|
|
128
|
+
out[g] = { checks: rs.length, passing: rs.filter((r) => r.pass).length,
|
|
129
|
+
search: mean(rs.filter((r) => r.kind === "search").map((r) => r.score)),
|
|
130
|
+
coverage: mean(rs.filter((r) => r.kind === "coverage").map((r) => r.score)) };
|
|
131
|
+
}
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
100
134
|
/** The candidate against the base. Worse on any one rule is worse. */
|
|
101
135
|
function compare(cand, base) {
|
|
102
136
|
const was = new Map(base.map((r) => [r.id, r.score]));
|
|
@@ -110,6 +144,8 @@ function compare(cand, base) {
|
|
|
110
144
|
failures.push(`${r.id}: answered before, nothing now`);
|
|
111
145
|
if (r.hard && b !== undefined && r.score > b)
|
|
112
146
|
failures.push(`${r.id}: ${b}% → ${r.score}%, and it may not rise`);
|
|
147
|
+
if (r.kind === "coverage" && b !== undefined && r.score < b)
|
|
148
|
+
failures.push(`${r.id}: coverage ${b} → ${r.score}; missing ${r.top?.join(", ")}`);
|
|
113
149
|
}
|
|
114
150
|
// Aggregates, over the checks both sides ran: one question moving a rank is noise.
|
|
115
151
|
const both = (kind) => cand.filter((r) => (kind === "known" ? r.id === "known@10" : r.kind === kind) && was.has(r.id));
|
|
@@ -143,8 +179,15 @@ async function main() {
|
|
|
143
179
|
console.error("usage: westmarch-eval <view> [--golden eval/golden.jsonl] [--base <view>] [--known 200] [--json] [--report out.json]");
|
|
144
180
|
process.exit(flags.help ? 0 : 2);
|
|
145
181
|
}
|
|
146
|
-
const
|
|
182
|
+
const golden = String(flags.golden ?? "eval/golden.jsonl");
|
|
183
|
+
const checks = readFileSync(golden, "utf8")
|
|
147
184
|
.split("\n").filter((l) => l.trim() && !l.startsWith("//")).map((l) => JSON.parse(l));
|
|
185
|
+
// A coverage list kept in its own file: a list, or {value: weight}.
|
|
186
|
+
for (const c of checks)
|
|
187
|
+
if (c.kind === "coverage" && typeof c.values === "string") {
|
|
188
|
+
const list = JSON.parse(readFileSync(join(dirname(golden), c.values), "utf8"));
|
|
189
|
+
Object.assign(c, Array.isArray(list) ? { values: list } : { values: Object.keys(list), weights: list });
|
|
190
|
+
}
|
|
148
191
|
const known = Number(flags.known ?? 200);
|
|
149
192
|
const { env } = await import("@huggingface/transformers");
|
|
150
193
|
env.cacheDir = `${homedir()}/.cache/fangorn-mcp/models`; // shared with fangorn-mcp and westmarch-view
|
|
@@ -163,7 +206,7 @@ async function main() {
|
|
|
163
206
|
const failures = cmp?.failures ?? cand.results.filter((r) => !r.pass).map((r) => `${r.id}: ${r.score} is outside its bounds`);
|
|
164
207
|
// What reproduces this grade: the model, the runtime, and the views (whose manifests hash every shard).
|
|
165
208
|
const report = { model: EMBED_MODEL, node: process.version, platform: `${process.platform}-${process.arch}`,
|
|
166
|
-
candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
|
|
209
|
+
goals: byGoal(cand.results), candidate: cand, base, verdict: cmp?.verdict ?? null, deltas: cmp?.deltas ?? [], failures, pass: !failures.length };
|
|
167
210
|
if (flags.report)
|
|
168
211
|
writeFileSync(String(flags.report), JSON.stringify(report, null, 1));
|
|
169
212
|
if (flags.json)
|
|
@@ -177,6 +220,8 @@ async function main() {
|
|
|
177
220
|
}
|
|
178
221
|
for (const d of report.deltas)
|
|
179
222
|
console.log(`\n${d.kind}: ${d.base} → ${d.cand}`);
|
|
223
|
+
for (const [g, s] of Object.entries(report.goals))
|
|
224
|
+
console.log(`\ngoal ${g}: ${s.passing}/${s.checks} passing${s.search === null ? "" : `, search ${s.search}`}${s.coverage === null ? "" : `, coverage ${s.coverage}`}`);
|
|
180
225
|
console.log(`\n${report.verdict ?? (report.pass ? "pass" : "fail")}${failures.length ? `\n ${failures.join("\n ")}` : ""}`);
|
|
181
226
|
}
|
|
182
227
|
process.exit(report.pass ? 0 : 1);
|
|
@@ -228,6 +273,14 @@ async function selfcheck() {
|
|
|
228
273
|
const [scoped] = await grade(rows, roles, [{ id: "s", kind: "records", of: { where: { city: "Stevens Point" } }, expect: { match: { text: "Rd" } }, max_pct: 100 }], async () => null, 0);
|
|
229
274
|
assert(scoped.score === 50, `\`of\` narrows the rows a record check counts: ${scoped.score}`);
|
|
230
275
|
assert(r["known@10"].score === 1, "every title finds its own row");
|
|
276
|
+
// Coverage, and goals scored apart.
|
|
277
|
+
const [cov] = await grade(rows, roles, [{ id: "towns", kind: "coverage", field: "city", values: ["plover", "Stevens Point", "Wausau"], min: 0.5, goal: "towns" }], async () => null, 0);
|
|
278
|
+
assert(cov.score === 0.667 && cov.pass && cov.top?.[0] === "Wausau", `coverage is the share present, the rest listed: ${JSON.stringify(cov)}`);
|
|
279
|
+
const [pop] = await grade(rows, roles, [{ id: "pop", kind: "coverage", field: "city", values: ["Plover", "Wausau", "Adell"],
|
|
280
|
+
weights: { Plover: 30, Wausau: 60, Adell: 10 } }], async () => null, 0);
|
|
281
|
+
assert(pop.score === 0.3 && pop.top?.join() === "Wausau,Adell", `weighted coverage, the heaviest gap first: ${JSON.stringify(pop)}`);
|
|
282
|
+
const goals = byGoal([cov, { ...r.rezoning, goal: "towns" }, r.plover]);
|
|
283
|
+
assert(goals.towns.checks === 2 && goals.towns.coverage === 0.667 && goals.towns.search === 1 && !("undefined" in goals), `goals: ${JSON.stringify(goals)}`);
|
|
231
284
|
// The verdict.
|
|
232
285
|
const res = (id, score, kind = "search", extra = {}) => ({ id, kind, score, pass: true, ...extra });
|
|
233
286
|
const base = [res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.1, "records", { hard: true })];
|
|
@@ -240,5 +293,6 @@ async function selfcheck() {
|
|
|
240
293
|
assert(compare([res("q1", 0.8), res("q2", 0.4), res("known@10", 0.9, "known"), res("pii", 0.2, "records", { hard: true })], base).verdict === "worse", "a hard record may not rise");
|
|
241
294
|
assert(compare([res("q1", 0.8, "search", { pass: false }), res("q2", 0.4)], base).verdict === "worse", "a floor fails whatever the base did");
|
|
242
295
|
assert(compare([res("q1", 0.8), res("q2", 0.4), res("q3", 0)], base).verdict === "unchanged", "a new question has no base to fall from");
|
|
296
|
+
assert(compare([res("cov", 0.9, "coverage")], [res("cov", 1, "coverage")]).verdict === "worse", "coverage may not fall");
|
|
243
297
|
console.log("eval: selfcheck ok");
|
|
244
298
|
}
|
package/docs/app-to-agent.md
CHANGED
|
@@ -123,6 +123,13 @@ What is asked lives in `eval/golden.jsonl`, one check per line, as data rather t
|
|
|
123
123
|
{"id":"addresses","kind":"records","expect":{"match":{"text":"/\\b\\d{2,6} [A-Z]\\w+ St\\b/"}},"max_pct":0.5,"hard":true}
|
|
124
124
|
```
|
|
125
125
|
|
|
126
|
+
Which questions go in is the owner's bet: who would pay for this, for which answers.
|
|
127
|
+
Write it down in `eval/goals.md` (the customer, their questions, the coverage they need, a
|
|
128
|
+
date, and what would count as failure), tag each check with the `goal` it serves, and the
|
|
129
|
+
report scores each goal on its own. A `coverage` check (`{"kind":"coverage","field":"city",
|
|
130
|
+
"values":[…]}`) measures how much of the list a goal needs is present at all; with a base,
|
|
131
|
+
any fall fails, because a recipe that loses a town has dropped its records.
|
|
132
|
+
|
|
126
133
|
`expect` is a predicate over a row: `where` (the whole-value match agents filter with),
|
|
127
134
|
`match` (a field, or several joined by `|`, to a regex; case-insensitive unless written
|
|
128
135
|
`/re/flags`), `not`. Relevance is a predicate and not a list of record ids, because ids
|
|
@@ -580,29 +587,20 @@ can keep an app current: a cron job, a CI workflow, or an agent that just publis
|
|
|
580
587
|
## Using Fangorn apps from an agent
|
|
581
588
|
|
|
582
589
|
`fangorn-mcp` is one MCP server for every Fangorn app: a new app needs no new
|
|
583
|
-
registration.
|
|
584
|
-
and search by meaning included.
|
|
585
|
-
|
|
586
|
-
```sh
|
|
587
|
-
curl -fLo fangorn-mcp https://github.com/fangorn-network/westmarch/releases/latest/download/fangorn-mcp-linux-x64
|
|
588
|
-
chmod +x fangorn-mcp
|
|
589
|
-
claude mcp add fangorn -- "$PWD/fangorn-mcp"
|
|
590
|
-
```
|
|
591
|
-
|
|
592
|
-
Or through npm, wherever node is:
|
|
590
|
+
registration. Install it through npm (node ≥ 20):
|
|
593
591
|
|
|
594
592
|
```sh
|
|
595
|
-
claude mcp add fangorn
|
|
593
|
+
claude mcp add fangorn -e FANGORN_LOG_WINDOW=100000 -- \
|
|
594
|
+
npx -y -p @fangorn-network/westmarch -p @huggingface/transformers fangorn-mcp
|
|
596
595
|
```
|
|
597
596
|
|
|
598
597
|
Without `-p @huggingface/transformers`, the install is ~500 MB lighter and `search` ranks by
|
|
599
|
-
words, not meaning.
|
|
600
|
-
|
|
601
|
-
```json
|
|
602
|
-
{ "mcpServers": { "fangorn": { "command": "/path/to/fangorn-mcp" } } }
|
|
603
|
-
```
|
|
598
|
+
words, not meaning. Without `FANGORN_LOG_WINDOW`, `list-apps` scans the chain 1,000 blocks
|
|
599
|
+
per call and takes minutes.
|
|
604
600
|
|
|
605
|
-
|
|
601
|
+
[docs/mcp.md](mcp.md) has the full setup: other MCP clients, every setting, running from
|
|
602
|
+
source, and troubleshooting. A single-file binary (`bun build-bin.js`) exists but is not
|
|
603
|
+
published yet.
|
|
606
604
|
|
|
607
605
|
It starts with three tools:
|
|
608
606
|
- **`list-apps`** reads the apps off the chain and keeps the ones whose cards verify.
|